June 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Add library for date and time related activity
by art＠torproject.org 02 Jun '12

02 Jun '12

commit 460d88ce95a4b759cd2284223829369059dfa535 Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu May 31 21:15:51 2012 +0200 Add library for date and time related activity --- ooni/date.py | 11 +++++++++++ 1 files changed, 11 insertions(+), 0 deletions(-) diff --git a/ooni/date.py b/ooni/date.py new file mode 100644 index 0000000..6f83191 --- /dev/null +++ b/ooni/date.py @@ -0,0 +1,11 @@ +from datetime import datetime + +def pretty_date(): + cur_time = datetime.utcnow() + d_format = "%d %B %Y %H:%M:%S" + pretty = cur_time.strftime(d_format) + return pretty + +def now(): + return datetime.utcnow() +

1 0

[ooni-probe/master] Remove unused import
by art＠torproject.org 02 Jun '12

02 Jun '12

commit 22e4ace5f34127fe866bf07d4013819ab5dfd871 Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu May 31 21:28:11 2012 +0200 Remove unused import --- ooni/plugoo/work.py | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index 8ba0a3f..24a2b46 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -19,8 +19,6 @@ from zope.interface import Interface, Attribute from twisted.python import failure from twisted.internet import reactor, defer -from ooni.plugoo.nodes import LocalNode - class Worker(object): """ This is the core of OONI. It takes as input Work Units and

1 0

[ooni-probe/master] Fix non blocking test launching.
by art＠torproject.org 02 Jun '12

02 Jun '12

commit 422e315dbd87234e2dc5317222a96c65c72cc592 Author: Arturo Filastò <hellais(a)torproject.org> Date: Fri Jun 1 22:04:17 2012 +0200 Fix non blocking test launching. Minor refactoring. --- ooni/plugins/httphost.py | 1 - ooni/plugoo/tests.py | 2 +- ooni/plugoo/work.py | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py index 2c7f8af..45ab587 100644 --- a/ooni/plugins/httphost.py +++ b/ooni/plugins/httphost.py @@ -107,7 +107,6 @@ class HTTPHostTest(OONITest): def experiment(self, args): control_server = self.local_options['controlserver'] - print args censored = self.httplib_test(control_server, args['asset']) return {'control': control_server, 'host': args['asset'], diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 8e0ab8e..0632670 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -53,7 +53,7 @@ class OONITest(object): if self.blocking: self.d = threads.deferToThread(self.experiment, args) else: - self.d = defer.maybeDeferred(self.experiment, args) + self.d = self.experiment(args) self.d.addCallback(self.control, args) self.d.addCallback(self.finished) diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index 24a2b46..16a6e2c 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -129,6 +129,7 @@ class WorkGenerator(object): self.Test = test if self.Test.assets and self.Test.assets.values()[0]: + print self.Test.assets self.assetGenerator = itertools.product(*self.Test.assets.values()) else: self.assetGenerator = None

1 0

[ooni-probe/master] Implement basic BridgeT test
by art＠torproject.org 02 Jun '12

02 Jun '12

commit 075d03666fa9a96ca951fad805a1c0664e5010e1 Author: Arturo Filastò <hellais(a)torproject.org> Date: Sat Jun 2 14:03:07 2012 +0200 Implement basic BridgeT test --- ooni/plugins/bridget.py | 72 +++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 72 insertions(+), 0 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py new file mode 100644 index 0000000..0614d5d --- /dev/null +++ b/ooni/plugins/bridget.py @@ -0,0 +1,72 @@ +""" +This is a self genrated test created by scaffolding.py. +you will need to fill it up with all your necessities. +Safe hacking :). +""" +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import reactor + +from ooni import log +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset + +class bridgetArgs(usage.Options): + optParameters = [['bridges', 'b', None, 'List of bridges to scan'], + ['relays', 'f', None, 'List of relays to scan'], + ['resume', 'r', 0, 'Resume at this index']] + +class bridgetTest(OONITest): + implements(IPlugin, ITest) + + shortName = "bridget" + description = "bridget" + requirements = None + options = bridgetArgs + blocking = False + + def experiment(self, args): + log.msg("Doing test") + # What you return here gets handed as input to control + from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState + from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor + def updates(prog, tag, summary): + log.msg("%d%%: %s" % (prog, summary)) + + def setup_failed(args): + log.msg("Setup Failed.") + report.update({'failed': args}) + return report + + def setup_complete(proto): + log.msg("Setup Complete.") + report.update({'success': args}) + return report + + config = TorConfig() + config.SocksPort = 9999 + config.OrPort = 1234 + if 'bridge' in args: + config.UseBridges = 1 + config.Bridge = args['bridge'] + config.save() + report = {'tor_config': config.config} + log.msg("Starting Tor") + d = launch_tor(config, reactor, progress_updates=updates) + d.addCallback(setup_complete) + d.addErrback(setup_failed) + return d + + def load_assets(self): + assets = {} + if self.local_options: + if self.local_options['bridges']: + assets.update({'bridge': Asset(self.local_options['bridges'])}) + elif self.local_options['relays']: + assets.update({'relay': Asset(self.local_options['relay'])}) + return assets + +# We need to instantiate it otherwise getPlugins does not detect it +# XXX Find a way to load plugins without instantiating them. +bridget = bridgetTest(None, None, None)

1 0

[ooni-probe/master] Add basic structure for test writing docs
by art＠torproject.org 02 Jun '12

02 Jun '12

commit a12510f14060b67121532c22e9d06df8d2c5b01f Author: Arturo Filastò <hellais(a)torproject.org> Date: Sat Jun 2 14:03:25 2012 +0200 Add basic structure for test writing docs --- docs/writing_tests.md | 17 +++++++++++++++++ 1 files changed, 17 insertions(+), 0 deletions(-) diff --git a/docs/writing_tests.md b/docs/writing_tests.md new file mode 100644 index 0000000..460620a --- /dev/null +++ b/docs/writing_tests.md @@ -0,0 +1,17 @@ += Writing OONI Tests = + +There are two modes of writing tests. The first way is by making the Test +blocking and have each instance of a test run in a separate thread. The other +is by making them non-blocking using the twisted deferred non blocking pattern. + +The first kind of mechanism relied of writing a test that uses blocking code +and each instance of it is run inside of a separate thread. Beware that your +blocking code must be thread safe to run properly (as is obvious :P) + +The other menthod invloves having some knowledge about twisted. The test you +will write will be written in twisted and should implement async style non +blocking architecture. + +It is recommended that tests are written using the second pattern and the first +should only be used for runnign tests that have been previously written. +

1 0

[ooni-probe/master] Add hack to make import of txtorcon work
by art＠torproject.org 02 Jun '12

02 Jun '12

commit b132993ed859b2c727edd2a6db08b08c011a8b29 Author: Arturo Filastò <hellais(a)torproject.org> Date: Sat Jun 2 14:05:49 2012 +0200 Add hack to make import of txtorcon work --- ooni/lib/__init__.py | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-) diff --git a/ooni/lib/__init__.py b/ooni/lib/__init__.py new file mode 100644 index 0000000..fac7035 --- /dev/null +++ b/ooni/lib/__init__.py @@ -0,0 +1,9 @@ +import sys +import os + +pwd = os.path.dirname(__file__) +#child_path = os.path.normpath(os.path.join(pwd, '..')) +#sys.path.append(child_path) +sys.path.append(pwd) + +

1 0

[ooni-probe/master] Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe
by art＠torproject.org 02 Jun '12

02 Jun '12

commit 447ab389f0ae25c7cfb652e03927b88e2c549ab1 Merge: baab66b eb6a0f7 Author: Arturo Filastò <hellais(a)torproject.org> Date: Sat Jun 2 19:37:24 2012 +0200 Merge branch 'master' of ssh://git-rw.torproject.org/ooni-probe .gitignore | 3 +- docs/writing_tests.txt | 12 -- ooni/__init__.py | 1 + ooni/log.py | 40 ++++++ ooni/oonicli.py | 139 -------------------- ooni/ooniprobe.log | 3 +- ooni/ooniprobe.py | 272 +++++++++++++++------------------------- ooni/plugins/.bridget.py.swp | Bin 12288 -> 0 bytes ooni/plugins/.dnstamper.py.swp | Bin 12288 -> 0 bytes ooni/plugins/blocking.py | 41 ++++++ ooni/plugins/bridget.py | 36 ------ ooni/plugins/dnstamper.py | 22 ---- ooni/plugins/dropin.cache | 76 ----------- ooni/plugins/httphost.py | 122 ++++++++++++++++++ ooni/plugins/skel.py | 14 ++- ooni/plugoo/interface.py | 21 +++ ooni/plugoo/reports.py | 43 +++---- ooni/plugoo/tests.py | 241 +++++++----------------------------- ooni/plugoo/work.py | 71 ++++++++--- ooni/scaffolding.py | 67 ++++++++++ ooni/skel.py | 10 -- ooni/tests/worker_test.py | 27 ---- oonib/oonibackend.py | 8 +- tests/test_tests.py | 33 +++++ tests/test_worker.py | 18 +++ 25 files changed, 572 insertions(+), 748 deletions(-)

1 0

[ooni-probe/master] Merge github.com:hellais/ooni-probe
by art＠torproject.org 02 Jun '12

02 Jun '12

commit f2d7c2935d2f44009a3810db0d68fc8385334180 Merge: 447ab38 b132993 Author: Arturo Filastò <hellais(a)torproject.org> Date: Sat Jun 2 20:36:43 2012 +0200 Merge github.com:hellais/ooni-probe docs/writing_tests.md | 17 +++++++++++ ooni/date.py | 11 +++++++ ooni/lib/__init__.py | 9 ++++++ ooni/ooniprobe.py | 6 ++-- ooni/plugins/bridget.py | 72 ++++++++++++++++++++++++++++++++++++++++++++++ ooni/plugins/httphost.py | 1 - ooni/plugoo/reports.py | 43 +++++++++++++++++++-------- ooni/plugoo/tests.py | 2 +- ooni/plugoo/work.py | 13 ++++++-- ooni/scaffolding.py | 19 +++++++---- tests/test_worker.py | 11 +++++++ 11 files changed, 175 insertions(+), 29 deletions(-)

1 0

[ooni-probe/master] Merge branch 'twisted' of github.com:hellais/ooni-probe
by art＠torproject.org 02 Jun '12

02 Jun '12

commit baab66b10eccb5a3bc40addc84de0e45f2897497 Merge: abfcf7c 59f1b4e Author: Arturo Filastò <hellais(a)torproject.org> Date: Tue May 29 03:23:01 2012 +0200 Merge branch 'twisted' of github.com:hellais/ooni-probe HACKING | 36 +- assets/example.txt | 2 - assets/tcpscan.txt | 1 - config.py | 53 - docs/writing_tests.txt | 12 + install.txt | 4 - lib/traceroute.py | 32 - lib/txtraceroute.py | 389 - logo.py | 214 - nodes.conf | 53 - old-to-be-ported-code/AUTHORS | 3 + old-to-be-ported-code/CHANGES.yaml | 5 + old-to-be-ported-code/HACKING | 21 + old-to-be-ported-code/INSTALL | 8 + old-to-be-ported-code/LICENSE | 28 + old-to-be-ported-code/MANIFEST.in | 6 + old-to-be-ported-code/Makefile | 182 + old-to-be-ported-code/README.API | 29 + old-to-be-ported-code/README.examples | 10 + old-to-be-ported-code/README.rst | 40 + old-to-be-ported-code/RFA.md | 101 + old-to-be-ported-code/TODO | 418 + old-to-be-ported-code/TODO.plgoons | 79 + old-to-be-ported-code/bin/ooni-probe | 10 + old-to-be-ported-code/keyword-lists/Makefile | 3 + old-to-be-ported-code/keyword-lists/keyfile | 256 + old-to-be-ported-code/list/list.txt | 2 + old-to-be-ported-code/ooni-probe.diff | 358 + old-to-be-ported-code/ooni/.DS_Store | Bin 0 -> 15364 bytes old-to-be-ported-code/ooni/__init__.py | 12 + old-to-be-ported-code/ooni/captive_portal.py | 62 + old-to-be-ported-code/ooni/command.py | 250 + old-to-be-ported-code/ooni/common.py | 139 + old-to-be-ported-code/ooni/dns_cc_check.py | 48 + old-to-be-ported-code/ooni/dns_poisoning.py | 43 + old-to-be-ported-code/ooni/dnsooni.py | 356 + old-to-be-ported-code/ooni/empty.txt | 9 + old-to-be-ported-code/ooni/helpers.py | 38 + old-to-be-ported-code/ooni/http.py | 306 + old-to-be-ported-code/ooni/input.py | 33 + old-to-be-ported-code/ooni/namecheck.py | 39 + old-to-be-ported-code/ooni/output.py | 21 + .../ooni/plugins/captiveportal_plgoo.py | 55 + .../ooni/plugins/dnstest_plgoo.py | 84 + old-to-be-ported-code/ooni/plugins/http_plgoo.py | 70 + old-to-be-ported-code/ooni/plugins/marco_plgoo.py | 377 + .../ooni/plugins/netalyzr_plgoo.py | 30 + .../ooni/plugins/old_stuff_to_convert/README | 47 + .../plugins/old_stuff_to_convert/README.automation | 11 + .../ooni/plugins/old_stuff_to_convert/TODO | 6 + .../plugins/old_stuff_to_convert/cached-consensus |11724 ++++++++++++++++++++ .../plugins/old_stuff_to_convert/connectback.sh | 56 + .../plugins/old_stuff_to_convert/dirconntest.sh | 52 + .../plugins/old_stuff_to_convert/dns-checker.sh | 7 + .../old_stuff_to_convert/generic-host-test.sh | 33 + .../ooni/plugins/old_stuff_to_convert/host-prep.sh | 20 + .../plugins/old_stuff_to_convert/install-probe.sh | 27 + .../ooni/plugins/old_stuff_to_convert/run-tests.sh | 11 + .../plugins/old_stuff_to_convert/twitter-test.sh | 33 + old-to-be-ported-code/ooni/plugins/proxy_plgoo.py | 69 + .../ooni/plugins/simple_dns_plgoo.py | 35 + old-to-be-ported-code/ooni/plugins/skel_plgoo.py | 17 + old-to-be-ported-code/ooni/plugins/skel_plgoo.yaml | 33 + old-to-be-ported-code/ooni/plugins/tcpcon_plgoo.py | 278 + old-to-be-ported-code/ooni/plugins/tor.py | 80 + old-to-be-ported-code/ooni/plugins/torrc | 9 + old-to-be-ported-code/ooni/plugooni.py | 106 + old-to-be-ported-code/ooni/report.py | 22 + old-to-be-ported-code/ooni/transparenthttp.py | 41 + old-to-be-ported-code/ooni/yamlooni.py | 40 + old-to-be-ported-code/proxy-lists/Makefile | 11 + old-to-be-ported-code/proxy-lists/README | 9 + old-to-be-ported-code/proxy-lists/italy-dns.txt | 46 + .../proxy-lists/italy-http-block-pages-notes.txt | 62 + .../proxy-lists/italy-http-block-pages.txt | 5 + .../proxy-lists/parse-trusted-xff.sh | 16 + old-to-be-ported-code/proxy-lists/trusted-xff.html | 7789 +++++++++++++ old-to-be-ported-code/reports/marco-1.yamlooni | 86 + old-to-be-ported-code/reports/marco-2.yamlooni | 60 + old-to-be-ported-code/reports/marco.log | 4672 ++++++++ old-to-be-ported-code/reports/marco.yamlooni | 241 + old-to-be-ported-code/reports/marco_certs-1-2.out | 307 + old-to-be-ported-code/reports/marco_certs-1.out | 5850 ++++++++++ old-to-be-ported-code/reports/marco_certs-2.out | 1182 ++ old-to-be-ported-code/reports/marco_certs.out | 1900 ++++ old-to-be-ported-code/setup.py | 10 + old-to-be-ported-code/spec/proxooni-spec.txt | 65 + old-to-be-ported-code/test-lists/Makefile | 7 + old-to-be-ported-code/tests/test_import.py | 10 + old-to-be-ported-code/third-party/Makefile | 3 + old-to-be-ported-code/third-party/README | 14 + old/AUTHORS | 3 - old/CHANGES.yaml | 5 - old/HACKING | 21 - old/INSTALL | 8 - old/LICENSE | 28 - old/MANIFEST.in | 6 - old/Makefile | 182 - old/README.API | 29 - old/README.examples | 10 - old/README.rst | 40 - old/RFA.md | 101 - old/TODO | 418 - old/TODO.plgoons | 79 - old/bin/ooni-probe | 10 - old/keyword-lists/Makefile | 3 - old/keyword-lists/keyfile | 256 - old/list/list.txt | 2 - old/ooni-probe.diff | 358 - old/ooni/.DS_Store | Bin 15364 -> 0 bytes old/ooni/__init__.py | 12 - old/ooni/captive_portal.py | 62 - old/ooni/command.py | 250 - old/ooni/common.py | 139 - old/ooni/dns_cc_check.py | 48 - old/ooni/dns_poisoning.py | 43 - old/ooni/dnsooni.py | 356 - old/ooni/empty.txt | 9 - old/ooni/helpers.py | 38 - old/ooni/http.py | 306 - old/ooni/input.py | 33 - old/ooni/namecheck.py | 39 - old/ooni/output.py | 21 - old/ooni/plugins/captiveportal_plgoo.py | 55 - old/ooni/plugins/dnstest_plgoo.py | 84 - old/ooni/plugins/http_plgoo.py | 70 - old/ooni/plugins/marco_plgoo.py | 377 - old/ooni/plugins/netalyzr_plgoo.py | 30 - old/ooni/plugins/old_stuff_to_convert/README | 47 - .../plugins/old_stuff_to_convert/README.automation | 11 - old/ooni/plugins/old_stuff_to_convert/TODO | 6 - .../plugins/old_stuff_to_convert/cached-consensus |11724 -------------------- .../plugins/old_stuff_to_convert/connectback.sh | 56 - .../plugins/old_stuff_to_convert/dirconntest.sh | 52 - .../plugins/old_stuff_to_convert/dns-checker.sh | 7 - .../old_stuff_to_convert/generic-host-test.sh | 33 - old/ooni/plugins/old_stuff_to_convert/host-prep.sh | 20 - .../plugins/old_stuff_to_convert/install-probe.sh | 27 - old/ooni/plugins/old_stuff_to_convert/run-tests.sh | 11 - .../plugins/old_stuff_to_convert/twitter-test.sh | 33 - old/ooni/plugins/proxy_plgoo.py | 69 - old/ooni/plugins/simple_dns_plgoo.py | 35 - old/ooni/plugins/skel_plgoo.py | 17 - old/ooni/plugins/skel_plgoo.yaml | 33 - old/ooni/plugins/tcpcon_plgoo.py | 278 - old/ooni/plugins/tor.py | 80 - old/ooni/plugins/torrc | 9 - old/ooni/plugooni.py | 106 - old/ooni/report.py | 22 - old/ooni/transparenthttp.py | 41 - old/ooni/yamlooni.py | 40 - old/proxy-lists/Makefile | 11 - old/proxy-lists/README | 9 - old/proxy-lists/italy-dns.txt | 46 - old/proxy-lists/italy-http-block-pages-notes.txt | 62 - old/proxy-lists/italy-http-block-pages.txt | 5 - old/proxy-lists/parse-trusted-xff.sh | 16 - old/proxy-lists/trusted-xff.html | 7789 ------------- old/reports/marco-1.yamlooni | 86 - old/reports/marco-2.yamlooni | 60 - old/reports/marco.log | 4672 -------- old/reports/marco.yamlooni | 241 - old/reports/marco_certs-1-2.out | 307 - old/reports/marco_certs-1.out | 5850 ---------- old/reports/marco_certs-2.out | 1182 -- old/reports/marco_certs.out | 1900 ---- old/setup.py | 10 - old/spec/proxooni-spec.txt | 65 - old/test-lists/Makefile | 7 - old/tests/test_import.py | 10 - old/third-party/Makefile | 3 - old/third-party/README | 14 - ooni-probe.conf | 77 - ooni/assets/bridgetests.txt | 3 + ooni/assets/example.txt | 2 + ooni/assets/tcpscan.txt | 1 + ooni/config.py | 53 + ooni/lib/Makefile | 14 + ooni/logo.py | 214 + ooni/nodes.conf | 53 + ooni/ooni-probe.conf | 77 + ooni/oonicli.py | 139 + ooni/ooniprobe.log | 2 + ooni/ooniprobe.py | 191 + ooni/oonitests/bridget.py | 373 + ooni/oonitests/dnstamper.py | 156 + ooni/oonitests/httphost.py | 135 + ooni/oonitests/squid.py | 113 + ooni/oonitests/tcpscan.py | 86 + ooni/oonitests/template.py | 65 + ooni/oonitests/traceroute.py | 110 + ooni/plugins/.bridget.py.swp | Bin 0 -> 12288 bytes ooni/plugins/.dnstamper.py.swp | Bin 0 -> 12288 bytes ooni/plugins/__init__.py | 3 + ooni/plugins/bridget.py | 36 + ooni/plugins/dnstamper.py | 22 + ooni/plugins/dropin.cache | 76 + ooni/plugins/skel.py | 21 + ooni/plugoo/__init__.py | 80 + ooni/plugoo/assets.py | 56 + ooni/plugoo/interface.py | 2 + ooni/plugoo/nodes.py | 180 + ooni/plugoo/reports.py | 175 + ooni/plugoo/tests.py | 229 + ooni/plugoo/work.py | 162 + ooni/skel.py | 10 + ooni/tests/worker_test.py | 27 + ooni/utils.py | 146 + oonib/dnsbackend.py | 16 + oonib/httpbackend.py | 74 + oonib/oonibackend.py | 34 + oonicli.py | 142 - ooniprobe.py | 191 - plugins/__init__.py | 3 - plugins/dropin.cache | 48 - plugins/skel.py | 20 - plugoo/__init__.py | 80 - plugoo/assets.py | 56 - plugoo/interface.py | 2 - plugoo/nodes.py | 180 - plugoo/reports.py | 175 - plugoo/tests.py | 213 - plugoo/work.py | 162 - skel.py | 10 - tests/bridget.py | 378 - tests/dnstamper.py | 156 - tests/httphost.py | 135 - tests/squid.py | 113 - tests/tcpscan.py | 86 - tests/template.py | 65 - tests/traceroute.py | 110 - unittest/tests.py | 27 - utils.py | 146 - 233 files changed, 41804 insertions(+), 41969 deletions(-) diff --cc ooni/ooni-probe.conf index 0000000,336af99..d95e410 mode 000000,100644..100644 --- a/ooni/ooni-probe.conf +++ b/ooni/ooni-probe.conf @@@ -1,0 -1,72 +1,77 @@@ + # ooni-probe + # + # These are the global configuration parameters necessary to + # make ooni-probe work + [main] + reportdir = reports/ + logfile = ooniprobe.log + assetdir = assets/ + testdir = oonitests/ + + loglevel = DEBUG + consoleloglevel = DEBUG + proxyaddress = 127.0.0.1:9050 + + # The following configurations are for searching for PlanetLab + # nodes, adding them to a slice, and PlanetLab general API + # authentication: + pl_username = yourusername + pl_password = yourpassword + + # These are configurations specific to the tests that should be + # run by ooni-probe + [tests] + run = dnstamper + ### DNS testing related config parameters + + # This is the list of hostnames that must be looked up + dns_experiment = top-1m.txt + + # This is the dns servers to be tested + dns_experiment_dns = dns_servers.txt + + # This is the control known good DNS server -dns_control_server = 8.8.8.8 ++dns_control_server = 91.191.136.152 ++ ++# Specify whether the dnstamper test should attempt to remove ++# GeoIP-based false positives by doing a reverse DNS resolve ++# on positive results. ++dns_reverse_lookup = true + + ### traceroute testing related config parameters + + # This is the list of ips to traceroute to + traceroute = example_exp_list.txt + + # This is the list of ports that should be used + # src_x,src_y,src_z|dst_x,dst_y,dst_z + traceroute_ports = 0,53,80,123,443|0,53,80,123,443 + + # The protocol to be used in the scan + traceroute_proto = UDP, TCP, ICMP + + ### keyword injection related tests + + # List of keywords + keywords = keywordlist.txt + + # hosts + keywords_hosts = hostslist.txt + + # Methods to be used for testing + keyword_method = http,telnet + + ### Tor bridge testing + + tor_bridges = bridgetests.txt + tor_bridges_timeout = 40 + + [report] + file = report.log + timestamp = true + #ssh = 127.0.0.1:22 + #ssh_user = theusername + #ssh_password = thepassword + #ssh_keyfile = ~/.ssh/mykey_rsa + #ssh_rpath = ~/ooni-probe/ + #tcp = "127.0.0.1:9088" diff --cc ooni/oonitests/dnstamper.py index 0000000,68be12d..498ba04 mode 000000,100644..100644 --- a/ooni/oonitests/dnstamper.py +++ b/ooni/oonitests/dnstamper.py @@@ -1,0 -1,70 +1,156 @@@ ++# -*- coding: utf-8 -*- ++""" ++ dnstamper ++ ********* ++ ++ This test resolves DNS for a list of domain names, one per line, in the ++ file specified in the ooni-config under the setting "dns_experiment". If ++ the file is top-1m.txt, the test will be run using Amazon's list of top ++ one million domains. The experimental dns servers to query should ++ be specified one per line in assets/dns_servers.txt. ++ ++ The test reports censorship if the cardinality of the intersection of ++ the query result set from the control server and the query result set ++ from the experimental server is zero, which is to say, if the two sets ++ have no matching results whatsoever. ++ ++ NOTE: This test frequently results in false positives due to GeoIP-based ++ load balancing on major global sites such as google, facebook, and ++ youtube, etc. ++ ++ :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft ++ :license: see LICENSE for more details ++""" ++ ++try: ++ from dns import resolver, reversename ++except: ++ print "Error: dnspython is not installed! (http://www.dnspython.org/)" + try: - from dns import resolver ++ import gevent + except: - print "Error dnspython is not installed! (http://www.dnspython.org/)" -import gevent ++ print "Error: gevent is not installed! (http://www.gevent.org/)" ++ + import os ++ + import plugoo + from plugoo.assets import Asset + from plugoo.tests import Test + - + __plugoo__ = "DNST" + __desc__ = "DNS censorship detection test" + ++class Top1MAsset(Asset): ++ """ ++ Class for parsing top-1m.txt as an asset. ++ """ ++ def __init__(self, file=None): ++ self = Asset.__init__(self, file) ++ ++ def parse_line(self, line): ++ self = Asset.parse_line(self, line) ++ return line.split(',')[1].replace('\n','') ++ + class DNSTAsset(Asset): ++ """ ++ Creates DNS testing specific Assets. ++ """ + def __init__(self, file=None): - self = asset.__init__(self, file) ++ self = Asset.__init__(self, file) + + class DNST(Test): + def lookup(self, hostname, ns): ++ """ ++ Resolves a hostname through a DNS nameserver, ns, to the corresponding ++ IP address(es). ++ """ + res = resolver.Resolver(configure=False) + res.nameservers = [ns] + answer = res.query(hostname) + + ret = [] + + for data in answer: + ret.append(data.address) + + return ret + ++ def reverse_lookup(self, ip, ns): ++ """ ++ Attempt to do a reverse DNS lookup to determine if the control and exp ++ sets from a positive result resolve to the same domain, in order to ++ remove false positives due to GeoIP load balancing. ++ """ ++ res = resolver.Resolver(configure=False) ++ res.nameservers = [ns] ++ n = reversename.from_address(ip) ++ revn = res.query(n, "PTR").__iter__().next().to_text()[:-1] ++ ++ return revn ++ + def experiment(self, *a, **kw): ++ """ ++ Compares the lookup() sets of the control and experiment groups. ++ """ + # this is just a dirty hack + address = kw['data'][0] + ns = kw['data'][1] + + config = self.config ++ ctrl_ns = config.tests.dns_control_server + + print "ADDRESS: %s" % address + print "NAMESERVER: %s" % ns + + exp = self.lookup(address, ns) - control = self.lookup(address, config.tests.dns_control_server) ++ control = self.lookup(address, ctrl_ns) ++ ++ result = [] + + if len(set(exp) & set(control)) > 0: - print "%s : no tampering on %s" % (address, ns) - return (address, ns, False) ++ print "Address %s has not tampered with on DNS server %s\n" % (address, ns) ++ result = (address, ns, exp, control, False) ++ return result + else: - print "%s : possible tampering on %s (%s, %s)" % (address, ns, exp, control) - return (address, ns, exp, control, True) ++ print "Address %s has possibly been tampered on %s:\nDNS resolution through %s yeilds:\n%s\nAlthough the control group DNS servers resolve to:\n%s" % (address, ns, ns, exp, control) ++ result = (address, ns, exp, control, True) ++ ++ if config.tests.dns_reverse_lookup: ++ ++ exprevn = [self.reverse_lookup(ip, ns) for ip in exp] ++ ctrlrevn = [self.reverse_lookup(ip, ctrl_ns) ++ for ip in control] ++ ++ if len(set(exprevn) & set(ctrlrevn)) > 0: ++ print "Further testing has eliminated this as a false positive." ++ else: ++ print "Reverse DNS on the results returned by %s returned:\n%s\nWhich does not match the expected domainname:\n%s\n" % (ns, exprevn, ctrlrevn) ++ return result ++ ++ else: ++ print "\n" ++ return result + + def run(ooni): - """Run the test ++ """ ++ Run the test. + """ + config = ooni.config + urls = [] + - dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, \ - config.tests.dns_experiment)) - dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, \ ++ if (config.tests.dns_experiment == "top-1m.txt"): ++ dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, ++ config.tests.dns_experiment)) ++ else: ++ dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, ++ config.tests.dns_experiment)) ++ dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, + config.tests.dns_experiment_dns)) + + assets = [dns_experiment, dns_experiment_dns] + + dnstest = DNST(ooni) - ooni.logger.info("starting test") - dnstest.run(assets) - ooni.logger.info("finished") - ++ ooni.logger.info("Beginning dnstamper test...") ++ dnstest.run(assets, {'index': 1}) ++ ooni.logger.info("Dnstamper test completed!") + diff --cc ooni/oonitests/httphost.py index 0000000,6446e1f..25adff2 mode 000000,100644..100644 --- a/ooni/oonitests/httphost.py +++ b/ooni/oonitests/httphost.py @@@ -1,0 -1,132 +1,135 @@@ + """ + HTTP Host based filtering + ************************* + + This test detect HTTP Host field + based filtering. + It is used to detect censorship on + performed with Web Guard (used by + T-Mobile US). + """ + import os + from datetime import datetime + from gevent import monkey + + import urllib2 + import httplib + # WARNING! Using gevent's socket + # introduces the 0x20 DNS "feature". + # This will result is weird DNS requests + # appearing on the wire. + monkey.patch_socket() + -from BeautifulSoup import BeautifulSoup ++try: ++ from BeautifulSoup import BeautifulSoup ++except: ++ print "BeautifulSoup-3.2.1 is missing. Please see https://crate.io/packages/BeautifulSoup/" + + from plugoo.assets import Asset + from plugoo.tests import Test + + __plugoo__ = "HTTP Host" + __desc__ = "This detects HTTP Host field based filtering" + + class HTTPHostAsset(Asset): + """ + This is the asset that should be used by the Test. It will + contain all the code responsible for parsing the asset file + and should be passed on instantiation to the test. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + return line.split(',')[1].replace('\n','') + + class HTTPHost(Test): + """ + The main Test class + """ + + def check_response(self, response): + soup = BeautifulSoup(response) + if soup.head.title.string == "Content Filtered": + # Response indicates censorship + return True + else: + # Response does not indicate censorship + return False + + + def is_censored(self, response): + if response: + soup = BeautifulSoup(response) + censored = self.check_response(response) + else: + censored = "unreachable" + return censored + + def urllib2_test(self, control_server, host): + req = urllib2.Request(control_server) + req.add_header('Host', host) + try: + r = urllib2.urlopen(req) + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + def httplib_test(self, control_server, host): + try: + conn = httplib.HTTPConnection(control_server) + conn.putrequest("GET", "", skip_host=True, skip_accept_encoding=True) + conn.putheader("Host", host) + conn.putheader("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6") + conn.endheaders() + r = conn.getresponse() + response = r.read() + censored = self.is_censored(response) + except Exception, e: + censored = "Error! %s" % e + + return censored + + + def experiment(self, *a, **kw): + """ + Try to connect to the control server with + the specified host field. + """ + host = kw['data'] + control_server = kw['control_server'] + self.logger.info("Testing %s (%s)" % (host, control_server)) + + #censored = self.urllib2_test(control_server, host) + censored = self.httplib_test(control_server, host) + + self.logger.info("%s: %s" % (host, censored)) + return {'Time': datetime.now(), + 'Host': host, + 'Censored': censored} + + + def run(ooni): + """ + This is the function that will be called by OONI + and it is responsible for instantiating and passing + the arguments to the Test class. + """ + config = ooni.config + + # This the assets array to be passed to the run function of + # the test + assets = [HTTPHostAsset(os.path.join(config.main.assetdir, \ + "top-1m.csv"))] + + # Instantiate the Test + thetest = HTTPHost(ooni) + ooni.logger.info("starting HTTP Host Test...") + # Run the test with argument assets + thetest.run(assets, {'index': 5825, 'control_server': '195.85.254.203:8080'}) + ooni.logger.info("finished.") + + diff --cc ooni/plugoo/reports.py index 0000000,c099456..c3e7c23 mode 000000,100644..100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@@ -1,0 -1,175 +1,175 @@@ + import os + from datetime import datetime + import yaml + + import logging + import itertools + import gevent + + class Report: + """This is the ooni-probe reporting mechanism. It allows + reporting to multiple destinations and file formats. + + :scp the string of <host>:<port> of an ssh server + + :yaml the filename of a the yaml file to write + + :file the filename of a simple txt file to write + + :tcp the <host>:<port> of a TCP server that will just listen for + inbound connection and accept a stream of data (think of it + as a `nc -l -p <port> > filename.txt`) + """ + def __init__(self, ooni, + scp="127.0.0.1:22", + file="test.report", + tcp="127.0.0.1:9000"): + + self.file = file + self.tcp = tcp + self.scp = scp + self.config = ooni.config.report + self.logger = ooni.logger + + if self.config.timestamp: + tmp = self.file.split('.') + self.file = '.'.join(tmp[:-1]) + "-" + \ + datetime.now().isoformat('-') + '.' + \ + tmp[-1] + print self.file + + try: + import paramiko + except: + self.scp = None - self.logger.warn("Could not import paramiko. SCP will not be disabled") ++ self.logger.warn("Could not import paramiko. SCP will be disabled") + + def __call__(self, data): + """ + This should be invoked every time you wish to write some + data to the reporting system + """ + #print "Writing report(s)" + #dump = '--- \n' + dump = yaml.dump([data]) + #dump += yaml.dump(data) + reports = [] + + if self.file: + reports.append("file") + + if self.tcp: + reports.append("tcp") + + if self.scp: + reports.append("scp") + + jobs = [gevent.spawn(self.send_report, *(dump, report)) for report in reports] + gevent.joinall(jobs) + ret = [] + for job in jobs: + #print job.value + ret.append(job.value) + return ret + + def file_report(self, data, file=None, mode='a+'): + """ + This reports to a file in YAML format + """ + if not file: + file = self.file + try: + f = open(file, mode) + f.write(data) + except Exception, e: + raise e + finally: + f.close() + + + def tcp_report(self, data): + """ + This connect to the specified tcp server + and writes the data passed as argument. + """ + host, port = self.tcp.split(":") + tcp = socket.getprotobyname('tcp') + send_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, tcp) + try: + send_socket.connect((host, int(port))) + send_socket.send(data) + + except Exception, e: + raise e + + finally: + send_socket.close() + + + def scp_report(self, data, rfile=None, mode='a+'): + """ + Push data to the remote ssh server. + + :rfile the remote filename to write + :data the raw data content that should be written + :mode in what mode the file should be created + """ + if not rfile: + rfile = self.file + host, port = self.scp.split(":") + transport = paramiko.Transport((host, port)) + + # The remote path of the remote file to write + rfpath = os.path.join(self.config.ssh_rpath, rfile) + + try: + username = self.config.ssh_username + except: + raise "No username provided" + + # Load the local known host key file + transport.load_host_keys(os.path.expanduser("~/.ssh/known_hosts")) + + # We prefer to use an ssh keyfile fo authentication + if self.config.ssh_keyfile: + keyfile = os.path.expanduser(self.config.ssh_keyfile) + key = paramiko.RSAKey.from_private_key_file(keylocfile) + try: + transport.connect(username=username, pkey=key) + except Exception, e: + raise e + + # If not even a password is fine + elif self.config.ssh_password: + try: + transport.connect(username=username, password=self.config.ssh_password) + except Exception, e: + raise e + + # ... but no authentication, that is madness! + else: + raise "No key or password provided for ssh" + + sftp = paramiko.SFTPClient.from_transport(transport) + try: + sftp = ssh.open_sftp() + remote_file = sftp.file(rfile, mode) + remote_file.set_pipelined(True) + remote_file.write(data) + + except Exception, e: + raise e + sftp.close() + transport.close() + + + def send_report(self, data, type): + """ + This sends the report using the + specified type. + """ + #print "Reporting %s to %s" % (data, type) + self.logger.info("Reporting to %s" % type) + getattr(self, type+"_report").__call__(data) + +

1 0

[ooni-probe/master] Make it run once if there are no Asset
by art＠torproject.org 02 Jun '12

02 Jun '12

commit a925c465b45907d675ca915e3b5c2832b39390e8 Author: Arturo Filastò <hellais(a)torproject.org> Date: Thu May 31 21:15:24 2012 +0200 Make it run once if there are no Asset * Write regression test for it --- ooni/ooniprobe.py | 6 +++--- ooni/plugoo/reports.py | 43 ++++++++++++++++++++++++++++++------------- ooni/plugoo/work.py | 10 ++++++++-- tests/test_worker.py | 11 +++++++++++ 4 files changed, 52 insertions(+), 18 deletions(-) diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index d75b9b9..7248e70 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -59,12 +59,12 @@ def runTest(test, options, global_options): parallelism = int(global_options['parallelism']) worker = work.Worker(parallelism) - test = plugoo[test].__class__ - report = reports.Report(global_options['output']) + test_class = plugoo[test].__class__ + report = reports.Report(test, global_options['output']) log.start(global_options['log'], 1) - wgen = work.WorkGenerator(test(options, global_options, report), + wgen = work.WorkGenerator(test_class(options, global_options, report), dict(options), start=options['resume']) diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index 099aff7..6140294 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -1,9 +1,8 @@ import os -from datetime import datetime import yaml import itertools -from ooni import log +from ooni import log, date class Report: """This is the ooni-probe reporting mechanism. It allows @@ -19,10 +18,11 @@ class Report: inbound connection and accept a stream of data (think of it as a `nc -l -p <port> > filename.txt`) """ - def __init__(self, file="test.report", + def __init__(self, testname=None, file="report.log", scp=None, tcp=None): + self.testname = testname self.file = file self.tcp = tcp self.scp = scp @@ -36,16 +36,21 @@ class Report: # print self.file self.scp = None - - def __call__(self, data): - """ - This should be invoked every time you wish to write some - data to the reporting system - """ - #print "Writing report(s)" - #dump = '--- \n' - dump = yaml.dump([data]) - #dump += yaml.dump(data) + self.write_header() + + def write_header(self): + pretty_date = date.pretty_date() + header = "# OONI Probe Report for Test %s\n" % self.testname + header += "# %s\n\n" % pretty_date + self._write_to_report(header) + # XXX replace this with something proper + test_details = {'start_time': date.now(), + 'asn': 'ASN-1234', + 'test_name': self.testname, + 'addr': '1234'} + self(test_details) + + def _write_to_report(self, dump): reports = [] if self.file: @@ -61,6 +66,18 @@ class Report: for report in reports: self.send_report(dump, report) + def __call__(self, data): + """ + This should be invoked every time you wish to write some + data to the reporting system + """ + #print "Writing report(s)" + #dump = '--- \n' + dump = yaml.dump([data]) + #dump += yaml.dump(data) + + self._write_to_report(dump) + def file_report(self, data, file=None, mode='a+'): """ This reports to a file in YAML format diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index b9fc6b8..8ba0a3f 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -130,7 +130,11 @@ class WorkGenerator(object): def __init__(self, test, arguments=None, start=None): self.Test = test - self.assetGenerator = itertools.product(*self.Test.assets.values()) + if self.Test.assets and self.Test.assets.values()[0]: + self.assetGenerator = itertools.product(*self.Test.assets.values()) + else: + self.assetGenerator = None + self.assetNames = self.Test.assets.keys() self.idx = 0 @@ -148,6 +152,9 @@ class WorkGenerator(object): self.idx += 1 def next(self): + if self.end: + raise StopIteration + if not self.assetGenerator: self.end = True return (self.assetNames, self.Test, self.idx) @@ -161,7 +168,6 @@ class WorkGenerator(object): raise StopIteration self.idx += 1 - print "IDX:%s" % self.idx return (ret, self.Test, self.idx) def p_next(self): diff --git a/tests/test_worker.py b/tests/test_worker.py index e53750b..3222f65 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -12,7 +12,18 @@ class WorkerTestCase(unittest.TestCase): asset.append(i) dummytest.assets['asset'] = asset wgen = work.WorkGenerator(dummytest) + for j, x in enumerate(wgen): pass self.assertEqual(i, j) + def testNoAssets(self): + class DummyTest: + assets = {'asset': None} + dummytest = DummyTest() + wgen = work.WorkGenerator(dummytest) + i = 0 + for j in wgen: + i += 1 + self.assertEqual(i, 1) +

1 0