- tor-commits - lists.torproject.org

[ooni-probe/master] * Changed runner.getTestOptions() to return the localoptions, in addition to setting them as an attribute.
by isis＠torproject.org 08 Nov '12

08 Nov '12

commit c342c61e8ef0f7d79300054823d788ef1f4af878 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Nov 5 14:02:49 2012 +0000 * Changed runner.getTestOptions() to return the localoptions, in addition to setting them as an attribute. --- ooni/runner.py | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index d711153..52c6596 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -104,11 +104,13 @@ def getTestOptions(cls, subargs): raise Exception("This test requires root to run") try: - cls.buildOptions(subargs) + local_opts = cls.buildOptions(subargs) except Exception, e: log.err(e) - return cls.local_options + log.debug("getTestOptions: local_options = %s" % local_opts) + + return local_opts def loadTestsAndOptions(classes, config): """ @@ -240,7 +242,6 @@ class ORunner(object): log.debug("ORunner: cases=%s" % type(cases)) log.debug("ORunner: options=%s" % options) - try: first = options.pop(0) except: @@ -258,8 +259,7 @@ class ORunner(object): filename = 'report_'+date.timestamp()+'.yaml' reportFile = open(filename, 'a+') self.reporterFactory = ReporterFactory( - reportFile, testSuite=self.baseSuite(self.cases) - ) + reportFile, testSuite=self.baseSuite(self.cases)) def runWithInputUnit(self, input_unit): idx = 0

1 0

[ooni-probe/master] * Changed the nettest inputParser to no longer be wrapped by @staticmethod, because that might be confusing to test writers.
by isis＠torproject.org 08 Nov '12

08 Nov '12

commit 6d971bc3c49c50084d047ea77546a2eec7a53d84 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Nov 5 14:04:53 2012 +0000 * Changed the nettest inputParser to no longer be wrapped by @staticmethod, because that might be confusing to test writers. --- nettests/echo.py | 8 ++++---- 1 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nettests/echo.py b/nettests/echo.py index bc47519..793f224 100644 --- a/nettests/echo.py +++ b/nettests/echo.py @@ -126,15 +126,15 @@ class EchoTest(ScapyTest): % (self.name, ''.join(self.__dict__))) @staticmethod - def inputParser(inputs): + def inputParser(self, one_input): log.debug("Removing possible ports from host addresses...") log.debug("Initial inputs:\n%s" % pprint(inputs)) - assert isinstance(inputs, list) - hosts = [h.rsplit(':', 1)[0] for h in inputs] + #host = [h.rsplit(':', 1)[0] for h in inputs] + host = h.rsplit(':', 1)[0] log.debug("Inputs converted to:\n%s" % hosts) - return hosts + return host def tryInterfaces(self, ifaces): try:

1 0

[ooni-probe/master] * Added brief doctrings to get pylint to shutup about it.
by isis＠torproject.org 08 Nov '12

08 Nov '12

commit 85f0da80a5fb78b877f29c47931923544933dc06 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Nov 5 14:06:06 2012 +0000 * Added brief doctrings to get pylint to shutup about it. --- bin/ooniprobe | 23 +++++++++++------------ nettests/simpletest.py | 23 +++++++++++++++++------ 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/bin/ooniprobe b/bin/ooniprobe index 1e73c85..0742da9 100755 --- a/bin/ooniprobe +++ b/bin/ooniprobe @@ -1,17 +1,16 @@ #!/usr/bin/env python # -*- encoding: utf-8 -*- -############################################################################### -# -# oonicli -# ----------------------- -# -# Run the Twisted Trial (unittest module) based oonicli. Startup script based -# on twisted trial, see http://twistedmatrix.com/ . -# -# :authors: Arturo Filastò -# :licence: see included LICENSE file -# :version: 0.0.1-pre-alpha -############################################################################### +''' + oonicli + ----------- + + Run the Twisted Trial (unittest module) based oonicli. Startup script based + on twisted trial, see http://twistedmatrix.com/ . + + :authors: Arturo Filastò, Isis Lovecruft + :licence: see included LICENSE file + :version: 0.0.1-pre-alpha +''' import os, sys import copy_reg diff --git a/nettests/simpletest.py b/nettests/simpletest.py index a83b8ce..2769a15 100644 --- a/nettests/simpletest.py +++ b/nettests/simpletest.py @@ -1,18 +1,29 @@ +#-*- coding: utf-8 -*- +"""Contains classes for testing the basic functionality of oonicli.""" + from ooni import nettest + class SimpleTest(nettest.NetTestCase): - inputs = range(1,100) + """ + A very simple test which prints integers, for testing that oonicli is + working correctly. + """ + inputs = range(1, 100) optParameters = [['asset', 'a', None, 'Asset file'], - ['controlserver', 'c', 'google.com', 'Specify the control server'], + ['controlserver', 'c', 'google.com', + 'Specify the control server'], ['resume', 'r', 0, 'Resume at this index'], ['other', 'o', None, 'Other arguments']] - def test_foo(self, *arg, **kw): + def test_foo(self): + """Test that tests are working.""" print "Running %s with %s" % ("test_foo", self.input) self.report['test_foo'] = 'Antani' self.report['shared'] = "sblinda" - self.assertEqual(1,1) + self.assertEqual(1, 1) - def test_f4oo(self, *arg, **kw): + def test_f4oo(self): + """Test that tests are working.""" print "Running %s with %s" % ("test_f4oo", self.input) self.report['test_f4oo'] = 'Antani' self.report['shared'] = "sblinda2" - self.assertEqual(1,1) + self.assertEqual(1, 1)

1 0

[ooni-probe/master] * Fixed oonicli.py trying to catch the wrong exception.
by isis＠torproject.org 08 Nov '12

08 Nov '12

commit 1739910de4b4744213961a00c9f0f56a76b52f7a Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Nov 5 13:58:26 2012 +0000 * Fixed oonicli.py trying to catch the wrong exception. * Added a docstring in oonicli.py. * Fixed oonicli.py not printing the help/info message when called without arguments. --- ooni/oonicli.py | 16 +++++++++++----- 1 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index b9f1dee..a61b3f9 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -77,7 +77,6 @@ class Options(usage.Options, app.ReactorSelectionMixin): def parseArgs(self, *args): try: self['test'] = args[0] - self['subArgs'] = args[1:] except: raise usage.UsageError("No test filename specified!") @@ -87,16 +86,23 @@ class Options(usage.Options, app.ReactorSelectionMixin): def run(): - log.start() + """ + Call me to begin testing a file or module. + """ - if len(sys.argv) == 1: - sys.argv.append("--help") config = Options() + + if len(sys.argv) == 1: + config.getUsage() + try: config.parseOptions() - except usage.error, ue: + except usage.UsageError, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) + log.start() + log.debug("oonicli.run: config set to %s" % config) + if config['debug-stacktraces']: defer.setDebugging(True)

1 0

[ooni-probe/master] * There are still bugs in the options parser, and inputs is a listiterater, which also needs to be fixed.
by isis＠torproject.org 08 Nov '12

08 Nov '12

commit 94d1260906017f58afabba2d2fefbf1bf4ac394f Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Nov 5 14:01:10 2012 +0000 * There are still bugs in the options parser, and inputs is a listiterater, which also needs to be fixed. --- ooni/nettest.py | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 94c8c56..84c3289 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -311,6 +311,7 @@ class NetTestAdaptor(unittest.TestCase): setattr(cls, "local_options", options) log.debug("NetTestAdaptor: getTestOptions: cls.local_options = %s" % cls.local_options) + return options @classmethod def setUpClass(cls): @@ -434,9 +435,11 @@ class NetTestCase(NetTestAdaptor): else: log.debug("could not find cls.localOptions!") - return {'inputs': self.parsed_inputs, - 'name': self.name, - 'version': self.version} + ret= {'inputs': self.parsed_inputs, + 'name': self.name, + 'version': self.version} + log.debug("%s" % ret) + # if options: # return options # else:

1 0

[ooni-probe/master] * Removed bug as a result of merging which reintroduced the bug for log
by isis＠torproject.org 08 Nov '12

08 Nov '12

commit c84cc0923243dbc7ffa4a0ecaa9d42505d448315 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 8 07:42:35 2012 +0000 * Removed bug as a result of merging which reintroduced the bug for log messages being printed twice. --- ooni/oonicli.py | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 4f5eac5..ae78583 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -91,8 +91,6 @@ def run(): except usage.UsageError, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) - log.start() - if cmd_line_options['debug-stacktraces']: defer.setDebugging(True)

1 0

[ooni-probe/master] * Adding new Makefile in top dir
by isis＠torproject.org 08 Nov '12

08 Nov '12

commit ae4b1cf1f94fd014d2cedb89ed93271a3db5eeee Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 8 06:47:36 2012 +0000 * Adding new Makefile in top dir * Adding saved copy of *.secdev.org SSL cert so we can correctly check for cert warning. * Updated installation documentation. --- Makefile | 198 +++++++++++++++++++++++++++++++++++++++++++++++ docs/source/install.rst | 12 +++- ooni/lib/secdev.org.pem | 20 +++++ 3 files changed, 228 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a965c00 --- /dev/null +++ b/Makefile @@ -0,0 +1,198 @@ +# +# .:| Open Observatory of Network Interference |:. +# +# /lib/Makefile +# ------------- +# For obtaining, building, and configuring dependencies for OONI installations. +# +# @authors: Isis Lovecruft, Arturo Filastó +# @version: 0.1.0-alpha +# @license: see included LICENSE file +# @copyright: 2012 Isis Lovecruft, Arturo Filastó +# +# XXX TODO eventually this should be converted into a distutils setup.py +# + +here_we_are = . +top_srcdir = ${here_we_are} + +srcdir_ooni = ${top_srcdir}/ooni +srcdir_ooni_kit = ${srcdir_ooni}/kit +srcdir_ooni_plugoo = ${srcdir_ooni}/plugoo +srcdir_ooni_protocols = ${srcdir_ooni}/protocols +srcdir_ooni_templates = ${srcdir_ooni}/templates + +srcdir_oonib = ${top_srcdir}/oonib +srcdir_oonib_lib = ${srcdir_oonib}/lib +srcdir_oonib_report = ${srcdir_oonib}/report +srcdir_oonib_report_db = ${srcdir_oonib_report}/db +srcdir_oonib_testhelpers = ${srcdir_oonib}/testhelpers + +nettests = ${top_srcdir}/nettests +nettests_core = ${nettests}/core +nettests_examples = ${nettests}/examples +nettests_experimental = ${nettests}/experimental +nettests_third_party = ${nettests}/third_party + +prefix = ${top_srcdir} +exec_prefix = ${srcdir_ooni} + +bindir = ${top_srcdir}/bin +docsdir = ${top_srcdir}/docs +libdir = ${srcdir_ooni}/lib +includedir = ${top_srcdir}/include +testsdir = ${top_srcdir}/tests + +#pkgtwisteddir = $(libdir)/Twisted-* +pkgscapydir = $(libdir)/scapy +pkgtxtorcondir = $(libdir)/txtorcon + +SUBDIRS = ${top_srcdir} \ + ${srcdir_ooni} \ + ${srcdir_ooni_kit} \ + ${srcdir_ooni_plugoo} \ + ${srcdir_ooni_protocols} \ + ${srcdir_ooni_templates} \ + ${srcdir_oonib} \ + ${srcdir_oonib_lib} \ + ${srcdir_oonib_report} \ + ${srcdir_oonib_report_db} \ + ${srcdir_oonib_testhelpers} \ + ${nettests} \ + ${nettests_core} \ + ${nettests_examples} \ + ${nettests_experimental} \ + ${nettests_third_party} \ + ${pkgscapydir} \ + ${pkgtxtorcondir} + +all: txtorcon + +dependencies: txtorcon non-pip + +non-pip: scapy-all + +clean: + rm -rf ${libdir}/txtorcon + +clean-all: + rm -rf ${libdir}/txtorcon + rm -rf ${libdir}/txtraceroute.py + rm -rf ${libdir}/txscapy.py + rm -rf ${libdir}/rfc3339.py + +clean-scapy-all: + rm -rf ${libdir}/scapy* + rm -rf ${libdir}/pypcap* + rm -rf ${libdir}/libdnet* + +clean-non-pip: clean-scapy-all + +clean-dependencies: + rm -rf ${libdir}/scapy* + rm -rf ${libdir}/pypcap* + rm -rf ${libdir}/libdnet* + rm -rf ${libdir}/txtorcon + +txtraceroute: + echo "Processing dependency txtraceroute..." + cd ${libdir} && \ + git clone https://github.com/hellais/txtraceroute.git txtraceroute.git && \ + mv txtraceroute.git/txtraceroute.py txtraceroute.py && \ + rm -rf txtraceroute.git && \ + cd ${top_srcdir} + +txtorcon: + echo "Processing dependency txtorcon..." + cd ${libdir} && \ + git clone https://github.com/meejah/txtorcon.git txtorcon.git && \ + mv txtorcon.git/txtorcon txtorcon && \ + rm -rf txtorcon.git && \ + cd ${top_srcdir} + +txscapy: + echo "Processing dependency txscapy" + cd ${libdir} && \ + git clone https://github.com/hellais/txscapy.git txscapy.git && \ + mv txscapy.git/txscapy.py txscapy.py && \ + rm -rf txscapy.git && \ + cd ${top_srcdir} + +rfc3339: + echo "Processing RFC3339 dependency" + cd ${libdir} && \ + hg clone https://bitbucket.org/henry/rfc3339 rfc3339 && \ + mv rfc3339/rfc3339.py rfc3339.py && \ + rm -rf rfc3339 \ + cd ${top_srcdir} + +## requires sudo for setup.py: +scapy: + echo "Processing dependency scapy..." + cd ${libdir} ; \ + wget --ca-certificate="secdev.org.pem" -O scapy.tar.gz https://www.secdev.org/projects/scapy/files/scapy-latest.tar.gz && \ + tar -xzf scapy.tar.gz && \ + cd scapy-* && sudo python setup.py install && \ + cd .. && rm scapy.tar.gz && cd ${top_srcdir} + +## requires sudo for setup.py +pypcap: + echo "Processing dependency pypcap..." + cd ${libdir} ; \ + wget -O pypcap.tar.gz https://pypcap.googlecode.com/files/pypcap-1.1.tar.gz && \ + tar -xzf pypcap.tar.gz && cd pypcap-* && \ + sudo python setup.py install && \ + cd .. && rm pypcap.tar.gz && \ + cd ${top_srcdir} +# make all && make build && make install && make test && \ + make cleandir distclean && cd ${libdir} && \ + rm -rf pypcap* + +libdnet: + echo "Processing dependency libdnet..." + cd ${libdir} ; \ + wget -O libdnet.tar.gz https://libdnet.googlecode.com/files/libdnet-1.12.tgz && \ + tar -xzf libdnet.tar.gz && cd libdnet-* && \ + configure && make && make install && make clean && \ + cd .. && rm -rf libdnet* && cd ${top_srcdir} + +scapy-all: scapy pypcap libdnet + +tags-recursive: + list='$(SUBDIRS)'; for subdir in $$list; do \ + test "$$subdir" = . || (cd $$subdir && make tags); \ + done + +ETAGS = etags +ETAGSFLAGS = +ETAGSARGS = + +tags: TAGS + +ID: $(HEADERS) $(SOURCES) $(LISP) $(TAGS_FILES) + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; fi; \ + done | \ + $(AWK) ' { files[$$0] = 1; } \ + END { for (i in files) print i; }'`; \ + mkid -fID $$unique + +TAGS: tags-recursive $(HEADERS) $(SOURCES) $(TAGS_DEPENDENCIES) \ + $(TAGS_FILES) $(LISP) + tags=; \ + here=`pwd`; \ + list='$(SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + test -f $$subdir/TAGS && tags="$$tags -i $$here/$$subdir/TAGS"; \ + fi; \ + done; \ + list='$(SOURCES) $(HEADERS) $(LISP) $(TAGS_FILES)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; fi; \ + done | \ + $(AWK) ' { files[$$0] = 1; } \ + END { for (i in files) print i; }'`; \ + test -z "$(ETAGS_ARGS)$$tags$$unique" \ + || $(ETAGS) $(ETAGSFLAGS) $(ETAGS_ARGS) \ + $$tags $$unique diff --git a/docs/source/install.rst b/docs/source/install.rst index 4651064..6cab566 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -19,10 +19,18 @@ OONI depends on the following pieces of software. *Optional* -* Dnspython[1]: http://www.dnspython.org/ -* Paramiko[2]: http://www.lag.net/paramiko/ +* Dnspython [1]: http://www.dnspython.org/ +* Paramiko [2]: http://www.lag.net/paramiko/ * Txtorcon: https://github.com/meejah/txtorcon +If you would like to use Scapy to plot graphs of network activity, +or be capable of dumping obtained information to a PostScript or PDF +file, then you'll need: + + * gnuplot-py + * numpy + * Pyx + [1][2] These dependencies will soon be removed completely. Debian packages diff --git a/ooni/lib/secdev.org.pem b/ooni/lib/secdev.org.pem new file mode 100644 index 0000000..6fdbb97 --- /dev/null +++ b/ooni/lib/secdev.org.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDVDCCAjwCCQD6iQnFvlSvNjANBgkqhkiG9w0BAQUFADBsMQswCQYDVQQGEwJG +UjEMMAoGA1UECBMDSWRGMQ4wDAYDVQQHEwVQYXJpczETMBEGA1UEChMKc2VjZGV2 +Lm9yZzETMBEGA1UECxMKc2VjZGV2Lm9yZzEVMBMGA1UEAxQMKi5zZWNkZXYub3Jn +MB4XDTA4MDUxOTIxMzAxNVoXDTE4MDUyMDIxMzAxNVowbDELMAkGA1UEBhMCRlIx +DDAKBgNVBAgTA0lkRjEOMAwGA1UEBxMFUGFyaXMxEzARBgNVBAoTCnNlY2Rldi5v +cmcxEzARBgNVBAsTCnNlY2Rldi5vcmcxFTATBgNVBAMUDCouc2VjZGV2Lm9yZzCC +ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMijlApVIOF86nIsPvIfKjkQ +qpw8DWtugsUQkspXGaJM5qM3CvoxQ3VQejIZiLIS/B57WtpwfhD63h+dswUZy1wI +Z4injE/uF4R7ylNammROjS1ycQbFM1fWX/1nzKFrxWpX3lU2YjwB9qIAlE3u/SyH +U10oq9ZJ5KlyOrjTPe3eb0KBwF5W0AJxcTiGQJhADZAaAivZRT880GYJAo3UaL/G +JaBYIYSFxvGnqmUVM9kbnGLFQEQahBpgmtCzMRVFXp/AccxCtXKY+LORtSGNKaB6 +ODDidG8jyb3S9GmjtgxwyWHvY/9YRW2BkB3AufRsOAWUN7jWDtRLKy6FCLbxE/sC +AwEAATANBgkqhkiG9w0BAQUFAAOCAQEAex0loqATvXxZEagphrLASUGpKIlTf3a1 +1adokzrKvbuDXcxNqUKEPxI09TjnT/zySLfVc18t+yy2baSstPFC9RrLPeu8rfzL +k+NTDmM3OfW60MCeEnyNxPvW0wCIrFLfH3t5XPT3J2DtYLmecg8Lf/sQOEWPyMVc +uCaFIYsAypGYi0wwG5VDQHEsKxkHC2nBRwGJdx9w70yy14H/JOAZl5yQpLHEc4Db +RUfNTIV2myXOIET2VbCN2Yc8Gegsclc506XVOQypp5Ndvy4GW2yRRE2ps1c1xH6P +OHENUp0JPyLeyibmoOCUfrlrq2KoSashFZmPCGYFFJvcKAYI45GcaQ== +-----END CERTIFICATE-----

1 0

[ooni-probe/master] Merge branch 'master' of git-rw.torproject.org:ooni-probe
by isis＠torproject.org 08 Nov '12

08 Nov '12

commit 5c29daa8c9d33f272f618d37d1ba76b5f7998640 Merge: 85f0da8 e10033c Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 8 06:40:55 2012 +0000 Merge branch 'master' of git-rw.torproject.org:ooni-probe Conflicts: ooni/nettest.py ooni/oonicli.py ooni/runner.py .gitignore | 2 + before_i_commit.sh | 37 ++ data/Makefile | 10 + docs/source/api/modules.rst | 7 + docs/source/api/ooni.kit.rst | 19 + docs/source/api/ooni.lib.rst | 36 ++ docs/source/api/ooni.rst | 15 +- docs/source/api/ooni.templates.rst | 15 +- docs/source/api/ooni.utils.rst | 114 ++++- docs/source/index.rst | 41 +- docs/source/writing_tests.rst | 3 +- inputs/Makefile | 3 + inputs/captive_portal_tests.txt.good | 4 - inputs/cctld.txt | 511 ------------------ inputs/dns_servers.txt.bak | 6 - inputs/dns_servers.txt.bak2 | 1 - inputs/example_exp_list.txt | 3 - inputs/major_isp_dns_servers.txt | 796 ----------------------------- inputs/short_hostname_list.txt | 7 - inputs/tld-list-cc.txt | 511 ------------------ inputs/tld-list-mozilla.txt | 5 - inputs/top-1m.txt.bak2 | 11 - nettests/bridge_reachability/bridget.py | 462 +++++++++++++++++ nettests/bridge_reachability/echo.py | 195 +++++++ nettests/bridget.py | 499 ------------------ nettests/core/captiveportal.py | 5 +- nettests/core/dnstamper.py | 97 +++-- nettests/core/echo.py | 1 - nettests/core/keyword_filtering.py | 39 ++ nettests/core/url_list.py | 23 + nettests/echo.py | 196 ------- nettests/examples/example_httpt.py | 9 +- nettests/examples/example_scapyt.py | 1 + ooni/__init__.py | 15 +- ooni/config.py | 38 ++- ooni/custodiet.py | 421 --------------- ooni/inputunit.py | 133 +----- ooni/lib/Makefile | 36 -- ooni/lib/txscapy.py | 55 ++- ooni/nettest.py | 391 +++------------ ooni/nodes.py | 176 +++++++ ooni/oonicli.py | 40 +- ooni/plugoo/__init__.py | 47 -- ooni/plugoo/assets.py | 62 --- ooni/plugoo/interface.py | 56 -- ooni/plugoo/nodes.py | 176 ------- ooni/plugoo/reports.py | 145 ------ ooni/plugoo/tests.py | 142 ----- ooni/plugoo/work.py | 148 ------ ooni/protocols/http.py | 141 ----- ooni/protocols/scapyproto.py | 55 -- ooni/reporter.py | 95 ++-- ooni/runner.py | 261 ++++------- ooni/scaffolding.py | 78 --- ooni/templates/httpt.py | 51 ++- ooni/templates/scapyt.py | 108 ++++- ooni/utils/geodata.py | 31 ++ ooni/utils/legacy.py | 459 ----------------- ooni/utils/log.py | 20 +- ooni/utils/onion.py | 4 +- oonib/oonibackend.py | 12 +- oonib/report/api.py | 63 +-- oonib/report/db/models.py | 83 --- ooniprobe.conf | 25 + requirements.txt | 27 +- test_inputs/README | 48 ++ test_inputs/dns_tamper_file.txt | 3 + test_inputs/dns_tamper_test_resolvers.txt | 2 + test_inputs/http_host_file.txt | 2 + test_inputs/keyword_filtering_file.txt | 2 + test_inputs/url_lists_file.txt | 2 + tests/test_dns.py | 24 + 72 files changed, 1868 insertions(+), 5493 deletions(-) diff --cc nettests/bridge_reachability/echo.py index 0000000,0c20a3f..e59eee5 mode 000000,100644..100644 --- a/nettests/bridge_reachability/echo.py +++ b/nettests/bridge_reachability/echo.py @@@ -1,0 -1,195 +1,195 @@@ + #!/usr/bin/env python + # -*- coding: utf-8 -*- + # + # +---------+ + # | echo.py | + # +---------+ + # A simple ICMP-8 ping test. + # + # :author: Isis Lovecruft + # :version: 0.0.1-pre-alpha + # :license: (c) 2012 Isis Lovecruft + # see attached LICENCE file + # + + import os + import sys + + from pprint import pprint + + from twisted.internet import reactor + from twisted.plugin import IPlugin + from twisted.python import usage + from ooni.nettest import NetTestCase + from ooni.utils import log, Storage + from ooni.utils.net import PermissionsError, IfaceError + + try: + from scapy.all import sr1, IP, ICMP ## XXX v4/v6? + from ooni.lib import txscapy + from ooni.lib.txscapy import txsr, txsend + from ooni.templates.scapyt import ScapyTest + except: + log.msg("This test requires scapy, see www.secdev.org/projects/scapy") + + ## xxx TODO: move these to a utility function for determining OSes + LINUX=sys.platform.startswith("linux") + OPENBSD=sys.platform.startswith("openbsd") + FREEBSD=sys.platform.startswith("freebsd") + NETBSD=sys.platform.startswith("netbsd") + DARWIN=sys.platform.startswith("darwin") + SOLARIS=sys.platform.startswith("sunos") + WINDOWS=sys.platform.startswith("win32") + + class EchoTest(ScapyTest): + """ + xxx fill me in + """ + name = 'echo' + author = 'Isis Lovecruft <isis(a)torproject.org>' + description = 'A simple ICMP-8 test to see if a host is reachable.' + version = '0.0.1' + inputFile = ['file', 'f', None, 'File of list of IPs to ping'] + requiresRoot = True + + optParameters = [ + ['interface', 'i', None, 'Network interface to use'], + ['count', 'c', 5, 'Number of packets to send', int], + ['size', 's', 56, 'Number of bytes to send in ICMP data field', int], + ['ttl', 'l', 25, 'Set the IP Time to Live', int], + ['timeout', 't', 2, 'Seconds until timeout if no response', int], + ['pcap', 'p', None, 'Save pcap to this file'], + ['receive', 'r', True, 'Receive response packets'] + ] + + def setUp(self, *a, **kw): + ''' + :ivar ifaces: + Struct returned from getifaddrs(3) and turned into a tuple in the + form (*ifa_name, AF_FAMILY, *ifa_addr) + ''' + + if self.localOptions: + log.debug("%s: local_options found" % self.name) + for key, value in self.localOptions.items(): + log.debug("setting self.%s = %s" % (key, value)) + setattr(self, key, value) + + ## xxx is this now .subOptions? + #self.inputFile = self.localOptions['file'] + self.timeout *= 1000 ## convert to milliseconds + + if not self.interface: + log.msg("No network interface specified!") + log.debug("OS detected: %s" % sys.platform) + if LINUX or OPENBSD or NETBSD or FREEBSD or DARWIN or SOLARIS: + from twisted.internet.test import _posixifaces + log.msg("Attempting to discover network interfaces...") + ifaces = _posixifaces._interfaces() + elif WINDOWS: + from twisted.internet.test import _win32ifaces + log.msg("Attempting to discover network interfaces...") + ifaces = _win32ifaces._interfaces() + else: + log.debug("Client OS %s not accounted for!" % sys.platform) + log.debug("Unable to discover network interfaces...") + ifaces = [('lo', '')] + + ## found = {'eth0': '1.1.1.1'} + found = [{i[0]: i[2]} for i in ifaces if i[0] != 'lo'] + log.info("Found interfaces:\n%s" % pprint(found)) + self.interfaces = self.tryInterfaces(found) + else: + ## xxx need a way to check that iface exists, is up, and + ## we have permissions on it + log.debug("Our interface has been set to %s" % self.interface) + + if self.pcap: + try: + self.pcapfile = open(self.pcap, 'a+') + except: + log.msg("Unable to write to pcap file %s" % self.pcap) + self.pcapfile = None + + try: + assert os.path.isfile(self.file) + fp = open(self.file, 'r') + except Exception, e: + hosts = ['8.8.8.8', '38.229.72.14'] + log.err(e) + else: + self.inputs = self.inputProcessor(fp) + self.removePorts(hosts) + + log.debug("Initialization of %s test completed with:\n%s" + % (self.name, ''.join(self.__dict__))) + + @staticmethod - def inputParser(inputs): ++ def inputParser(self, one_input): + log.debug("Removing possible ports from host addresses...") + log.debug("Initial inputs:\n%s" % pprint(inputs)) + - assert isinstance(inputs, list) - hosts = [h.rsplit(':', 1)[0] for h in inputs] ++ #host = [h.rsplit(':', 1)[0] for h in inputs] ++ host = h.rsplit(':', 1)[0] + log.debug("Inputs converted to:\n%s" % hosts) + - return hosts ++ return host + + def tryInterfaces(self, ifaces): + try: + from scapy.all import sr1 ## we want this check to be blocking + except: + log.msg("This test requires scapy: www.secdev.org/projects/scapy") + raise SystemExit + + ifup = {} + while ifaces: + for ifname, ifaddr in ifaces: + log.debug("Currently testing network capabilities of interface" + + "%s by sending a packet to our address %s" + % (ifname, ifaddr)) + try: + pkt = IP(dst=ifaddr)/ICMP() + ans, unans = sr(pkt, iface=ifname, timeout=self.timeout) + except Exception, e: + raise PermissionsError if e.find("Errno 1") else log.err(e) + else: + ## xxx i think this logic might be wrong + log.debug("Interface test packet\n%s\n\n%s" + % (pkt.summary(), pkt.show2())) + if ans.summary(): + log.info("Received answer for test packet on interface" + +"%s :\n%s" % (ifname, ans.summary())) + ifup.update(ifname, ifaddr) + else: + log.info("Our interface test packet was unanswered:\n%s" + % unans.summary()) + + if len(ifup) > 0: + log.msg("Discovered the following working network interfaces: %s" + % ifup) + return ifup + else: + raise IfaceError("Could not find a working network interface.") + + def buildPackets(self): + log.debug("self.input is %s" % self.input) + log.debug("self.hosts is %s" % self.hosts) + for addr in self.input: + packet = IP(dst=self.input)/ICMP() + self.request.append(packet) + return packet + + def test_icmp(self): + if self.recieve: + self.buildPackets() + all = [] + for packet in self.request: + d = self.sendReceivePackets(packets=packet) + all.append(d) + self.response.update({packet: d}) + d_list = defer.DeferredList(all) + return d_list + else: + d = self.sendPackets() + return d diff --cc ooni/nettest.py index 84c3289,3534a32..27e733c --- a/ooni/nettest.py +++ b/ooni/nettest.py @@@ -1,17 -1,8 +1,16 @@@ # -*- encoding: utf-8 -*- # -# :authors: Arturo "hellais" Filastò <art(a)fuffa.org> +# nettest.py +# -------------------> +# +# :authors: Arturo "hellais" Filastò <art(a)fuffa.org>, +# Isis Lovecruft <isis(a)torproject.org> # :licence: see LICENSE +# :copyright: 2012 Arturo Filasto, Isis Lovecruft +# :version: 0.1.0-alpha +# +# <------------------- - from functools import partial import sys import os import itertools @@@ -392,10 -118,10 +127,10 @@@ class NetTestCase(unittest.TestCase) """ name = "I Did Not Change The Name" author = "Jane Doe <foo(a)example.com>" - version = "0" + version = "0.0.0" + inputs = [None] inputFile = None - inputs = [None] report = {} report['errors'] = [] diff --cc ooni/oonicli.py index a61b3f9,5ee6deb..4f5eac5 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@@ -41,21 -41,19 +41,15 @@@ class Options(usage.Options, app.Reacto optFlags = [["help", "h"], ['debug-stacktraces', 'B', -- 'Report deferred creation and callback stack traces'], -- ] ++ 'Report deferred creation and callback stack traces'],] -- optParameters = [ -- ["reportfile", "o", None, "report file name"], - ["logfile", "l", "test.log", "log file name"], - ['temp-directory', None, '_ooni_temp', - 'Path to use as working directory for tests.'] - ] - ["logfile", "l", None, "log file name"], - ] ++ optParameters = [["reportfile", "o", None, "report file name"], ++ ["logfile", "l", None, "log file name"],] compData = usage.Completions( extraActions=[usage.CompleteFiles( "*.py", descr="file | module | package | TestCase | testMethod", -- repeat=True)], -- ) ++ repeat=True)],) tracer = None @@@ -86,29 -85,22 +80,27 @@@ def run(): - if len(sys.argv) == 1: - sys.argv.append("--help") + """ + Call me to begin testing a file or module. + """ - - config = Options() - + cmd_line_options = Options() + if len(sys.argv) == 1: - config.getUsage() - ++ cmd_line_options.getUsage() try: - config.parseOptions() + cmd_line_options.parseOptions() - except usage.error, ue: + except usage.UsageError, ue: raise SystemExit, "%s: %s" % (sys.argv[0], ue) + log.start() - log.debug("oonicli.run: config set to %s" % config) + - if config['debug-stacktraces']: + if cmd_line_options['debug-stacktraces']: defer.setDebugging(True) - classes = runner.findTestClassesFromConfig(config) - casesList, options = runner.loadTestsAndOptions(classes, config) + classes = runner.findTestClassesFromConfig(cmd_line_options) + casesList, options = runner.loadTestsAndOptions(classes, cmd_line_options) for idx, cases in enumerate(casesList): - orunner = runner.ORunner(cases, options[idx], config) + orunner = runner.ORunner(cases, options[idx], cmd_line_options) + log.start(cmd_line_options['logfile']) orunner.run() + diff --cc ooni/runner.py index 52c6596,797a9d4..c8c6812 --- a/ooni/runner.py +++ b/ooni/runner.py @@@ -239,27 -149,31 +149,34 @@@ class ORunner(object) self.cases = cases self.options = options + log.debug("ORunner: cases=%s" % type(cases)) + log.debug("ORunner: options=%s" % options) + try: - first = options.pop(0) - except: - first = options - - if 'inputs' in first: - self.inputs = self.options['inputs'] + assert len(options) != 0, "Length of options is zero!" + except AssertionError, ae: + self.inputs = [] + log.err(ae) else: - log.msg("Could not find inputs!") - self.inputs = [None] + try: + first = options.pop(0) + except: + first = options + + if 'inputs' in first: + self.inputs = options['inputs'] + else: + log.msg("Could not find inputs!") + log.msg("options[0] = %s" % first) + self.inputs = [None] try: - reportFile = open(config['reportfile'], 'a+') - except: + reportFile = open(cmd_line_options['reportfile'], 'a+') + except TypeError: filename = 'report_'+date.timestamp()+'.yaml' reportFile = open(filename, 'a+') - self.reporterFactory = ReporterFactory( - reportFile, testSuite=self.baseSuite(self.cases)) + self.reporterFactory = ReporterFactory(reportFile, + testSuite=self.baseSuite(self.cases)) def runWithInputUnit(self, input_unit): idx = 0

1 0

[ooni-probe/master] Merge branch 'master' of git-rw.torproject.org:ooni-probe
by isis＠torproject.org 08 Nov '12

08 Nov '12

commit 16835b07f9cc925fc0dd2e673b18d419d3ffdf98 Merge: 5c29daa e05f529 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 8 06:41:33 2012 +0000 Merge branch 'master' of git-rw.torproject.org:ooni-probe before_i_commit.sh | 1 + nettests/bridge_reachability/bridget.py | 2 +- nettests/bridge_reachability/echo.py | 35 +---- nettests/core/dnstamper.py | 28 ++-- nettests/core/http_host.py | 11 +- nettests/examples/example_scapyt.py | 25 +--- ooni/nettest.py | 44 +++++- ooni/runner.py | 24 ++-- ooni/templates/scapyt.py | 103 +++++++------- ooni/utils/assertions.py | 180 ----------------------- ooni/utils/config.py | 121 ---------------- ooni/utils/meta.py | 241 ------------------------------- ooni/utils/onion.py | 17 ++- ooni/utils/process.py | 73 ---------- ooni/utils/timer.py | 107 -------------- wtf/README | 9 ++ wtf/assertions.py | 180 +++++++++++++++++++++++ wtf/config.py | 66 +++++++++ wtf/meta.py | 241 +++++++++++++++++++++++++++++++ wtf/process.py | 73 ++++++++++ wtf/timer.py | 107 ++++++++++++++ 21 files changed, 829 insertions(+), 859 deletions(-)

1 0

[ooni-probe/master] * Made the submit-patch script create the /patches directory if it doesn't already exist.
by isis＠torproject.org 08 Nov '12

08 Nov '12

commit d02cb70f6e788c37273f8953be827a0adb0a8185 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Nov 8 07:14:43 2012 +0000 * Made the submit-patch script create the /patches directory if it doesn't already exist. --- submit-patch | 7 ++++++- 1 files changed, 6 insertions(+), 1 deletions(-) diff --git a/submit-patch b/submit-patch index f794a2f..4768deb 100755 --- a/submit-patch +++ b/submit-patch @@ -87,7 +87,12 @@ echo; echo -e "\033[40m\033[0;32m THIS SCRIPT DOES NOT SEND THE PATCH FILES. \033[0m" echo -e "\033[40m\033[0;32m You'll have to handle that bit on your own. \033[0m" echo; - +if test ! -d "patches" ; then + echo; echo -e "\033[40m\033[0;32m Creating '/patches' directory... \033[0m" + mkdir patches +else + echo; echo -e "\033[40m\033[0;32m Using '/patches' directory... \033[0m" +fi git format-patch --full-index -o "./patches" --stat -l10 --ignore-submodules \ --binary --cover-letter --numbered --ignore-if-in-upstream \ --suffix=".patch" --to="$DEVELOPERS" --cc="$CC" master

1 0