commit bae19b558fcfa618778dea948e292c393ab49ed8 Author: Arturo Filastò art@fuffa.org Date: Sat Nov 10 21:40:22 2012 +0100
Do some cleaning up * Remove Readme.md file * Update TODO with lists of tickets * Remove no longer used unittests --- README.md | 108 ++++++++++++++---------------------------- TODO | 105 +++------------------------------------- tests/assets/ipports.txt | 2 - tests/assets/urllist.txt | 3 - tests/legacy/test_plugins.py | 40 --------------- tests/legacy/test_tests.py | 45 ----------------- tests/test_worker.py | 29 ----------- 7 files changed, 43 insertions(+), 289 deletions(-)
diff --git a/README.md b/README.md index 4837275..137386a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# ooni-probe - Open Observatory of Network Interference +# ooniprobe - Open Observatory of Network Interference
"The Net interprets censorship as damage and routes around it." - John Gilmore; TIME magazine (6 December 1993) @@ -15,103 +15,65 @@ To run OONI-probe without having to install it you must tell python that it can import modules from the root of ooni-probe, as well as initialize the included submodules.
-From the root directory of the repo (.../ooni-probe/), initialize the submodules by doing: +## Getting started
- $ git submodule init && git submodule update +Requirements:
-Next, you will need to tell Python that OONI is part of its path: + * Git: http://git-scm.com/book/en/Getting-Started-Installing-Git + * Python >= 2.6: http://www.python.org/download/releases/ + * pip: http://www.pip-installer.org/en/latest/
- $ export PYTHONPATH=$PYTHONPATH:`pwd` +On debian based systems these can be installed with:
-Then to see what tests are available: + apt-get install git-core python python-pip python-dev
- $ cd ooni - $ python ooniprobe.py +The python dependencies required for running ooniprobe are:
-If you see some errors see INSTALL to install the missing dependencies. + * Twisted + * Scapy >= 2.2.0 + * txtorcon
-To list the help for a specific test: +They can be installed from the requirements.txt with:
- $ python ooniprobe.py httpt --help + pip install -r requirements.txt
-## Virtualenv way (Recommended) +You are highly recommended to do so from inside of a virtual environment, since +pip does not download the packages via SSL and you will need to install it +system wide.
- virtualenv2 ENV/ - source ENV/bin/activate - pip install twisted Scapy pyyaml +This will require you to have installed virtualenv.
-The setup.py script of pyOpenSSL has some issues and it is therefore -recommended to install it system wide with: + apt-get install python-virtualenv
- sudo pip install pyOpenSSL +To create a new virtual environment do
-Note: be sure to do this over a secure network since pip does not by default -use SSL... + virtualenv env
-To install the most up to date scapy version (requires mercurial): +Then install OONI with:
- pip install hg+http://hg.secdev.org/scapy + pip install -r requirements.txt
+## Running some tests
-# More details +To see the possible command line options run:
-With the belief that unfettered access to information is a intrinsic human right, -OONI seeks to observe levels of surveillance, censorship, and network discrimination -in order for people worldwide to have a clearer understanding of the ways in -which their access to information is filtered. + ./bin/ooniprobe --help
-The end goal of OONI is to collect data which can show an accurate -topology of network interference and censorship. Through this topology, it will be -possible to see what the internet looks like from nearly any location, including -what sites are censored, or have been tampered with, and by whom. We're calling -it filternet. +For interesting tests to run look in the nettests/core/ directory.
-OONI uses open methodologies and the data will be provided in raw -format to allow any researcher to indipendently draw their conclusions -from the results OONI tests. +To run a test you can do so with:
-There are currently projects aimed at measuring censorship in one -way or another but they either use non open methodologies or their -tools are not open sources. OONI aims at filling up this gap by -creating the first open source framework for developing network -tests and collecting data on censorship. + ./bin/ooniprobe -o report_file_name path/to/test.py
-OONI revolves around three major concepts: Assets, Tests and -Reports. +Normally tests take options, you can see them with:
-## Assets + ./bin/ooniprobe -o report_file_name path/to/test.py --help
-Assets are the inputs used inside Tests to detect censorship events. -These can be URL lists, keywords, ip addresses, packets or any kind -of set of data. -In the python specific implementation this is represented as a python -iterable object. This means that the Testing framework will be able -to iterate through every element in the Asset. +## Configuration
-## Tests +By default ooniprobe will not include personal identifying information in the +test result, nor create a pcap file. This behavior can be personalized by +editing your ooniprobe.conf configuration file.
-This is the core of OONI. These are the actual tests that will be run -using as input (if an input is required) the Assets. -Tests can be summarized as an experiment and a control. The control -represents the expected result and the experiment is the network operation -being performed on the live network. If the experiment does not match up -with the control then a censorship event had occured.
-OONI probe provides some useful functionality to the application developer -that may be useful when developing censorship detection tests. For example -it is possible to make a request over the Tor network easily or use a fast -and flexible non-blocking HTTP client implementation. - -## Reports - -This is the data that is collected from the test. OONI probe provides a -flexible means of storing results and uploading this data to a remote -server or a flat file. - -The Test developer should include in the report as much data as possible -and can contain raw packet dumps as well as structured synthetic results. - -In future on top of ooni-probe Reports it will be possible to develop -flexible post-processing tools to allow data-visualization guru's to -properly visualize and contextualize the resulting data.
diff --git a/TODO b/TODO index b8524f3..6a989fd 100644 --- a/TODO +++ b/TODO @@ -1,108 +1,19 @@ -This is a list of things to be done on ooni-probe. +You can find things that need to be fixed by running from the root of the project:
-Once you have completed something you should add a brief note to this file -stating what you have done under the item. If you discover needed tasks, feel -free to add them, but also keep in mind that OONI is mostly using the Tor Trac -instance, and the main ticket for OONI which all tests should be organized -under is here: + grep -rn XXX *
- https://trac.torproject.org/projects/tor/ticket/5869 +# The full list of tickets related to OONI can be found by doing this query on trac:
-New things to test ------------------- + https://trac.torproject.org/projects/tor/query?status=accepted&status=as...
-* Some of the tests still give errors. This is top priority. +# Things related to the OONIB can be found with this query:
-* The submit-patch script is still a bit janky, and could use - some sprucing up. + https://trac.torproject.org/projects/tor/query?status=accepted&status=as...
-* The Makefile that I made to serve a quick check for whether or not the tests - are broken could use some more tests added to it. One thing is that some - tests take certain types of input files, others take none at all, and - nowhere is this presented to someone trying to run a test. So, the informing - users/testers bit can be worked on, and the testing. Obviously we're going - to want something more robust that a 20 LOC Makefile pretty fast. +# Tests that still need to be implemented can be found with:
-Finalization of API design --------------------------- + https://trac.torproject.org/projects/tor/query?status=accepted&status=as...
-* The nettest.TestCase should have an interface.
- I know that there is a push away from using zope.interfaces, but I think - it is actually *highly* necessary for ensuring that subclasses implement - the required functions, and also that they do not improperly override - necessary functions, for them to run.
- Personally, I am quite annoyed when I subclass a class from Twisted and - override a public method, and it breaks things (when nothing in their - documentation informed me that it would break things) and I have to spend - half an hour digging through their code to figure out precisely what is - needed externally from the function I'm overriding. Others should not have - to do this with our code.
-* The nettest.TestCase should have a twisted.python.usage.Options subclass and - interface as well, even if the instantiation of that subclass is handled by - the ooni.oonicli or the ooni.runner. There is more functionality to - usage.Options that we should expose than merely "optParameters", for - instance the "coerceOptions" parameter validation methods, or the - "postOptions" configuration. - -New things to develop ---------------------- - -These are either components specific to the new refactor of ooni -or that we haven't yet figured out how they should work. - -* Finish implementing the backend collection code. - - o PCAP READER/WRITER: - This should be quite simple...see scapy.all.wrpcap and - scapy.all.rdpcap. However, we have been warned by other projects that - this does *not* scale well. For example, see: - https://github.com/isislovecruft/switzerland/blob/master/switzerland/client/... - Which is a circular ring buffer specifically for libpcap, to avoid kernel - buffer overflows due to a high number of incoming packets. I expect this - to only be an issue on substantially high-bandwidth nodes...though that - is what we'll be dealing with when we deploy on Mlab. - - o PCAP UPLOADER: - This also sounds simple, and is, until you begin to deal with things like - persistence. What we really need is rsync, written in python, or at least - some cross-platform implementation. I (Isis speaking) am the current - maintainer of pyrsync, BUT DO NOT USE PYRSYNC. It is only an - implementation of the rsync *algorithm* for diffs, it is not rsync the - program. Also, it is BROKEN AND I DO NOT MAINTAIN IT. If you want to - maintain it, please take it off my hands. - -* Useability: - - o UNITTESTS. Pronto. - - o DOCUMENTATION. If you found something that confused you, or still - confuses you, and you couldn't find the answer within fifteen seconds, - then that thing is not well documented. Make it better, or at least mark - it with an "XXX document me!" tag. - -* Persistence: - - o We need some type of scheduler/cron thing which will background the tests - so that they don't take up a terminal, and can be configured to run - certain tests at timed intervals. - - o The Reporter will probably need to be updated to handle knowing when *a - test* has completed, but that the scheduler is still running. - -Migrate code from old ---------------------- - -Migrate all the interesting parts of the old code to the new. This is mostly -finished, but there still are things in the /old-to-be-ported directory which -might be of use. At this point, because we have gone through several version -of the API design, many of them are entirely unusable, and merely the general -idea remains. - -It's important to make the new code asych and based on Twisted. It should -respect the design goals of the new ooni-probe model. Also, importing new, -non-standard libraries should be discussed first, if the new test is to be -used in the core of OONI (packaging scapy and twisted already makes our -codebase quite large). diff --git a/tests/assets/ipports.txt b/tests/assets/ipports.txt deleted file mode 100644 index ade757f..0000000 --- a/tests/assets/ipports.txt +++ /dev/null @@ -1,2 +0,0 @@ -127.0.0.1:80 -8.8.8.8:53 diff --git a/tests/assets/urllist.txt b/tests/assets/urllist.txt deleted file mode 100644 index dc38e22..0000000 --- a/tests/assets/urllist.txt +++ /dev/null @@ -1,3 +0,0 @@ -http://google.com/ -http://127.0.0.1/ -http://dio.it/ diff --git a/tests/legacy/test_plugins.py b/tests/legacy/test_plugins.py deleted file mode 100644 index 26bee51..0000000 --- a/tests/legacy/test_plugins.py +++ /dev/null @@ -1,40 +0,0 @@ -from twisted.internet import defer, reactor -from twisted.trial import unittest - -from ooni.ooniprobe import retrieve_plugoo, runTest, Options -from ooni.plugoo import work, tests - -def asset_file(filename): - import os - file_dir = os.path.normpath(os.path.join(__file__, '..')) - return os.path.join(file_dir, 'assets', filename) - -class PluginsTestCase(unittest.TestCase): - def test_plugin_blocking(self): - suboptions = {'asset': asset_file('urllist.txt')} - runTest('blocking', suboptions, Options(), reactor) - return - - def test_plugin_tcpconnect(self): - suboptions = {'asset': asset_file('ipports.txt')} - runTest('tcpconnect', suboptions, Options(), reactor) - return - - - def test_plugin_captivep(self): - runTest('blocking', None, Options(), reactor) - return - - - def test_plugin_httphost(self): - suboptions = {'asset': asset_file('urllist.txt')} - runTest('httphost', suboptions, Options(), reactor) - return - - - def test_plugin_httpt(self): - suboptions = {'urls': asset_file('urllist.txt')} - runTest('httpt', suboptions, Options(), reactor) - return - - diff --git a/tests/legacy/test_tests.py b/tests/legacy/test_tests.py deleted file mode 100644 index dc89b98..0000000 --- a/tests/legacy/test_tests.py +++ /dev/null @@ -1,45 +0,0 @@ -from twisted.internet import defer -from twisted.trial import unittest - -from ooni.plugoo import work, tests - -class TestsTestCase(unittest.TestCase): - def setUp(self): - class dummyReport: - def __call__(self, *args, **kw): - pass - self.dummyreport = dummyReport() - self.callbackResults = None - self.errbackResults = None - - def _callback(self, *args, **kw): - #print args, kw - self.callbackResults = args, kw - - def _errback(self, *args, **kw): - pass - - @defer.inlineCallbacks - def test_fallThrough(self): - """ - This tests to make sure that what is returned from the experiment - method falls all the way through to control and finish. - """ - test_dict = {"hello": "world"} - class DummyTest(tests.OONITest): - blocking = False - def experiment(self, args): - def bla(a): - print a - return test_dict - d2 = defer.Deferred() - d2.addCallback(bla) - from twisted.internet import reactor - reactor.callLater(0.1, d2.callback, None) - return d2 - - test = DummyTest(None, None, self.dummyreport) - yield test.startTest(None).addCallback(self._callback) - self.assertEqual(self.callbackResults[0][0]['return_value'], test_dict) - return - diff --git a/tests/test_worker.py b/tests/test_worker.py deleted file mode 100644 index 3222f65..0000000 --- a/tests/test_worker.py +++ /dev/null @@ -1,29 +0,0 @@ -from twisted.trial import unittest - -from ooni.plugoo import work, tests - -class WorkerTestCase(unittest.TestCase): - def testWorkGenerator(self): - class DummyTest: - assets = {} - dummytest = DummyTest() - asset = [] - for i in range(10): - asset.append(i) - dummytest.assets['asset'] = asset - wgen = work.WorkGenerator(dummytest) - - for j, x in enumerate(wgen): - pass - self.assertEqual(i, j) - - def testNoAssets(self): - class DummyTest: - assets = {'asset': None} - dummytest = DummyTest() - wgen = work.WorkGenerator(dummytest) - i = 0 - for j in wgen: - i += 1 - self.assertEqual(i, 1) -