- tor-commits - lists.torproject.org

[ooni-probe/master] Add documentation for the DNSSpoof test
by art＠torproject.org 06 Dec '12

06 Dec '12

commit 6f7e91543e9b11a47efa6ca03a0f2df4fcd438d9 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Dec 6 15:25:46 2012 +0000 Add documentation for the DNSSpoof test --- docs/source/tests/dnsspoof.rst | 111 ++++++++++++++++++++++++++++++++++++++++ 1 files changed, 111 insertions(+), 0 deletions(-) diff --git a/docs/source/tests/dnsspoof.rst b/docs/source/tests/dnsspoof.rst new file mode 100644 index 0000000..abb08e4 --- /dev/null +++ b/docs/source/tests/dnsspoof.rst @@ -0,0 +1,111 @@ +Details +======= + +*Test Name*: DNS Spoof + +*Current version*: 0.1 + +*NetTest*: DNSSpoof (https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/nettests/manipulati…) + +*Test Helper*: DNS Test Helper (https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/oonib/testhelpers/d…) + +*Test Type*: Traffic Manipulation + +*Requires Root*: Yes + +Description +=========== + +This test performs A queries to a test resolver and a known good control resolver. The query is considered tampered with if the two responses match. + +How to run the test +=================== + +`./bin/ooniprobe nettests/manipulation/dnsspoof.py [-s] [-k] [-i] -r <test resolver> -h <hostname> -b IP:PORT` + +*test resolver* is a single test resolver (IP address). +*hostname* is the hostname to query. +*IP:PORT* is the address of the known good "control" resolver. +*-s, --ipsrc* Do *not* check if IP src and ICMP IP citation match +*-k, --seqack* Check if TCP sequence number and ACK match in the ICMP citation +*-i, --ipid* Check if the IPID matches when processing answers + + +Sample report +============= + +From running: +`./bin/ooniprobe nettests/manipulation/dnsspoof.py -h torproject.org -r 4.2.2.2:53` + +:: + + ########################################### + # OONI Probe Report for DNS Spoof test + # Thu Dec 6 11:10:38 2012 + ########################################### + --- + options: + collector: null + help: 0 + logfile: null + pcapfile: null + reportfile: null + resume: 0 + subargs: [-h, torproject.org, -r, '4.2.2.2:53'] + test: nettests/manipulation/dnsspoof.py + probe_asn: null + probe_cc: null + probe_ip: 127.0.0.1 + software_name: ooniprobe + software_version: 0.0.7.1-alpha + start_time: 1354828238.0 + test_name: DNS Spoof + test_version: 0.10000000000000001 + ... + --- + input: null + report: + answer_flags: [ipsrc] + answered_packets: + - - raw_packet: !!binary | + RQAAfDj1AAA4EZJIBAICAn8AAAEANQA1AGjH/wAAgYAAAQAEAAAAAAp0b3Jwcm9qZWN0A29yZwAA + AQABCnRvcnByb2plY3QDb3JnAAABAAEAAADnAAQm5UgQCnRvcnByb2plY3QDb3JnAAABAAEAAADn + AARSw0tlCnRvcnByb2plY3QDb3JnAAABAAEAAADnAARWOx4oCnRvcnByb2plY3QDb3JnAAABAAEA + AADnAAQm5UgO + summary: 'IP / UDP / DNS Ans "38.229.72.16" ' + sent_packets: + - - raw_packet: !!binary | + RQAAPAABAABAEfWrfwAAAQQCAgIANQA1AChvjwAAAQAAAQAAAAAAAAp0b3Jwcm9qZWN0A29yZwAA + AQAB + summary: 'IP / UDP / DNS Qry "torproject.org" ' + test_name: test_a_lookup + test_runtime: 0.23476505279541016 + test_started: 1354810238.400979 + ... + --- + input: null + report: + answer_flags: [ipsrc] + answered_packets: + - - raw_packet: !!binary | + RQAAfGQmAAAvEWYLCAgICH8AAAEANQA1AGizfwAAgYAAAQAEAAAAAAp0b3Jwcm9qZWN0A29yZwAA + AQABCnRvcnByb2plY3QDb3JnAAABAAEAAAOEAAQm5UgQCnRvcnByb2plY3QDb3JnAAABAAEAAAOE + AARSw0tlCnRvcnByb2plY3QDb3JnAAABAAEAAAOEAARWOx4oCnRvcnByb2plY3QDb3JnAAABAAEA + AAOEAAQm5UgO + summary: 'IP / UDP / DNS Ans "38.229.72.16" ' + sent_packets: + - - raw_packet: !!binary | + RQAAPAABAABAEeuffwAAAQgICAgANQA1AChlgwAAAQAAAQAAAAAAAAp0b3Jwcm9qZWN0A29yZwAA + AQAB + summary: 'IP / UDP / DNS Qry "torproject.org" ' + test_name: test_control_a_lookup + test_runtime: 0.23965692520141602 + test_started: 1354810238.625988 + ... + --- + input: null + report: {spoofing: false} + test_name: summary + test_runtime: 0.00017499923706054688 + test_started: 1354810238.8703561 + ...

1 0

[ooni-probe/master] Merge pull request #17 from aagbsn/update_readme_with_https_urls
by art＠torproject.org 06 Dec '12

06 Dec '12

commit 0b40aa8923dfa1b1469421050a66cb25bf248580 Merge: 87a3099 b55355f Author: Arturo Filastò <hellais(a)gmail.com> Date: Thu Dec 6 09:44:12 2012 -0800 Merge pull request #17 from aagbsn/update_readme_with_https_urls Use HTTPS where available README.md | 14 +++++++------- 1 files changed, 7 insertions(+), 7 deletions(-)

1 0

[ooni-probe/master] Add references to classes in writing_tests
by art＠torproject.org 06 Dec '12

06 Dec '12

commit 23c0d737fe60cac3947eeeb20453ea4ccfaee6c8 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Dec 6 18:03:01 2012 +0000 Add references to classes in writing_tests --- docs/source/writing_tests.rst | 103 ++++++++++++++++++++--------------------- 1 files changed, 50 insertions(+), 53 deletions(-) diff --git a/docs/source/writing_tests.rst b/docs/source/writing_tests.rst index e31d37f..4ce6268 100644 --- a/docs/source/writing_tests.rst +++ b/docs/source/writing_tests.rst @@ -3,7 +3,7 @@ Writing OONI tests The OONI testing API is heavily influenced and partially based on the python -:class:`unittest` module and :class:`twisted.trial`. +:mod:`unittest` module and :mod:`twisted.trial`. Test Cases @@ -32,8 +32,9 @@ Inputs are what is given as input to every iteration of the Test Case. Iflyou have 100 inputs, then every test case will be run 100 times. To configure a static set of inputs you should define the -:class:`ooni.nettest.TestCase` attribute ``inputs``. The test will be run -``len(inputs)`` times. Any iterable object is a valid ``inputs`` attribute. +:class:`ooni.nettest.NetTestCase` attribute ``inputs``. The test will be +run ``len(inputs)`` times. Any iterable object is a valid ``inputs`` +attribute. If you would like to have inputs be determined from a user specified input file, then you must set the ``inputFile`` attribute. This is an array that @@ -56,12 +57,12 @@ this:: Setup and command line passing ------------------------------ -Tests may define the `setUp` method that will be called every time the Test -Case object is instantiated, in here you may place some common logic to all your -Test Methods that should be run before any testing occurs. +Tests may define the `setUp` method that will be called every time the +Test Case object is instantiated, in here you may place some common logic +to all your Test Methods that should be run before any testing occurs. Command line arguments can be parsed thanks to the twisted -`twisted.python.usage.UsageOptions` class. +:class:`twisted.python.usage.UsageOptions` class. You will have to subclass this and define the NetTestCase attribute usageOptions to point to a subclass of this. @@ -73,7 +74,7 @@ usageOptions to point to a subclass of this. 'URL of the test backend to use'] ] - class MyTestCase(nettest.TestCase): + class MyTestCase(nettest.NetTestCase): usageOptions = UsageOptions inputFile = ['file', 'f', None, "Some foo file"] @@ -83,30 +84,31 @@ usageOptions to point to a subclass of this. self.localOptions['backend'] -You will then be able to access the parsed command line arguments via the class -attribute localOptions. +You will then be able to access the parsed command line arguments via the +class attribute localOptions. The `requiredOptions` attributes specifies an array of parameters that are required for the test to run properly. -`inputFile` is a special class attribute that will be used for processing of -the inputFile. The filename that is read here will be given to the -`ooni.nettest.NetTestCase.inputProcessor` method that will yield, by default, -one line of the file at a time. +`inputFile` is a special class attribute that will be used for processing +of the inputFile. The filename that is read here will be given to the +:class:`ooni.nettest.NetTestCase.inputProcessor` method that will yield, by +default, one line of the file at a time. Test Methods ------------ -These shall be defined inside of your :class:`ooni.nettest.TestCase` subclass. -These will be class methods. +These shall be defined inside of your :class:`ooni.nettest.NetTestCase` +subclass. These will be class methods. -All class methods that are prefixed with test\_ shall be run. Functions that -are relevant to your test should be all lowercase separated by underscore. +All class methods that are prefixed with test\_ shall be run. Functions +that are relevant to your test should be all lowercase separated by +underscore. -To add data to the test report you may write directly to the report object like -so:: +To add data to the test report you may write directly to the report object +like so:: def test_my_function(): result = do_something() @@ -126,36 +128,31 @@ input. Test Templates -------------- -Test templates assist you in writing tests. They already contain all the common -functionality that is useful to running a test of that type. They also take -care of writing the data they collect that is relevant to the test run to the -report file. +Test templates assist you in writing tests. They already contain all the +common functionality that is useful to running a test of that type. They +also take care of writing the data they collect that is relevant to the +test run to the report file. -Currently implemented test templates are `ooni.templates.scapyt -<https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/ooni/templates/scap…>` -for tests based on Scapy, `ooni.templates.tcpt -<https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/ooni/templates/tcpt…>` -for tests based on TCP, `ooni.templates.httpt -<https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/ooni/templates/http…>` -for tests based on HTTP, `ooni.templates.dnst -<https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/ooni/templates/dnst…>` -for tests based on DNS. +Currently implemented test templates are :mod:`ooni.templates.scapyt` for +tests based on Scapy, :mod:`ooni.templates.tcpt` for tests based on TCP, +:mod:`ooni.templates.httpt` for tests based on HTTP, and +:mod:`ooni.templates.dnst` for tests based on DNS. Scapy based tests ................. -Scapy based tests will be a subclass of `ooni.templates.scapyt.BaseScapyTest`. +Scapy based tests will be a subclass of :class:`ooni.templates.scapyt.BaseScapyTest`. It provides a wrapper around the scapy send and receive function that will -write the sent and received packets to the report with sanitization of the src -and destination IP addresses. +write the sent and received packets to the report with sanitization of the +src and destination IP addresses. -It has the same syntax as the Scapy sr function, except that it will return a -deferred. +It has the same syntax as the Scapy sr function, except that it will +return a deferred. To implement a simple ICMP ping based on this function you can do like so -(taken from nettest/examples/example_scapyt.py): +(Taken from :class:`nettest.examples.example_scapyt.ExampleICMPPingScapy`) :: @@ -191,7 +188,7 @@ receive function, the only difference is that instead of using the regular scapy super socket it uses our twisted driven wrapper around it. Alternatively this test can also be written using the -`twisted.defer.inlineCallbacks` decorator, that makes it look more similar to +:func:`twisted.defer.inlineCallbacks` decorator, that makes it look more similar to regular sequential code. :: @@ -253,7 +250,7 @@ Report Format TCP based tests ............... -TCP based tests will subclass `ooni.templates.tcpt.TCPTest`. +TCP based tests will subclass :class:`ooni.templates.tcpt.TCPTest`. This test template facilitates the sending of TCP payloads to the wire and recording the response. @@ -284,12 +281,12 @@ recording the response. The possible failures for a TCP connection are: -`twisted.internet.error.NoRouteError` that corresponds to errno.ENETUNREACH +:class:`twisted.internet.error.NoRouteError` that corresponds to errno.ENETUNREACH -`twisted.internet.error.ConnectionRefusedError` that corresponds to +:class:`twisted.internet.error.ConnectionRefusedError` that corresponds to errno.ECONNREFUSED -`twisted.internet.error.TCPTimedOutError` that corresponds to errno.ETIMEDOUT +:class:`twisted.internet.error.TCPTimedOutError` that corresponds to errno.ETIMEDOUT Report format ************* @@ -323,14 +320,14 @@ TODO finish this with more details HTTP based tests ................ -HTTP based tests will be a subclass of `ooni.templates.httpt.HTTPTest`. +HTTP based tests will be a subclass of :class:`ooni.templates.httpt.HTTPTest`. -It provides methods `ooni.templates.httpt.HTTPTest.processResponseBody` and -`ooni.templates.httpt.HTTPTest.processResponseHeaders` for interacting with the +It provides methods :func:`ooni.templates.httpt.HTTPTest.processResponseBody` and +:func:`ooni.templates.httpt.HTTPTest.processResponseHeaders` for interacting with the response body and headers respectively. For example, to implement a HTTP test that returns the sha256 hash of the -response body (based on nettests/examples/example_httpt.py): +response body (based on :mod:`nettests.examples.example_httpt`): :: @@ -436,12 +433,12 @@ Report format DNS based tests ............... -DNS based tests will be a subclass of `ooni.templates.dnst.DNSTest`. +DNS based tests will be a subclass of :class:`ooni.templates.dnst.DNSTest`. -It provides methods `ooni.templates.dnst.DNSTest.performPTRLookup` and -`ooni.templates.dnst.DNSTest.performALookup` +It provides methods :func:`ooni.templates.dnst.DNSTest.performPTRLookup` +and :func:`ooni.templates.dnst.DNSTest.performALookup` -For example (taken from nettets/examples/example_dnst.py): +For example (taken from :mod:`nettests.examples.example_dnst`): :: @@ -507,4 +504,4 @@ Report format test_started: 1354801371.980114 ... -For a more complex example, see: `DNS Tamper Test <https://gitweb.torproject.org/ooni-probe.git/blob/HEAD:/nettests/blocking/d…>`_ +For a more complex example, see: :mod:`nettests.blocking.dnstamper`

1 0

[ooni-probe/master] Merge pull request #16 from aagbsn/read_tor_path_from_config
by art＠torproject.org 06 Dec '12

06 Dec '12

commit 87a30994789e699782552d7342033a2359e400d7 Merge: 4648dd5 3fd67d2 Author: Arturo Filastò <hellais(a)gmail.com> Date: Thu Dec 6 09:43:58 2012 -0800 Merge pull request #16 from aagbsn/read_tor_path_from_config call launch_tor with the tor_binary from config ooni/runner.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-)

1 0

[ooni-probe/master] Merge branch 'master' of github.com:hellais/ooni-probe
by art＠torproject.org 06 Dec '12

06 Dec '12

commit b829278e5f94b7b0faa261f466318e3c5efb39fa Merge: 9dcd60b 3b830b6 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Dec 6 22:39:44 2012 +0100 Merge branch 'master' of github.com:hellais/ooni-probe * 'master' of github.com:hellais/ooni-probe: Make nettests/* python modules Add nettest apidoc Set release version to 0.1-alpha Add references to classes in writing_tests Add documentation for the DNSSpoof test Add documentation for HTTPHeaderFieldManipulation Clean up writing_tests.rst Remove redundant heading Add automodule for tcpt to ooni.templates.rst Use section headers Add documentation and examples templates httpt and dnst Clean up index.rst Add docstrings to ooni/runner.py Remove automodule for ooni/__init__.py Add docstrings to network helper functions Add dnst to ooni.templates.rst Fix spelling in test documentation call launch_tor with the tor_binary from config Use HTTPS where available README.md | 14 +- docs/source/api/modules.rst | 7 - docs/source/api/ooni.rst | 8 - docs/source/api/ooni.templates.rst | 22 +- docs/source/api/ooni.utils.rst | 2 - docs/source/conf.py | 4 +- docs/source/index.rst | 5 +- docs/source/nettests/modules.rst | 8 + docs/source/nettests/nettests.blocking.rst | 26 ++ docs/source/nettests/nettests.examples.rst | 58 ++++ docs/source/nettests/nettests.experimental.rst | 58 ++++ docs/source/nettests/nettests.manipulation.rst | 58 ++++ docs/source/nettests/nettests.scanning.rst | 10 + docs/source/nettests/nettests.third_party.rst | 10 + docs/source/tests/dnsspoof.rst | 111 +++++++ docs/source/tests/dnstamper.rst | 6 +- .../tests/http_header_field_manipulation.rst | 346 ++++++++++++++++++++ docs/source/tests/http_host.rst | 10 +- docs/source/tests/http_invalid_request_line.rst | 4 +- docs/source/tests/http_requests.rst | 4 +- docs/source/tests/tcpconnect.rst | 2 +- docs/source/tests/traceroute.rst | 2 +- docs/source/writing_tests.rst | 271 +++++++++++++--- nettests/blocking/__init__.py | 1 + nettests/examples/example_http_checksum.py | 27 ++ ooni/runner.py | 8 + ooni/utils/txscapy.py | 5 + 27 files changed, 993 insertions(+), 94 deletions(-)

1 0

[ooni-probe/master] Make nettests/* python modules
by art＠torproject.org 06 Dec '12

06 Dec '12

commit f2504854b778899fdabdcf63ae555a86fc7882da Author: aagbsn <aagbsn(a)extc.org> Date: Thu Dec 6 19:29:41 2012 +0000 Make nettests/* python modules --- nettests/blocking/__init__.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/nettests/__init__.py b/nettests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nettests/blocking/__init__.py b/nettests/blocking/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/nettests/blocking/__init__.py @@ -0,0 +1 @@ + diff --git a/nettests/examples/__init__.py b/nettests/examples/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nettests/experimental/__init__.py b/nettests/experimental/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nettests/manipulation/__init__.py b/nettests/manipulation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nettests/scanning/__init__.py b/nettests/scanning/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nettests/third_party/__init__.py b/nettests/third_party/__init__.py new file mode 100644 index 0000000..e69de29

1 0

[ooni-probe/master] Merge pull request #18 from aagbsn/update_documentation
by art＠torproject.org 06 Dec '12

06 Dec '12

commit 3b830b654312ff6e47863fcde119f760514ccfb0 Merge: 0b40aa8 f250485 Author: Arturo Filastò <hellais(a)gmail.com> Date: Thu Dec 6 12:16:17 2012 -0800 Merge pull request #18 from aagbsn/update_documentation Update documentation docs/source/api/modules.rst | 7 - docs/source/api/ooni.rst | 8 - docs/source/api/ooni.templates.rst | 22 +- docs/source/api/ooni.utils.rst | 2 - docs/source/conf.py | 4 +- docs/source/index.rst | 5 +- docs/source/nettests/modules.rst | 8 + docs/source/nettests/nettests.blocking.rst | 26 ++ docs/source/nettests/nettests.examples.rst | 58 ++++ docs/source/nettests/nettests.experimental.rst | 58 ++++ docs/source/nettests/nettests.manipulation.rst | 58 ++++ docs/source/nettests/nettests.scanning.rst | 10 + docs/source/nettests/nettests.third_party.rst | 10 + docs/source/tests/dnsspoof.rst | 111 +++++++ docs/source/tests/dnstamper.rst | 6 +- .../tests/http_header_field_manipulation.rst | 346 ++++++++++++++++++++ docs/source/tests/http_host.rst | 10 +- docs/source/tests/http_invalid_request_line.rst | 4 +- docs/source/tests/http_requests.rst | 4 +- docs/source/tests/tcpconnect.rst | 2 +- docs/source/tests/traceroute.rst | 2 +- docs/source/writing_tests.rst | 271 +++++++++++++--- nettests/blocking/__init__.py | 1 + nettests/examples/example_http_checksum.py | 27 ++ ooni/runner.py | 7 + ooni/utils/txscapy.py | 5 + 26 files changed, 985 insertions(+), 87 deletions(-)

1 0

[ooni-probe/master] Add nettest apidoc
by art＠torproject.org 06 Dec '12

06 Dec '12

commit c0be39e758b52f797c88bd67782d6cae57838ab2 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Dec 6 19:29:12 2012 +0000 Add nettest apidoc --- docs/source/api/modules.rst | 7 --- docs/source/index.rst | 2 + docs/source/nettests/modules.rst | 8 +++ docs/source/nettests/nettests.blocking.rst | 26 +++++++++++ docs/source/nettests/nettests.examples.rst | 58 ++++++++++++++++++++++++ docs/source/nettests/nettests.experimental.rst | 58 ++++++++++++++++++++++++ docs/source/nettests/nettests.manipulation.rst | 58 ++++++++++++++++++++++++ docs/source/nettests/nettests.scanning.rst | 10 ++++ docs/source/nettests/nettests.third_party.rst | 10 ++++ 9 files changed, 230 insertions(+), 7 deletions(-) diff --git a/docs/source/api/modules.rst b/docs/source/api/modules.rst deleted file mode 100644 index a9f85c8..0000000 --- a/docs/source/api/modules.rst +++ /dev/null @@ -1,7 +0,0 @@ -ooni -==== - -.. toctree:: - :maxdepth: 4 - - ooni diff --git a/docs/source/index.rst b/docs/source/index.rst index 1abb517..711440f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -118,8 +118,10 @@ More developer documentation .. toctree:: :maxdepth: 2 + :glob: oonib writing_tests api/* + nettests/modules glossary diff --git a/docs/source/nettests/modules.rst b/docs/source/nettests/modules.rst new file mode 100644 index 0000000..408bbad --- /dev/null +++ b/docs/source/nettests/modules.rst @@ -0,0 +1,8 @@ +nettests +======== + +.. toctree:: + :glob: + :maxdepth: 2 + + * diff --git a/docs/source/nettests/nettests.blocking.rst b/docs/source/nettests/nettests.blocking.rst new file mode 100644 index 0000000..03a08b6 --- /dev/null +++ b/docs/source/nettests/nettests.blocking.rst @@ -0,0 +1,26 @@ +nettests blocking Package +========================= + +:mod:`nettests.blocking.dnstamper` Module +----------------------------------------- + +.. automodule:: nettests.blocking.dnstamper + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.blocking.http_requests` Module +--------------------------------------------- + +.. automodule:: nettests.blocking.http_requests + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.blocking.tcpconnect` Module +------------------------------------------ + +.. automodule:: nettests.blocking.tcpconnect + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/nettests/nettests.examples.rst b/docs/source/nettests/nettests.examples.rst new file mode 100644 index 0000000..6027455 --- /dev/null +++ b/docs/source/nettests/nettests.examples.rst @@ -0,0 +1,58 @@ +nettests examples Package +========================= + +:mod:`nettests.examples.example_dnst` Module +-------------------------------------------- + +.. automodule:: nettests.examples.example_dnst + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.examples.example_http_checksum` Module +----------------------------------------------------- + +.. automodule:: nettests.examples.example_http_checksum + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.examples.example_httpt` Module +--------------------------------------------- + +.. automodule:: nettests.examples.example_httpt + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.examples.example_myip` Module +-------------------------------------------- + +.. automodule:: nettests.examples.example_myip + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.examples.example_scapyt` Module +---------------------------------------------- + +.. automodule:: nettests.examples.example_scapyt + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.examples.example_scapyt_yield` Module +---------------------------------------------------- + +.. automodule:: nettests.examples.example_scapyt_yield + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.examples.example_tcpt` Module +-------------------------------------------- + +.. automodule:: nettests.examples.example_tcpt + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/nettests/nettests.experimental.rst b/docs/source/nettests/nettests.experimental.rst new file mode 100644 index 0000000..110cf14 --- /dev/null +++ b/docs/source/nettests/nettests.experimental.rst @@ -0,0 +1,58 @@ +nettests experimental Package +============================= + +:mod:`nettests.experimental.chinatrigger` Module +------------------------------------------------ + +.. automodule:: nettests.experimental.chinatrigger + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.experimental.domclass_collector` Module +------------------------------------------------------ + +.. automodule:: nettests.experimental.domclass_collector + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.experimental.http_keyword_filtering` Module +---------------------------------------------------------- + +.. automodule:: nettests.experimental.http_keyword_filtering + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.experimental.http_uk_mobile_networks` Module +----------------------------------------------------------- + +.. automodule:: nettests.experimental.http_uk_mobile_networks + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.experimental.keyword_filtering` Module +----------------------------------------------------- + +.. automodule:: nettests.experimental.keyword_filtering + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.experimental.parasitictraceroute` Module +------------------------------------------------------- + +.. automodule:: nettests.experimental.parasitictraceroute + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.experimental.squid` Module +----------------------------------------- + +.. automodule:: nettests.experimental.squid + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/nettests/nettests.manipulation.rst b/docs/source/nettests/nettests.manipulation.rst new file mode 100644 index 0000000..c05f7a7 --- /dev/null +++ b/docs/source/nettests/nettests.manipulation.rst @@ -0,0 +1,58 @@ +nettests manipulation Package +============================= + +:mod:`nettests.manipulation.captiveportal` Module +-------------------------------------------------- + +.. automodule:: nettests.manipulation.captiveportal + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.manipulation.daphne` Module +------------------------------------------ + +.. automodule:: nettests.manipulation.daphne + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.manipulation.dnsspoof` Module +-------------------------------------------- + +.. automodule:: nettests.manipulation.dnsspoof + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.manipulation.http_header_field_manipulation` Module +------------------------------------------------------------------ + +.. automodule:: nettests.manipulation.http_header_field_manipulation + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.manipulation.http_host` Module +--------------------------------------------- + +.. automodule:: nettests.manipulation.http_host + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.manipulation.http_invalid_request_line` Module +------------------------------------------------------------- + +.. automodule:: nettests.manipulation.http_invalid_request_line + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettests.manipulation.traceroute` Module +---------------------------------------------- + +.. automodule:: nettests.manipulation.traceroute + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/nettests/nettests.scanning.rst b/docs/source/nettests/nettests.scanning.rst new file mode 100644 index 0000000..8bad39b --- /dev/null +++ b/docs/source/nettests/nettests.scanning.rst @@ -0,0 +1,10 @@ +nettests scanning Package +========================= + +:mod:`nettests.scanning.http_url_list` Module +--------------------------------------------- + +.. automodule:: nettests.scanning.http_url_list + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/nettests/nettests.third_party.rst b/docs/source/nettests/nettests.third_party.rst new file mode 100644 index 0000000..6d0e78a --- /dev/null +++ b/docs/source/nettests/nettests.third_party.rst @@ -0,0 +1,10 @@ +nettests third_party Package +============================ + +:mod:`nettests.third_party.netalyzr` Module +------------------------------------------- + +.. automodule:: nettests.third_party.netalyzr + :members: + :undoc-members: + :show-inheritance:

1 0

[ooni-probe/master] Set release version to 0.1-alpha
by art＠torproject.org 06 Dec '12

06 Dec '12

commit 4601552d7805786b4401c9bda85d06c091bf0e7c Author: aagbsn <aagbsn(a)extc.org> Date: Thu Dec 6 18:10:26 2012 +0000 Set release version to 0.1-alpha --- docs/source/conf.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 570e4e6..1362e06 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -51,9 +51,9 @@ copyright = u'2012, The Tor Project' # built documents. # # The short X.Y version. -version = 'v0.0.7.1' +version = '0.1' # The full version, including alpha/beta/rc tags. -release = 'v0.0.7.1-alpha' +release = '0.1-alpha' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.

1 0

[ooni-probe/master] Use twisted fdesc when writing to files
by art＠torproject.org 06 Dec '12

06 Dec '12

commit 9dcd60b51dc692c6e3d042a0ea9b2c580aa803d8 Author: Arturo Filastò <art(a)fuffa.org> Date: Thu Dec 6 22:37:15 2012 +0100 Use twisted fdesc when writing to files * Add support for processing the ASN number of the probe Test reports shall follow the specification detailed inside of docs/reports.rst --- oonib/report/__init__.py | 3 -- oonib/report/file_collector.py | 61 +++++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/oonib/report/__init__.py b/oonib/report/__init__.py index 96667b4..fcbf220 100644 --- a/oonib/report/__init__.py +++ b/oonib/report/__init__.py @@ -1,6 +1,3 @@ -def generateReportID(): - return otime.timestamp() + '_' + randomStr(20) - class MissingField(Exception): pass diff --git a/oonib/report/file_collector.py b/oonib/report/file_collector.py index 9896784..584a11d 100644 --- a/oonib/report/file_collector.py +++ b/oonib/report/file_collector.py @@ -4,8 +4,10 @@ import json import re import os -from oonib.report import generateReportID +from twisted.internet import fdesc + from oonib.report import MissingField, InvalidRequestField +from ooni.utils import randomStr from cyclone import web @@ -15,11 +17,14 @@ def parseNewReportRequest(request): """ version_string = re.compile("[0-9A-Za-z_\-\.]+$") name = re.compile("[a-zA-Z0-9_\- ]+$") + probe_asn = re.compile("AS[0-9]+$") - expected_request = {'software_name': name, + expected_request = { + 'software_name': name, 'software_version': version_string, 'test_name': name, - 'test_version': version_string + 'test_version': version_string, + 'probe_asn': probe_asn } parsed_request = json.loads(request) @@ -28,17 +33,20 @@ def parseNewReportRequest(request): value_to_check = parsed_request[k] except KeyError: raise MissingField(k) + print "Matching %s with %s | %s" % (regexp, value_to_check, k) if re.match(regexp, str(value_to_check)): continue else: raise InvalidRequestField(k) + return parsed_request class NewReportHandlerFile(web.RequestHandler): """ Responsible for creating and updating reports by writing to flat file. """ + def post(self): """ Creates a new report with the input @@ -49,7 +57,7 @@ class NewReportHandlerFile(web.RequestHandler): 'software_version': 'XXX', 'test_name': 'XXX', 'test_version': 'XXX', - 'progress': 'XXX', + 'probe_asn': 'XXX' 'content': 'XXX' } @@ -91,49 +99,58 @@ class NewReportHandlerFile(web.RequestHandler): software_version = report_data['software_version'] test_name = report_data['test_name'] test_version = report_data['test_version'] + probe_asn = report_data['test_version'] content = report_data['content'] - report_id = generateReportID() + if not probe_asn: + probe_asn = "AS0" - #report_filename = '_'.join((report_id, - # report_data['software_name'], - # report_data['software_version'], - # report_data['test_name'], - # report_data['test_version'])) + report_id = otime.timestamp() + '_' \ + + probe_asn + '_' \ + + randomStr(50) # The report filename contains the timestamp of the report plus a # random nonce report_filename = os.path.join(config.main.report_dir, report_id) - report_filename += '.yamloo' response = {'backend_version': backend_version, 'report_id': report_id - } + } + + self.writeToReport(report_filename, + report_data['content']) - fp = open(report_filename, 'w+') - fp.write(report_data['content']) - fp.close() self.write(response) + def writeToReport(report_filename, data): + with open(report_filename, 'w+') as fd: + fdesc.setNonBlocking(fd) + fdesc.writeToFD(data) + def put(self): """ Update an already existing report. - {'report_id': 'XXX', + { + 'report_id': 'XXX', 'content': 'XXX' } """ parsed_request = parseUpdateReportRequest(self.request.body) + report_id = parsed_request['report_id'] + print "Got this request %s" % parsed_request + report_filename = os.path.join(config.main.report_dir, + report_id) - report_filename = os.path.join(config.main.report_dir, report_id) - report_filename += '.yamloo' + self.updateReport(report_filename, parsed_request['content']) + + def updateReport(report_filename, data): try: - with open(report_filename, 'a+') as f: - # XXX-Twisted change this to use t.i.a.fdesc and perhaps make a - # nice little object for it. - f.write(parsed_request['content']) + with open(report_filename, 'a+') as fd: + fdesc.setNonBlocking(fd) + fdesc.writeToFD(data) except IOError as e: web.HTTPError(404, "Report not found")

1 0