March 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Update imports in http_requests.py
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit 1710ca546a6262707bdf3c69da0be75bf01a1d0e Author: aagbsn <aagbsn(a)extc.org> Date: Wed Feb 27 03:36:22 2013 +0100 Update imports in http_requests.py --- nettests/blocking/http_requests.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/nettests/blocking/http_requests.py b/nettests/blocking/http_requests.py index 4d99fc3..9208739 100644 --- a/nettests/blocking/http_requests.py +++ b/nettests/blocking/http_requests.py @@ -10,7 +10,7 @@ from twisted.python import usage from ooni.utils import log from ooni.utils.net import userAgents from ooni.templates import httpt -from ooni.nettest import failureToString, handleAllFailures +from ooni.errors import failureToString, handleAllFailures class UsageOptions(usage.Options): optParameters = [

1 0

[ooni-probe/master] Open YAML report in createReport
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit f94ad6bc7a01b78c3b6a63f8ad965bb5dea23f07 Author: aagbsn <aagbsn(a)extc.org> Date: Thu Feb 28 15:57:23 2013 +0100 Open YAML report in createReport Also adds reporter exceptions ReportAlreadyClosed and ReportNotCreated --- ooni/errors.py | 6 ++++++ ooni/reporter.py | 12 +++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 2023dbf..36a042f 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -126,3 +126,9 @@ class InvalidOONIBCollectorAddress(Exception): class AllReportersFailed(Exception): pass +class ReportNotCreated(Exception): + pass + +class ReportAlreadyClosed(Exception): + pass + diff --git a/ooni/reporter.py b/ooni/reporter.py index 69d4892..c01e904 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -27,6 +27,7 @@ except ImportError: from ooni.errors import InvalidOONIBCollectorAddress +from ooni.errors import ReportNotCreated, ReportAlreadyClosed from ooni import otime from ooni.utils import geodata, pushFilenameStack @@ -184,15 +185,17 @@ class YAMLReporter(OReporter): log.msg("Report already exists with filename %s" % report_path) pushFilenameStack(report_path) - log.debug("Creating %s" % report_path) - self._stream = open(report_path, 'w+') - + self.report_path = report_path OReporter.__init__(self, test_details) def _writeln(self, line): self._write("%s\n" % line) def _write(self, format_string, *args): + if not self._stream: + raise ReportNotCreated + if self._stream.closed: + raise ReportAlreadyClosed s = str(format_string) assert isinstance(s, type('')) if args: @@ -214,6 +217,9 @@ class YAMLReporter(OReporter): """ Writes the report header and fire callbacks on self.created """ + log.debug("Creating %s" % self.report_path) + self._stream = open(self.report_path, 'w+') + self._writeln("###########################################") self._writeln("# OONI Probe Report for %s (%s)" % (self.testDetails['test_name'],

1 0

[ooni-probe/master] Add DNS injection test for detecting censorship when DNS inject happens
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit c32514d224704cdd247a3b1da3519af277065d8e Author: Arturo Filastò <art(a)fuffa.org> Date: Sat Mar 2 20:36:10 2013 +0100 Add DNS injection test for detecting censorship when DNS inject happens --- nettests/experimental/dns_injection.py | 63 ++++++++++++++++++++++++++++++++ 1 files changed, 63 insertions(+), 0 deletions(-) diff --git a/nettests/experimental/dns_injection.py b/nettests/experimental/dns_injection.py new file mode 100644 index 0000000..1424f7f --- /dev/null +++ b/nettests/experimental/dns_injection.py @@ -0,0 +1,63 @@ +# -*- encoding: utf-8 -*- +from twisted.python import usage +from twisted.internet import defer + +from ooni.templates import dnst +from ooni import nettest +from ooni.utils import log + +class UsageOptions(usage.Options): + optParameters = [ + ['resolver', 'r', '8.8.8.1', 'an invalid DNS resolver'], + ['timeout', 't', 3, 'timeout after which we should consider the query failed'] + ] + +class DNSInjectionTest(dnst.DNSTest): + """ + This test detects DNS spoofed DNS responses by performing UDP based DNS + queries towards an invalid DNS resolver. + + For it to work we must be traversing the network segment of a machine that + is actively injecting DNS query answers. + """ + name = "DNS Injection" + description = "Checks for injection of spoofed DNS answers" + version = "0.1" + authors = "Arturo Filastò" + + inputFile = ['file', 'f', None, + 'Input file of list of hostnames to attempt to resolve'] + + usageOptions = UsageOptions + requiredOptions = ['resolver', 'file'] + + def setUp(self): + self.resolver = (self.localOptions['resolver'], 53) + self.queryTimeout = [self.localOptions['timeout']] + + def inputProcessor(self, filename): + fp = open(filename) + for line in fp: + if line.startswith('http://'): + yield line.replace('http://', '').replace('/', '').strip() + else: + yield x.strip() + fp.close() + + def test_injection(self): + self.report['injected'] = None + + d = self.performALookup(self.input, self.resolver) + @d.addCallback + def cb(res): + log.msg("The DNS query for %s is injected" % self.input) + self.report['injected'] = True + + @d.addErrback + def err(err): + err.trap(defer.TimeoutError) + log.msg("The DNS query for %s is not injected" % self.input) + self.report['injected'] = False + + return d +

1 0

[ooni-probe/master] Add tests for YAML reporter
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit 5da092dd0097cd4ffa40de2e6c1a2e99646a516e Author: aagbsn <aagbsn(a)extc.org> Date: Tue Feb 26 15:51:39 2013 +0100 Add tests for YAML reporter --- tests/test_reporter.py | 115 ++++++++++++++++++++++++++++++++++++++---------- 1 files changed, 91 insertions(+), 24 deletions(-) diff --git a/tests/test_reporter.py b/tests/test_reporter.py index d2fcc4e..e21b7a1 100644 --- a/tests/test_reporter.py +++ b/tests/test_reporter.py @@ -1,9 +1,10 @@ from twisted.internet import defer from twisted.trial import unittest -from ooni.reporter import Report, YAMLReporter, OONIBReporter +from ooni.reporter import Report, YAMLReporter, OONIBReporter, safe_dump from ooni.managers import ReportEntryManager, TaskManager from ooni.nettest import NetTest, NetTestState +from ooni.errors import ReportNotCreated, ReportAlreadyClosed from ooni.tasks import TaskWithTimeout from tests.mocks import MockOReporter, MockTaskManager @@ -13,6 +14,7 @@ from tests.mocks import MockOReporterThatFailsWriteOnce from tests.mocks import MockOReporterThatFailsOpen from twisted.python import failure +import yaml class TestReport(unittest.TestCase): def setUp(self): @@ -120,29 +122,94 @@ class TestReport(unittest.TestCase): d.addErrback(f) return d -#class TestYAMLReporter(unittest.TestCase): -# def setUp(self): -# pass -# def tearDown(self): -# pass -# def test_create_yaml_reporter(self): -# raise NotImplementedError -# def test_open_yaml_report_and_succeed(self): -# raise NotImplementedError -# def test_open_yaml_report_and_fail(self): -# raise NotImplementedError -# def test_write_yaml_report_entry(self): -# raise NotImplementedError -# def test_write_multiple_yaml_report_entry(self): -# raise NotImplementedError -# def test_close_yaml_report(self): -# raise NotImplementedError -# def test_write_yaml_report_after_close(self): -# raise NotImplementedError -# def test_write_yaml_report_before_open(self): -# raise NotImplementedError -# def test_close_yaml_report_after_task_complete(self): -# raise NotImplementedError +class TestYAMLReporter(unittest.TestCase): + def setUp(self): + self.testDetails = {'software_name': 'ooniprobe', 'options': + {'pcapfile': None, 'help': 0, 'subargs': ['-f', 'alexa_10'], 'resume': + 0, 'parallelism': '10', 'no-default-reporter': 0, 'testdeck': None, + 'test': 'nettests/blocking/http_requests.py', 'logfile': None, + 'collector': None, 'reportfile': None}, 'test_version': '0.2.3', + 'software_version': '0.0.10', 'test_name': 'http_requests_test', + 'start_time': 1362054343.0, 'probe_asn': 'AS0', 'probe_ip': + '127.0.0.1', 'probe_cc': 'US'} + + def tearDown(self): + pass + def test_create_yaml_reporter(self): + self.assertIsInstance(YAMLReporter(self.testDetails), + YAMLReporter) + + def test_open_yaml_report_and_succeed(self): + r = YAMLReporter(self.testDetails) + r.createReport() + # verify that testDetails was written to report properly + def f(r): + r._stream.seek(0) + details, = yaml.safe_load_all(r._stream) + self.assertEqual(details, self.testDetails) + r.created.addCallback(f) + return r.created + + #def test_open_yaml_report_and_fail(self): + # #XXX: YAMLReporter does not handle failures of this type + # pass + + def test_write_yaml_report_entry(self): + r = YAMLReporter(self.testDetails) + r.createReport() + + report_entry = {'foo':'bar', 'bin':'baz'} + r.writeReportEntry(report_entry) + + # verify that details and entry were written to report + def f(r): + r._stream.seek(0) + report = yaml.safe_load_all(r._stream) + details, entry = report + self.assertEqual(details, self.testDetails) + self.assertEqual(entry, report_entry) + r.created.addCallback(f) + return r.created + + def test_write_multiple_yaml_report_entry(self): + r = YAMLReporter(self.testDetails) + r.createReport() + def reportEntry(): + for x in xrange(10): + yield {'foo':'bar', 'bin':'baz', 'item':x} + for entry in reportEntry(): + r.writeReportEntry(entry) + # verify that details and multiple entries were written to report + def f(r): + r._stream.seek(0) + report = yaml.safe_load_all(r._stream) + details = report.next() + self.assertEqual(details, self.testDetails) + self.assertEqual([r for r in report], [r for r in reportEntry()]) + r.created.addCallback(f) + return r.created + + def test_close_yaml_report(self): + r = YAMLReporter(self.testDetails) + r.createReport() + r.finish() + self.assertTrue(r._stream.closed) + + def test_write_yaml_report_after_close(self): + r = YAMLReporter(self.testDetails) + r.createReport() + r.finish() + def f(r): + r.writeReportEntry("foo") + r.created.addCallback(f) + self.assertFailure(r.created, ReportAlreadyClosed) + + def test_write_yaml_report_before_open(self): + r = YAMLReporter(self.testDetails) + def f(r): + r.writeReportEntry("foo") + r.created.addCallback(f) + self.assertFailure(r.created, ReportNotCreated) #class TestOONIBReporter(unittest.TestCase): # def setUp(self):

1 0

[ooni-probe/master] Merge pull request #53 from aagbsn/task_manager
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit a3f42467534259f85415f65d39971e8d80296cfe Merge: 997a747 5da092d Author: Jacob Appelbaum <jacob(a)appelbaum.net> Date: Mon Mar 4 15:55:26 2013 -0800 Merge pull request #53 from aagbsn/task_manager Add a task scheduler for tests and reports bin/ooniprobe | 14 +- decks/short_no_root.deck | 60 ++ example_inputs/alexa-top-1k.txt | 1000 +++++++++++++++++++++++++++++++ example_inputs/tcpconnect_host_file.txt | 3 + nettests/blocking/http_requests.py | 2 +- nettests/blocking/tcpconnect.py | 2 +- nettests/examples/example_simple.py | 8 + ooni/__init__.py | 2 - ooni/director.py | 294 +++++++++ ooni/errors.py | 134 ++++ ooni/inputunit.py | 85 --- ooni/managers.py | 156 +++++ ooni/nettest.py | 504 ++++++++++++---- ooni/oonicli.py | 168 +++--- ooni/ratelimiting.py | 68 +++ ooni/reporter.py | 326 +++++++---- ooni/runner.py | 357 +----------- ooni/tasks.py | 138 +++++ ooni/templates/dnst.py | 4 +- ooni/templates/httpt.py | 2 +- ooni/templates/tcpt.py | 3 +- tests/mocks.py | 168 ++++++ tests/test_director.py | 58 ++ tests/test_managers.py | 215 +++++++ tests/test_nettest.py | 264 ++++++++ tests/test_reporter.py | 238 ++++++++ tests/test_safe_represent.py | 14 + 27 files changed, 3507 insertions(+), 780 deletions(-) diff --cc ooni/reporter.py index 1f6c3e6,c01e904..594c6b8 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@@ -183,13 -148,19 +148,16 @@@ class OReporter(object) else: test_input = test.input - test_started = test._start_time - test_runtime = time.time() - test_started - - report = {'input': test_input, - 'test_name': test_name, - 'test_started': test_started, - 'test_runtime': test_runtime, - 'report': test_report} - return defer.maybeDeferred(self.writeReportEntry, report) + test_report['input'] = test_input + test_report['test_name'] = test_name + test_report['test_started'] = test._start_time + test_report['test_runtime'] = time.time() - test._start_time + + return defer.maybeDeferred(self.writeReportEntry, test_report) + class InvalidDestination(ReporterException): + pass + class YAMLReporter(OReporter): """ These are useful functions for reporting to YAML format.

1 0

[ooni-probe/master] Update ooniprobe.conf.sample
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit 30204f40d06aed92c2955e8ce4239f1a76652ff4 Author: samsmith <github.com(a)msmith.net> Date: Mon Mar 4 15:58:07 2013 +0000 Update ooniprobe.conf.sample correct comments. --- ooniprobe.conf.sample | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooniprobe.conf.sample b/ooniprobe.conf.sample index 8eac848..51c60f5 100644 --- a/ooniprobe.conf.sample +++ b/ooniprobe.conf.sample @@ -10,9 +10,9 @@ privacy: includeip: false # Should we include the ASN of the probe in the report? includeasn: false - # Should we include the ASN of the probe in the report? + # Should we include the country as reported by GeoIP in the report? includecountry: false - # Should we include the ASN of the probe in the report? + # Should we include the city as reported by GeoIP in the report? includecity: false # Should we collect a full packet capture on the client? includepcap: false

1 0

[ooni-probe/master] Fix typo in dns injection test input processor
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit 989c83bce551122c7f5c4002c2e461ec5bb4435b Author: Arturo Filastò <art(a)fuffa.org> Date: Mon Mar 4 21:37:52 2013 +0100 Fix typo in dns injection test input processor --- nettests/experimental/dns_injection.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/nettests/experimental/dns_injection.py b/nettests/experimental/dns_injection.py index 1424f7f..97233cf 100644 --- a/nettests/experimental/dns_injection.py +++ b/nettests/experimental/dns_injection.py @@ -41,7 +41,7 @@ class DNSInjectionTest(dnst.DNSTest): if line.startswith('http://'): yield line.replace('http://', '').replace('/', '').strip() else: - yield x.strip() + yield line.strip() fp.close() def test_injection(self):

1 0

[ooni-probe/master] Add changelog for ooni
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit 1b67e0597ebfce98118abddfc3ee7cd373f4e164 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Mar 5 01:53:17 2013 +0100 Add changelog for ooni --- ChangeLog.md | 253 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 253 insertions(+), 0 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md new file mode 100644 index 0000000..7d22967 --- /dev/null +++ b/ChangeLog.md @@ -0,0 +1,253 @@ +## v0.0.10 + +### ooniprobe + +* Fix bug that made HTTP based tests stall + +* Update DNS Test example to not import the DNS Test template If you import the + DNS Test template it will be considered a valid test case and command line + argument parsing will not work as expected. see: + #7795 for more details + +* Fix major bug in DNS test template that prevented PTR lookups from working + properly I was calling the queryUDP function with the arguments in the wrong + order. Twisted, why you API no consistent? + +* Add support for specifying the level of parallelism in tests (aka router + melt mode) + +* Do not swallow failures when a test instance fails to run fixes #7714 + +### scripts + +* Add report archival script + +Fix bug in TCP connect test that made it not properly log errors + +* Refactor failure handling code in nettest Add function that traps all the + supported failure and outputs the failure string representing it. + +### documentation + +* Add birdseye view of the ooniprobe architecture + +* Add details on the current implementation status of ooni* + +* Add draft ooniprobe API specification + +* Add instructions for supervisord configuration and clean up README.md + +## 0.0.9 + +### ooniprobe + +* Set the default ASN to 0 + +* Make Beautiful soup a soft depedency + +* Add support for sending the ASN number of the probe: + the ASN number will get sent when creating a new report + +* Add support for obtaining the probes IP address via getinfo address as per + https://trac.torproject.org/projects/tor/ticket/7447 + +* Fix bug in ooniprobe test decks + https://trac.torproject.org/projects/tor/ticket/7664 + +### oonib + +* Use twisted fdesc when writing to files + +* Add support for processing the ASN number of the probe + +* Test reports shall follow the specification detailed inside of docs/reports.rst + +* Add support for setting the tor binary path in oonib/config.py + +### scripts + +* Add a very simple example on how to securely parse the ooniprobe reports + +### documentation + +* Add documentation for the DNSSpoof test + +* Add documentation for HTTPHeaderFieldManipulation + +* Clean up writing_tests.rst + +* Properly use the power of sphinx! + +### Tests + +* fixup Netalyzr third party plugin + +## v0.0.8-alpha + +### ooniprobe + +* Allow test resolver file to have comments. + +* Autostart Tor in default configuration. + +* Add support for starting Tor via txtorcon. + +* Make the sniffer not run in a separate thread, but use a non blocking fdesc. + Do some refactoring of scapy testing, following Factory creational pattern + and a pub-sub pattern for the readers and writers. + +* Extend TrueHeaders to support calculation of difference between two HTTP headers respectful of + capitalization + +* Implement test deck system for automating the specification of command line + arguments for tests + +* Implement sr1 in txscapy + +* Include socksproxy address in HTTP based tests + +* Include the resolver IP:Port in the report + +* Changes to the report format of HTTP Test template derived tests: + Requests are now stored inside of an array to allow + the storing of multiple request/response pairs. + +* Fix bug that lead to httpt based reports to not have the url attribute set + properly. + +* twisted Headers() class edited to avoid header fix in reference to: + https://trac.torproject.org/projects/tor/ticket/7432 + +* Parametrize tor socksport for usage with modified HTTP Agent + +* Update URL List test to take as input also a single URL + +* Clean up filenames of reports generated by ooni-probe: + they now follow the format $testName_report_$timestamp.yamloo + +* Add ooniprobe prefix to logs + +* Respect the includeip = false option in ooniprobe.conf for scapyt derivate + tests: + If the option to not include the IP address of the probe is set, + change the source and destination ip address of the sent and received + packets to 127.0.0.1. + +#### Tests + +* Implement basic keyword filtering detection test. + +* Add ICMP support to multi protocol traceroute test + +* parametrize max_ttl and timeout + +* make max_ttl and timeout be included in the report + +* Port UK Mobile Network test to new API + +* Port daphn3 test + +* Randomize source port by default in traceroute test and include source port in + report + +* Test and Implement HTTP Header Field Manipulation Test (rename it to what we + had originally called it since it made most sense) + +* Implement test that detects DNS spoofing + +* Implement TCP payload sending test template: + Example test based on this test template + +* Make report IDs include the timestamp of the report + +* Add test that detects censorship in HTTP pages based on HTTP body length + +* Add socks proxy support to HTTP Test + +* Create DNS Test template: + Use such template for DNS Tamper test. + Add example usage of DNS Test Template. + +* Refactor captive portal test to run tests in threads + +### oonib + +* Implement basic collector for ooniprobe reports. + Reports can be submitted over the network via http to a remote collector. + Implement the backend component of the collector that writes submitted + reports to flat files, following the report_id naming convention. + +* Implement very simple HTTP Request backend that does only the part of HTTP we + need for testing + +* Make oonib a daemon + +* Loosen up the oonib regexp to support the timestamp report format + +* Add Tor Hidden Service support + +* Make the reporting directory of the collector configurable + +* Implement TCP Echo test helper. + +### scripts + +* Add fabfile for automatic deployment of ooni-probe to remote sites + +### documentation + +* Update documentation on how to setup ooniprobe. + +## v0.0.7.1-alpha + +* Add software version to the report + +* Implement basic oonib reporting to flat files containing the report ID. + +* Improve HTTP Host test to work with the HTTP Requests test backend + +## v0.0.7-alpha + +* Add test_name key to ooniprobe reports + +* Port TCP connect test to the new API + +## v0.0.4-alpha + +* Add multi protocol multi port traceroute for UDP and TCP + +* Implement basic HTTP request test that does capitalization variations on the +HTTP method. + +* Bugfixing and refactoring of txscapy for sending and receiving of scapy +packets. + +## v0.0.3-alpha + +* Implement logging to PCAP file support + +* Remove dependency on trial + +* Port china trigger to new API + +* Rename keyword filtering test to HTTP keyword filtering + +* Refactor install documentation. + +* Convert header of ooniprobe script to a non docstring + +* Add Makefile to fetch Maxmind geoip database files + +* Implement GeoIP lookup support + +* From configuration options it is possible to choice what level of privacy + the prober is willing to accept. Implement config file support You are able + to specify basic and advanced options in YAML format + +* Remove raw inputs and move them to a separate repository and add Makefile to + fetch such lists + +## 0.0.1-alpha + +First release of ooni-probe. woot! +

1 0

[ooni-probe/master] Merge pull request #61 from samsmith/patch-1
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit a677f33fa59fd3838881c67f526642a2adce9fec Merge: a3f4246 30204f4 Author: aagbsn <aagbsn(a)extc.org> Date: Tue Mar 5 05:47:44 2013 -0800 Merge pull request #61 from samsmith/patch-1 Update ooniprobe.conf.sample ooniprobe.conf.sample | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-)

1 0

[ooni-probe/master] Update http_header_field_manipulation.py
by isis＠torproject.org 09 Mar '13

09 Mar '13

commit 5f585c26a26be84b02bc85038b75bbcdf0cf12bb Author: samsmith <github.com(a)msmith.net> Date: Thu Mar 7 11:22:38 2013 +0000 Update http_header_field_manipulation.py correct the (inverted) test description (and rewrite to be clearer) --- .../manipulation/http_header_field_manipulation.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nettests/manipulation/http_header_field_manipulation.py b/nettests/manipulation/http_header_field_manipulation.py index c831814..509f4ef 100644 --- a/nettests/manipulation/http_header_field_manipulation.py +++ b/nettests/manipulation/http_header_field_manipulation.py @@ -38,8 +38,8 @@ class UsageOptions(usage.Options): class HTTPHeaderFieldManipulation(httpt.HTTPTest): """ It performes HTTP requests with request headers that vary capitalization - towards a backend. If we detect that the headers the backend received - matches the ones we have sent then we have detected tampering. + towards a backend. If the headers reported by the server differ from + the ones we sent, then we have detected tampering. """ name = "HTTP Header Field Manipulation" author = "Arturo Filastò"

1 0