commit 283ed262507377a0a14c7c8445831485a43cab06 Merge: 01058a9 aa45aca Author: Arturo Filastò <art@fuffa.org> Date: Sat Mar 1 12:59:44 2014 +0100 Merge branch 'master' into feature/bridget * master: (510 commits) Install pyrex-real so that new pip does not give error Update index of documentation page. Bump to v1.0.0-rc-7 Add WARNING message to every run of ooniprobe. Expand the disclaimer in the README.md Update vagrant link to point to https site. Bump to version 1.0.0 Add instructions on setting up development dependencies. Make the geoip related tests make a less stronger assertion. Make GeoIP AS and GeoIP City soft dependencies. Minor fixes to comments DNS-host-resolution-for-tls_handshake Bug-Fixes-for-tls_handshake Properly expand the home directory to that of the user running ooniprobe. Fix import error in test_managers. Properly handle the cancelled failure. Silence the dummy protocol Write test that reproduces HTTP test stalling condition. Call connectionLost to avoid dirty reactor. Pass the mock super socket in the ScapyFactory constructor ... .coveragerc | 4 + .gitignore | 5 +- .travis.yml | 16 +- ChangeLog.md | 30 + HACKING | 2 +- LICENSE | 5 +- MANIFEST.in | 7 + Makefile | 202 - README.md | 272 +- Vagrantfile | 44 + bin/oonib | 36 - bin/oonid | 7 + collector | 1 - data/Makefile | 12 +- data/decks/complete.deck | 72 + data/decks/complete_no_root.deck | 60 + data/decks/fast.deck | 36 + data/decks/fast_no_root.deck | 24 + data/decks/mlab.deck | 60 + data/decks/mlab_no_root.deck | 48 + data/inputs/Makefile | 13 + data/inputs/README | 47 + data/nettests/examples/example_dns_http.py | 11 + data/nettests/examples/example_dnst.py | 13 + data/nettests/examples/example_http_checksum.py | 27 + data/nettests/examples/example_httpt.py | 36 + data/nettests/examples/example_myip.py | 21 + data/nettests/examples/example_scapyt.py | 29 + data/nettests/examples/example_scapyt_yield.py | 25 + data/nettests/examples/example_simple.py | 8 + data/nettests/examples/example_tcpt.py | 21 + data/ooniprobe.conf.sample | 64 + data/ui/.bowerrc | 5 + data/ui/app/index.html | 39 + data/ui/app/libs/angular-resource/README.md | 4 + .../app/libs/angular-resource/angular-resource.js | 445 + data/ui/app/libs/angular-resource/component.json | 17 + data/ui/app/libs/angular/angular.js |14733 ++++++++++++++++++++ data/ui/app/libs/angular/component.json | 14 + data/ui/app/libs/bootstrap/component.json | 9 + .../libs/bootstrap/css/bootstrap-responsive.css | 1109 ++ data/ui/app/libs/bootstrap/css/bootstrap.css | 6158 ++++++++ .../bootstrap/img/glyphicons-halflings-white.png | Bin 0 -> 8777 bytes .../libs/bootstrap/img/glyphicons-halflings.png | Bin 0 -> 12799 bytes data/ui/app/libs/bootstrap/js/bootstrap.js | 2276 +++ data/ui/app/libs/jquery/component.json | 14 + data/ui/app/libs/jquery/composer.json | 23 + data/ui/app/libs/jquery/jquery.js | 9472 +++++++++++++ data/ui/app/libs/ng-upload/component.json | 23 + data/ui/app/libs/ng-upload/ng-upload.js | 107 + data/ui/app/scripts/app.js | 30 + data/ui/app/scripts/controllers.js | 96 + data/ui/app/scripts/directives.js | 5 + data/ui/app/scripts/filters.js | 5 + data/ui/app/scripts/services.js | 27 + data/ui/app/styles/app.css | 21 + data/ui/app/views/inputs.html | 31 + data/ui/app/views/sidebar.html | 10 + data/ui/app/views/test.html | 48 + data/ui/component.json | 9 + debian/README | 5 + debian/changelog | 5 + debian/compat | 1 + debian/control | 54 + debian/copyright | 51 + debian/rules | 6 + debian/source/format | 1 + decks/before_i_commit.testdeck | 36 - decks/short_no_root.deck | 60 - docs/fabfile.py | 22 + docs/source/architecture.rst | 40 + docs/source/index.rst | 18 +- docs/source/reports.rst | 15 +- docs/source/tests/dnsconsistency.rst | 4 +- docs/source/tests/dnsspoof.rst | 2 +- docs/source/tests/http_requests.rst | 6 +- fabfile.py | 47 +- inputs/Makefile | 11 - inputs/README | 47 - nettests/NETTESTS_HAVE_MOVED.txt | 1 + nettests/blocking/__init__.py | 1 - nettests/blocking/dnsconsistency.py | 173 - nettests/blocking/http_requests.py | 130 - nettests/blocking/tcpconnect.py | 45 - nettests/examples/example_dnst.py | 13 - nettests/examples/example_http_checksum.py | 27 - nettests/examples/example_httpt.py | 36 - nettests/examples/example_myip.py | 17 - nettests/examples/example_scapyt.py | 29 - nettests/examples/example_scapyt_yield.py | 25 - nettests/examples/example_simple.py | 8 - nettests/examples/example_tcpt.py | 21 - .../experimental/bridge_reachability/bridget.py | 462 - nettests/experimental/bridge_reachability/echo.py | 132 - nettests/experimental/chinatrigger.py | 108 - nettests/experimental/dns_injection.py | 63 - nettests/experimental/domclass_collector.py | 33 - nettests/experimental/http_keyword_filtering.py | 45 - nettests/experimental/http_uk_mobile_networks.py | 85 - nettests/experimental/keyword_filtering.py | 52 - nettests/experimental/parasitictraceroute.py | 129 - nettests/experimental/squid.py | 117 - nettests/manipulation/captiveportal.py | 650 - nettests/manipulation/daphne.py | 119 - nettests/manipulation/dnsspoof.py | 69 - .../manipulation/http_header_field_manipulation.py | 189 - nettests/manipulation/http_host.py | 109 - nettests/manipulation/http_invalid_request_line.py | 109 - nettests/manipulation/traceroute.py | 143 - nettests/scanning/http_url_list.py | 98 - nettests/third_party/Makefile | 3 - nettests/third_party/README | 14 - nettests/third_party/netalyzr.py | 58 - nettests/tls-handshake.py | 32 - ooni/__init__.py | 12 +- ooni/api/spec.py | 253 + ooni/config.py | 118 - ooni/deck.py | 214 + ooni/director.py | 182 +- ooni/errors.py | 99 +- ooni/geoip.py | 194 + ooni/managers.py | 82 +- ooni/nettest.py | 455 +- ooni/nettests/blocking/__init__.py | 1 + ooni/nettests/blocking/dnsconsistency.py | 180 + ooni/nettests/blocking/http_requests.py | 124 + ooni/nettests/blocking/tcp_connect.py | 70 + .../experimental/bridge_reachability/bridget.py | 462 + .../experimental/bridge_reachability/echo.py | 132 + ooni/nettests/experimental/chinatrigger.py | 108 + ooni/nettests/experimental/dns_injection.py | 63 + ooni/nettests/experimental/domclass_collector.py | 33 + .../experimental/http_filtering_bypassing.py | 84 + .../experimental/http_keyword_filtering.py | 45 + ooni/nettests/experimental/http_trix.py | 47 + .../experimental/http_uk_mobile_networks.py | 85 + ooni/nettests/experimental/keyword_filtering.py | 52 + ooni/nettests/experimental/parasitictraceroute.py | 129 + ooni/nettests/experimental/script.py | 90 + ooni/nettests/experimental/squid.py | 117 + ooni/nettests/experimental/tls_handshake.py | 837 ++ ooni/nettests/manipulation/captiveportal.py | 650 + ooni/nettests/manipulation/daphne.py | 120 + ooni/nettests/manipulation/dnsspoof.py | 76 + .../manipulation/http_header_field_manipulation.py | 160 + ooni/nettests/manipulation/http_host.py | 160 + .../manipulation/http_invalid_request_line.py | 110 + ooni/nettests/manipulation/traceroute.py | 135 + ooni/nettests/scanning/http_url_list.py | 98 + ooni/nettests/third_party/Makefile | 3 + ooni/nettests/third_party/README | 14 + ooni/nettests/third_party/netalyzr.py | 58 + ooni/oonibclient.py | 252 + ooni/oonicli.py | 223 +- ooni/oonid.py | 20 + ooni/otime.py | 2 + ooni/reporter.py | 281 +- ooni/runner.py | 241 - ooni/settings.py | 99 + ooni/tasks.py | 61 +- ooni/templates/dnst.py | 10 +- ooni/templates/httpt.py | 55 +- ooni/templates/scapyt.py | 8 +- ooni/templates/tcpt.py | 2 + ooni/tests/__init__.py | 3 + ooni/tests/disable_test_dns.py | 24 + ooni/tests/mocks.py | 172 + ooni/tests/test_deck.py | 130 + ooni/tests/test_director.py | 47 + ooni/tests/test_geoip.py | 22 + ooni/tests/test_managers.py | 228 + ooni/tests/test_mutate.py | 15 + ooni/tests/test_nettest.py | 315 + ooni/tests/test_oonibclient.py | 150 + ooni/tests/test_otime.py | 15 + ooni/tests/test_reporter.py | 91 + ooni/tests/test_safe_represent.py | 14 + ooni/tests/test_templates.py | 43 + ooni/tests/test_trueheaders.py | 41 + ooni/tests/test_txscapy.py | 47 + ooni/tests/test_utils.py | 22 + ooni/utils/__init__.py | 16 +- ooni/utils/geodata.py | 40 - ooni/utils/log.py | 27 +- ooni/utils/logo.py | 214 - ooni/utils/net.py | 36 +- ooni/utils/onion.py | 704 - ooni/utils/trueheaders.py | 159 + ooni/utils/txagentwithsocks.py | 215 - ooni/utils/txscapy.py | 21 +- oonib/INSTALL | 4 - oonib/README.md | 117 - oonib/__init__.py | 24 - oonib/config.py | 58 - oonib/db/__init__.py | 30 - oonib/db/tables.py | 123 - oonib/models.py | 129 - oonib/oonibackend.py | 74 - oonib/report/__init__.py | 5 - oonib/report/api.py | 106 - oonib/report/file_collector.py | 193 - oonib/requirements.txt | 24 - oonib/runner.py | 81 - oonib/testhelpers/__init__.py | 5 - oonib/testhelpers/dns_helpers.py | 16 - oonib/testhelpers/http_helpers.py | 154 - oonib/testhelpers/ssl_helpers.py | 9 - oonib/testhelpers/tcp_helpers.py | 72 - ooniprobe.conf.sample | 42 - requirements-dev.txt | 1 + requirements.txt | 41 +- scripts/archive_reports.py | 174 - scripts/build-ooniprobe.sh | 15 + setup-dependencies.sh | 1002 +- setup.py | 118 +- tests/mocks.py | 168 - tests/test-class-design.py | 101 - tests/test_director.py | 58 - tests/test_dns.py | 24 - tests/test_inputunit.py | 29 - tests/test_managers.py | 215 - tests/test_mutate.py | 15 - tests/test_nettest.py | 264 - tests/test_otime.py | 15 - tests/test_reporter.py | 238 - tests/test_runner.py | 80 - tests/test_safe_represent.py | 14 - tests/test_trueheaders.py | 41 - tests/test_utils.py | 20 - 229 files changed, 44851 insertions(+), 8729 deletions(-) diff --cc ooni/reporter.py index 3da1eea,c8d0748..cf35a87 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@@ -24,20 -26,17 +26,18 @@@ try from scapy.packet import Packet except ImportError: log.err("Scapy is not installed.") - + class Packet(object): + pass - from ooni.errors import InvalidOONIBCollectorAddress - from ooni.errors import ReportNotCreated, ReportAlreadyClosed + from ooni import errors from ooni import otime - from ooni.utils import geodata, pushFilenameStack + from ooni.utils import pushFilenameStack from ooni.utils.net import BodyReceiver, StringProducer, userAgents - from ooni import config - - from ooni.tasks import ReportEntry, TaskTimedOut + from ooni.settings import config + from ooni.tasks import ReportEntry, ReportTracker class ReporterException(Exception): pass