December 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Updated ooniprobe.conf with the default_timeout setting for general maximum
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit a07df95df0c79b989098e898b95f8d86ae2615ae Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Dec 13 19:50:00 2012 +0000 Updated ooniprobe.conf with the default_timeout setting for general maximum connection timeout, and changed the old 'checktimeout' to be 'default_timeout'. --- ooni/reporter.py | 14 ++++++-------- ooniprobe.conf.sample | 9 +++++---- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index 37cf9d1..89a62ba 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -121,15 +121,13 @@ def getTestDetails(options): config.privacy.includecountry or \ config.privacy.includecity: log.msg("Running geoIP lookup via check.torproject.org") - if config.privacy.checktimeout is not None and \ - isinstance(config.privacy.checktimeout, int): - my_ip_timeout = config.privacy.checktimeout + if config.advanced.default_timeout is not None and \ + isinstance(config.advanced.default_timeout, int): + my_ip_timeout = float(config.advanced.default_timeout) else: - log.debug( - "reporter.getTestDetails(): bad config.privacy.checktimeout %s" - % str(config.privacy.checktimeout) - ) - my_ip_timeout = 15 + log.debug("getTestDetails(): bad config: default_timeout %s" + % str(config.advanced.default_timeout)) + my_ip_timeout = float(120) client_ip = yield geodata.myIP(connectTimeout=my_ip_timeout) client_location = geodata.IPToLocation(client_ip) else: diff --git a/ooniprobe.conf.sample b/ooniprobe.conf.sample index 67da610..2d6ef47 100644 --- a/ooniprobe.conf.sample +++ b/ooniprobe.conf.sample @@ -17,13 +17,14 @@ privacy: # Should we collect a full packet capture on the client? includepcap: false advanced: - # XXX change this to point to the directory where you have stored the GeoIP - # database file. This should be the directory in which OONI is installed - # /path/to/ooni-probe/data/ + # XXX change this to point to the directory where you have stored the + # GeoIP database file. This should be the directory in which OONI is + # installed /path/to/ooni-probe/data/ geoip_data_dir: /usr/share/GeoIP/ + # Should we display debug messages: debug: true # How many seconds should we wait for connections before timing out?: - checktimeout: 15 + default_timeout: 30 # Location where Tor is installed: tor_binary: '/usr/sbin/tor' # For auto detection

1 0

[ooni-probe/master] Updated .gitignore to exclude *.resume files.
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit bef5d093b88d173165b8f3f77a73ae15342b9e9d Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Dec 13 19:53:15 2012 +0000 Updated .gitignore to exclude *.resume files. --- .gitignore | 14 ++++++++------ 1 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 4cb0775..6bd4da2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,12 @@ *.pyc assets/* -ENV/* build/* +docs/build/* +data/*.dat +ENV/* +fluff/ package/* +pcaps/* proxy-lists/ip-cc.txt proxy-lists/ips.txt proxy-lists/italy-dns-ips.txt @@ -14,9 +18,7 @@ ooni/plugins/dropin.cache ooni/report*.yaml oonib/oonibackend.conf oonib/data/* -fluff/ -pcaps/ -report*.yaml -docs/build/* -data/*.dat +*.resume ooniprobe.conf +report*.yaml +report*.yamloo

1 0

[ooni-probe/master] Patched upstream scapy, see /ooni/lib/001-scapy_missing-exc.sh.patch. Modified
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 73fe964c1979e16672c14ccc652ef115634704ac Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Dec 13 21:17:28 2012 +0000 Patched upstream scapy, see /ooni/lib/001-scapy_missing-exc.sh.patch. Modified txscapy so that user's running unpatched versions will also work. * Fixed timeout settings for ScapySender. * Fixed an error where scapy.utils.PcapWriter wasn't expecting unicode... * Cleaned up extra imports in /ooni/utils/txscapy.py. * Cleaned up a log statement which concatenated to strings (this causes errors for twisted sometimes). * Moved exception classes to beginning of txscapy.py file. --- ooni/lib/001-scapy_missing-exc.sh.patch | 78 +++++++++++++++++++++++++++++++ ooni/utils/txscapy.py | 68 ++++++++++++++++----------- 2 files changed, 119 insertions(+), 27 deletions(-) diff --git a/ooni/lib/001-scapy_missing-exc.sh.patch b/ooni/lib/001-scapy_missing-exc.sh.patch new file mode 100644 index 0000000..3f5095c --- /dev/null +++ b/ooni/lib/001-scapy_missing-exc.sh.patch @@ -0,0 +1,78 @@ +# +# Add a missing Exception class to /scapy/arch/linux.py +# +# To apply this patch: +# STEP 1: Chdir to the source directory. If you have scapy installed, this is +# likely located at /usr/share/pyshared. +# STEP 2: Run the 'patch' program with this file as input, i.e.: +# +# /usr/share/pyshared$ patch -p1 ./scapy/arch/linux.py </path/to/this/file> +# +# To apply this patch with the use of 'applypatch': +# STEP 1: Chdir to the source directory. +# STEP 2: Run the 'applypatch' program with this patch file as input. +# +# @authors: Isis Lovecruft +# @license: This file is part of ooniprobe, see LICENSE file for details. +# @copyright: 2012 Isis Lovecruft +# +#### End of Preamble #### + +#### Patch data follows #### +diff -c 'scapy/arch/linux.py' 'scapy/arch/linux.py-patched' +Index: ./scapy/arch/linux.py +*** ./scapy/arch/linux.py Thu Dec 13 21:26:59 2012 +--- ./scapy/arch/linux.py-patched Thu Dec 13 21:27:53 2012 +*************** +*** 14,21 **** + from scapy.data import * + from scapy.supersocket import SuperSocket + import scapy.arch +! from scapy.error import warning +! + + + # From bits/ioctls.h +--- 14,20 ---- + from scapy.data import * + from scapy.supersocket import SuperSocket + import scapy.arch +! from scapy.error import warning,Scapy_Exception + + + # From bits/ioctls.h +#### End of Patch data #### + +#### ApplyPatch data follows #### +# Data version : 1.0 +# Date generated : Thu Dec 13 21:33:12 2012 +# Generated by : makepatch 2.03 +# Recurse directories : Yes +# Excluded files : (\A|/).*\~\Z +# (\A|/).*\.a\Z +# (\A|/).*\.bak\Z +# (\A|/).*\.BAK\Z +# (\A|/).*\.elc\Z +# (\A|/).*\.exe\Z +# (\A|/).*\.gz\Z +# (\A|/).*\.ln\Z +# (\A|/).*\.o\Z +# (\A|/).*\.obj\Z +# (\A|/).*\.olb\Z +# (\A|/).*\.old\Z +# (\A|/).*\.orig\Z +# (\A|/).*\.rej\Z +# (\A|/).*\.so\Z +# (\A|/).*\.Z\Z +# (\A|/)\.del\-.*\Z +# (\A|/)\.make\.state\Z +# (\A|/)\.nse_depinfo\Z +# (\A|/)core\Z +# (\A|/)tags\Z +# (\A|/)TAGS\Z +# p 'scapy/arch/linux.py' 16563 1355434073 0100644 +#### End of ApplyPatch data #### + +#### End of Patch kit [created: Thu Dec 13 21:33:12 2012] #### +#### Patch checksum: 56 1830 50583 #### +#### Checksum: 74 2507 41533 #### diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py index e91da09..647f20e 100644 --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@ -18,35 +18,33 @@ from twisted.internet import reactor, threads, error from twisted.internet import defer, abstract from zope.interface import implements -from scapy.config import conf -from scapy.all import PcapWriter, MTU from scapy.all import BasePacketList, conf, PcapReader -from scapy.all import Gen, SetGen +from scapy.all import Gen, SetGen, MTU +from scapy.error import Scapy_Exception -from ooni.utils import log from ooni import config +from ooni.utils import log try: - conf.use_pcap = True - conf.use_dnet = True - from scapy.all import PcapWriter from scapy.arch import pcapdnet - config.pcap_dnet = True - -except ImportError, e: - log.err("pypcap or dnet not installed. " - "Certain tests may not work.") - + conf.use_pcap = True + conf.use_dnet = True +except ImportError: + log.err("pypcap or dnet not installed. Certain tests may not work.") config.pcap_dnet = False conf.use_pcap = False conf.use_dnet = False - from scapy.all import PcapWriter -from scapy.all import BasePacketList, PcapReader -from scapy.all import Gen, SetGen, MTU + +class ProtocolNotRegistered(Exception): + pass + +class ProtocolAlreadyRegistered(Exception): + pass + def getNetworksFromRoutes(): from scapy.all import conf, ltoa, read_routes @@ -72,11 +70,6 @@ def getDefaultIface(): return net.iface raise IfaceError -class ProtocolNotRegistered(Exception): - pass - -class ProtocolAlreadyRegistered(Exception): - pass class ScapyFactory(abstract.FileDescriptor): """ @@ -88,9 +81,23 @@ class ScapyFactory(abstract.FileDescriptor): if interface == 'auto': interface = getDefaultIface() if not super_socket: - super_socket = conf.L3socket(iface=interface, - promisc=True, filter='') - #super_socket = conf.L2socket(iface=interface) + try: + # scapy is missing an import in /scapy/arch/linux.py + # see /ooni/lib/000-scapy-missing-exc.patch + super_socket = conf.L3socket(iface=interface, + promisc=True, filter='') + #super_socket = conf.L2socket(iface=interface) + except NameError, ne: + raise Scapy_Exception("Filter parse error") + except Scapy_Exception, se: + log.err("txscapy: %s" % se.message) + log.debug("txscapy: Trying socket setup again without filter") + try: + super_socket = conf.L3socket(iface=interface, + promisc=True) + except: + log.err("txscapy: Socket setup failed, giving up...") + raise sys.exit(1) self.protocols = [] fdesc._setCloseOnExec(super_socket.ins.fileno()) @@ -149,9 +156,13 @@ class ScapyProtocol(object): raise NotImplementedError class ScapySender(ScapyProtocol): - timeout = 5 - # This deferred will fire when we have finished sending a receiving packets. + # This deferred will fire when we have finished sending and receiving + # packets. + timeout = 5 + if config.advanced.default_timeout: + timeout = int(config.advanced.default_timeout) + # Should we look for multiple answers for the same sent packet? multi = False @@ -232,7 +243,10 @@ class ScapySender(ScapyProtocol): class ScapySniffer(ScapyProtocol): def __init__(self, pcap_filename, *arg, **kw): - self.pcapwriter = PcapWriter(pcap_filename, *arg, **kw) + # The "str(pcap_filename)" explicit typing is due to an error where + # scapy.utils.PcapWriter expects strings, and it's getting unicode + # due to the "# -*- coding: utf-8 -*-"...this might be a problem... + self.pcapwriter = PcapWriter(str(pcap_filename), *arg, **kw) def packetReceived(self, packet): self.pcapwriter.write(packet)

1 0

[ooni-probe/master] Fixed a misnamed variable, and a call to log.err when there was not an error
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 4df3c8d2d0d55f4dd1ca5cce809f31e13c961aa6 Author: Isis Lovecruft <isis(a)torproject.org> Date: Mon Dec 17 09:27:18 2012 +0000 Fixed a misnamed variable, and a call to log.err when there was not an error followed by an unhandled exception in ooni/utils/geodata.py. --- ooni/utils/geodata.py | 10 ++++++---- 1 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ooni/utils/geodata.py b/ooni/utils/geodata.py index 07cc55d..d089490 100644 --- a/ooni/utils/geodata.py +++ b/ooni/utils/geodata.py @@ -35,7 +35,7 @@ def myIP(*args, **kwargs): target_site = 'https://check.torproject.org/' regexp = "Your IP address appears to be: <b>(.+?)<\/b>" if 'connectTimeout' in kwargs: - myAgent = Agent(reactor, connectTimeout=connectTimeout) + myAgent = Agent(reactor, connectTimeout=kwargs['connectTimeout']) else: myAgent = Agent(reactor, *args, **kwargs) @@ -73,9 +73,11 @@ def IPToLocation(ipaddr): asn_dat = pygeoip.GeoIP(asn_file) location['asn'] = asn_dat.org_by_addr(ipaddr) except IOError: - log.err("Could not find GeoIP data files. Go into data/ " - "and run make geoip") - raise GeoIPDataFilesNotFound + try: + raise GeoIPDataFilesNotFound( + "Couldn't find GeoIP files. Go to ./data and run \"make geoip\".") + except GeoIPDataFilesNotFound, gnf: + log.err(gnf) return location

1 0

[ooni-probe/master] Apply patch from Dominic at Measurement Labs; adds check for missing file to
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 2e939403f73b39372030e63ac051b8f6233d6c31 Author: Dominic Hamon <dominic(a)measurementlab.net> Date: Mon Dec 17 11:19:05 2012 +0000 Apply patch from Dominic at Measurement Labs; adds check for missing file to commithook script. --- before_i_commit.sh | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/before_i_commit.sh b/before_i_commit.sh index 8b8180f..fda323e 100755 --- a/before_i_commit.sh +++ b/before_i_commit.sh @@ -10,7 +10,7 @@ # rm *.yamloo; rm before_i_commit.log # -rm before_i_commit.log +rm -f before_i_commit.log find . -type f -name "*.py[co]" -delete @@ -18,7 +18,7 @@ find . -type f -name "*.py[co]" -delete echo "Below you should not see anything" echo "---------------------------------" -grep "Error: " before_i_commit.log +[ -f before_i_commit.log ] && grep "Error: " before_i_commit.log echo "---------------------------------" echo "If you do, it means something is wrong." echo "Read through the log file and fix it."

1 0

[ooni-probe/master] Merge remote-tracking branch 'tpo-common/master'
by isis＠torproject.org 18 Dec '12

18 Dec '12

1 0

[ooni-probe/master] Merge branch 'dev-missing-test'
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 705587ae5d974bca56d0ac4c407bec4c312c37f4 Merge: d8d286a d38df33 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Dec 18 04:07:47 2012 +0000 Merge branch 'dev-missing-test' ooni/runner.py | 30 +++++++++++++++++++++--------- 1 files changed, 21 insertions(+), 9 deletions(-)

1 0

[ooni-probe/master] Merge branch 'master' into tcp-flags
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit d8d286ac341dfc1a9cb56e8d5d4d1c49e7fee53a Merge: 2e93940 8d3a668 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Dec 18 03:55:51 2012 +0000 Merge branch 'master' into tcp-flags Conflicts: ooni/nettest.py ooni/oonicli.py ooni/reporter.py ooni/runner.py ooni/utils/__init__.py ooni/utils/geodata.py ooni/utils/hacks.py ooni/utils/log.py ooni/utils/net.py ooni/utils/txscapy.py scripts/before_i_commit.sh README.md | 51 +-- before_i_commit.sh | 29 -- before_i_commit.testdeck | 33 -- bin/INSTRUCTIONS | 15 - bin/Makefile | 54 --- bin/canary | 27 -- bin/old_ooniprobe | 80 ---- bin/oonib | 4 - decks/before_i_commit.testdeck | 33 ++ docs/source/api/modules.rst | 7 - docs/source/api/ooni.rst | 8 - docs/source/api/ooni.templates.rst | 22 +- docs/source/api/ooni.utils.rst | 2 - docs/source/architecture.rst | 204 ++++++++++ docs/source/conf.py | 4 +- docs/source/index.rst | 6 +- docs/source/nettests/modules.rst | 8 + docs/source/nettests/nettests.blocking.rst | 26 ++ docs/source/nettests/nettests.examples.rst | 58 +++ docs/source/nettests/nettests.experimental.rst | 58 +++ docs/source/nettests/nettests.manipulation.rst | 58 +++ docs/source/nettests/nettests.scanning.rst | 10 + docs/source/nettests/nettests.third_party.rst | 10 + docs/source/reports.rst | 9 +- docs/source/tests/dnsspoof.rst | 111 ++++++ docs/source/tests/dnstamper.rst | 6 +- .../tests/http_header_field_manipulation.rst | 343 ++++++++++++++++ docs/source/tests/http_host.rst | 10 +- docs/source/tests/http_invalid_request_line.rst | 4 +- docs/source/tests/http_requests.rst | 4 +- docs/source/tests/tcpconnect.rst | 2 +- docs/source/tests/traceroute.rst | 2 +- docs/source/writing_tests.rst | 275 +++++++++++--- nettests/blocking/__init__.py | 1 + nettests/blocking/dnstamper.py | 6 +- nettests/examples/example_http_checksum.py | 27 ++ nettests/third_party/netalyzr.py | 1 - ooni/__init__.py | 3 +- ooni/config.py | 37 +- ooni/kit/daphn3.py | 4 +- ooni/kit/domclass.py | 60 ++-- ooni/lib/__init__.py | 5 - ooni/lib/secdev.org.pem | 20 - ooni/nettest.py | 17 + ooni/nodes.py | 174 -------- ooni/oonicli.py | 23 +- ooni/otime.py | 3 - ooni/reporter.py | 97 +++-- ooni/runner.py | 54 ++- ooni/templates/dnst.py | 57 ++-- ooni/templates/httpt.py | 4 - ooni/templates/scapyt.py | 5 - ooni/utils/__init__.py | 35 ++- ooni/utils/geodata.py | 33 -- ooni/utils/hacks.py | 4 - ooni/utils/log.py | 34 +- ooni/utils/txscapy.py | 13 +- oonib/config.py | 10 + oonib/oonibackend.py | 4 +- oonib/report/__init__.py | 3 - oonib/report/api.py | 6 +- oonib/report/file_collector.py | 96 ++++-- oonib/requirements.txt | 40 +-- oonib/runner.py | 3 + oonib/testhelpers/ssl_helpers.py | 2 +- requirements.txt | 2 - scripts/before_i_commit.sh | 43 ++ scripts/example_parser.py | 22 + scripts/submit-patch | 100 +++++ setup.py | 37 ++ submit-patch | 100 ----- tests/test_runner.py | 20 +- tests/test_utils.py | 20 + to-be-ported/TODO | 418 -------------------- to-be-ported/spec/proxooni-spec.txt | 65 --- to-be-ported/very-old/TODO.plgoons | 79 ---- to-be-ported/very-old/TO_BE_PORTED | 14 - to-be-ported/very-old/ooni-probe.diff | 358 ----------------- to-be-ported/very-old/ooni/#namecheck.py# | 39 -- to-be-ported/very-old/ooni/.DS_Store | Bin 15364 -> 0 bytes to-be-ported/very-old/ooni/__init__.py | 12 - to-be-ported/very-old/ooni/command.py | 250 ------------ to-be-ported/very-old/ooni/dns_poisoning.py | 43 -- to-be-ported/very-old/ooni/dnsooni.py | 356 ----------------- to-be-ported/very-old/ooni/helpers.py | 38 -- to-be-ported/very-old/ooni/http.py | 306 -------------- to-be-ported/very-old/ooni/input.py | 33 -- to-be-ported/very-old/ooni/namecheck.py | 39 -- .../very-old/ooni/plugins/dnstest_plgoo.py | 84 ---- to-be-ported/very-old/ooni/plugins/http_plgoo.py | 70 ---- to-be-ported/very-old/ooni/plugins/marco_plgoo.py | 377 ------------------ to-be-ported/very-old/ooni/plugins/proxy_plgoo.py | 69 ---- .../very-old/ooni/plugins/simple_dns_plgoo.py | 35 -- to-be-ported/very-old/ooni/plugins/tcpcon_plgoo.py | 278 ------------- to-be-ported/very-old/ooni/plugins/tor.py | 80 ---- to-be-ported/very-old/ooni/plugins/torrc | 9 - to-be-ported/very-old/ooni/plugooni.py | 106 ----- to-be-ported/very-old/ooni/transparenthttp.py | 41 -- var/old_notes.txt | 418 ++++++++++++++++++++ var/proxooni-spec.txt | 65 +++ var/secdev.org.pem | 20 + 101 files changed, 2327 insertions(+), 4173 deletions(-) diff --cc ooni/oonicli.py index 0bf4d55,b06bde9..6afc453 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@@ -105,34 -75,21 +105,23 @@@ def updateStatusBar(stop_func) # moment. eta = config.state[test_filename].eta() progress = config.state[test_filename].progress() - progress_bar_frmt = "[%s] %s%%" % (test_filename, progress) - print progress_bar_frmt + while progress is not None: + print "[%s] %s%%" % (test_filename, progress) + else: + print "[%s] All tests in file completed." % test_filename + stop_func() def testsEnded(*arg, **kw): - """ - You can place here all the post shutdown tasks. - """ - log.debug("testsEnded: Finished running all tests") + """You can place here all the post shutdown tasks.""" + log.debug("Finished running all tests") config.start_reactor = False - try: reactor.stop() - except: pass + if not reactor.running: + try: reactor.stop() + except: reactor.runUntilCurrent() - def startSniffing(): - from ooni.utils.txscapy import ScapyFactory, ScapySniffer - try: - checkForRoot() - except PermissionsError: - print "[!] Includepcap options requires root priviledges to run" - print " you should run ooniprobe as root or disable the options in ooniprobe.conf" - sys.exit(1) - - print "Starting sniffer" - config.scapyFactory = ScapyFactory(config.advanced.interface) - - sniffer = ScapySniffer(config.reports.pcap) - config.scapyFactory.registerProtocol(sniffer) + def testFailed(failure): + log.err("Failed in running a test inside a test list") + failure.printTraceback() def runTestList(none, test_list): """ @@@ -149,16 -106,12 +138,17 @@@ deck_dl.append(d1) d2 = defer.DeferredList(deck_dl) - d2.addBoth(testsEnded) + d2.addCallback(testsEnded) + d2.addErrback(testFailed) - # Print every 5 second the list of current tests running - l = task.LoopingCall(updateStatusBar) - l.start(5.0) + try: + # Print every 5 second the list of current tests running + coop = task.Cooperator(started=False) + coop.cooperate(updateStatusBar) #this will need a .next() method + coop.start() + except StopIteration: + return d2 + return d2 def errorRunningTests(failure): @@@ -177,18 -132,13 +168,20 @@@ def run() log.start(cmd_line_options['logfile']) + config.cmd_line_options = cmd_line_options + if config.privacy.includepcap: - log.msg("Starting") - if not config.reports.pcap: - config.generatePcapFilename() - runner.startSniffing() + try: + checkForRoot() + except PermissionsError, pe: + m = ("Capturing packets requires administrator/root privileges. ", + "Run ooniprobe as root or set 'includepcap = false' in ", + "ooniprobe.conf .") + log.warn("%s" % m) + sys.exit(1) + else: + log.msg("Starting packet capture") + runner.startSniffing() resume = cmd_line_options['resume'] diff --cc ooni/reporter.py index 89a62ba,728c3f5..aac163d --- a/ooni/reporter.py +++ b/ooni/reporter.py @@@ -1,20 -1,12 +1,21 @@@ +#-*- coding: utf-8 -*- +# +# reporter.py +# ----------- +# In here goes the logic for the creation of ooniprobe reports. +# +# :authors: Arturo Filastò, Isis Lovecruft +# :license: see included LICENSE file + + import traceback import itertools import logging +import sys +import os import time import yaml import json - import traceback -import sys -import os + import re from yaml.representer import * from yaml.emitter import * @@@ -25,15 -17,20 +26,16 @@@ from twisted.trial import reporte from twisted.internet import defer, reactor from twisted.internet.error import ConnectionRefusedError -from ooni.utils import log +from ooni import config, otime - from ooni.utils import log, geodata ++from ooni.utils import log, geodata, pushFilenameStack +from ooni.utils.net import BodyReceiver, StringProducer, userAgents try: from scapy.packet import Packet except ImportError: log.err("Scapy is not installed.") + -from ooni import otime -from ooni.utils import geodata, pushFilenameStack -from ooni.utils.net import BodyReceiver, StringProducer, userAgents - -from ooni import config - def createPacketReport(packet_list): """ Takes as input a packet a list. @@@ -159,8 -152,8 +157,8 @@@ def getTestDetails(options) 'test_version': options['version'], 'software_name': 'ooniprobe', 'software_version': software_version - } + } - defer.returnValue(test_details) + return test_details class OReporter(object): def __init__(self, cmd_line_options): @@@ -307,15 -310,12 +315,12 @@@ class OONIBReporter(OReporter) """ Creates a report on the oonib collector. """ - test_name = options['name'] - test_version = options['version'] - - url = self.backend_url + '/report/new' + url = self.backend_url + '/report' try: - test_details = yield getTestDetails(options) + test_details = getTestDetails(options) - except Exception, e: - log.exception(e) + except Exception, ex: + log.exception(ex) test_details['options'] = self.cmd_line_options diff --cc ooni/runner.py index 19dc171,4ebfa0b..7488d76 --- a/ooni/runner.py +++ b/ooni/runner.py @@@ -27,50 -15,18 +27,54 @@@ from twisted.trial.unittest import Skip from txtorcon import TorProtocolFactory, TorConfig from txtorcon import TorState, launch_tor -from ooni import config - +from ooni import config, nettest, reporter +from ooni.inputunit import InputUnitFactory from ooni.reporter import OONIBReporter, YAMLReporter, OONIBReportError + + from ooni.inputunit import InputUnitFactory + from ooni.nettest import NetTestCase, NoPostProcessor + -from ooni.utils import log, checkForRoot, pushFilenameStack -from ooni.utils import NotRootError, Storage +from ooni.utils import log, checkForRoot +from ooni.utils import PermissionsError, Storage from ooni.utils.net import randomFreePort + +class NoTestCasesFound(Exception): + pass + +class InvalidResumeFile(Exception): + pass + +class noResumeSession(Exception): + pass + +class InvalidConfigFile(Exception): + message = "Invalid setting in ooniprobe.conf: " + +class UnableToStartTor(Exception): + pass + + +def isTestCase(obj): + """Return True if obj is a subclass of NetTestCase, False otherwise.""" + try: + return issubclass(obj, nettest.NetTestCase) + except TypeError: + return False + +def checkRequiredOptions(test_instance): + """ + If test_instance has an attribute 'requiredOptions', then check that + those options were utilised on the commandline. + """ + required = getattr(test_instance, 'requiredOptions', None) + if required: + for required_option in required: + log.debug("Checking if %s is present" % required_option) + if not test_instance.localOptions[required_option]: + raise usage.UsageError("%s not specified!" % required_option) + - def processTest(obj): + def processTest(obj, cmd_line_options): """ Process the parameters and :class:`twisted.python.usage.Options` of a :class:`ooni.nettest.Nettest`. @@@ -101,29 -55,43 +105,29 @@@ obj.usageOptions.optFlags = [] for flag in obj.baseFlags: obj.usageOptions.optFlags.append(flag) + if obj.inputFile: # inputFile is the optParameters list + obj.usageOptions.optParameters.append(obj.inputFile) options = obj.usageOptions() - options.parseOptions(config.cmd_line_options['subargs']) - + options.parseOptions(cmd_line_options['subargs']) obj.localOptions = options - if obj.inputFile: + if obj.inputFile: # inputFilename is the actual filename obj.inputFilename = options[obj.inputFile[0]] try: - log.debug("processing options") - tmp_test_case_object = obj() - tmp_test_case_object._checkRequiredOptions() - - except usage.UsageError, e: - test_name = tmp_test_case_object.name - log.err("There was an error in running %s!" % test_name) - log.err("%s" % e) + log.debug("Parsing commandline options") + tmp_test_instance = obj() + checkRequiredOptions(tmp_test_instance) + except usage.UsageError, ue: + log.err("%s" % ue) options.opt_help() - raise usage.UsageError("Error in parsing command line args for %s" % test_name) - - if obj.requiresRoot: - try: - checkForRoot() - except NotRootError: - log.err("%s requires root to run" % obj.name) - sys.exit(1) - - return obj - -def isTestCase(obj): - try: - return issubclass(obj, NetTestCase) - except TypeError: - return False + raise usage.UsageError("Error parsing command line args for %s" + % tmp_test_case_object.name) + else: + return obj - def findTestClassesFromFile(filename): + def findTestClassesFromFile(cmd_line_options): """ Takes as input the command line config parameters and returns the test case classes. @@@ -456,9 -353,11 +462,8 @@@ def increaseInputUnitIdx(test_filename) config.stateDict[test_filename] += 1 yield updateResumeFile(test_filename) -def updateProgressMeters(test_filename, input_unit_factory, - test_case_number): - """ - Update the progress meters for keeping track of test state. - """ +def updateProgressMeters(test_filename, input_unit_factory, test_case_number): + """Update the progress meters for keeping track of test state.""" - log.msg("Setting up progress meters") if not config.state.test_filename: config.state[test_filename] = Storage() @@@ -560,8 -451,16 +565,13 @@@ def runTestCases(test_cases, options, c except Exception: log.exception("Problem in running test") + yaml_reporter.finish() -class UnableToStartTor(Exception): - pass - def startTor(): + """ Starts Tor + Launches a Tor with :param: socks_port :param: control_port + :param: tor_binary set in ooniprobe.conf + """ @defer.inlineCallbacks def state_complete(state): config.tor_state = state @@@ -631,6 -539,11 +650,12 @@@ def startSniffing() print "Starting sniffer" config.scapyFactory = ScapyFactory(config.advanced.interface) - if os.path.exists(config.reports.pcap): ++ pcapfile = config.reports.pcap ++ if pcapfile and os.path.exists(pcapfile): + print "Report PCAP already exists with filename %s" % config.reports.pcap + print "Renaming files with such name..." + pushFilenameStack(config.reports.pcap) + sniffer = ScapySniffer(config.reports.pcap) config.scapyFactory.registerProtocol(sniffer) diff --cc ooni/utils/__init__.py index be5b38a,8510a3b..efa609a --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@@ -1,13 -1,10 +1,10 @@@ - """ - - """ - +import imp +import os import logging import string import random + import glob import yaml -import imp -import os class Storage(dict): """ @@@ -88,3 -82,34 +85,33 @@@ def randomStr(length, num=True) if num: chars += string.digits return ''.join(random.choice(chars) for x in range(length)) + - + def pushFilenameStack(filename): + """ + Takes as input a target filename and checks to see if a file by such name + already exists. If it does exist then it will attempt to rename it to .1, + if .1 exists it will rename .1 to .2 if .2 exists then it will rename it to + .3, etc. + This is similar to pushing into a LIFO stack. + + XXX: This will not work with stacks bigger than 10 elements because + glob.glob(".*") will return them in the wrong order (a.1, a.11, a.2, a.3, + etc.) + This is probably not an issue since the only thing it causes is that files + will be renamed in the wrong order and you shouldn't have the same report + filename for more than 10 reports anyways, because you should be making + ooniprobe generate the filename for you. + + Args: + filename (str): the path to filename that you wish to create. + """ + stack = glob.glob(filename+".*") + for f in reversed(stack): + c_idx = f.split(".")[-1] + c_filename = '.'.join(f.split(".")[:-1]) + new_idx = int(c_idx) + 1 + new_filename = "%s.%s" % (c_filename, new_idx) + os.rename(f, new_filename) + os.rename(filename, filename+".1") + + diff --cc ooni/utils/hacks.py index 4cf94d0,64b5a53..4bbdf48 --- a/ooni/utils/hacks.py +++ b/ooni/utils/hacks.py @@@ -1,17 -1,5 +1,13 @@@ +# -*- encoding: utf-8 -*- +# +# hacks.py +# ******** # When some software has issues and we need to fix it in a # hackish way, we put it in here. This one day will be empty. +# +# :authors: Arturo Filastò, Isis Lovecruft +# :licence: see LICENSE + - from yaml.representer import * - from yaml.emitter import * - from yaml.serializer import * - from yaml.resolver import * import copy_reg diff --cc ooni/utils/log.py index 2721807,0740c10..70543ce --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@@ -1,24 -1,28 +1,31 @@@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from functools import wraps import sys import os -import logging import traceback +import logging from twisted.python import log as txlog + from twisted.python import util from twisted.python.failure import Failure from twisted.python.logfile import DailyLogFile from ooni import otime from ooni import config --## Get rid of the annoying "No route found for --## IPv6 destination warnings": --logging.getLogger("scapy.runtime").setLevel(logging.ERROR) + + class LogWithNoPrefix(txlog.FileLogObserver): + def emit(self, eventDict): + text = txlog.textFromEventDict(eventDict) + if text is None: + return + + util.untilConcludes(self.write, "%s\n" % text) + util.untilConcludes(self.flush) # Hoorj! def start(logfile=None, application_name="ooniprobe"): daily_logfile = None @@@ -46,21 -42,17 +45,22 @@@ txlog.addObserver(txlog.FileLogObserver(daily_logfile).emit) def stop(): - txlog.msg("Stopping OONI") + print "Stopping OONI" def msg(msg, *arg, **kw): - txlog.msg(msg, logLevel=logging.INFO, *arg, **kw) + print "%s" % msg def debug(msg, *arg, **kw): - txlog.msg(msg, logLevel=logging.DEBUG, *arg, **kw) + if config.advanced.debug: + print "[D] %s" % msg +def warn(msg, *arg, **kw): + txlog.logging.captureWarnings('true') + txlog.logging.warn(msg) + #txlog.showwarning() + def err(msg, *arg, **kw): - txlog.err("Error: " + str(msg), logLevel=logging.ERROR, *arg, **kw) + print "[!] %s" % msg def exception(error): """ diff --cc ooni/utils/txscapy.py index 647f20e,62bde94..7fa31fa --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@@ -36,17 -31,20 +36,24 @@@ except ImportError config.pcap_dnet = False conf.use_pcap = False conf.use_dnet = False - from scapy.all import PcapWriter + + class DummyPcapWriter: + def __init__(self, pcap_filename, *arg, **kw): + log.err("Initializing DummyPcapWriter. We will not actually write to a pcapfile") - + def write(self): + pass - + PcapWriter = DummyPcapWriter +class ProtocolNotRegistered(Exception): + pass + +class ProtocolAlreadyRegistered(Exception): + pass + def getNetworksFromRoutes(): + """ Return a list of networks from the routing table """ from scapy.all import conf, ltoa, read_routes from ipaddr import IPNetwork, IPAddress diff --cc scripts/before_i_commit.sh index 0000000,918b137..a504ad8 mode 000000,100755..100755 --- a/scripts/before_i_commit.sh +++ b/scripts/before_i_commit.sh @@@ -1,0 -1,42 +1,43 @@@ -#!/bin/bash ++#!/bin/sh + # This script should be run before you commit to verify that the basic tests + # are working as they should + # Once you have run it you can inspect the log file via + # + # $ less before_i_commit.log + # To clean up everything that is left by the running of this tool, do as + # following: + # + # rm *.yamloo; rm before_i_commit.log + # + + if [ -f before_i_commit.log ]; + then + # this is technically the date it was moved, not the date it was created + mv before_i_commit.log before_i_commit-`date +%s`.log; + touch before_i_commit.log; + else + touch before_i_commit.log; + fi + + find . -type f -name "*.py[co]" -delete + + if [ -f env/bin/activate ]; + then + source env/bin/activate; + else + echo "Assuming that your virtual environment is pre-configured..."; + fi + + ./bin/ooniprobe -i decks/before_i_commit.testdeck + + echo "Below you should not see anything" + echo "---------------------------------" -grep "Error: " before_i_commit.log ++[ -f before_i_commit.log ] && grep "Error: " before_i_commit.log + echo "---------------------------------" + echo "If you do, it means something is wrong." + echo "Read through the log file and fix it." + echo "If you are having some problems fixing some things that have to do with" + echo "the core of OONI, let's first discuss it on IRC, or open a ticket" + read -#cat *.yamloo | less ++cat *yamloo | less ++rm -f *yamloo

1 0

[ooni-probe/master] Cleaned up handling of missing file in /nettest or /test.
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit d38df3373f9c56f191d17422e90f0703947d0093 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Dec 18 01:13:12 2012 +0000 Cleaned up handling of missing file in /nettest or /test. --- ooni/runner.py | 30 +++++++++++++++++++++--------- 1 files changed, 21 insertions(+), 9 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index 4ebfa0b..94b2986 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -103,13 +103,19 @@ def findTestClassesFromFile(cmd_line_options): A list of class objects found in a file or module given on the commandline. """ - filename = cmd_line_options['test'] classes = [] - module = filenameToModule(filename) - for name, val in inspect.getmembers(module): - if isTestCase(val): - classes.append(processTest(val, cmd_line_options)) - return classes + filename = cmd_line_options['test'] + relative = filename.rsplit('/', 1)[1] + try: + module = filenameToModule(filename) + except ValueError, ve: + log.fail("%r doesn't exist." % relative) + else: + for name, val in inspect.getmembers(module): + if isTestCase(val): + classes.append(processTest(val, cmd_line_options)) + finally: + return classes def makeTestCases(klass, tests, method_prefix): """ @@ -556,6 +562,12 @@ def loadTest(cmd_line_options): # Ideally this would get all wrapped in a nice little class that get's # instanced with it's cmd_line_options as an instance attribute classes = findTestClassesFromFile(cmd_line_options) - test_cases, options = loadTestsAndOptions(classes, cmd_line_options) - - return test_cases, options, cmd_line_options + try: + test_cases, options = loadTestsAndOptions(classes, cmd_line_options) + return test_cases, options, cmd_line_options + except NoTestCasesFound, ntcf: + log.warn(ntcf) + if not 'testdeck' in cmd_line_options: # exit if this was this only test + sys.exit(1) # file and there aren't any tests + else: + pass # there are more tests, so continue

1 0

[ooni-probe/master] Fixed an error with the test_timeout callback taking the wrong parameters, and
by isis＠torproject.org 18 Dec '12

18 Dec '12

commit 623eeca5b719be980c8ddb0444c89eecaf5cadd6 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Dec 18 05:18:44 2012 +0000 Fixed an error with the test_timeout callback taking the wrong parameters, and cleaned up a couple things due to the ooni logger formatting changes. --- ooni/oonicli.py | 2 +- ooni/runner.py | 11 +++++++---- ooni/utils/log.py | 15 +++++++-------- ooniprobe.conf.sample | 8 +++++--- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 6afc453..582a6ae 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -215,5 +215,5 @@ def run(): d.addErrback(errorRunningTests) # XXX I believe we don't actually need this: - reactor.run() + #reactor.run() diff --git a/ooni/runner.py b/ooni/runner.py index 4bb3e9f..a0710cd 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -248,10 +248,10 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, # This is used to store a copy of all the test reports tests_report = {} - def test_timeout(d): + def test_timeout(d, test_instance): timeout_error = defer.TimeoutError( "%s test for %s timed out after %s seconds" - % (test_name, test_instance.input, test_instance.timeout)) + % (test_instance.name, test_instance.input, test_instance.timeout)) timeout_fail = failure.Failure(err) try: d.errback(timeout_fail) @@ -323,8 +323,10 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, # call setups on the test test_instance._setUp() test_instance.setUp() + # get the timeout and _parents, in case it was set in setUp() test_instance.timeout = getTestTimeout(test_instance, test_method) + test_instance.timedOut = False test = getattr(test_instance, test_method) test_instance._testMethod = test @@ -332,7 +334,8 @@ def runTestCasesWithInput(test_cases, test_input, yaml_reporter, d = defer.maybeDeferred(test) # register the timer with the reactor - call_timeout = reactor.callLater(test_instance.timeout, test_timeout, d) + call_timeout = reactor.callLater(test_instance.timeout, test_timeout, d, + test_instance) d.addBoth(lambda x: call_timeout.active() and call_timeout.cancel() or x) # check if anything has been aborted or marked as 'skip' @@ -678,7 +681,7 @@ def loadTest(cmd_line_options): test_cases, options = loadTestsAndOptions(classes, cmd_line_options) return test_cases, options, cmd_line_options except NoTestCasesFound, ntcf: - log.warn(ntcf) + log.err(ntcf) if not 'testdeck' in cmd_line_options: # exit if this was this only test sys.exit(1) # file and there aren't any tests else: diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 70543ce..95163f3 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -50,17 +50,16 @@ def stop(): def msg(msg, *arg, **kw): print "%s" % msg -def debug(msg, *arg, **kw): +def debug(message, *arg, **kw): if config.advanced.debug: - print "[D] %s" % msg + print "[D] %s" % message -def warn(msg, *arg, **kw): - txlog.logging.captureWarnings('true') - txlog.logging.warn(msg) - #txlog.showwarning() +def warn(message, *arg, **kw): + if config.advanced.show_warnings: + print "[W] %s" % message -def err(msg, *arg, **kw): - print "[!] %s" % msg +def err(message, *arg, **kw): + print "[!] %s" % message def exception(error): """ diff --git a/ooniprobe.conf.sample b/ooniprobe.conf.sample index 2d6ef47..99faa4b 100644 --- a/ooniprobe.conf.sample +++ b/ooniprobe.conf.sample @@ -21,15 +21,17 @@ advanced: # GeoIP database file. This should be the directory in which OONI is # installed /path/to/ooni-probe/data/ geoip_data_dir: /usr/share/GeoIP/ - # Should we display debug messages: + # Should we display debug messages?: debug: true + # Should we display warning messages?: + show_warnings: true # How many seconds should we wait for connections before timing out?: default_timeout: 30 # Location where Tor is installed: tor_binary: '/usr/sbin/tor' - # For auto detection + # To automatically detect the system default networking interface, use: interface: auto - # Of specify a specific interface + # Or specify a specific interface: #interface: wlan0 # If you do not specify start_tor, you will have to have Tor running and # explicitly set the control port and orport.

1 0