commit d8d286ac341dfc1a9cb56e8d5d4d1c49e7fee53a Merge: 2e93940 8d3a668 Author: Isis Lovecruft isis@torproject.org Date: Tue Dec 18 03:55:51 2012 +0000

Merge branch 'master' into tcp-flags

Conflicts: ooni/nettest.py ooni/oonicli.py ooni/reporter.py ooni/runner.py ooni/utils/__init__.py ooni/utils/geodata.py ooni/utils/hacks.py ooni/utils/log.py ooni/utils/net.py ooni/utils/txscapy.py scripts/before_i_commit.sh

README.md | 51 +-- before_i_commit.sh | 29 -- before_i_commit.testdeck | 33 -- bin/INSTRUCTIONS | 15 - bin/Makefile | 54 --- bin/canary | 27 -- bin/old_ooniprobe | 80 ---- bin/oonib | 4 - decks/before_i_commit.testdeck | 33 ++ docs/source/api/modules.rst | 7 - docs/source/api/ooni.rst | 8 - docs/source/api/ooni.templates.rst | 22 +- docs/source/api/ooni.utils.rst | 2 - docs/source/architecture.rst | 204 ++++++++++ docs/source/conf.py | 4 +- docs/source/index.rst | 6 +- docs/source/nettests/modules.rst | 8 + docs/source/nettests/nettests.blocking.rst | 26 ++ docs/source/nettests/nettests.examples.rst | 58 +++ docs/source/nettests/nettests.experimental.rst | 58 +++ docs/source/nettests/nettests.manipulation.rst | 58 +++ docs/source/nettests/nettests.scanning.rst | 10 + docs/source/nettests/nettests.third_party.rst | 10 + docs/source/reports.rst | 9 +- docs/source/tests/dnsspoof.rst | 111 ++++++ docs/source/tests/dnstamper.rst | 6 +- .../tests/http_header_field_manipulation.rst | 343 ++++++++++++++++ docs/source/tests/http_host.rst | 10 +- docs/source/tests/http_invalid_request_line.rst | 4 +- docs/source/tests/http_requests.rst | 4 +- docs/source/tests/tcpconnect.rst | 2 +- docs/source/tests/traceroute.rst | 2 +- docs/source/writing_tests.rst | 275 +++++++++++--- nettests/blocking/__init__.py | 1 + nettests/blocking/dnstamper.py | 6 +- nettests/examples/example_http_checksum.py | 27 ++ nettests/third_party/netalyzr.py | 1 - ooni/__init__.py | 3 +- ooni/config.py | 37 +- ooni/kit/daphn3.py | 4 +- ooni/kit/domclass.py | 60 ++-- ooni/lib/__init__.py | 5 - ooni/lib/secdev.org.pem | 20 - ooni/nettest.py | 17 + ooni/nodes.py | 174 -------- ooni/oonicli.py | 23 +- ooni/otime.py | 3 - ooni/reporter.py | 97 +++-- ooni/runner.py | 54 ++- ooni/templates/dnst.py | 57 ++-- ooni/templates/httpt.py | 4 - ooni/templates/scapyt.py | 5 - ooni/utils/__init__.py | 35 ++- ooni/utils/geodata.py | 33 -- ooni/utils/hacks.py | 4 - ooni/utils/log.py | 34 +- ooni/utils/txscapy.py | 13 +- oonib/config.py | 10 + oonib/oonibackend.py | 4 +- oonib/report/__init__.py | 3 - oonib/report/api.py | 6 +- oonib/report/file_collector.py | 96 ++++-- oonib/requirements.txt | 40 +-- oonib/runner.py | 3 + oonib/testhelpers/ssl_helpers.py | 2 +- requirements.txt | 2 - scripts/before_i_commit.sh | 43 ++ scripts/example_parser.py | 22 + scripts/submit-patch | 100 +++++ setup.py | 37 ++ submit-patch | 100 ----- tests/test_runner.py | 20 +- tests/test_utils.py | 20 + to-be-ported/TODO | 418 -------------------- to-be-ported/spec/proxooni-spec.txt | 65 --- to-be-ported/very-old/TODO.plgoons | 79 ---- to-be-ported/very-old/TO_BE_PORTED | 14 - to-be-ported/very-old/ooni-probe.diff | 358 ----------------- to-be-ported/very-old/ooni/#namecheck.py# | 39 -- to-be-ported/very-old/ooni/.DS_Store | Bin 15364 -> 0 bytes to-be-ported/very-old/ooni/__init__.py | 12 - to-be-ported/very-old/ooni/command.py | 250 ------------ to-be-ported/very-old/ooni/dns_poisoning.py | 43 -- to-be-ported/very-old/ooni/dnsooni.py | 356 ----------------- to-be-ported/very-old/ooni/helpers.py | 38 -- to-be-ported/very-old/ooni/http.py | 306 -------------- to-be-ported/very-old/ooni/input.py | 33 -- to-be-ported/very-old/ooni/namecheck.py | 39 -- .../very-old/ooni/plugins/dnstest_plgoo.py | 84 ---- to-be-ported/very-old/ooni/plugins/http_plgoo.py | 70 ---- to-be-ported/very-old/ooni/plugins/marco_plgoo.py | 377 ------------------ to-be-ported/very-old/ooni/plugins/proxy_plgoo.py | 69 ---- .../very-old/ooni/plugins/simple_dns_plgoo.py | 35 -- to-be-ported/very-old/ooni/plugins/tcpcon_plgoo.py | 278 ------------- to-be-ported/very-old/ooni/plugins/tor.py | 80 ---- to-be-ported/very-old/ooni/plugins/torrc | 9 - to-be-ported/very-old/ooni/plugooni.py | 106 ----- to-be-ported/very-old/ooni/transparenthttp.py | 41 -- var/old_notes.txt | 418 ++++++++++++++++++++ var/proxooni-spec.txt | 65 +++ var/secdev.org.pem | 20 + 101 files changed, 2327 insertions(+), 4173 deletions(-)

diff --cc ooni/oonicli.py index 0bf4d55,b06bde9..6afc453 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@@ -105,34 -75,21 +105,23 @@@ def updateStatusBar(stop_func) # moment. eta = config.state[test_filename].eta() progress = config.state[test_filename].progress() - progress_bar_frmt = "[%s] %s%%" % (test_filename, progress) - print progress_bar_frmt + while progress is not None: + print "[%s] %s%%" % (test_filename, progress) + else: + print "[%s] All tests in file completed." % test_filename + stop_func()

def testsEnded(*arg, **kw): - """ - You can place here all the post shutdown tasks. - """ - log.debug("testsEnded: Finished running all tests") + """You can place here all the post shutdown tasks.""" + log.debug("Finished running all tests") config.start_reactor = False - try: reactor.stop() - except: pass + if not reactor.running: + try: reactor.stop() + except: reactor.runUntilCurrent()

- def startSniffing(): - from ooni.utils.txscapy import ScapyFactory, ScapySniffer - try: - checkForRoot() - except PermissionsError: - print "[!] Includepcap options requires root priviledges to run" - print " you should run ooniprobe as root or disable the options in ooniprobe.conf" - sys.exit(1) - - print "Starting sniffer" - config.scapyFactory = ScapyFactory(config.advanced.interface) - - sniffer = ScapySniffer(config.reports.pcap) - config.scapyFactory.registerProtocol(sniffer) + def testFailed(failure): + log.err("Failed in running a test inside a test list") + failure.printTraceback()

def runTestList(none, test_list): """ @@@ -149,16 -106,12 +138,17 @@@ deck_dl.append(d1)

d2 = defer.DeferredList(deck_dl) - d2.addBoth(testsEnded) + d2.addCallback(testsEnded) + d2.addErrback(testFailed)

- # Print every 5 second the list of current tests running - l = task.LoopingCall(updateStatusBar) - l.start(5.0) + try: + # Print every 5 second the list of current tests running + coop = task.Cooperator(started=False) + coop.cooperate(updateStatusBar) #this will need a .next() method + coop.start() + except StopIteration: + return d2 + return d2

def errorRunningTests(failure): @@@ -177,18 -132,13 +168,20 @@@ def run()

log.start(cmd_line_options['logfile'])

+ config.cmd_line_options = cmd_line_options + if config.privacy.includepcap: - log.msg("Starting") - if not config.reports.pcap: - config.generatePcapFilename() - runner.startSniffing() + try: + checkForRoot() + except PermissionsError, pe: + m = ("Capturing packets requires administrator/root privileges. ", + "Run ooniprobe as root or set 'includepcap = false' in ", + "ooniprobe.conf .") + log.warn("%s" % m) + sys.exit(1) + else: + log.msg("Starting packet capture") + runner.startSniffing()

resume = cmd_line_options['resume']

diff --cc ooni/reporter.py index 89a62ba,728c3f5..aac163d --- a/ooni/reporter.py +++ b/ooni/reporter.py @@@ -1,20 -1,12 +1,21 @@@ +#-*- coding: utf-8 -*- +# +# reporter.py +# ----------- +# In here goes the logic for the creation of ooniprobe reports. +# +# :authors: Arturo Filastò, Isis Lovecruft +# :license: see included LICENSE file + + import traceback import itertools import logging +import sys +import os import time import yaml import json - import traceback -import sys -import os + import re

from yaml.representer import * from yaml.emitter import * @@@ -25,15 -17,20 +26,16 @@@ from twisted.trial import reporte from twisted.internet import defer, reactor from twisted.internet.error import ConnectionRefusedError

-from ooni.utils import log +from ooni import config, otime - from ooni.utils import log, geodata ++from ooni.utils import log, geodata, pushFilenameStack +from ooni.utils.net import BodyReceiver, StringProducer, userAgents

try: from scapy.packet import Packet except ImportError: log.err("Scapy is not installed.")

+ -from ooni import otime -from ooni.utils import geodata, pushFilenameStack -from ooni.utils.net import BodyReceiver, StringProducer, userAgents - -from ooni import config - def createPacketReport(packet_list): """ Takes as input a packet a list. @@@ -159,8 -152,8 +157,8 @@@ def getTestDetails(options) 'test_version': options['version'], 'software_name': 'ooniprobe', 'software_version': software_version - } + } - defer.returnValue(test_details) + return test_details

class OReporter(object): def __init__(self, cmd_line_options): @@@ -307,15 -310,12 +315,12 @@@ class OONIBReporter(OReporter) """ Creates a report on the oonib collector. """ - test_name = options['name'] - test_version = options['version'] - - url = self.backend_url + '/report/new' + url = self.backend_url + '/report'

try: - test_details = yield getTestDetails(options) + test_details = getTestDetails(options) - except Exception, e: - log.exception(e) + except Exception, ex: + log.exception(ex)

test_details['options'] = self.cmd_line_options

diff --cc ooni/runner.py index 19dc171,4ebfa0b..7488d76 --- a/ooni/runner.py +++ b/ooni/runner.py @@@ -27,50 -15,18 +27,54 @@@ from twisted.trial.unittest import Skip from txtorcon import TorProtocolFactory, TorConfig from txtorcon import TorState, launch_tor

-from ooni import config - +from ooni import config, nettest, reporter +from ooni.inputunit import InputUnitFactory from ooni.reporter import OONIBReporter, YAMLReporter, OONIBReportError + + from ooni.inputunit import InputUnitFactory + from ooni.nettest import NetTestCase, NoPostProcessor + -from ooni.utils import log, checkForRoot, pushFilenameStack -from ooni.utils import NotRootError, Storage +from ooni.utils import log, checkForRoot +from ooni.utils import PermissionsError, Storage from ooni.utils.net import randomFreePort

+ +class NoTestCasesFound(Exception): + pass + +class InvalidResumeFile(Exception): + pass + +class noResumeSession(Exception): + pass + +class InvalidConfigFile(Exception): + message = "Invalid setting in ooniprobe.conf: " + +class UnableToStartTor(Exception): + pass + + +def isTestCase(obj): + """Return True if obj is a subclass of NetTestCase, False otherwise.""" + try: + return issubclass(obj, nettest.NetTestCase) + except TypeError: + return False + +def checkRequiredOptions(test_instance): + """ + If test_instance has an attribute 'requiredOptions', then check that + those options were utilised on the commandline. + """ + required = getattr(test_instance, 'requiredOptions', None) + if required: + for required_option in required: + log.debug("Checking if %s is present" % required_option) + if not test_instance.localOptions[required_option]: + raise usage.UsageError("%s not specified!" % required_option) + - def processTest(obj): + def processTest(obj, cmd_line_options): """ Process the parameters and :class:`twisted.python.usage.Options` of a :class:`ooni.nettest.Nettest`. @@@ -101,29 -55,43 +105,29 @@@ obj.usageOptions.optFlags = [] for flag in obj.baseFlags: obj.usageOptions.optFlags.append(flag) + if obj.inputFile: # inputFile is the optParameters list + obj.usageOptions.optParameters.append(obj.inputFile)

options = obj.usageOptions() - options.parseOptions(config.cmd_line_options['subargs']) - + options.parseOptions(cmd_line_options['subargs']) obj.localOptions = options

- if obj.inputFile: + if obj.inputFile: # inputFilename is the actual filename obj.inputFilename = options[obj.inputFile[0]]

try: - log.debug("processing options") - tmp_test_case_object = obj() - tmp_test_case_object._checkRequiredOptions() - - except usage.UsageError, e: - test_name = tmp_test_case_object.name - log.err("There was an error in running %s!" % test_name) - log.err("%s" % e) + log.debug("Parsing commandline options") + tmp_test_instance = obj() + checkRequiredOptions(tmp_test_instance) + except usage.UsageError, ue: + log.err("%s" % ue) options.opt_help() - raise usage.UsageError("Error in parsing command line args for %s" % test_name) - - if obj.requiresRoot: - try: - checkForRoot() - except NotRootError: - log.err("%s requires root to run" % obj.name) - sys.exit(1) - - return obj - -def isTestCase(obj): - try: - return issubclass(obj, NetTestCase) - except TypeError: - return False + raise usage.UsageError("Error parsing command line args for %s" + % tmp_test_case_object.name) + else: + return obj

- def findTestClassesFromFile(filename): + def findTestClassesFromFile(cmd_line_options): """ Takes as input the command line config parameters and returns the test case classes. @@@ -456,9 -353,11 +462,8 @@@ def increaseInputUnitIdx(test_filename) config.stateDict[test_filename] += 1 yield updateResumeFile(test_filename)

-def updateProgressMeters(test_filename, input_unit_factory, - test_case_number): - """ - Update the progress meters for keeping track of test state. - """ +def updateProgressMeters(test_filename, input_unit_factory, test_case_number): + """Update the progress meters for keeping track of test state.""" - log.msg("Setting up progress meters") if not config.state.test_filename: config.state[test_filename] = Storage()

@@@ -560,8 -451,16 +565,13 @@@ def runTestCases(test_cases, options, c

except Exception: log.exception("Problem in running test") + yaml_reporter.finish()

-class UnableToStartTor(Exception): - pass - def startTor(): + """ Starts Tor + Launches a Tor with :param: socks_port :param: control_port + :param: tor_binary set in ooniprobe.conf + """ @defer.inlineCallbacks def state_complete(state): config.tor_state = state @@@ -631,6 -539,11 +650,12 @@@ def startSniffing() print "Starting sniffer" config.scapyFactory = ScapyFactory(config.advanced.interface)

- if os.path.exists(config.reports.pcap): ++ pcapfile = config.reports.pcap ++ if pcapfile and os.path.exists(pcapfile): + print "Report PCAP already exists with filename %s" % config.reports.pcap + print "Renaming files with such name..." + pushFilenameStack(config.reports.pcap) + sniffer = ScapySniffer(config.reports.pcap) config.scapyFactory.registerProtocol(sniffer)

diff --cc ooni/utils/__init__.py index be5b38a,8510a3b..efa609a --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@@ -1,13 -1,10 +1,10 @@@ - """ - - """ - +import imp +import os import logging import string import random + import glob import yaml -import imp -import os

class Storage(dict): """ @@@ -88,3 -82,34 +85,33 @@@ def randomStr(length, num=True) if num: chars += string.digits return ''.join(random.choice(chars) for x in range(length)) + - + def pushFilenameStack(filename): + """ + Takes as input a target filename and checks to see if a file by such name + already exists. If it does exist then it will attempt to rename it to .1, + if .1 exists it will rename .1 to .2 if .2 exists then it will rename it to + .3, etc. + This is similar to pushing into a LIFO stack. + + XXX: This will not work with stacks bigger than 10 elements because + glob.glob(".*") will return them in the wrong order (a.1, a.11, a.2, a.3, + etc.) + This is probably not an issue since the only thing it causes is that files + will be renamed in the wrong order and you shouldn't have the same report + filename for more than 10 reports anyways, because you should be making + ooniprobe generate the filename for you. + + Args: + filename (str): the path to filename that you wish to create. + """ + stack = glob.glob(filename+".*") + for f in reversed(stack): + c_idx = f.split(".")[-1] + c_filename = '.'.join(f.split(".")[:-1]) + new_idx = int(c_idx) + 1 + new_filename = "%s.%s" % (c_filename, new_idx) + os.rename(f, new_filename) + os.rename(filename, filename+".1") + + diff --cc ooni/utils/hacks.py index 4cf94d0,64b5a53..4bbdf48 --- a/ooni/utils/hacks.py +++ b/ooni/utils/hacks.py @@@ -1,17 -1,5 +1,13 @@@ +# -*- encoding: utf-8 -*- +# +# hacks.py +# ******** # When some software has issues and we need to fix it in a # hackish way, we put it in here. This one day will be empty. +# +# :authors: Arturo Filastò, Isis Lovecruft +# :licence: see LICENSE + - from yaml.representer import * - from yaml.emitter import * - from yaml.serializer import * - from yaml.resolver import *

import copy_reg

diff --cc ooni/utils/log.py index 2721807,0740c10..70543ce --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@@ -1,24 -1,28 +1,31 @@@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from functools import wraps import sys import os -import logging import traceback +import logging

from twisted.python import log as txlog + from twisted.python import util from twisted.python.failure import Failure from twisted.python.logfile import DailyLogFile

from ooni import otime from ooni import config

--## Get rid of the annoying "No route found for --## IPv6 destination warnings": --logging.getLogger("scapy.runtime").setLevel(logging.ERROR) + + class LogWithNoPrefix(txlog.FileLogObserver): + def emit(self, eventDict): + text = txlog.textFromEventDict(eventDict) + if text is None: + return + + util.untilConcludes(self.write, "%s\n" % text) + util.untilConcludes(self.flush) # Hoorj!

def start(logfile=None, application_name="ooniprobe"): daily_logfile = None @@@ -46,21 -42,17 +45,22 @@@ txlog.addObserver(txlog.FileLogObserver(daily_logfile).emit)

def stop(): - txlog.msg("Stopping OONI") + print "Stopping OONI"

def msg(msg, *arg, **kw): - txlog.msg(msg, logLevel=logging.INFO, *arg, **kw) + print "%s" % msg

def debug(msg, *arg, **kw): - txlog.msg(msg, logLevel=logging.DEBUG, *arg, **kw) + if config.advanced.debug: + print "[D] %s" % msg

+def warn(msg, *arg, **kw): + txlog.logging.captureWarnings('true') + txlog.logging.warn(msg) + #txlog.showwarning() + def err(msg, *arg, **kw): - txlog.err("Error: " + str(msg), logLevel=logging.ERROR, *arg, **kw) + print "[!] %s" % msg

def exception(error): """ diff --cc ooni/utils/txscapy.py index 647f20e,62bde94..7fa31fa --- a/ooni/utils/txscapy.py +++ b/ooni/utils/txscapy.py @@@ -36,17 -31,20 +36,24 @@@ except ImportError config.pcap_dnet = False conf.use_pcap = False conf.use_dnet = False - from scapy.all import PcapWriter + + class DummyPcapWriter: + def __init__(self, pcap_filename, *arg, **kw): + log.err("Initializing DummyPcapWriter. We will not actually write to a pcapfile") - + def write(self): + pass - + PcapWriter = DummyPcapWriter

+class ProtocolNotRegistered(Exception): + pass + +class ProtocolAlreadyRegistered(Exception): + pass +

def getNetworksFromRoutes(): + """ Return a list of networks from the routing table """ from scapy.all import conf, ltoa, read_routes from ipaddr import IPNetwork, IPAddress

diff --cc scripts/before_i_commit.sh index 0000000,918b137..a504ad8 mode 000000,100755..100755 --- a/scripts/before_i_commit.sh +++ b/scripts/before_i_commit.sh @@@ -1,0 -1,42 +1,43 @@@ -#!/bin/bash ++#!/bin/sh + # This script should be run before you commit to verify that the basic tests + # are working as they should + # Once you have run it you can inspect the log file via + # + # $ less before_i_commit.log + # To clean up everything that is left by the running of this tool, do as + # following: + # + # rm *.yamloo; rm before_i_commit.log + # + + if [ -f before_i_commit.log ]; + then + # this is technically the date it was moved, not the date it was created + mv before_i_commit.log before_i_commit-`date +%s`.log; + touch before_i_commit.log; + else + touch before_i_commit.log; + fi + + find . -type f -name "*.py[co]" -delete + + if [ -f env/bin/activate ]; + then + source env/bin/activate; + else + echo "Assuming that your virtual environment is pre-configured..."; + fi + + ./bin/ooniprobe -i decks/before_i_commit.testdeck + + echo "Below you should not see anything" + echo "---------------------------------" -grep "Error: " before_i_commit.log ++[ -f before_i_commit.log ] && grep "Error: " before_i_commit.log + echo "---------------------------------" + echo "If you do, it means something is wrong." + echo "Read through the log file and fix it." + echo "If you are having some problems fixing some things that have to do with" + echo "the core of OONI, let's first discuss it on IRC, or open a ticket" + read -#cat *.yamloo | less ++cat *yamloo | less ++rm -f *yamloo