[tor-commits] [ooni-probe/master] Merge branch 'master' into tcp-flags
isis at torproject.org
isis at torproject.org
Tue Dec 18 05:53:46 UTC 2012
commit d8d286ac341dfc1a9cb56e8d5d4d1c49e7fee53a
Merge: 2e93940 8d3a668
Author: Isis Lovecruft <isis at torproject.org>
Date: Tue Dec 18 03:55:51 2012 +0000
Merge branch 'master' into tcp-flags
Conflicts:
ooni/nettest.py
ooni/oonicli.py
ooni/reporter.py
ooni/runner.py
ooni/utils/__init__.py
ooni/utils/geodata.py
ooni/utils/hacks.py
ooni/utils/log.py
ooni/utils/net.py
ooni/utils/txscapy.py
scripts/before_i_commit.sh
README.md | 51 +--
before_i_commit.sh | 29 --
before_i_commit.testdeck | 33 --
bin/INSTRUCTIONS | 15 -
bin/Makefile | 54 ---
bin/canary | 27 --
bin/old_ooniprobe | 80 ----
bin/oonib | 4 -
decks/before_i_commit.testdeck | 33 ++
docs/source/api/modules.rst | 7 -
docs/source/api/ooni.rst | 8 -
docs/source/api/ooni.templates.rst | 22 +-
docs/source/api/ooni.utils.rst | 2 -
docs/source/architecture.rst | 204 ++++++++++
docs/source/conf.py | 4 +-
docs/source/index.rst | 6 +-
docs/source/nettests/modules.rst | 8 +
docs/source/nettests/nettests.blocking.rst | 26 ++
docs/source/nettests/nettests.examples.rst | 58 +++
docs/source/nettests/nettests.experimental.rst | 58 +++
docs/source/nettests/nettests.manipulation.rst | 58 +++
docs/source/nettests/nettests.scanning.rst | 10 +
docs/source/nettests/nettests.third_party.rst | 10 +
docs/source/reports.rst | 9 +-
docs/source/tests/dnsspoof.rst | 111 ++++++
docs/source/tests/dnstamper.rst | 6 +-
.../tests/http_header_field_manipulation.rst | 343 ++++++++++++++++
docs/source/tests/http_host.rst | 10 +-
docs/source/tests/http_invalid_request_line.rst | 4 +-
docs/source/tests/http_requests.rst | 4 +-
docs/source/tests/tcpconnect.rst | 2 +-
docs/source/tests/traceroute.rst | 2 +-
docs/source/writing_tests.rst | 275 +++++++++++---
nettests/blocking/__init__.py | 1 +
nettests/blocking/dnstamper.py | 6 +-
nettests/examples/example_http_checksum.py | 27 ++
nettests/third_party/netalyzr.py | 1 -
ooni/__init__.py | 3 +-
ooni/config.py | 37 +-
ooni/kit/daphn3.py | 4 +-
ooni/kit/domclass.py | 60 ++--
ooni/lib/__init__.py | 5 -
ooni/lib/secdev.org.pem | 20 -
ooni/nettest.py | 17 +
ooni/nodes.py | 174 --------
ooni/oonicli.py | 23 +-
ooni/otime.py | 3 -
ooni/reporter.py | 97 +++--
ooni/runner.py | 54 ++-
ooni/templates/dnst.py | 57 ++--
ooni/templates/httpt.py | 4 -
ooni/templates/scapyt.py | 5 -
ooni/utils/__init__.py | 35 ++-
ooni/utils/geodata.py | 33 --
ooni/utils/hacks.py | 4 -
ooni/utils/log.py | 34 +-
ooni/utils/txscapy.py | 13 +-
oonib/config.py | 10 +
oonib/oonibackend.py | 4 +-
oonib/report/__init__.py | 3 -
oonib/report/api.py | 6 +-
oonib/report/file_collector.py | 96 ++++--
oonib/requirements.txt | 40 +--
oonib/runner.py | 3 +
oonib/testhelpers/ssl_helpers.py | 2 +-
requirements.txt | 2 -
scripts/before_i_commit.sh | 43 ++
scripts/example_parser.py | 22 +
scripts/submit-patch | 100 +++++
setup.py | 37 ++
submit-patch | 100 -----
tests/test_runner.py | 20 +-
tests/test_utils.py | 20 +
to-be-ported/TODO | 418 --------------------
to-be-ported/spec/proxooni-spec.txt | 65 ---
to-be-ported/very-old/TODO.plgoons | 79 ----
to-be-ported/very-old/TO_BE_PORTED | 14 -
to-be-ported/very-old/ooni-probe.diff | 358 -----------------
to-be-ported/very-old/ooni/#namecheck.py# | 39 --
to-be-ported/very-old/ooni/.DS_Store | Bin 15364 -> 0 bytes
to-be-ported/very-old/ooni/__init__.py | 12 -
to-be-ported/very-old/ooni/command.py | 250 ------------
to-be-ported/very-old/ooni/dns_poisoning.py | 43 --
to-be-ported/very-old/ooni/dnsooni.py | 356 -----------------
to-be-ported/very-old/ooni/helpers.py | 38 --
to-be-ported/very-old/ooni/http.py | 306 --------------
to-be-ported/very-old/ooni/input.py | 33 --
to-be-ported/very-old/ooni/namecheck.py | 39 --
.../very-old/ooni/plugins/dnstest_plgoo.py | 84 ----
to-be-ported/very-old/ooni/plugins/http_plgoo.py | 70 ----
to-be-ported/very-old/ooni/plugins/marco_plgoo.py | 377 ------------------
to-be-ported/very-old/ooni/plugins/proxy_plgoo.py | 69 ----
.../very-old/ooni/plugins/simple_dns_plgoo.py | 35 --
to-be-ported/very-old/ooni/plugins/tcpcon_plgoo.py | 278 -------------
to-be-ported/very-old/ooni/plugins/tor.py | 80 ----
to-be-ported/very-old/ooni/plugins/torrc | 9 -
to-be-ported/very-old/ooni/plugooni.py | 106 -----
to-be-ported/very-old/ooni/transparenthttp.py | 41 --
var/old_notes.txt | 418 ++++++++++++++++++++
var/proxooni-spec.txt | 65 +++
var/secdev.org.pem | 20 +
101 files changed, 2327 insertions(+), 4173 deletions(-)
diff --cc ooni/oonicli.py
index 0bf4d55,b06bde9..6afc453
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@@ -105,34 -75,21 +105,23 @@@ def updateStatusBar(stop_func)
# moment.
eta = config.state[test_filename].eta()
progress = config.state[test_filename].progress()
- progress_bar_frmt = "[%s] %s%%" % (test_filename, progress)
- print progress_bar_frmt
+ while progress is not None:
+ print "[%s] %s%%" % (test_filename, progress)
+ else:
+ print "[%s] All tests in file completed." % test_filename
+ stop_func()
def testsEnded(*arg, **kw):
- """
- You can place here all the post shutdown tasks.
- """
- log.debug("testsEnded: Finished running all tests")
+ """You can place here all the post shutdown tasks."""
+ log.debug("Finished running all tests")
config.start_reactor = False
- try: reactor.stop()
- except: pass
+ if not reactor.running:
+ try: reactor.stop()
+ except: reactor.runUntilCurrent()
- def startSniffing():
- from ooni.utils.txscapy import ScapyFactory, ScapySniffer
- try:
- checkForRoot()
- except PermissionsError:
- print "[!] Includepcap options requires root priviledges to run"
- print " you should run ooniprobe as root or disable the options in ooniprobe.conf"
- sys.exit(1)
-
- print "Starting sniffer"
- config.scapyFactory = ScapyFactory(config.advanced.interface)
-
- sniffer = ScapySniffer(config.reports.pcap)
- config.scapyFactory.registerProtocol(sniffer)
+ def testFailed(failure):
+ log.err("Failed in running a test inside a test list")
+ failure.printTraceback()
def runTestList(none, test_list):
"""
@@@ -149,16 -106,12 +138,17 @@@
deck_dl.append(d1)
d2 = defer.DeferredList(deck_dl)
- d2.addBoth(testsEnded)
+ d2.addCallback(testsEnded)
+ d2.addErrback(testFailed)
- # Print every 5 second the list of current tests running
- l = task.LoopingCall(updateStatusBar)
- l.start(5.0)
+ try:
+ # Print every 5 second the list of current tests running
+ coop = task.Cooperator(started=False)
+ coop.cooperate(updateStatusBar) #this will need a .next() method
+ coop.start()
+ except StopIteration:
+ return d2
+
return d2
def errorRunningTests(failure):
@@@ -177,18 -132,13 +168,20 @@@ def run()
log.start(cmd_line_options['logfile'])
+ config.cmd_line_options = cmd_line_options
+
if config.privacy.includepcap:
- log.msg("Starting")
- if not config.reports.pcap:
- config.generatePcapFilename()
- runner.startSniffing()
+ try:
+ checkForRoot()
+ except PermissionsError, pe:
+ m = ("Capturing packets requires administrator/root privileges. ",
+ "Run ooniprobe as root or set 'includepcap = false' in ",
+ "ooniprobe.conf .")
+ log.warn("%s" % m)
+ sys.exit(1)
+ else:
+ log.msg("Starting packet capture")
+ runner.startSniffing()
resume = cmd_line_options['resume']
diff --cc ooni/reporter.py
index 89a62ba,728c3f5..aac163d
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@@ -1,20 -1,12 +1,21 @@@
+#-*- coding: utf-8 -*-
+#
+# reporter.py
+# -----------
+# In here goes the logic for the creation of ooniprobe reports.
+#
+# :authors: Arturo Filastò, Isis Lovecruft
+# :license: see included LICENSE file
+
+ import traceback
import itertools
import logging
+import sys
+import os
import time
import yaml
import json
- import traceback
-import sys
-import os
+ import re
from yaml.representer import *
from yaml.emitter import *
@@@ -25,15 -17,20 +26,16 @@@ from twisted.trial import reporte
from twisted.internet import defer, reactor
from twisted.internet.error import ConnectionRefusedError
-from ooni.utils import log
+from ooni import config, otime
- from ooni.utils import log, geodata
++from ooni.utils import log, geodata, pushFilenameStack
+from ooni.utils.net import BodyReceiver, StringProducer, userAgents
try:
from scapy.packet import Packet
except ImportError:
log.err("Scapy is not installed.")
+
-from ooni import otime
-from ooni.utils import geodata, pushFilenameStack
-from ooni.utils.net import BodyReceiver, StringProducer, userAgents
-
-from ooni import config
-
def createPacketReport(packet_list):
"""
Takes as input a packet a list.
@@@ -159,8 -152,8 +157,8 @@@ def getTestDetails(options)
'test_version': options['version'],
'software_name': 'ooniprobe',
'software_version': software_version
- }
+ }
- defer.returnValue(test_details)
+ return test_details
class OReporter(object):
def __init__(self, cmd_line_options):
@@@ -307,15 -310,12 +315,12 @@@ class OONIBReporter(OReporter)
"""
Creates a report on the oonib collector.
"""
- test_name = options['name']
- test_version = options['version']
-
- url = self.backend_url + '/report/new'
+ url = self.backend_url + '/report'
try:
- test_details = yield getTestDetails(options)
+ test_details = getTestDetails(options)
- except Exception, e:
- log.exception(e)
+ except Exception, ex:
+ log.exception(ex)
test_details['options'] = self.cmd_line_options
diff --cc ooni/runner.py
index 19dc171,4ebfa0b..7488d76
--- a/ooni/runner.py
+++ b/ooni/runner.py
@@@ -27,50 -15,18 +27,54 @@@ from twisted.trial.unittest import Skip
from txtorcon import TorProtocolFactory, TorConfig
from txtorcon import TorState, launch_tor
-from ooni import config
-
+from ooni import config, nettest, reporter
+from ooni.inputunit import InputUnitFactory
from ooni.reporter import OONIBReporter, YAMLReporter, OONIBReportError
+
+ from ooni.inputunit import InputUnitFactory
+ from ooni.nettest import NetTestCase, NoPostProcessor
+
-from ooni.utils import log, checkForRoot, pushFilenameStack
-from ooni.utils import NotRootError, Storage
+from ooni.utils import log, checkForRoot
+from ooni.utils import PermissionsError, Storage
from ooni.utils.net import randomFreePort
+
+class NoTestCasesFound(Exception):
+ pass
+
+class InvalidResumeFile(Exception):
+ pass
+
+class noResumeSession(Exception):
+ pass
+
+class InvalidConfigFile(Exception):
+ message = "Invalid setting in ooniprobe.conf: "
+
+class UnableToStartTor(Exception):
+ pass
+
+
+def isTestCase(obj):
+ """Return True if obj is a subclass of NetTestCase, False otherwise."""
+ try:
+ return issubclass(obj, nettest.NetTestCase)
+ except TypeError:
+ return False
+
+def checkRequiredOptions(test_instance):
+ """
+ If test_instance has an attribute 'requiredOptions', then check that
+ those options were utilised on the commandline.
+ """
+ required = getattr(test_instance, 'requiredOptions', None)
+ if required:
+ for required_option in required:
+ log.debug("Checking if %s is present" % required_option)
+ if not test_instance.localOptions[required_option]:
+ raise usage.UsageError("%s not specified!" % required_option)
+
- def processTest(obj):
+ def processTest(obj, cmd_line_options):
"""
Process the parameters and :class:`twisted.python.usage.Options` of a
:class:`ooni.nettest.Nettest`.
@@@ -101,29 -55,43 +105,29 @@@
obj.usageOptions.optFlags = []
for flag in obj.baseFlags:
obj.usageOptions.optFlags.append(flag)
+ if obj.inputFile: # inputFile is the optParameters list
+ obj.usageOptions.optParameters.append(obj.inputFile)
options = obj.usageOptions()
- options.parseOptions(config.cmd_line_options['subargs'])
-
+ options.parseOptions(cmd_line_options['subargs'])
obj.localOptions = options
- if obj.inputFile:
+ if obj.inputFile: # inputFilename is the actual filename
obj.inputFilename = options[obj.inputFile[0]]
try:
- log.debug("processing options")
- tmp_test_case_object = obj()
- tmp_test_case_object._checkRequiredOptions()
-
- except usage.UsageError, e:
- test_name = tmp_test_case_object.name
- log.err("There was an error in running %s!" % test_name)
- log.err("%s" % e)
+ log.debug("Parsing commandline options")
+ tmp_test_instance = obj()
+ checkRequiredOptions(tmp_test_instance)
+ except usage.UsageError, ue:
+ log.err("%s" % ue)
options.opt_help()
- raise usage.UsageError("Error in parsing command line args for %s" % test_name)
-
- if obj.requiresRoot:
- try:
- checkForRoot()
- except NotRootError:
- log.err("%s requires root to run" % obj.name)
- sys.exit(1)
-
- return obj
-
-def isTestCase(obj):
- try:
- return issubclass(obj, NetTestCase)
- except TypeError:
- return False
+ raise usage.UsageError("Error parsing command line args for %s"
+ % tmp_test_case_object.name)
+ else:
+ return obj
- def findTestClassesFromFile(filename):
+ def findTestClassesFromFile(cmd_line_options):
"""
Takes as input the command line config parameters and returns the test
case classes.
@@@ -456,9 -353,11 +462,8 @@@ def increaseInputUnitIdx(test_filename)
config.stateDict[test_filename] += 1
yield updateResumeFile(test_filename)
-def updateProgressMeters(test_filename, input_unit_factory,
- test_case_number):
- """
- Update the progress meters for keeping track of test state.
- """
+def updateProgressMeters(test_filename, input_unit_factory, test_case_number):
+ """Update the progress meters for keeping track of test state."""
- log.msg("Setting up progress meters")
if not config.state.test_filename:
config.state[test_filename] = Storage()
@@@ -560,8 -451,16 +565,13 @@@ def runTestCases(test_cases, options, c
except Exception:
log.exception("Problem in running test")
+ yaml_reporter.finish()
-class UnableToStartTor(Exception):
- pass
-
def startTor():
+ """ Starts Tor
+ Launches a Tor with :param: socks_port :param: control_port
+ :param: tor_binary set in ooniprobe.conf
+ """
@defer.inlineCallbacks
def state_complete(state):
config.tor_state = state
@@@ -631,6 -539,11 +650,12 @@@ def startSniffing()
print "Starting sniffer"
config.scapyFactory = ScapyFactory(config.advanced.interface)
- if os.path.exists(config.reports.pcap):
++ pcapfile = config.reports.pcap
++ if pcapfile and os.path.exists(pcapfile):
+ print "Report PCAP already exists with filename %s" % config.reports.pcap
+ print "Renaming files with such name..."
+ pushFilenameStack(config.reports.pcap)
+
sniffer = ScapySniffer(config.reports.pcap)
config.scapyFactory.registerProtocol(sniffer)
diff --cc ooni/utils/__init__.py
index be5b38a,8510a3b..efa609a
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@@ -1,13 -1,10 +1,10 @@@
- """
-
- """
-
+import imp
+import os
import logging
import string
import random
+ import glob
import yaml
-import imp
-import os
class Storage(dict):
"""
@@@ -88,3 -82,34 +85,33 @@@ def randomStr(length, num=True)
if num:
chars += string.digits
return ''.join(random.choice(chars) for x in range(length))
+
-
+ def pushFilenameStack(filename):
+ """
+ Takes as input a target filename and checks to see if a file by such name
+ already exists. If it does exist then it will attempt to rename it to .1,
+ if .1 exists it will rename .1 to .2 if .2 exists then it will rename it to
+ .3, etc.
+ This is similar to pushing into a LIFO stack.
+
+ XXX: This will not work with stacks bigger than 10 elements because
+ glob.glob(".*") will return them in the wrong order (a.1, a.11, a.2, a.3,
+ etc.)
+ This is probably not an issue since the only thing it causes is that files
+ will be renamed in the wrong order and you shouldn't have the same report
+ filename for more than 10 reports anyways, because you should be making
+ ooniprobe generate the filename for you.
+
+ Args:
+ filename (str): the path to filename that you wish to create.
+ """
+ stack = glob.glob(filename+".*")
+ for f in reversed(stack):
+ c_idx = f.split(".")[-1]
+ c_filename = '.'.join(f.split(".")[:-1])
+ new_idx = int(c_idx) + 1
+ new_filename = "%s.%s" % (c_filename, new_idx)
+ os.rename(f, new_filename)
+ os.rename(filename, filename+".1")
+
+
diff --cc ooni/utils/hacks.py
index 4cf94d0,64b5a53..4bbdf48
--- a/ooni/utils/hacks.py
+++ b/ooni/utils/hacks.py
@@@ -1,17 -1,5 +1,13 @@@
+# -*- encoding: utf-8 -*-
+#
+# hacks.py
+# ********
# When some software has issues and we need to fix it in a
# hackish way, we put it in here. This one day will be empty.
+#
+# :authors: Arturo Filastò, Isis Lovecruft
+# :licence: see LICENSE
+
- from yaml.representer import *
- from yaml.emitter import *
- from yaml.serializer import *
- from yaml.resolver import *
import copy_reg
diff --cc ooni/utils/log.py
index 2721807,0740c10..70543ce
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@@ -1,24 -1,28 +1,31 @@@
+# -*- encoding: utf-8 -*-
+#
+# :authors: Arturo Filastò
+# :licence: see LICENSE
+
+from functools import wraps
import sys
import os
-import logging
import traceback
+import logging
from twisted.python import log as txlog
+ from twisted.python import util
from twisted.python.failure import Failure
from twisted.python.logfile import DailyLogFile
from ooni import otime
from ooni import config
--## Get rid of the annoying "No route found for
--## IPv6 destination warnings":
--logging.getLogger("scapy.runtime").setLevel(logging.ERROR)
+
+ class LogWithNoPrefix(txlog.FileLogObserver):
+ def emit(self, eventDict):
+ text = txlog.textFromEventDict(eventDict)
+ if text is None:
+ return
+
+ util.untilConcludes(self.write, "%s\n" % text)
+ util.untilConcludes(self.flush) # Hoorj!
def start(logfile=None, application_name="ooniprobe"):
daily_logfile = None
@@@ -46,21 -42,17 +45,22 @@@
txlog.addObserver(txlog.FileLogObserver(daily_logfile).emit)
def stop():
- txlog.msg("Stopping OONI")
+ print "Stopping OONI"
def msg(msg, *arg, **kw):
- txlog.msg(msg, logLevel=logging.INFO, *arg, **kw)
+ print "%s" % msg
def debug(msg, *arg, **kw):
- txlog.msg(msg, logLevel=logging.DEBUG, *arg, **kw)
+ if config.advanced.debug:
+ print "[D] %s" % msg
+def warn(msg, *arg, **kw):
+ txlog.logging.captureWarnings('true')
+ txlog.logging.warn(msg)
+ #txlog.showwarning()
+
def err(msg, *arg, **kw):
- txlog.err("Error: " + str(msg), logLevel=logging.ERROR, *arg, **kw)
+ print "[!] %s" % msg
def exception(error):
"""
diff --cc ooni/utils/txscapy.py
index 647f20e,62bde94..7fa31fa
--- a/ooni/utils/txscapy.py
+++ b/ooni/utils/txscapy.py
@@@ -36,17 -31,20 +36,24 @@@ except ImportError
config.pcap_dnet = False
conf.use_pcap = False
conf.use_dnet = False
- from scapy.all import PcapWriter
+
+ class DummyPcapWriter:
+ def __init__(self, pcap_filename, *arg, **kw):
+ log.err("Initializing DummyPcapWriter. We will not actually write to a pcapfile")
-
+ def write(self):
+ pass
-
+ PcapWriter = DummyPcapWriter
+class ProtocolNotRegistered(Exception):
+ pass
+
+class ProtocolAlreadyRegistered(Exception):
+ pass
+
def getNetworksFromRoutes():
+ """ Return a list of networks from the routing table """
from scapy.all import conf, ltoa, read_routes
from ipaddr import IPNetwork, IPAddress
diff --cc scripts/before_i_commit.sh
index 0000000,918b137..a504ad8
mode 000000,100755..100755
--- a/scripts/before_i_commit.sh
+++ b/scripts/before_i_commit.sh
@@@ -1,0 -1,42 +1,43 @@@
-#!/bin/bash
++#!/bin/sh
+ # This script should be run before you commit to verify that the basic tests
+ # are working as they should
+ # Once you have run it you can inspect the log file via
+ #
+ # $ less before_i_commit.log
+ # To clean up everything that is left by the running of this tool, do as
+ # following:
+ #
+ # rm *.yamloo; rm before_i_commit.log
+ #
+
+ if [ -f before_i_commit.log ];
+ then
+ # this is technically the date it was moved, not the date it was created
+ mv before_i_commit.log before_i_commit-`date +%s`.log;
+ touch before_i_commit.log;
+ else
+ touch before_i_commit.log;
+ fi
+
+ find . -type f -name "*.py[co]" -delete
+
+ if [ -f env/bin/activate ];
+ then
+ source env/bin/activate;
+ else
+ echo "Assuming that your virtual environment is pre-configured...";
+ fi
+
+ ./bin/ooniprobe -i decks/before_i_commit.testdeck
+
+ echo "Below you should not see anything"
+ echo "---------------------------------"
-grep "Error: " before_i_commit.log
++[ -f before_i_commit.log ] && grep "Error: " before_i_commit.log
+ echo "---------------------------------"
+ echo "If you do, it means something is wrong."
+ echo "Read through the log file and fix it."
+ echo "If you are having some problems fixing some things that have to do with"
+ echo "the core of OONI, let's first discuss it on IRC, or open a ticket"
+ read
-#cat *.yamloo | less
++cat *yamloo | less
++rm -f *yamloo
More information about the tor-commits
mailing list