[tor-commits] [ooni-probe/master] Squashed commit of the following:
art at torproject.org
art at torproject.org
Fri Apr 29 09:42:21 UTC 2016
commit 6d2728ca795a97f6929c37579432ebcf909097d8
Author: Daniel Ramsay <daniel at dretzq.org.uk>
Date: Mon Aug 31 23:14:52 2015 +0100
Squashed commit of the following:
Add support for ORG's blocked.org.uk scheduler
Add:
* ORG Queue integration
* Support for additional values to send to ORG backend
Refactor director startup
* Moved collector setup to shared scope
* Moved deck setup to shared scope
* Moved annotations setup to shared scope
* Moved global options to shared scope
* fix some naming convention errors
* move global counter to more sensible scope
* fixes from pull request code review
* remove another mobile browser
* remove iPhone useragent to prevent mobile versions being returned to client
* add HTTPS support for reporter
* add authentication info from environment
* import PROBE_ vars from environment into report
* allow https collector in regex
* AMQP connection details as cmdline parameter
* more queue state, looping listen call
* remove old bin script
* exit after bounded random number of test decks
* improve error handling
* use proper custom exception for queue timeout reason
* reduce duplicate serialization
* add flag to prevent response body being written to report
* add subargs to queue createdeck
* fix global args passing
* reuse cached geoip
* reuse existing TOR connection
* simplify for one-shot use
* extend and report lifetime
* probe exit on AMQP error
* remove ooniresource from setup script
---
Vagrantfile | 2 +
bin/ooniprobequeue | 21 ++
ooni/director.py | 2 +-
ooni/geoip.py | 57 +++--
ooni/nettests/blocking/http_requests.py | 3 +
ooni/oonicli.py | 426 +++++++++++++++++++++++---------
ooni/reporter.py | 19 +-
ooni/templates/httpt.py | 2 +-
ooni/utils/net.py | 4 +-
setup.py | 5 +-
10 files changed, 384 insertions(+), 157 deletions(-)
diff --git a/Vagrantfile b/Vagrantfile
index 16a6ba0..a06dc41 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -12,6 +12,8 @@ Vagrant.configure("2") do |config|
# Place the ooni-backend source code in ../ooni-backend to sync it with the vagrant instance
config.vm.synced_folder "../ooni-backend", "/backend"
+ config.vm.network :private_network, ip: "192.168.38.20"
+
end
$script = <<SCRIPT
diff --git a/bin/ooniprobequeue b/bin/ooniprobequeue
new file mode 100755
index 0000000..f81f156
--- /dev/null
+++ b/bin/ooniprobequeue
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+import sys
+import copy_reg
+from twisted.internet import reactor
+
+# This is a hack to overcome a bug in python
+from ooni.utils.hacks import patched_reduce_ex
+copy_reg._reduce_ex = patched_reduce_ex
+
+code=0
+
+from ooni.oonicli import runWithDaemonDirector
+d = runWithDaemonDirector()
+ at d.addBoth
+def cb(result):
+ global code
+ if result is not None:
+ code=1
+ reactor.stop()
+reactor.run()
+sys.exit(code)
diff --git a/ooni/director.py b/ooni/director.py
index 5ffee97..9bac49d 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -132,7 +132,7 @@ class Director(object):
yield config.check_tor()
if config.advanced.start_tor:
yield self.startTor()
- elif config.tor.control_port:
+ elif config.tor.control_port and config.tor_state is None:
log.msg("Connecting to Tor Control Port...")
yield self.getTorState()
diff --git a/ooni/geoip.py b/ooni/geoip.py
index 0ab2475..cb66675 100644
--- a/ooni/geoip.py
+++ b/ooni/geoip.py
@@ -190,34 +190,37 @@ class ProbeIP(object):
@defer.inlineCallbacks
def lookup(self):
- try:
- yield self.askTor()
- log.msg("Found your IP via Tor %s" % self.address)
- self.resolveGeodata()
+ if self.address:
defer.returnValue(self.address)
- except errors.TorStateNotFound:
- log.debug("Tor is not running. Skipping IP lookup via Tor.")
- except Exception:
- log.msg("Unable to lookup the probe IP via Tor.")
-
- try:
- yield self.askTraceroute()
- log.msg("Found your IP via Traceroute %s" % self.address)
- self.resolveGeodata()
- defer.returnValue(self.address)
- except errors.InsufficientPrivileges:
- log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient privileges")
- except:
- log.msg("Unable to lookup the probe IP via traceroute")
-
- try:
- yield self.askGeoIPService()
- log.msg("Found your IP via a GeoIP service: %s" % self.address)
- self.resolveGeodata()
- defer.returnValue(self.address)
- except Exception:
- log.msg("Unable to lookup the probe IP via GeoIPService")
- raise
+ else:
+ try:
+ yield self.askTor()
+ log.msg("Found your IP via Tor %s" % self.address)
+ self.resolveGeodata()
+ defer.returnValue(self.address)
+ except errors.TorStateNotFound:
+ log.debug("Tor is not running. Skipping IP lookup via Tor.")
+ except Exception:
+ log.msg("Unable to lookup the probe IP via Tor.")
+
+ try:
+ yield self.askTraceroute()
+ log.msg("Found your IP via Traceroute %s" % self.address)
+ self.resolveGeodata()
+ defer.returnValue(self.address)
+ except errors.InsufficientPrivileges:
+ log.debug("Cannot determine the probe IP address with a traceroute, becase of insufficient priviledges")
+ except:
+ log.msg("Unable to lookup the probe IP via traceroute")
+
+ try:
+ yield self.askGeoIPService()
+ log.msg("Found your IP via a GeoIP service: %s" % self.address)
+ self.resolveGeodata()
+ defer.returnValue(self.address)
+ except Exception:
+ log.msg("Unable to lookup the probe IP via GeoIPService")
+ raise
@defer.inlineCallbacks
def askGeoIPService(self):
diff --git a/ooni/nettests/blocking/http_requests.py b/ooni/nettests/blocking/http_requests.py
index 997fb4e..f21b041 100644
--- a/ooni/nettests/blocking/http_requests.py
+++ b/ooni/nettests/blocking/http_requests.py
@@ -17,6 +17,9 @@ class UsageOptions(usage.Options):
['url', 'u', None, 'Specify a single URL to test.'],
['factor', 'f', 0.8,
'What factor should be used for triggering censorship (0.8 == 80%)']]
+ optFlags = [
+ ["withoutbody","B", "don't include HTTP response body inside of the report"],
+ ]
class HTTPRequestsTest(httpt.HTTPTest):
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 030014e..47aacfb 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -1,10 +1,14 @@
import sys
+
import os
+import json
import yaml
+import random
+import urlparse
from twisted.python import usage
from twisted.python.util import spewer
-from twisted.internet import defer
+from twisted.internet import defer, reactor, protocol, task
from ooni import errors, __version__
@@ -17,6 +21,9 @@ from ooni.utils import log
from ooni.utils.net import hasRawSocketPermission
+
+class LifetimeExceeded(Exception): pass
+
class Options(usage.Options):
synopsis = """%s [options] [path to test].py
""" % (os.path.basename(sys.argv[0]),)
@@ -49,7 +56,8 @@ class Options(usage.Options):
["datadir", "d", None,
"Specify a path to the ooniprobe data directory"],
["annotations", "a", None,
- "Annotate the report with a key:value[, key:value] format."]]
+ "Annotate the report with a key:value[, key:value] format."],
+ ["queue", "Q", None, "AMQP Queue URL amqp://user:pass@host:port/vhost/queue"]]
compData = usage.Completions(
extraActions=[usage.CompleteFiles(
@@ -102,11 +110,63 @@ def parseOptions():
return dict(cmd_line_options)
-def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
- """
- Instance the director, parse command line options and start an ooniprobe
- test!
- """
+def director_startup_handled_failures(failure):
+ log.err("Could not start the director")
+ failure.trap(errors.TorNotRunning,
+ errors.InvalidOONIBCollectorAddress,
+ errors.UnableToLoadDeckInput,
+ errors.CouldNotFindTestHelper,
+ errors.CouldNotFindTestCollector,
+ errors.ProbeIPUnknown,
+ errors.InvalidInputFile,
+ errors.ConfigFileIncoherent)
+
+ if isinstance(failure.value, errors.TorNotRunning):
+ log.err("Tor does not appear to be running")
+ log.err("Reporting with the collector %s is not possible" %
+ global_options['collector'])
+ log.msg(
+ "Try with a different collector or disable collector reporting with -n")
+
+ elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
+ log.err("Invalid format for oonib collector address.")
+ log.msg(
+ "Should be in the format http://<collector_address>:<port>")
+ log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
+
+ elif isinstance(failure.value, errors.UnableToLoadDeckInput):
+ log.err("Unable to fetch the required inputs for the test deck.")
+ log.msg(
+ "Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues")
+
+ elif isinstance(failure.value, errors.CouldNotFindTestHelper):
+ log.err("Unable to obtain the required test helpers.")
+ log.msg(
+ "Try with a different bouncer or check that Tor is running properly.")
+
+ elif isinstance(failure.value, errors.CouldNotFindTestCollector):
+ log.err("Could not find a valid collector.")
+ log.msg(
+ "Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.")
+
+ elif isinstance(failure.value, errors.ProbeIPUnknown):
+ log.err("Failed to lookup probe IP address.")
+ log.msg("Check your internet connection.")
+
+ elif isinstance(failure.value, errors.InvalidInputFile):
+ log.err("Invalid input file \"%s\"" % failure.value)
+
+ elif isinstance(failure.value, errors.ConfigFileIncoherent):
+ log.err("Incoherent config file")
+
+ if config.advanced.debug:
+ log.exception(failure)
+
+def director_startup_other_failures(failure):
+ log.err("An unhandled exception occurred while starting the director!")
+ log.exception(failure)
+
+def setupGlobalOptions(logging, start_tor, check_incoherences):
global_options = parseOptions()
config.global_options = global_options
config.set_paths()
@@ -133,49 +193,44 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
log.err("Insufficient Privileges to capture packets."
" See ooniprobe.conf privacy.includepcap")
sys.exit(2)
-
- director = Director()
- if global_options['list']:
- print "# Installed nettests"
- for net_test_id, net_test in director.getNetTests().items():
- print "* %s (%s/%s)" % (net_test['name'],
- net_test['category'],
- net_test['id'])
- print " %s" % net_test['description']
-
- sys.exit(0)
-
- elif global_options['printdeck']:
- del global_options['printdeck']
- print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
- print yaml.safe_dump([{'options': global_options}]).strip()
-
- sys.exit(0)
-
- if global_options.get('annotations') is not None:
- annotations = {}
- for annotation in global_options["annotations"].split(","):
- pair = annotation.split(":")
- if len(pair) == 2:
- key = pair[0].strip()
- value = pair[1].strip()
- annotations[key] = value
- else:
- log.err("Invalid annotation: %s" % annotation)
- sys.exit(1)
- global_options["annotations"] = annotations
-
- if global_options['no-collector']:
- log.msg("Not reporting using a collector")
- global_options['collector'] = None
- start_tor = False
- else:
- start_tor = True
+ return global_options
+
+def setupAnnotations(global_options):
+ annotations={}
+ for annotation in global_options["annotations"].split(","):
+ pair = annotation.split(":")
+ if len(pair) == 2:
+ key = pair[0].strip()
+ value = pair[1].strip()
+ annotations[key] = value
+ else:
+ log.err("Invalid annotation: %s" % annotation)
+ sys.exit(1)
+ global_options["annotations"] = annotations
+ return annotations
+
+def setupCollector(global_options, net_test_loader):
+ collector = None
+ if not global_options['no-collector']:
+ if global_options['collector']:
+ collector = global_options['collector']
+ elif 'collector' in config.reports \
+ and config.reports['collector']:
+ collector = config.reports['collector']
+ elif net_test_loader.collector:
+ collector = net_test_loader.collector
+
+ if collector and collector.startswith('httpo:') \
+ and (not (config.tor_state or config.tor.socks_port)):
+ raise errors.TorNotRunning
+ return collector
+
+
+def createDeck(global_options,url=None,filename=None):
+ log.msg("Creating deck for: %s" %(url or filename,) )
deck = Deck(no_collector=global_options['no-collector'])
deck.bouncer = global_options['bouncer']
- if global_options['collector']:
- start_tor |= True
try:
if global_options['testdeck']:
@@ -183,7 +238,15 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
else:
log.debug("No test deck detected")
test_file = nettest_to_path(global_options['test_file'], True)
- net_test_loader = NetTestLoader(global_options['subargs'],
+ if url is not None:
+ args = ('-u',url)
+ elif filename is not None:
+ args = ('-f',filename)
+ else:
+ args = tuple()
+ if any(global_options['subargs']):
+ args = global_options['subargs'] + args
+ net_test_loader = NetTestLoader(args,
test_file=test_file)
if global_options['collector']:
net_test_loader.collector = global_options['collector']
@@ -205,6 +268,48 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
log.exception(e)
log.err(e)
sys.exit(5)
+ return deck
+
+def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
+ """
+ Instance the director, parse command line options and start an ooniprobe
+ test!
+ """
+
+ global_options = setupGlobalOptions(logging, start_tor, check_incoherences)
+
+ director = Director()
+ if global_options['list']:
+ print "# Installed nettests"
+ for net_test_id, net_test in director.getNetTests().items():
+ print "* %s (%s/%s)" % (net_test['name'],
+ net_test['category'],
+ net_test['id'])
+ print " %s" % net_test['description']
+
+ sys.exit(0)
+
+ elif global_options['printdeck']:
+ del global_options['printdeck']
+ print "# Copy and paste the lines below into a test deck to run the specified test with the specified arguments"
+ print yaml.safe_dump([{'options': global_options}]).strip()
+
+ sys.exit(0)
+
+ if global_options.get('annotations') is not None:
+ annotations = setupAnnotations(global_options)
+
+ if global_options['no-collector']:
+ log.msg("Not reporting using a collector")
+ global_options['collector'] = None
+ start_tor = False
+ else:
+ start_tor = True
+
+ if global_options['collector']:
+ start_tor |= True
+
+ deck = createDeck(global_options)
start_tor |= deck.requiresTor
d = director.start(start_tor=start_tor,
@@ -216,61 +321,6 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
except errors.UnableToLoadDeckInput as error:
return defer.failure.Failure(error)
- def director_startup_handled_failures(failure):
- log.err("Could not start the director")
- failure.trap(errors.TorNotRunning,
- errors.InvalidOONIBCollectorAddress,
- errors.UnableToLoadDeckInput,
- errors.CouldNotFindTestHelper,
- errors.CouldNotFindTestCollector,
- errors.ProbeIPUnknown,
- errors.InvalidInputFile,
- errors.ConfigFileIncoherent)
-
- if isinstance(failure.value, errors.TorNotRunning):
- log.err("Tor does not appear to be running")
- log.err("Reporting with the collector %s is not possible" %
- global_options['collector'])
- log.msg(
- "Try with a different collector or disable collector reporting with -n")
-
- elif isinstance(failure.value, errors.InvalidOONIBCollectorAddress):
- log.err("Invalid format for oonib collector address.")
- log.msg(
- "Should be in the format http://<collector_address>:<port>")
- log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion")
-
- elif isinstance(failure.value, errors.UnableToLoadDeckInput):
- log.err("Unable to fetch the required inputs for the test deck.")
- log.msg(
- "Please file a ticket on our issue tracker: https://github.com/thetorproject/ooni-probe/issues")
-
- elif isinstance(failure.value, errors.CouldNotFindTestHelper):
- log.err("Unable to obtain the required test helpers.")
- log.msg(
- "Try with a different bouncer or check that Tor is running properly.")
-
- elif isinstance(failure.value, errors.CouldNotFindTestCollector):
- log.err("Could not find a valid collector.")
- log.msg(
- "Try with a different bouncer, specify a collector with -c or disable reporting to a collector with -n.")
-
- elif isinstance(failure.value, errors.ProbeIPUnknown):
- log.err("Failed to lookup probe IP address.")
- log.msg("Check your internet connection.")
-
- elif isinstance(failure.value, errors.InvalidInputFile):
- log.err("Invalid input file \"%s\"" % failure.value)
-
- elif isinstance(failure.value, errors.ConfigFileIncoherent):
- log.err("Incoherent config file")
-
- if config.advanced.debug:
- log.exception(failure)
-
- def director_startup_other_failures(failure):
- log.err("An unhandled exception occurred while starting the director!")
- log.exception(failure)
# Wait until director has started up (including bootstrapping Tor)
# before adding tests
@@ -285,19 +335,7 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
# deck is a singleton, the default collector set in
# ooniprobe.conf will be used
- collector = None
- if not global_options['no-collector']:
- if global_options['collector']:
- collector = global_options['collector']
- elif 'collector' in config.reports \
- and config.reports['collector']:
- collector = config.reports['collector']
- elif net_test_loader.collector:
- collector = net_test_loader.collector
-
- if collector and collector.startswith('httpo:') \
- and (not (config.tor_state or config.tor.socks_port)):
- raise errors.TorNotRunning
+ collector = setupCollector(global_options, net_test_loader)
test_details = net_test_loader.testDetails
test_details['annotations'] = global_options['annotations']
@@ -307,11 +345,163 @@ def runWithDirector(logging=True, start_tor=True, check_incoherences=True):
collector)
return director.allTestsDone
- def start():
+
+ d.addCallback(setup_nettest)
+ d.addCallback(post_director_start)
+ d.addErrback(director_startup_handled_failures)
+ d.addErrback(director_startup_other_failures)
+ return d
+
+
+
+# this variant version of runWithDirector splits the process in two,
+# allowing a single director instance to be reused with multiple decks.
+
+def runWithDaemonDirector(logging=True, start_tor=True, check_incoherences=True):
+ """
+ Instance the director, parse command line options and start an ooniprobe
+ test!
+ """
+
+ try:
+ import pika
+ from pika import exceptions
+ from pika.adapters import twisted_connection
+ except ImportError:
+ print "Pika is required for queue connection."
+ print "Install with \"pip install pika\"."
+ sys.exit(7)
+
+
+
+ global_options = setupGlobalOptions(logging, start_tor, check_incoherences)
+
+ director = Director()
+
+ if global_options.get('annotations') is not None:
+ annotations = setupAnnotations(global_options)
+
+ if global_options['no-collector']:
+ log.msg("Not reporting using a collector")
+ global_options['collector'] = None
+ start_tor = False
+ else:
+ start_tor = True
+
+
+ def run_test(global_options, url=None, filename=None):
+ assert url is not None or filename is not None
+
+ deck = createDeck(global_options, url=url, filename=filename)
+
+ d = director.start(start_tor=True,
+ check_incoherences=check_incoherences)
+
+ def setup_nettest(_):
+ try:
+ return deck.setup()
+ except errors.UnableToLoadDeckInput as error:
+ return defer.failure.Failure(error)
+
+
+
+ # Wait until director has started up (including bootstrapping Tor)
+ # before adding tests
+ def post_director_start(_):
+ for net_test_loader in deck.netTestLoaders:
+ # Decks can specify different collectors
+ # for each net test, so that each NetTest
+ # may be paired with a test_helper and its collector
+ # However, a user can override this behavior by
+ # specifying a collector from the command-line (-c).
+ # If a collector is not specified in the deck, or the
+ # deck is a singleton, the default collector set in
+ # ooniprobe.conf will be used
+
+ collector = setupCollector(global_options, net_test_loader)
+
+ test_details = net_test_loader.testDetails
+ test_details['annotations'] = global_options['annotations']
+
+ director.startNetTest(net_test_loader,
+ global_options['reportfile'],
+ collector)
+ return director.allTestsDone
+
d.addCallback(setup_nettest)
d.addCallback(post_director_start)
d.addErrback(director_startup_handled_failures)
d.addErrback(director_startup_other_failures)
return d
- return start()
+ finished = defer.Deferred()
+
+ @defer.inlineCallbacks
+ def readmsg(_,channel, queue_object, consumer_tag, counter):
+
+ # Wait for a message and decode it.
+ if counter >= lifetime:
+ queue_object.close(LifetimeExceeded())
+ yield channel.basic_cancel(consumer_tag=consumer_tag)
+ finished.callback(None)
+
+ else:
+ log.msg("Waiting for message")
+
+ try:
+ ch, method, properties, body = yield queue_object.get()
+ log.msg("Got message")
+ data = json.loads(body)
+ counter += 1
+
+ log.msg("Received %d/%d: %s" %(counter, lifetime, data['url'],))
+ # acknowledge the message
+ ch.basic_ack(delivery_tag=method.delivery_tag)
+
+ d = run_test(global_options, url=data['url'].encode('utf8'))
+ # When the test has been completed, go back to waiting for a message.
+ d.addCallback(readmsg, channel, queue_object, consumer_tag, counter+1)
+ except exceptions.AMQPError,v:
+ finished.errback(v)
+
+
+
+ @defer.inlineCallbacks
+ def runQueue(connection, name, qos):
+ # Set up the queue consumer. When a message is received, run readmsg
+ channel = yield connection.channel()
+ yield channel.basic_qos(prefetch_count=qos)
+ queue_object, consumer_tag = yield channel.basic_consume(
+ queue=name,
+ no_ack=False)
+ readmsg(None,channel,queue_object,consumer_tag, 0)
+
+
+
+ # Create the AMQP connection. This could be refactored to allow test URLs
+ # to be submitted through an HTTP server interface or something.
+ urlp = urlparse.urlparse(config.global_options['queue'])
+ urlargs = dict(urlparse.parse_qsl(urlp.query))
+
+ # random lifetime requests counter
+ lifetime = random.randint(820,1032)
+
+ # AMQP connection details are sent through the cmdline parameter '-Q'
+
+ creds = pika.PlainCredentials(urlp.username or 'guest',
+ urlp.password or 'guest')
+ parameters = pika.ConnectionParameters(urlp.hostname,
+ urlp.port or 5672,
+ urlp.path.rsplit('/',1)[0] or '/',
+ creds,
+ heartbeat_interval=120,
+ )
+ cc = protocol.ClientCreator(reactor,
+ twisted_connection.TwistedProtocolConnection,
+ parameters)
+ d = cc.connectTCP(urlp.hostname, urlp.port or 5672)
+ d.addCallback(lambda protocol: protocol.ready)
+ # start the wait/process sequence.
+ d.addCallback(runQueue, urlp.path.rsplit('/',1)[-1], int(urlargs.get('qos',1)))
+
+ return finished
diff --git a/ooni/reporter.py b/ooni/reporter.py
index 36a0e88..5e702bb 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -240,7 +240,7 @@ class OONIBReporter(OReporter):
Will raise :class:ooni.errors.InvalidOONIBCollectorAddress an exception
if the oonib reporter is not valid.
"""
- regexp = '^(http|httpo):\/\/[a-zA-Z0-9\-\.]+(:\d+)?$'
+ regexp = '^(http|https|httpo):\/\/[a-zA-Z0-9\-\.]+(:\d+)?$'
if not re.match(regexp, self.collectorAddress):
raise errors.InvalidOONIBCollectorAddress
@@ -265,7 +265,7 @@ class OONIBReporter(OReporter):
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
- bodyProducer = StringProducer(json.dumps(request))
+ bodyProducer = StringProducer(request_json)
try:
yield self.agent.request("PUT", url,
@@ -288,6 +288,7 @@ class OONIBReporter(OReporter):
# tor is started.
from ooni.utils.hacks import SOCKS5Agent
+ from twisted.web.client import Agent
from twisted.internet import reactor
if self.collectorAddress.startswith('httpo://'):
@@ -298,8 +299,13 @@ class OONIBReporter(OReporter):
self.agent = SOCKS5Agent(reactor, proxyEndpoint=proxyEndpoint)
elif self.collectorAddress.startswith('https://'):
- # XXX add support for securely reporting to HTTPS collectors.
- log.err("HTTPS based collectors are currently not supported.")
+ # not sure if there's something else it needs. Seems to work.
+ # Very difficult to get it to work with self-signed certs.
+ self.agent = Agent(reactor)
+
+ elif self.collectorAddress.startswith('http://'):
+ log.msg("Warning using unencrypted collector")
+ self.agent = Agent(reactor)
url = self.collectorAddress + '/report'
@@ -319,12 +325,15 @@ class OONIBReporter(OReporter):
# the backend.
'content': content
}
+ # import values from the environment
+ request.update([(k.lower(),v) for (k,v) in os.environ.iteritems()
+ if k.startswith('PROBE_')])
log.msg("Reporting %s" % url)
request_json = json.dumps(request)
log.debug("Sending %s" % request_json)
- bodyProducer = StringProducer(json.dumps(request))
+ bodyProducer = StringProducer(request_json)
log.msg("Creating report with OONIB Reporter. Please be patient.")
log.msg("This may take up to 1-2 minutes...")
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index e77913d..a753955 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -136,7 +136,7 @@ class HTTPTest(NetTestCase):
if response:
request_response['response'] = {
'headers': list(response.headers.getAllRawHeaders()),
- 'body': response_body,
+ 'body': response_body if self.localOptions.get('withoutbody',1) == 0 else '',
'code': response.code
}
if failure_string:
diff --git a/ooni/utils/net.py b/ooni/utils/net.py
index 014020c..96d62a1 100644
--- a/ooni/utils/net.py
+++ b/ooni/utils/net.py
@@ -27,8 +27,6 @@ except ImportError:
# agents with largest anonymity set.
userAgents = ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.7) Gecko/20091221 Firefox/3.5.7",
- "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3 1 2 like Mac OS X; en-us)"
- "AppleWebKit/528.18 (KHTML, like Gecko) Mobile/7D11",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6",
@@ -169,4 +167,4 @@ def hasRawSocketPermission():
socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_RAW)
return True
except socket.error:
- return False
\ No newline at end of file
+ return False
diff --git a/setup.py b/setup.py
index 9b874f3..d99a90f 100644
--- a/setup.py
+++ b/setup.py
@@ -156,7 +156,7 @@ class install(_st_install):
prefix = os.path.abspath(self.prefix)
self.set_data_files(prefix)
self.do_egg_install()
- self.ooniresources()
+ #self.ooniresources()
install_requires = []
dependency_links = []
@@ -204,7 +204,8 @@ setup(
packages=packages,
include_package_data=True,
scripts=["bin/oonideckgen", "bin/ooniprobe",
- "bin/oonireport", "bin/ooniresources"],
+ "bin/oonireport", "bin/ooniresources",
+ "bin/ooniprobequeue"],
dependency_links=dependency_links,
install_requires=install_requires,
cmdclass={"install": install},
More information about the tor-commits
mailing list