March 2013 - tor-commits - lists.torproject.org

[ooni-probe/master] Refactoring of inputProcessor
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit cc3882df9b7d3d40eb2d3e1d918b3bba475417b1 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Jan 16 20:59:48 2013 +0100 Refactoring of inputProcessor --- ooni/nettest.py | 55 +++++++++++++++++++++++++++++++++---------------------- 1 files changed, 33 insertions(+), 22 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 728d093..2da1c8f 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -419,11 +419,14 @@ class NetTestCase(object): """ raise NoPostProcessor - def inputProcessor(self, filename=None): + def inputProcessor(self, filename): """ You may replace this with your own custom input processor. It takes as input a file name. + An inputProcessor is an iterator that will yield one item from the file + and takes as argument a filename. + This can be useful when you have some input data that is in a certain format and you want to set the input attribute of the test to something that you will be able to properly process. @@ -441,36 +444,44 @@ class NetTestCase(object): Other fun stuff is also possible. """ log.debug("Running default input processor") - if filename: - fp = open(filename) - for x in fp.xreadlines(): - yield x.strip() - fp.close() + with open(filename) as f: + for line in f: + yield line.strip() + + @property + def inputFileSpecified(self): + """ + Returns: + True + when inputFile is supported and is specified + False + when input is either not support or not specified + """ + if not self.inputFile: + return False + + k = self.inputFile[0] + if self.localOptions.get(k): + return True else: - pass + return False def getInputProcessor(self): """ This method must be called after all options are validated by _checkValidOptions and _checkRequiredOptions, which ensure that if the inputFile is a required option it will be present. - """ - if self.inputFile: - if self.inputFile[0] in self.localOptions: - self.inputFilename = self.localOptions[self.inputFile[0]] - - inputProcessor = self.inputProcessor - inputFilename = self.inputFilename - class inputProcessorIterator(object): - """ - Here we convert the input processor generator into an iterator - so that we can run it twice. - """ - def __iter__(self): - return inputProcessor(inputFilename) + We check to see if it's possible to have an input file and if the user + has specified such file. - return inputProcessorIterator() + Returns: + a generator that will yield one item from the file based on the + inputProcessor. + """ + if self.inputFileSpecified: + self.inputFilename = self.localOptions[self.inputFile[0]] + return self.inputProcessor(self.inputFilename) return None

1 0

[ooni-probe/master] Add a shutdown hook for when everything is done
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit c577255a1d82332c4ca4c0b3a12620449f55e340 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Jan 16 21:02:55 2013 +0100 Add a shutdown hook for when everything is done --- ooni/oonicli.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/ooni/oonicli.py b/ooni/oonicli.py index d11a9f7..405f8ac 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -158,7 +158,7 @@ def shutdown(result): This will get called once all the operations that need to be done in the current oonicli session have been completed. """ - reator.stop() + reactor.stop() def runWithDirector(): """

1 0

[ooni-probe/master] Remove unused imports inside of ooni, oonicli and runner
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit 8eff9940590130c98bc83caf46c76580d9e4eff7 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Jan 16 21:23:05 2013 +0100 Remove unused imports inside of ooni, oonicli and runner --- ooni/runner.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/ooni/runner.py b/ooni/runner.py index 9b884a9..080db18 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -7,6 +7,9 @@ import yaml from twisted.internet import defer from twisted.internet import reactor +from txtorcon import TorConfig +from txtorcon import TorState, launch_tor + from ooni import config from ooni.reporter import OONIBReporter, YAMLReporter, OONIBReportError from ooni.inputunit import InputUnitFactory

1 0

[ooni-probe/master] Refactor NetTestLoader and add checkOptions, and add Tor support
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit 6254642808851599890c7219c1025a096b486b5c Author: aagbsn <aagbsn(a)extc.org> Date: Wed Jan 16 22:36:58 2013 +0000 Refactor NetTestLoader and add checkOptions, and add Tor support --- ooni/director.py | 25 ++++++++++++++++----- ooni/nettest.py | 62 +++++++++++++++++++++++++++--------------------------- ooni/oonicli.py | 28 ++++++++++++++---------- 3 files changed, 66 insertions(+), 49 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 9a20491..2dc4e36 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -78,6 +78,22 @@ class Director(object): self.failures = [] + self.torControlProtocol = None + + def start(self): + if config.privacy.includepcap: + log.msg("Starting") + if not config.reports.pcap: + config.generatePcapFilename() + self.startSniffing() + + if config.advanced.start_tor: + log.msg("Starting Tor...") + d = self.startTor() + else: + d = defer.succeed(None) + return d + @property def measurementSuccessRatio(self): if self.totalMeasurements == 0: @@ -154,7 +170,7 @@ class Director(object): def netTestDone(self, result, net_test): self.activeNetTests.remove(net_test) - def startNetTest(self, net_test_loader, options): + def startNetTest(self, _, net_test_loader): """ Create the Report for the NetTest and start the report NetTest. @@ -162,14 +178,11 @@ class Director(object): net_test_loader: an instance of :class:ooni.nettest.NetTestLoader - options: - is a dict containing the options to be passed to the chosen net - test. + _: #XXX very dirty hack """ report = Report(self.reporters, self.reportEntryManager) - net_test = NetTest(net_test_loader, options, report) - net_test.setUpNetTestCases() + net_test = NetTest(net_test_loader, report) net_test.director = self self.measurementManager.schedule(net_test.generateMeasurements()) diff --git a/ooni/nettest.py b/ooni/nettest.py index 4aa013c..464d91a 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -20,7 +20,8 @@ class NoTestCasesFound(Exception): class NetTestLoader(object): method_prefix = 'test' - def __init__(self, net_test_file): + def __init__(self, net_test_file, net_test_args): + self.netTestArgs = net_test_args self.testCases = self.loadNetTest(net_test_file) @property @@ -145,9 +146,33 @@ class NetTestLoader(object): test_class, _ = test_cases[0] self.testVersion = test_class.version self.testName = os.path.basename(net_test_file).strip('.py') - return test_cases + def checkOptions(self): + """ + Call processTest and processOptions methods of each NetTestCase + """ + test_classes = set([]) + for test_class, test_method in self.testCases: + test_classes.add(test_class) + + for klass in test_classes: + options = self.usageOptions() + options.parseOptions(self.netTestArgs) + if options: + klass.localOptions = options + + test_instance = klass() + if test_instance.requiresRoot: + checkForRoot() + test_instance._checkRequiredOptions() + test_instance._checkValidOptions() + + inputs = test_instance.getInputProcessor() + if not inputs: + inputs = [None] + klass.inputs = inputs + def _loadNetTestFromFileObject(self, net_test_string): """ Load NetTest from a string @@ -231,15 +256,12 @@ class NetTestState(object): class NetTest(object): director = None - def __init__(self, net_test_loader, options, report): + def __init__(self, net_test_loader, report): """ - net_test_file: - is a file object containing the test to be run. - - options: - is a dict containing the options to be passed to the net test. + net_test_loader: + an instance of :class:ooni.nettest.NetTestLoader containing + the test to be run. """ - self.options = options self.report = report self.testCases = net_test_loader.testCases @@ -322,28 +344,6 @@ class NetTest(object): self.state.allTasksScheduled() - def setUpNetTestCases(self): - """ - Call processTest and processOptions methods of each NetTestCase - """ - test_classes = set([]) - for test_class, test_method in self.testCases: - test_classes.add(test_class) - - for klass in test_classes: - klass.localOptions = self.options - - test_instance = klass() - if test_instance.requiresRoot: - checkForRoot() - test_instance._checkRequiredOptions() - test_instance._checkValidOptions() - - inputs = test_instance.getInputProcessor() - if not inputs: - inputs = [None] - klass.inputs = inputs - class NetTestCase(object): """ This is the base of the OONI nettest universe. When you write a nettest diff --git a/ooni/oonicli.py b/ooni/oonicli.py index e307d35..78389f4 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -174,12 +174,18 @@ def runWithDirector(): net_test_args = global_options.pop('subargs') net_test_file = global_options['test'] + net_test_loader = NetTestLoader(net_test_file, net_test_args) - net_test_loader = NetTestLoader(net_test_file) - options = net_test_loader.usageOptions() - options.parseOptions(net_test_args) - - net_test_options = dict(options) + try: + net_test_loader.checkOptions() + except MissingRequiredOption, option_name: + log.err('Missing required option: "%s"' % option_name) + print net_test_loader.usageOptions().getUsage() + sys.exit(2) + except usage.UsageError, e: + log.err(e) + print net_test_loader.usageOptions().getUsage() + sys.exit(2) # reporters = [YAMLReporter, OONIBReporter] @@ -198,13 +204,11 @@ def runWithDirector(): sys.exit(1) director = Director(reporters) - try: - d = director.startNetTest(net_test_loader, net_test_options) - d.addBoth(shutdown) - reactor.run() - except MissingRequiredOption, option_name: - log.err('Missing required option: "%s"' % option_name) - print options.getUsage() + d = director.start() + + d.addCallback(director.startNetTest, net_test_loader) + d.addBoth(shutdown) + reactor.run() def run(): """

1 0

[ooni-probe/master] Move startTor and startSniffing methods into Director
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit 5f589c687d42b7f24049081df9e0f64967e1c8ff Author: aagbsn <aagbsn(a)extc.org> Date: Wed Jan 16 20:34:30 2013 +0000 Move startTor and startSniffing methods into Director Removes unused imports in runner and adds exceptions to errors --- ooni/director.py | 110 +++++++++++++++++++++++++++++++++++++++++++++++++++++- ooni/errors.py | 5 ++ ooni/runner.py | 111 +----------------------------------------------------- 3 files changed, 116 insertions(+), 110 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 80cf22c..b3dfba3 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -1,7 +1,16 @@ +import sys +from ooni import config from ooni.managers import ReportEntryManager, MeasurementManager from ooni.reporter import Report -from ooni.utils import log +from ooni.utils import log, checkForRoot, NotRootError +from ooni.utils.net import randomFreePort from ooni.nettest import NetTest +from ooni.errors import UnableToStartTor + +from txtorcon import TorConfig +from txtorcon import TorState, launch_tor + +from twisted.internet import defer, reactor class Director(object): """ @@ -161,3 +170,102 @@ class Director(object): net_test.done.addBoth(self.netTestDone, net_test) return net_test.done + def startSniffing(self): + """ Start sniffing with Scapy. Exits if required privileges (root) are not + available. + """ + from ooni.utils.txscapy import ScapyFactory, ScapySniffer + try: + checkForRoot() + except NotRootError: + print "[!] Includepcap options requires root priviledges to run" + print " you should run ooniprobe as root or disable the options in ooniprobe.conf" + sys.exit(1) + + print "Starting sniffer" + config.scapyFactory = ScapyFactory(config.advanced.interface) + + if os.path.exists(config.reports.pcap): + print "Report PCAP already exists with filename %s" % config.reports.pcap + print "Renaming files with such name..." + pushFilenameStack(config.reports.pcap) + + sniffer = ScapySniffer(config.reports.pcap) + config.scapyFactory.registerProtocol(sniffer) + + + def startTor(self): + """ Starts Tor + Launches a Tor with :param: socks_port :param: control_port + :param: tor_binary set in ooniprobe.conf + """ + @defer.inlineCallbacks + def state_complete(state): + config.tor_state = state + log.msg("Successfully bootstrapped Tor") + log.debug("We now have the following circuits: ") + for circuit in state.circuits.values(): + log.debug(" * %s" % circuit) + + socks_port = yield state.protocol.get_conf("SocksPort") + control_port = yield state.protocol.get_conf("ControlPort") + client_ip = yield state.protocol.get_info("address") + + config.tor.socks_port = int(socks_port.values()[0]) + config.tor.control_port = int(control_port.values()[0]) + + config.probe_ip = client_ip.values()[0] + + log.debug("Obtained our IP address from a Tor Relay %s" % config.privacy.client_ip) + + def setup_failed(failure): + log.exception(failure) + raise UnableToStartTor + + def setup_complete(proto): + """ + Called when we read from stdout that Tor has reached 100%. + """ + log.debug("Building a TorState") + state = TorState(proto.tor_protocol) + state.post_bootstrap.addCallback(state_complete) + state.post_bootstrap.addErrback(setup_failed) + return state.post_bootstrap + + def updates(prog, tag, summary): + log.debug("%d%%: %s" % (prog, summary)) + + tor_config = TorConfig() + if config.tor.control_port: + tor_config.ControlPort = config.tor.control_port + else: + control_port = int(randomFreePort()) + tor_config.ControlPort = control_port + config.tor.control_port = control_port + + if config.tor.socks_port: + tor_config.SocksPort = config.tor.socks_port + else: + socks_port = int(randomFreePort()) + tor_config.SocksPort = socks_port + config.tor.socks_port = socks_port + + if config.tor.data_dir: + data_dir = os.path.expanduser(config.tor.data_dir) + + if not os.path.exists(data_dir): + log.msg("%s does not exist. Creating it." % data_dir) + os.makedirs(data_dir) + tor_config.DataDirectory = data_dir + + tor_config.save() + + log.debug("Setting control port as %s" % tor_config.ControlPort) + log.debug("Setting SOCKS port as %s" % tor_config.SocksPort) + + d = launch_tor(tor_config, reactor, + tor_binary=config.advanced.tor_binary, + progress_updates=updates) + d.addCallback(setup_complete) + d.addErrback(setup_failed) + return d diff --git a/ooni/errors.py b/ooni/errors.py index cd4a136..6bb2144 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -114,3 +114,8 @@ def failureToString(failure): return string +class DirectorException(Exception): + pass + +class UnableToStartTor(DirectorException): + pass diff --git a/ooni/runner.py b/ooni/runner.py index d56dc26..bf51068 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -1,5 +1,4 @@ import os -import sys import time import random @@ -8,18 +7,14 @@ import yaml from twisted.internet import defer from twisted.internet import reactor -from txtorcon import TorConfig -from txtorcon import TorState, launch_tor - from ooni import config from ooni.reporter import OONIBReporter, YAMLReporter, OONIBReportError from ooni.inputunit import InputUnitFactory -from ooni.utils import log, checkForRoot, pushFilenameStack -from ooni.utils import NotRootError, Storage -from ooni.utils.net import randomFreePort +from ooni.utils import log +from ooni.utils import Storage class InvalidResumeFile(Exception): pass @@ -230,108 +225,6 @@ def runTestCases(test_cases, options, cmd_line_options): log.exception("Problem in running test") yaml_reporter.finish() -class UnableToStartTor(Exception): - pass - -def startTor(): - """ Starts Tor - Launches a Tor with :param: socks_port :param: control_port - :param: tor_binary set in ooniprobe.conf - """ - @defer.inlineCallbacks - def state_complete(state): - config.tor_state = state - log.msg("Successfully bootstrapped Tor") - log.debug("We now have the following circuits: ") - for circuit in state.circuits.values(): - log.debug(" * %s" % circuit) - - socks_port = yield state.protocol.get_conf("SocksPort") - control_port = yield state.protocol.get_conf("ControlPort") - client_ip = yield state.protocol.get_info("address") - - config.tor.socks_port = int(socks_port.values()[0]) - config.tor.control_port = int(control_port.values()[0]) - - config.probe_ip = client_ip.values()[0] - - log.debug("Obtained our IP address from a Tor Relay %s" % config.privacy.client_ip) - - def setup_failed(failure): - log.exception(failure) - raise UnableToStartTor - - def setup_complete(proto): - """ - Called when we read from stdout that Tor has reached 100%. - """ - log.debug("Building a TorState") - state = TorState(proto.tor_protocol) - state.post_bootstrap.addCallback(state_complete) - state.post_bootstrap.addErrback(setup_failed) - return state.post_bootstrap - - def updates(prog, tag, summary): - log.debug("%d%%: %s" % (prog, summary)) - - tor_config = TorConfig() - if config.tor.control_port: - tor_config.ControlPort = config.tor.control_port - else: - control_port = int(randomFreePort()) - tor_config.ControlPort = control_port - config.tor.control_port = control_port - - if config.tor.socks_port: - tor_config.SocksPort = config.tor.socks_port - else: - socks_port = int(randomFreePort()) - tor_config.SocksPort = socks_port - config.tor.socks_port = socks_port - - if config.tor.data_dir: - data_dir = os.path.expanduser(config.tor.data_dir) - - if not os.path.exists(data_dir): - log.msg("%s does not exist. Creating it." % data_dir) - os.makedirs(data_dir) - tor_config.DataDirectory = data_dir - - tor_config.save() - - log.debug("Setting control port as %s" % tor_config.ControlPort) - log.debug("Setting SOCKS port as %s" % tor_config.SocksPort) - - d = launch_tor(tor_config, reactor, - tor_binary=config.advanced.tor_binary, - progress_updates=updates) - d.addCallback(setup_complete) - d.addErrback(setup_failed) - return d - -def startSniffing(): - """ Start sniffing with Scapy. Exits if required privileges (root) are not - available. - """ - from ooni.utils.txscapy import ScapyFactory, ScapySniffer - try: - checkForRoot() - except NotRootError: - print "[!] Includepcap options requires root priviledges to run" - print " you should run ooniprobe as root or disable the options in ooniprobe.conf" - sys.exit(1) - - print "Starting sniffer" - config.scapyFactory = ScapyFactory(config.advanced.interface) - - if os.path.exists(config.reports.pcap): - print "Report PCAP already exists with filename %s" % config.reports.pcap - print "Renaming files with such name..." - pushFilenameStack(config.reports.pcap) - - sniffer = ScapySniffer(config.reports.pcap) - config.scapyFactory.registerProtocol(sniffer) - def loadTest(cmd_line_options): """ Takes care of parsing test command line arguments and loading their

1 0

[ooni-probe/master] Validate that the collector command line option
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit ead15c4bf8cfe05e204f94b94d5e9f976e27f9dd Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Jan 16 22:38:35 2013 +0100 Validate that the collector command line option --- ooni/errors.py | 1 + ooni/nettest.py | 14 +++++++------- ooni/oonicli.py | 13 +++++++++++++ ooni/reporter.py | 54 +++++++++++++++++++++++++++++++----------------------- 4 files changed, 52 insertions(+), 30 deletions(-) diff --git a/ooni/errors.py b/ooni/errors.py index 6bb2144..9281c6c 100644 --- a/ooni/errors.py +++ b/ooni/errors.py @@ -118,4 +118,5 @@ class DirectorException(Exception): pass class UnableToStartTor(DirectorException): +class InvalidOONIBCollectorAddress(Exception): pass diff --git a/ooni/nettest.py b/ooni/nettest.py index 2da1c8f..e2c17f1 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -59,13 +59,13 @@ class NetTestLoader(object): client_geodata['countrycode'] = None test_details = {'start_time': otime.utcTimeNow(), - 'probe_asn': client_geodata['asn'], - 'probe_cc': client_geodata['countrycode'], - 'probe_ip': client_geodata['ip'], - 'test_name': self.testName, - 'test_version': self.testVersion, - 'software_name': 'ooniprobe', - 'software_version': software_version + 'probe_asn': client_geodata['asn'], + 'probe_cc': client_geodata['countrycode'], + 'probe_ip': client_geodata['ip'], + 'test_name': self.testName, + 'test_version': self.testVersion, + 'software_name': 'ooniprobe', + 'software_version': software_version } return test_details diff --git a/ooni/oonicli.py b/ooni/oonicli.py index 405f8ac..e307d35 100644 --- a/ooni/oonicli.py +++ b/ooni/oonicli.py @@ -9,6 +9,8 @@ from twisted.internet import defer, reactor, task from twisted.python import usage from twisted.python.util import spewer +from ooni.errors import InvalidOONIBCollectorAddress + from ooni import config from ooni.director import Director from ooni.reporter import YAMLReporter, OONIBReporter @@ -184,6 +186,17 @@ def runWithDirector(): yaml_reporter = YAMLReporter(net_test_loader.testDetails) reporters = [yaml_reporter] + if global_options['collector']: + try: + oonib_reporter = OONIBReporter(net_test_loader.testDetails, + global_options['collector']) + reporters.append(oonib_reporter) + except InvalidOONIBCollectorAddress: + log.err("Invalid format for oonib collector address.") + log.msg("Should be in the format http://<collector_address>:<port>") + log.msg("for example: ooniprobe -c httpo://nkvphnp3p6agi5qq.onion") + sys.exit(1) + director = Director(reporters) try: d = director.startNetTest(net_test_loader, net_test_options) diff --git a/ooni/reporter.py b/ooni/reporter.py index 30c4817..237bae7 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -25,6 +25,8 @@ except ImportError: log.err("Scapy is not installed.") +from ooni.errors import InvalidOONIBCollectorAddress + from ooni import otime from ooni.utils import geodata, pushFilenameStack from ooni.utils.net import BodyReceiver, StringProducer, userAgents @@ -234,10 +236,11 @@ class OONIBTestDetailsLookupError(OONIBReportError): pass class OONIBReporter(OReporter): - collector_address = '' def __init__(self, test_details, collector_address): - self.collector_address = collector_address - self.report_id = None + self.collectorAddress = collector_address + self.validateCollectorAddress() + + self.reportID = None from ooni.utils.txagentwithsocks import Agent from twisted.internet import reactor @@ -249,6 +252,15 @@ class OONIBReporter(OReporter): OReporter.__init__(self, test_details) + def validateCollectorAddress(self): + """ + Will raise :class:ooni.errors.InvalidOONIBCollectorAddress an exception + if the oonib reporter is not valid. + """ + regexp = re.compile('^(http|httpo):\/\/\w+(:\d+)?$') + if not regexp.match(self.collectorAddress): + raise InvalidOONIBCollectorAddress + @defer.inlineCallbacks def writeReportEntry(self, entry): log.debug("Writing report with OONIB reporter") @@ -256,12 +268,12 @@ class OONIBReporter(OReporter): content += safe_dump(entry) content += '...\n' - url = self.collector_address + '/report' + url = self.collectorAddress + '/report' - request = {'report_id': self.report_id, + request = {'report_id': self.reportID, 'content': content} - log.debug("Updating report with id %s (%s)" % (self.report_id, url)) + log.debug("Updating report with id %s (%s)" % (self.reportID, url)) request_json = json.dumps(request) log.debug("Sending %s" % request_json) @@ -277,28 +289,24 @@ class OONIBReporter(OReporter): raise OONIBReportUpdateError @defer.inlineCallbacks - def createReport(self, options): + def createReport(self): """ Creates a report on the oonib collector. """ - url = self.collector_address + '/report' - - test_details['options'] = self.cmd_line_options - - log.debug("Obtained test_details: %s" % test_details) + url = self.collectorAddress + '/report' content = '---\n' - content += safe_dump(test_details) + content += safe_dump(self.testDetails) content += '...\n' - test_name = options['name'] - test_version = options['version'] - - request = {'software_name': test_details['software_name'], - 'software_version': test_details['software_version'], - 'probe_asn': test_details['probe_asn'], - 'test_name': test_details['test_name'], - 'test_version': test_details['test_version'], + request = {'software_name': self.testDetails['software_name'], + 'software_version': self.testDetails['software_version'], + 'probe_asn': self.testDetails['probe_asn'], + 'test_name': self.testDetails['test_name'], + 'test_version': self.testDetails['test_version'], + # XXX there is a bunch of redundancy in the arguments getting sent + # to the backend. This may need to get changed in the client and the + # backend. 'content': content } @@ -335,8 +343,8 @@ class OONIBReporter(OReporter): log.exception(e) raise OONIBReportCreationError - self.report_id = parsed_response['report_id'] - self.backend_version = parsed_response['backend_version'] + self.reportID = parsed_response['report_id'] + self.backendVersion = parsed_response['backend_version'] log.debug("Created report with id %s" % parsed_response['report_id']) class ReportClosed(Exception):

1 0

[ooni-probe/master] Add missing import os
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit e51c45412126958b10a2272513667d189613286e Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 18 00:14:40 2013 +0100 Add missing import os --- ooni/director.py | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/ooni/director.py b/ooni/director.py index 2dc4e36..93e39bd 100644 --- a/ooni/director.py +++ b/ooni/director.py @@ -1,4 +1,6 @@ import sys +import os + from ooni import config from ooni.managers import ReportEntryManager, MeasurementManager from ooni.reporter import Report

1 0

[ooni-probe/master] Add the input of the Measurement to the ReportEntries
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit e2e4bddbf4af60637d80a06c7d781f7ebcd879e9 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 18 00:33:51 2013 +0100 Add the input of the Measurement to the ReportEntries --- ooni/tasks.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/ooni/tasks.py b/ooni/tasks.py index 4f7da4c..a73ce11 100644 --- a/ooni/tasks.py +++ b/ooni/tasks.py @@ -105,7 +105,7 @@ class Measurement(TaskWithTimeout): """ self.testInstance = test_class() self.testInstance.input = test_input - self.testInstance.report = {} + self.testInstance.report = {'input': test_input} self.testInstance._start_time = time.time() self.testInstance._setUp() self.testInstance.setUp()

1 0

[ooni-probe/master] Fix regexp for validating collector address
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit 6366721d21b09df38c33e65e42acd81c97598e19 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 18 00:24:59 2013 +0100 Fix regexp for validating collector address --- ooni/reporter.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index c9fb61d..b01c99e 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -249,8 +249,8 @@ class OONIBReporter(OReporter): Will raise :class:ooni.errors.InvalidOONIBCollectorAddress an exception if the oonib reporter is not valid. """ - regexp = re.compile('^(http|httpo):\/\/\w+(:\d+)?$') - if not regexp.match(self.collectorAddress): + regexp = '^(http|httpo):\/\/[a-zA-Z0-9\-\.]+(:\d+)?$' + if not re.match(regexp, self.collectorAddress): raise InvalidOONIBCollectorAddress @defer.inlineCallbacks

1 0

[ooni-probe/master] Move the instancing of the t.i.w.Agent into createReport
by isis＠torproject.org 10 Mar '13

10 Mar '13

commit 5bf37ab039a64937cdd97959069145148d5d36d2 Author: Arturo Filastò <art(a)fuffa.org> Date: Fri Jan 18 00:24:20 2013 +0100 Move the instancing of the t.i.w.Agent into createReport Add XXX about possible cleaner solutions to the problem --- ooni/reporter.py | 22 ++++++++++++++-------- 1 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ooni/reporter.py b/ooni/reporter.py index bfd2330..c9fb61d 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -242,14 +242,6 @@ class OONIBReporter(OReporter): self.reportID = None - from ooni.utils.txagentwithsocks import Agent - from twisted.internet import reactor - try: - self.agent = Agent(reactor, sockshost="127.0.0.1", - socksport=int(config.tor.socks_port)) - except Exception, e: - log.exception(e) - OReporter.__init__(self, test_details) def validateCollectorAddress(self): @@ -293,6 +285,20 @@ class OONIBReporter(OReporter): """ Creates a report on the oonib collector. """ + # XXX we should probably be setting this inside of the constructor, + # however config.tor.socks_port is not set until Tor is started and the + # reporter is instantiated before Tor is started. We probably want to + # do this with some deferred kung foo or instantiate the reporter after + # tor is started. + + from ooni.utils.txagentwithsocks import Agent + from twisted.internet import reactor + try: + self.agent = Agent(reactor, sockshost="127.0.0.1", + socksport=int(config.tor.socks_port)) + except Exception, e: + log.exception(e) + url = self.collectorAddress + '/report' content = '---\n'

1 0