September 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Moving bootstrap_tor() functions to experiment()
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 0b892accffdd5a09c5bba87b9c75adfcdc50eea2 Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Sep 11 11:38:45 2012 +0000 Moving bootstrap_tor() functions to experiment() --- ooni/assets/bridgetests.txt | 8 +- ooni/plugins/bridget.py | 362 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 315 insertions(+), 55 deletions(-) diff --git a/ooni/assets/bridgetests.txt b/ooni/assets/bridgetests.txt index c837988..7bba841 100644 --- a/ooni/assets/bridgetests.txt +++ b/ooni/assets/bridgetests.txt @@ -1,3 +1,5 @@ -213.151.89.102:9001 -108.166.106.156:443 -217.150.224.213:443 +#213.151.89.102:9001 +#108.166.106.156:443 +#217.150.224.213:443 +85.193.252.111:443 +#68.98.41.14:9001 diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 5562fb6..f665290 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -8,92 +8,350 @@ # | connection to a list of bridges or relays. | # +----------------------------------------------+ # -# :authors: Arturo Filasto, Isis Lovecruft, Jacob Appelbaum +# :authors: Arturo Filasto, Isis Lovecruft # :licence: see included LICENSE # :version: 0.1.0-alpha -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import reactor +from __future__ import with_statement +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import reactor, error -from ooni.utils import log -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset +import random -class bridgetArgs(usage.Options): - optParameters = [['bridges', 'b', None, 'List of bridges to scan'], - ['relays', 'f', None, 'List of relays to scan'], - ['resume', 'r', 0, 'Resume at this index']] +try: + from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher +except: + print "BridgeT requires txtorcon: https://github.com/meejah/txtorcon.git" + print "Your copy of OONI should have it included, if you're seeing this" + print "message, please file a bug report." + log.msg ("Bridget: Unable to import from ooni.lib.txtorcon") -class bridgetTest(OONITest): +from ooni.utils import log +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset + + +class BridgetArgs(usage.Options): + optParameters = [ + ['bridges', 'b', None, + 'List of bridges to scan (IP:ORport)'], + ['relays', 'f', None, + 'List of relays to scan (IP)'], + ['socks', 's', 9049, + 'Tor SocksPort to use'], + ['control', 'c', 9052, + 'Tor ControlPort to use'], + ['random', 'x', False, + 'Randomize control and socks ports'], + ['tor-path', 'p', '/usr/sbin/tor', + 'Path to the Tor binary to use'], + ['transport', 't', None, + 'Tor ClientTransportPlugin string. Requires -b.'], + ['resume', 'r', 0, + 'Resume at this index']] + +class CustomCircuit(CircuitListenerMixin): + implements(IStreamAttacher) + + from txtorcon.interface import IRouterContainer, ICircuitContainer + + def __init__(self, state): + self.state = state + self.waiting_circuits = [] + + def waiting_on(self, circuit): + for (circid, d) in self.waiting_circuits: + if circuit.id == circid: + return true + return False + + def circuit_extend(self, circuit, router): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + if self.waiting_on(circuit): + log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) + + def circuit_built(self, circuit): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + + log.msg("Circuit %s built ..." % circuit.id) + log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) + + for (circid, d) in self.waiting_circuits: + if circid == circuit.id: + self.waiting_circuits.remove(circid, d) + d.callback(circuit) + + def circuit_failed(self, circuit, reason): + if self.waiting_on(circuit): + log.msg("A circuit we requested %s failed for reason %s" % + (circuit.id, reason)) + circid, d = None, None + for x in self.waiting_circuits: + if x[0] == circuit.id: + circid, d, stream_cc = x + if d is None: + raise Exception("Expected to find circuit.") + + self.waiting_circuits.remove((circid, d)) + log.msg("Trying to build a circuit for %s" % circid) + self.request_circuit_build(d) + + def check_circuit_route(self, circuit, router): + if router in circuit.path: + #router.update() ## XXX can i use without args? no. + TorInfo.dump(self) + + def request_circuit_build(self, deferred): + entries = self.state.entry_guards.value() + relays = self.state.routers.values() + + log.msg("We have these nodes listed as entry guards:") + log.msg("%s" % entries) + log.msg("We have these nodes listed as relays:") + log.msg("%s" % relays) + + path = [random.choice(entries), + random.choice(relays), + random.choice(relays)] + + log.msg("Requesting a circuit: %s" + % '-->'.join(map(lambda x: x.location.countrycode, path))) + + class AppendWaiting: + def __init__(self, attacher, deferred): + self.attacher = attacher + self.d = deferred + + def __call__(self, circuit): + """ + Return from build_circuit is a Circuit, however, we want to + wait until it is built before we can issue an attach on it and + callback to the Deferred we issue here. + """ + log.msg("Circuit %s is in progress ..." % circuit.id) + self.attacher.waiting_circuits.append((circuit.id, self.d)) + + return self.state.build_circuit(path).addCallback(AppendWaiting(self, deferred_to_callback)).addErrback(log.err) + +class BridgetTest(OONITest): + """ + XXX fill me in + + :ivar config: + An :class:`ooni.lib.txtorcon.TorConfig` instance. + :ivar use_bridges: + A boolean integer [0|1]. + :ivar entry_nodes: + A string of all provided relays to test. We have to do this + because txtorcon.TorState().entry_guards won't build a custom + circuit if the first hop isn't in the torrc's EntryNodes. + :ivar relay_list: + The same as :ivar entry_nodes: but in list form. + :ivar socks_port: + Integer for Tor's SocksPort. + :ivar control_port: + Integer for Tor's ControlPort. + :ivar plug_transport: + String defining the Tor's ClientTransportPlugin, for testing + a bridge's pluggable transport functionality. + :ivar tor_binary: + Path to the Tor binary to use, e.g. \'/usr/sbin/tor\' + """ implements(IPlugin, ITest) shortName = "bridget" - description = "Use a slave Tor process to test RELAY_EXTEND to bridges/relays" + description = "Use a Tor process to test connecting to bridges/relays" requirements = None - options = bridgetArgs + options = BridgetArgs blocking = False - def experiment(self, args): - log.msg("BridgeT: initiating test ... ") + def load_assets(self): + """ + Load bridges from file given in user options. Bridges should be given + in the form IP:ORport. We don't want to load relays as assets, because + it's inefficient to test them one at a time. + """ + assets = {} + if self.local_options: + if self.local_options['bridges']: + assets.update({'bridge': Asset(self.local_options['bridges'])}) + return assets + def initialize(self): + """ + Extra initialization steps. We only want one child Tor process + running, so we need to deal with the creation of the torrc only once, + before the experiment runs. + """ + self.relay_list = [] + ## XXX why doesn't the default set in the options work? + self.socks_port = 9049 + self.control_port = 9052 + + if self.local_options: + try: + from ooni.lib.txtorcon import TorConfig + except: + log.msg("Could not import TorConfig class from txtorcon") + raise + + options = self.local_options + self.config = TorConfig() + self.socks_port = options['socks'] + self.control_port = options['control'] + + if options['bridges']: + log.msg("Using Bridges ...") + self.config.UseBridges = 1 + + if options['relays']: + ''' + Stupid hack for when testing only relays (and not bridges): + Tor doesn't use EntryNodes when UseBridges is enabled, but + txtorcon requires config.state.entry_guards to make a custom + circuit, so we should list them as EntryNodes anyway. + ''' + log.msg("Using relays ...") + + with open(options['relays']) as relay_file: + for line in relay_file.readlines(): + if line.startswith('#'): + continue + else: + relay = line.replace('\n', '') ## not assets because + self.relay_list.append(relay) ## we don't want to + ## test one at a time + self.config.EntryNodes = ',' + self.config.EntryNodes.join(relay_list) + + if options['random']: + log.msg("Using randomized ControlPort and SocksPort ...") + self.socks_port = random.randint(1024, 2**16) + self.control_port = random.randint(1024, 2**16) + + if options['tor-path']: + self.tor_binary = options['tor-path'] + + if options['transport']: + ''' + ClientTransportPlugin transport socks4|socks5 IP:PORT + ClientTransportPlugin transport exec path-to-binary [options] + ''' + if not options['bridges']: + e = "To test pluggable transports, you must provide a file" + e = e+"with a list of bridge IP:ORPorts. See \'-b' option." + raise usage.UsageError("%s" % e) + + log.msg("Using pluggable transport ...") + ## XXX fixme there's got to be a better way to check the exec + assert type(options['transport']) is str + self.config.ClientTransportPlugin = options['transport'] + + self.config.SocksPort = self.socks_port + self.config.ControlPort = self.control_port + self.config.save() + + print self.config.create_torrc() + report = {'tor_config': self.config.config} + #log.msg("Starting Tor") + # + #self.tor_process_protocol = self.bootstrap_tor(self.config) + #self.d = self.bootstrap_tor(self.d, self.config, + # self.reactor, self.report) + #return self.d + return self.config + else: + return None + + def bootstrap_tor(self, config, args): + """ + Launch a Tor process with the TorConfig instance returned from + initialize(). + + Returns a Deferred which callbacks with a TorProcessProtocol connected + to the fully-bootstrapped Tor; this has a txtorcon.TorControlProtocol + instance as .protocol. + """ from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor - reactor = self.reactor + log.msg("Tor config: %s" % config) + log.msg("Starting Tor ...") def setup_failed(args): log.msg("Setup Failed.") report.update({'failed': args}) reactor.stop() - #return report def setup_complete(proto): log.msg("Setup Complete: %s" % proto) state = TorState(proto.tor_protocol) state.post_bootstrap.addCallback(state_complete).addErrback(setup_failed) report.update({'success': args}) - #return report def bootstrap(c): conf = TorConfig(c) conf.post_bootstrap.addCallback(setup_complete).addErrback(setup_failed) - log.msg("Slave Tor process connected, bootstrapping ...") - - config = TorConfig() - import random - config.SocksPort = random.randint(1024, 2**16) - config.ControlPort = random.randint(1024, 2**16) - #config.SocksPort = 12345 - #config.ControlPort = 12346 - - if 'bridge' in args: - config.UseBridges = 1 - config.Bridge = args['bridge'] - config.save() - print config.create_torrc() - report = {'tor_config': config.config} - log.msg("Starting Tor") + log.msg("Tor process connected, bootstrapping ...") def updates(prog, tag, summary): log.msg("%d%%: %s" % (prog, summary)) - #return - d = launch_tor(config, self.reactor, progress_updates=updates) - d.addCallback(setup_complete) - d.addErrback(setup_failed) - return d + ## :return: a Deferred which callbacks with a TorProcessProtocol + ## connected to the fully-bootstrapped Tor; this has a + ## txtorcon.TorControlProtocol instance as .protocol. + deferred = launch_tor(config, reactor, progress_updates=updates, + tor_binary=self.tor_binary) + deferred.addCallback(bootstrap, config) + deferred.addErrback(setup_failed) - def load_assets(self): - assets = {} - if self.local_options: - if self.local_options['bridges']: - assets.update({'bridge': Asset(self.local_options['bridges'])}) - elif self.local_options['relays']: - assets.update({'relay': Asset(self.local_options['relays'])}) - return assets + #print "Tor process ID: %s" % d.transport.pid + return deferred + + def experiment(self, args): + """ + XXX fill me in + """ + log.msg("BridgeT: initiating test ... ") + + def reconfigure_failed(args): + + def reconfigure_controller(proto, args): + ## if bridges and relays, use one bridge then build a circuit + ## from the relays + if args['bridge']: + print args + print args['bridge'] + #d.addCallback(CustomCircuit(state)) + proto.set_conf('Bridge', args['bridge']) + ## if bridges only, try one bridge at a time, but don't build + ## circuits, just return + ## if relays only, build circuits from relays + + ## XXX see txtorcon.TorControlProtocol.add_event_listener + ## we may not need full CustomCircuit class + + d = defer.Deferred() ## 1 make deferred + d.addCallback(self.bootstrap_tor, self.config) ## 2 blastoff + ## 3 reconfigure + ## 4 build circuits + + #d = self.bootstrap_tor(self.config, args).addCallback(configure, + ##c = CustomCircuit(state) + #d.addCallback(configure, d.protocol) + #d.addErrback(err) + #return d + +## So that getPlugins() can register the Test: +bridget = BridgetTest(None, None, None) -# We need to instantiate it otherwise getPlugins does not detect it -# XXX Find a way to load plugins without instantiating them. -bridget = bridgetTest(None, None, None) +## ISIS' NOTES +## +## +##

1 0

[ooni-probe/master] changed assets to a list for experiment to localize asynchronicity to local experiment
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 1f6520aa55b87b3bf76a558a4dee2b82afc1287a Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Sep 11 12:33:48 2012 +0000 changed assets to a list for experiment to localize asynchronicity to local experiment --- ooni/plugins/bridget.py | 33 +++++++++++++++++++++++---------- 1 files changed, 23 insertions(+), 10 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 822763f..b57d7c1 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -139,6 +139,19 @@ class CustomCircuit(CircuitListenerMixin): return self.state.build_circuit(path).addCallback(AppendWaiting(self, deferred_to_callback)).addErrback(log.err) +class BridgetAsset(Asset): + """ + Class for parsing bridge assets so that they can be commented out. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + if line.startswith('#'): + return + else: + return line.replace('\n','') + class BridgetTest(OONITest): """ XXX fill me in @@ -177,10 +190,18 @@ class BridgetTest(OONITest): in the form IP:ORport. We don't want to load relays as assets, because it's inefficient to test them one at a time. """ - assets = {} + assets = [] if self.local_options: if self.local_options['bridges']: - assets.update({'bridge': Asset(self.local_options['bridges'])}) + #assets.update({'bridge': + # BridgetAsset(self.local_options['bridges'])}) + with open(self.local_options['bridges'] as bridge_file): + for line in bridge_file.readlines(): + if line.startswith('#'): + continue + else: + bridge = line.replace('\n','') + assets.append(bridge) return assets def initialize(self): @@ -259,12 +280,6 @@ class BridgetTest(OONITest): print self.config.create_torrc() report = {'tor_config': self.config.config} - #log.msg("Starting Tor") - # - #self.tor_process_protocol = self.bootstrap_tor(self.config) - #self.d = self.bootstrap_tor(self.d, self.config, - # self.reactor, self.report) - #return self.d return self.config else: return None @@ -338,8 +353,6 @@ class BridgetTest(OONITest): d.addCallback(bootstrap, self.config) d.addErrback(setup_failed) - d.addCallback(self.bootstrap_tor, self.config) ## 2 blastoff - ## 3 reconfigure ## 4 build circuits #print "Tor process ID: %s" % d.transport.pid

1 0

[ooni-probe/master] error with bridges not respecting being commented out
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit b190bb198dd6e2abe09fa5c96a3d30c82ba1a3dd Author: Isis Lovecruft <isis(a)torproject.org> Date: Tue Sep 11 12:11:39 2012 +0000 error with bridges not respecting being commented out --- ooni/plugins/bridget.py | 75 ++++++++++++++++++++++------------------------ 1 files changed, 36 insertions(+), 39 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index f665290..822763f 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -269,19 +269,15 @@ class BridgetTest(OONITest): else: return None - def bootstrap_tor(self, config, args): + def experiment(self, args): """ - Launch a Tor process with the TorConfig instance returned from - initialize(). - - Returns a Deferred which callbacks with a TorProcessProtocol connected - to the fully-bootstrapped Tor; this has a txtorcon.TorControlProtocol - instance as .protocol. + XXX fill me in """ + log.msg("BridgeT: initiating test ... ") + from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor - log.msg("Tor config: %s" % config) log.msg("Starting Tor ...") def setup_failed(args): @@ -296,6 +292,14 @@ class BridgetTest(OONITest): report.update({'success': args}) def bootstrap(c): + """ + Launch a Tor process with the TorConfig instance returned from + initialize(). + + Returns a Deferred which callbacks with a TorProcessProtocol connected + to the fully-bootstrapped Tor; this has a txtorcon.TorControlProtocol + instance as .protocol. + """ conf = TorConfig(c) conf.post_bootstrap.addCallback(setup_complete).addErrback(setup_failed) log.msg("Tor process connected, bootstrapping ...") @@ -303,50 +307,43 @@ class BridgetTest(OONITest): def updates(prog, tag, summary): log.msg("%d%%: %s" % (prog, summary)) - ## :return: a Deferred which callbacks with a TorProcessProtocol - ## connected to the fully-bootstrapped Tor; this has a - ## txtorcon.TorControlProtocol instance as .protocol. - deferred = launch_tor(config, reactor, progress_updates=updates, - tor_binary=self.tor_binary) - deferred.addCallback(bootstrap, config) - deferred.addErrback(setup_failed) - - #print "Tor process ID: %s" % d.transport.pid - return deferred - - def experiment(self, args): - """ - XXX fill me in - """ - log.msg("BridgeT: initiating test ... ") - - def reconfigure_failed(args): - def reconfigure_controller(proto, args): ## if bridges and relays, use one bridge then build a circuit ## from the relays - if args['bridge']: - print args - print args['bridge'] - #d.addCallback(CustomCircuit(state)) - proto.set_conf('Bridge', args['bridge']) + print args + print args['bridge'] + #d.addCallback(CustomCircuit(state)) + proto.set_conf('Bridge', args['bridge']) + ## if bridges only, try one bridge at a time, but don't build ## circuits, just return ## if relays only, build circuits from relays + + def reconfigure_failed(proto, args): + log.msg("Reconfiguring Tor config with args %s failed" % args) + reactor.stop() ## XXX see txtorcon.TorControlProtocol.add_event_listener ## we may not need full CustomCircuit class - d = defer.Deferred() ## 1 make deferred + ## :return: a Deferred which callbacks with a TorProcessProtocol + ## connected to the fully-bootstrapped Tor; this has a + ## txtorcon.TorControlProtocol instance as .protocol. + + print args + d = launch_tor(self.config, reactor, progress_updates=updates, + tor_binary=self.tor_binary) + d.addCallback(reconfigure_controller, args) + d.addErrback(reconfigure_failed, args) + d.addCallback(bootstrap, self.config) + d.addErrback(setup_failed) + d.addCallback(self.bootstrap_tor, self.config) ## 2 blastoff ## 3 reconfigure ## 4 build circuits - - #d = self.bootstrap_tor(self.config, args).addCallback(configure, - ##c = CustomCircuit(state) - #d.addCallback(configure, d.protocol) - #d.addErrback(err) - #return d + + #print "Tor process ID: %s" % d.transport.pid + return d ## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None)

1 0

[ooni-probe/master] Marked the extra import in ooni/plugoo/tests.py, but did not remove it yet.
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 983b8827237e7d4885dad4bc5725d3b5c8e40bbe Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Sep 12 02:20:34 2012 +0000 Marked the extra import in ooni/plugoo/tests.py, but did not remove it yet. --- ooni/plugoo/tests.py | 7 ++++--- 1 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 7c190eb..13c06ed 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -5,6 +5,7 @@ from zope.interface import Interface, Attribute import logging import itertools from twisted.internet import reactor, defer, threads +## XXX why is this imported and not used? from twisted.python import failure from ooni.utils import log @@ -51,14 +52,14 @@ class OONITest(object): def load_assets(self): """ - This method should be overriden by the test writer to provide the logic - for loading their assets. + This method should be overriden by the test writer to provide the + logic for loading their assets. """ return {} def __repr__(self): return "<OONITest %s %s %s>" % (self.options, self.global_options, - self.assets) + self.assets) def end(self): """

1 0

[ooni-probe/master] There's still a problem with Tor's DataDirectory not being found, but it only asks for the consensus once per list of bridges/relays, i.e. once for every time bridget is run. This is still not optimal and should be fixed.
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 859f9399455975a5d828427ede1d7e3662f1e591 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 13 12:41:39 2012 +0000 There's still a problem with Tor's DataDirectory not being found, but it only asks for the consensus once per list of bridges/relays, i.e. once for every time bridget is run. This is still not optimal and should be fixed. The CustomCircuit class needs to be called as path=[] in reconf_controller(), but this should only be relevant when testing relays. --- ooni/plugins/bridget.py | 191 +++++++++++++++++++++++++---------------------- 1 files changed, 101 insertions(+), 90 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 2a7d573..a53630c 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -16,9 +16,10 @@ from __future__ import with_statement from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin -from twisted.internet import reactor, error +from twisted.internet import defer, error, reactor import random +import sys try: from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher @@ -61,9 +62,11 @@ class BridgetArgs(usage.Options): optFlags = [['random', 'x', 'Randomize control and socks ports']] def postOptions(self): + ## We can't test pluggable transports without bridges if self['transport'] and not self['bridges']: e = "Pluggable transport requires the bridges option" raise usage.UsageError, e + ## We can't use random and static port simultaneously if self['socks'] and self['control']: if self['random']: e = "Unable to use random and specific ports simultaneously" @@ -199,43 +202,6 @@ class BridgetTest(OONITest): options = BridgetArgs blocking = False - def load_assets(self): - """ - Load bridges and/or relays from files given in user options. Bridges - should be given in the form IP:ORport. We don't want to load these as - assets, because it's inefficient to start a Tor process for each one. - """ - assets = {} - self.bridge_list = [] - self.relay_list = [] - - ## XXX fix me - ## we should probably find a more memory nice way to load addresses, - ## in case the files are really large - if self.local_options: - if self.local_options['bridges']: - log.msg("Loading bridge information from %s ..." - % self.local_options['bridges']) - with open(self.local_options['bridges']) as bridge_file: - for line in bridge_file.readlines(): - if line.startswith('#'): - continue - else: - self.bridge_list.append(line.replace('\n','')) - assets.update({'bridges': self.bridge_list}) - - if self.local_options['relays']: - log.msg("Loading relay information from %s ..." - % self.local_options['relays']) - with open(options['relays']) as relay_file: - for line in relay_file.readlines(): - if line.startswith('#'): - continue - else: - self.relay_list.append(line.replace('\n','')) - assets.update({'relays': self.relay_list}) - return assets - def initialize(self): """ Extra initialization steps. We only want one child Tor process @@ -257,6 +223,10 @@ class BridgetTest(OONITest): options = self.local_options self.config = TorConfig() + ## Don't run the experiment if we don't have anything to test + if not options['bridges'] and not options['relays']: + self.suicide = True + if options['bridges']: self.config.UseBridges = 1 @@ -300,8 +270,42 @@ class BridgetTest(OONITest): self.config.ControlPort = self.control_port self.config.save() - print self.config.create_torrc() - report = {'tor_config': self.config.config} + def load_assets(self): + """ + Load bridges and/or relays from files given in user options. Bridges + should be given in the form IP:ORport. We don't want to load these as + assets, because it's inefficient to start a Tor process for each one. + """ + assets = {} + self.bridge_list = [] + self.relay_list = [] + + ## XXX fix me + ## we should probably find a more memory nice way to load addresses, + ## in case the files are really large + if self.local_options: + if self.local_options['bridges']: + log.msg("Loading bridge information from %s ..." + % self.local_options['bridges']) + with open(self.local_options['bridges']) as bridge_file: + for line in bridge_file.readlines(): + if line.startswith('#'): + continue + else: + self.bridge_list.append(line.replace('\n','')) + assets.update({'bridges': self.bridge_list}) + + if self.local_options['relays']: + log.msg("Loading relay information from %s ..." + % self.local_options['relays']) + with open(options['relays']) as relay_file: + for line in relay_file.readlines(): + if line.startswith('#'): + continue + else: + self.relay_list.append(line.replace('\n','')) + assets.update({'relays': self.relay_list}) + return assets def experiment(self, args): """ @@ -317,75 +321,83 @@ class BridgetTest(OONITest): this has a :class:`ooni.lib.txtorcon.torcontol.TorControlProtocol <TorControlProtocol>` instance as .protocol. """ - log.msg("Bridget: initiating test ... ") - from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor - log.msg("Starting Tor ...") - - def setup_failed(args): - log.msg("Setup Failed.") - report.update({'failed': args}) - reactor.stop() - - def setup_complete(proto): - log.msg("Setup Complete: %s" % proto) - state = TorState(proto.tor_protocol) - state.post_bootstrap.addCallback(state_complete).addErrback(setup_failed) - report.update({'success': args}) - def bootstrap(ctrl): """ Launch a Tor process with the TorConfig instance returned from initialize(). """ conf = TorConfig(ctrl) - conf.post_bootstrap.addCallback(setup_complete).addErrback(setup_failed) + conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) log.msg("Tor process connected, bootstrapping ...") - def updates(prog, tag, summary): - log.msg("%d%%: %s" % (prog, summary)) - - def reconfigure_controller(conf, bridge): + def reconf_controller(conf, bridge): ## if bridges and relays, use one bridge then build a circuit ## from three relays conf.Bridge = bridge ## XXX do we need a SIGHUP to restart? - ## XXX see txtorcon.TorControlProtocol.add_event_listener we may not - ## need full CustomCircuit class - - ## if bridges only, try one bridge at a time, but don't build - ## circuits, just return + ## XXX see txtorcon.TorControlProtocol.add_event_listener we + ## may not need full CustomCircuit class - ## if relays only, build circuits from relays + ## if bridges only, try one bridge at a time, but don't build + ## circuits, just return + ## if relays only, build circuits from relays - def reconfigure_failed(args): + def reconf_fail(args): log.msg("Reconfiguring Tor config with args %s failed" % args) reactor.stop() - - if len(self.bridge_list) >= 1: - for bridge in self.bridge_list: - try: - print "BRIDGE IS %s" % bridge - reconfigure_controller(self.config, bridge) - except: - reconfigure_failed(bridge) - - ## :return: a Deferred which callbacks with a TorProcessProtocol - ## connected to the fully-bootstrapped Tor; this has a - ## txtorcon.TorControlProtocol instance as .protocol. - d = launch_tor(self.config, - reactor, - progress_updates=updates, - tor_binary=self.tor_binary) - d.addCallback(bootstrap, self.config) - d.addErrback(setup_failed) - ## 4 build circuits - - #print "Tor process ID: %s" % d.transport.pid - return d + + def setup_fail(args): + log.msg("Setup Failed.") + report.update({'failed': args}) + reactor.stop() + + def setup_done(proto): + log.msg("Setup Complete: %s" % proto) + state = TorState(proto.tor_protocol) + state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail) + report.update({'success': args}) + + def updates(prog, tag, summary): + log.msg("%d%%: %s" % (prog, summary)) + + if len(args) == 0: + log.msg("Bridget needs lists of bridges and/or relays to test!") + log.msg("Exiting ...") + d = sys.exit() + return d + + else: + if len(self.bridge_list) >= 1: + for bridge in self.bridge_list: + try: + print "BRIDGE IS %s" % bridge + reconf_controller(self.config, bridge) + except: + reconf_fail(bridge) + + log.msg("Bridget: initiating test ... ") + log.msg("Using the following as our torrc:\n%s" + % self.config.create_torrc()) + report = {'tor_config': self.config.config} + log.msg("Starting Tor ...") + + ## :return: a Deferred which callbacks with a TorProcessProtocol + ## connected to the fully-bootstrapped Tor; this has a + ## txtorcon.TorControlProtocol instance as .protocol. + d = launch_tor(self.config, + reactor, + progress_updates=updates, + tor_binary=self.tor_binary) + d.addCallback(bootstrap, self.config) + d.addErrback(setup_fail) + ## now build circuits + + #print "Tor process ID: %s" % d.transport.pid + return d ## So that getPlugins() can register the Test: bridget = BridgetTest(None, None, None) @@ -399,7 +411,6 @@ bridget = BridgetTest(None, None, None) ## ## TODO: ## o add option for any kwarg=arg self.config setting -## o add DataDirectory option? ## o cleanup documentation ## o check if bridges are public relays ## o take bridge_desc file as input, also be able to give same

1 0

[ooni-probe/master] bridget is functional again, but DataDirectory still doesn't work so it takes forever and is hammering the DirAuths.
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit b49b70ebcdb1073e7a6fddf9a384f7e9d9c7398a Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Sep 12 02:22:55 2012 +0000 bridget is functional again, but DataDirectory still doesn't work so it takes forever and is hammering the DirAuths. --- ooni/plugins/bridget.py | 229 ++++++++++++++++++++++++++++------------------- 1 files changed, 136 insertions(+), 93 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index b57d7c1..2a7d573 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -34,23 +34,40 @@ from ooni.plugoo.assets import Asset class BridgetArgs(usage.Options): + + def portCheck(number): + number = int(number) + if number not in range(1024, 65535): + raise ValueError("Port out of range") + portCheck.coerceDoc = "Ports must be between 1024 and 65535" + optParameters = [ - ['bridges', 'b', None, - 'List of bridges to scan (IP:ORport)'], - ['relays', 'f', None, - 'List of relays to scan (IP)'], - ['socks', 's', 9049, + ['bridges', 'b', None, + 'List of bridges to scan <IP>:<ORport>'], + ['relays', 'f', None, + 'List of relays to scan <IP>'], + ['socks', 's', 9049, portCheck, 'Tor SocksPort to use'], - ['control', 'c', 9052, + ['control', 'c', 9052, portCheck, 'Tor ControlPort to use'], - ['random', 'x', False, - 'Randomize control and socks ports'], - ['tor-path', 'p', '/usr/sbin/tor', + ['tor-path', 'p', None, 'Path to the Tor binary to use'], + ['data-dir', 'd', None, + 'Tor DataDirectory to use'], ['transport', 't', None, - 'Tor ClientTransportPlugin string. Requires -b.'], - ['resume', 'r', 0, + 'Tor ClientTransportPlugin'], + ['resume', 'r', 0, 'Resume at this index']] + optFlags = [['random', 'x', 'Randomize control and socks ports']] + + def postOptions(self): + if self['transport'] and not self['bridges']: + e = "Pluggable transport requires the bridges option" + raise usage.UsageError, e + if self['socks'] and self['control']: + if self['random']: + e = "Unable to use random and specific ports simultaneously" + raise usageError, e class CustomCircuit(CircuitListenerMixin): implements(IStreamAttacher) @@ -158,14 +175,12 @@ class BridgetTest(OONITest): :ivar config: An :class:`ooni.lib.txtorcon.TorConfig` instance. - :ivar use_bridges: - A boolean integer [0|1]. - :ivar entry_nodes: - A string of all provided relays to test. We have to do this - because txtorcon.TorState().entry_guards won't build a custom - circuit if the first hop isn't in the torrc's EntryNodes. :ivar relay_list: - The same as :ivar entry_nodes: but in list form. + A list of all provided relays to test. We have to do this because + txtorcon.TorState().entry_guards won't build a custom circuit if the + first hop isn't in the torrc's EntryNodes. + :ivar bridge_list: + A list of all provided bridges to test. :ivar socks_port: Integer for Tor's SocksPort. :ivar control_port: @@ -179,77 +194,84 @@ class BridgetTest(OONITest): implements(IPlugin, ITest) shortName = "bridget" - description = "Use a Tor process to test connecting to bridges/relays" + description = "Use a Tor process to test connecting to bridges and relays" requirements = None options = BridgetArgs blocking = False def load_assets(self): """ - Load bridges from file given in user options. Bridges should be given - in the form IP:ORport. We don't want to load relays as assets, because - it's inefficient to test them one at a time. + Load bridges and/or relays from files given in user options. Bridges + should be given in the form IP:ORport. We don't want to load these as + assets, because it's inefficient to start a Tor process for each one. """ - assets = [] + assets = {} + self.bridge_list = [] + self.relay_list = [] + + ## XXX fix me + ## we should probably find a more memory nice way to load addresses, + ## in case the files are really large if self.local_options: if self.local_options['bridges']: - #assets.update({'bridge': - # BridgetAsset(self.local_options['bridges'])}) - with open(self.local_options['bridges'] as bridge_file): + log.msg("Loading bridge information from %s ..." + % self.local_options['bridges']) + with open(self.local_options['bridges']) as bridge_file: for line in bridge_file.readlines(): if line.startswith('#'): continue else: - bridge = line.replace('\n','') - assets.append(bridge) + self.bridge_list.append(line.replace('\n','')) + assets.update({'bridges': self.bridge_list}) + + if self.local_options['relays']: + log.msg("Loading relay information from %s ..." + % self.local_options['relays']) + with open(options['relays']) as relay_file: + for line in relay_file.readlines(): + if line.startswith('#'): + continue + else: + self.relay_list.append(line.replace('\n','')) + assets.update({'relays': self.relay_list}) return assets def initialize(self): """ Extra initialization steps. We only want one child Tor process - running, so we need to deal with the creation of the torrc only once, - before the experiment runs. + running, so we need to deal with the creation of TorConfig() only + once, before the experiment runs. """ - self.relay_list = [] - ## XXX why doesn't the default set in the options work? self.socks_port = 9049 self.control_port = 9052 + self.tor_binary = '/usr/sbin/tor' + self.data_directory = None if self.local_options: try: from ooni.lib.txtorcon import TorConfig except: - log.msg("Could not import TorConfig class from txtorcon") - raise + e = "Could not import TorConfig class from txtorcon!" + raise ImportError, e options = self.local_options self.config = TorConfig() - self.socks_port = options['socks'] - self.control_port = options['control'] if options['bridges']: - log.msg("Using Bridges ...") self.config.UseBridges = 1 if options['relays']: - ''' - Stupid hack for when testing only relays (and not bridges): - Tor doesn't use EntryNodes when UseBridges is enabled, but - txtorcon requires config.state.entry_guards to make a custom - circuit, so we should list them as EntryNodes anyway. - ''' - log.msg("Using relays ...") - - with open(options['relays']) as relay_file: - for line in relay_file.readlines(): - if line.startswith('#'): - continue - else: - relay = line.replace('\n', '') ## not assets because - self.relay_list.append(relay) ## we don't want to - ## test one at a time - self.config.EntryNodes = ',' - self.config.EntryNodes.join(relay_list) + ## Stupid hack for testing only relays: + ## Tor doesn't use EntryNodes when UseBridges is enabled, but + ## config.state.entry_guards needs to include the first hop to + ## build a custom circuit. + self.config.EntryNodes = ','.join(relay_list) + + if options['socks']: + self.socks_port = options['socks'] + + if options['control']: + self.control_port = options['control'] if options['random']: log.msg("Using randomized ControlPort and SocksPort ...") @@ -259,14 +281,14 @@ class BridgetTest(OONITest): if options['tor-path']: self.tor_binary = options['tor-path'] + if options['data-dir']: + self.config.DataDirectory = options['data-dir'] + if options['transport']: - ''' - ClientTransportPlugin transport socks4|socks5 IP:PORT - ClientTransportPlugin transport exec path-to-binary [options] - ''' + ## ClientTransportPlugin transport socks4|socks5 IP:PORT + ## ClientTransportPlugin transport exec path-to-binary [options] if not options['bridges']: - e = "To test pluggable transports, you must provide a file" - e = e+"with a list of bridge IP:ORPorts. See \'-b' option." + e = "You must use the bridge option to test a transport." raise usage.UsageError("%s" % e) log.msg("Using pluggable transport ...") @@ -280,15 +302,22 @@ class BridgetTest(OONITest): print self.config.create_torrc() report = {'tor_config': self.config.config} - return self.config - else: - return None def experiment(self, args): """ XXX fill me in + + :param args: + The :class:`ooni.plugoo.asset.Asset <Asset>` line currently being + used. + :meth launch_tor: + Returns a Deferred which callbacks with a + :class:`ooni.lib.txtorcon.torproto.TorProcessProtocol + <TorProcessProtocol>` connected to the fully-bootstrapped Tor; + this has a :class:`ooni.lib.txtorcon.torcontol.TorControlProtocol + <TorControlProtocol>` instance as .protocol. """ - log.msg("BridgeT: initiating test ... ") + log.msg("Bridget: initiating test ... ") from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor @@ -306,54 +335,54 @@ class BridgetTest(OONITest): state.post_bootstrap.addCallback(state_complete).addErrback(setup_failed) report.update({'success': args}) - def bootstrap(c): + def bootstrap(ctrl): """ Launch a Tor process with the TorConfig instance returned from initialize(). - - Returns a Deferred which callbacks with a TorProcessProtocol connected - to the fully-bootstrapped Tor; this has a txtorcon.TorControlProtocol - instance as .protocol. """ - conf = TorConfig(c) + conf = TorConfig(ctrl) conf.post_bootstrap.addCallback(setup_complete).addErrback(setup_failed) log.msg("Tor process connected, bootstrapping ...") def updates(prog, tag, summary): log.msg("%d%%: %s" % (prog, summary)) - def reconfigure_controller(proto, args): + def reconfigure_controller(conf, bridge): ## if bridges and relays, use one bridge then build a circuit - ## from the relays - print args - print args['bridge'] - #d.addCallback(CustomCircuit(state)) - proto.set_conf('Bridge', args['bridge']) - - ## if bridges only, try one bridge at a time, but don't build - ## circuits, just return - ## if relays only, build circuits from relays + ## from three relays + conf.Bridge = bridge + ## XXX do we need a SIGHUP to restart? - def reconfigure_failed(proto, args): + ## XXX see txtorcon.TorControlProtocol.add_event_listener we may not + ## need full CustomCircuit class + + ## if bridges only, try one bridge at a time, but don't build + ## circuits, just return + + ## if relays only, build circuits from relays + + def reconfigure_failed(args): log.msg("Reconfiguring Tor config with args %s failed" % args) reactor.stop() - ## XXX see txtorcon.TorControlProtocol.add_event_listener - ## we may not need full CustomCircuit class + if len(self.bridge_list) >= 1: + for bridge in self.bridge_list: + try: + print "BRIDGE IS %s" % bridge + reconfigure_controller(self.config, bridge) + except: + reconfigure_failed(bridge) ## :return: a Deferred which callbacks with a TorProcessProtocol ## connected to the fully-bootstrapped Tor; this has a ## txtorcon.TorControlProtocol instance as .protocol. - - print args - d = launch_tor(self.config, reactor, progress_updates=updates, + d = launch_tor(self.config, + reactor, + progress_updates=updates, tor_binary=self.tor_binary) - d.addCallback(reconfigure_controller, args) - d.addErrback(reconfigure_failed, args) d.addCallback(bootstrap, self.config) d.addErrback(setup_failed) - - ## 4 build circuits + ## 4 build circuits #print "Tor process ID: %s" % d.transport.pid return d @@ -362,6 +391,20 @@ class BridgetTest(OONITest): bridget = BridgetTest(None, None, None) ## ISIS' NOTES +## ----------- +## self.config.save() only needs to be called if Tor is already running. ## -## -## +## need to add transport type to torrc Bridge line: +## Bridge <transport> IP:ORPort <fingerprint> +## +## TODO: +## o add option for any kwarg=arg self.config setting +## o add DataDirectory option? +## o cleanup documentation +## o check if bridges are public relays +## o take bridge_desc file as input, also be able to give same +## format as output +## o change the stupid name +## +## FIX: +## data directory is not found, or permissions aren't right

1 0

[ooni-probe/master] Still need to fix problem with getHostByName() only returned one address
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 891d16a70e59ea3a4324822a0b602743613530b1 Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Fri Jun 15 17:55:32 2012 -0700 Still need to fix problem with getHostByName() only returned one address --- ooni/plugins/dnstamper.py | 195 +++++++++++++++++++++++++++++++++++++++++++++ 1 files changed, 195 insertions(+), 0 deletions(-) diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py new file mode 100644 index 0000000..34aaa01 --- /dev/null +++ b/ooni/plugins/dnstamper.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +""" + dnstamper + ********* + + This test resolves DNS for a list of domain names, one per line, in the + file specified in the ooni-config under the setting "dns_experiment". If + the file is top-1m.txt, the test will be run using Amazon's list of top + one million domains. The experimental dns servers to query should + be specified one per line in assets/dns_servers.txt. + + The test reports censorship if the cardinality of the intersection of + the query result set from the control server and the query result set + from the experimental server is zero, which is to say, if the two sets + have no matching results whatsoever. + + NOTE: This test frequently results in false positives due to GeoIP-based + load balancing on major global sites such as google, facebook, and + youtube, etc. + + :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft + :license: see LICENSE for more details + + TODO: + * Switch to using Twisted's DNS builtins instead of dnspython + * +""" + +import os + +from twisted.names import client +from twisted.internet import reactor +from twisted.internet.protocol import Factory, Protocol +from twisted.python import usage +from twisted.plugin import IPlugin +from zope.interface import implements + +from ooni.plugoo.assets import Asset +from ooni.plugoo.tests import ITest, OONITest +from ooni import log + +class Top1MAsset(Asset): + """ + Class for parsing the Alexa top-1m.txt as an asset. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + self = Asset.parse_line(self, line) + return line.split(',')[1].replace('\n','') + +class DNSTamperArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file of hostnames to resolve'], + ['controlserver', 'c', '8.8.8.8', 'Known good DNS server'], + ['testservers', 't', None, 'Asset file of DNS servers to test'], + ['usereverse', 'r', False, 'Also try reverse DNS resolves'], + ['resume', 's', 0, 'Resume at this index in the asset file']] + +class DNSTamperTest(OONITest): + implements(IPlugin, ITest) + + shortName = "dnstamper" + description = "DNS censorship detection test" + requirements = None + options = DNSTamperArgs + blocking = False + + def load_assets(self): + assets = {} + if self.local_options: + if self.local_options['asset']: + assetf = self.local_options['asset'] + if assetf == 'top-1m.txt': + assets.update({'asset': Top1MAsset(assetf)}) + else: + assets.update({'asset': Asset(assetf)}) + elif self.local_options['testservers']: + assets.update({'testservers': + Asset(self.local_options['testservers'])}) + return assets + + def lookup(self, hostname, nameserver): + """ + Resolves a hostname through a DNS nameserver to the corresponding + IP addresses. + """ + def got_result(result): + log.msg('Resolved %s through %s to %s' + % (hostname, nameserver, result)) + reactor.stop() + return {'resolved': True, + 'domain': hostname, + 'nameserver': nameserver, + 'address': result} + + def got_error(err): + log.msg(err.printTraceback()) + reactor.stop() + return {'resolved': False, + 'domain': hostname, + 'nameserver': nameserver, + 'address': err} + + res = client.createResolver(servers=[(nameserver, 53)]) + d = res.getHostByName(hostname) + d.addCallbacks(got_result, got_error) + return d + + ## XXX MAY ALSO BE: + #answer = res.getAddress(servers=[('nameserver', 53)]) + + def reverse_lookup(self, address, nameserver): + """ + Attempt to do a reverse DNS lookup to determine if the control and exp + sets from a positive result resolve to the same domain, in order to + remove false positives due to GeoIP load balancing. + """ + res = client.createResolver(servers=[(nameserver, 53)]) + ptr = '.'.join(addr.split('.')[::-1]) + '.in-addr.arpa' + d = res.lookupPointer(ptr) + d.addCallback(lambda (ans, auth, add): util.println(ans[0].payload.name)) + d.addErrback(log.err) + d.addBoth(lambda r: reactor.stop()) + return d + + def experiment(self, args): + """ + Compares the lookup() sets of the control and experiment groups. + """ + test_server = self.local_options['testservers'] + hostname = args['asset'] + exp_address = self.lookup(hostname, test_server) + + #return {'control': control_server, + # 'domain': args['asset'], + # 'experiment_address': address} + + if self.local_options['usereverse']: + exp_reversed = self.reverse_lookup(exp_address, test_server) + return exp_address, hostname, test_server, exp_reversed + else: + return exp_address, hostname, test_server, False + + def control(self, experiment_result): + (exp_address, hostname, test_server, exp_reversed) = experiment_result + control_server = self.local_options['controlserver'] + ctrl_address = self.lookup(hostname, control_server) + + ## XXX getHostByName() appears to be returning only one IP... + + if len(set(exp_address) & set(ctrl_address)) > 0: + log.msg("Address %s has not tampered with on DNS server %s" + % (hostname, test_server)) + return {'hostname': hostname, + 'test-nameserver': test_server, + 'test-address': exp_address, + 'control-nameserver': control_server, + 'control-address': ctrl_address, + 'tampering-detected': False} + else: + log.msg("Address %s has possibly been tampered on %s:" + % (hostname, test_server)) + log.msg("DNS resolution through testserver %s yeilds: %s" + % (test_server, exp_address)) + log.msg("However, DNS resolution through controlserver %s yeilds: %s" + % (control_server, ctrl_address)) + + if self.local_options['usereverse']: + ctrl_reversed = self.reverse_lookup(experiment_result, control_server) + if len(set(ctrl_reversed) & set(exp_reversed)) > 0: + log.msg("Further testing has eliminated false positives") + else: + log.msg("Reverse DNS on the results returned by %s returned:" + % (test_server)) + log.msg("%s" % exp_reversed) + log.msg("which does not match the expected domainname: %s" + % ctrl_reversed) + return {'hostname': hostname, + 'test-nameserver': test_server, + 'test-address': exp_address, + 'test-reversed': exp_reversed, + 'control-nameserver': control_server, + 'control-address': ctrl_address, + 'control-reversed': ctrl_reversed, + 'tampering-detected': True} + else: + return {'hostname': hostname, + 'test-nameserver': test_server, + 'test-address': exp_address, + 'control-nameserver': control_server, + 'control-address': ctrl_address, + 'tampering-detected': False} + +dnstamper = DNSTamperTest(None, None, None)

1 0

[ooni-probe/master] Implemented logging verbosity levels with global options. There's also an option now to suppress stdout. By default, all emit() messages still go the oonicli.log, regardless of logging level -- this is easily changed, however, by default we want all the info we can get.
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit df762643f4df5e9caec69b3b649d26f1e51c2210 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Sep 13 12:47:40 2012 +0000 Implemented logging verbosity levels with global options. There's also an option now to suppress stdout. By default, all emit() messages still go the oonicli.log, regardless of logging level -- this is easily changed, however, by default we want all the info we can get. --- ooni/ooniprobe.py | 14 +++++- ooni/plugoo/tests.py | 10 +++- ooni/utils/log.py | 116 +++++++++++++++++++++++++++++++++++++------------ 3 files changed, 106 insertions(+), 34 deletions(-) diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index 4b5c086..03b98b8 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -72,7 +72,13 @@ def runTest(test, options, global_options, reactor=reactor): test_class = plugoo[test].__class__ report = reports.Report(test, global_options['output']) - log.start(global_options['log'], 1) + log_to_stdout = True + if global_options['quiet']: + log_to_stdout = False + + log.start(log_to_stdout, + global_options['log'], + global_options['verbosity']) resume = 0 if not options: options = {} @@ -103,7 +109,8 @@ class Options(usage.Options): optFlags = [ #['remote', 'r', "If the test should be run remotely (not supported)"], #['status', 'x', 'Show current state'], - #['restart', 'r', 'Restart OONI'] + #['restart', 'r', 'Restart OONI'], + ['quiet', 'q', "Don't log to stdout"] ] optParameters = [ @@ -111,6 +118,7 @@ class Options(usage.Options): #['target-node', 't', 'localhost:31415', 'Select target node'], ['output', 'o', 'report.log', "Specify output report file"], ['log', 'l', 'oonicli.log', "Specify output log file"], + ['verbosity', 'v', 1, "Specify the logging level"], #['password', 'p', 'opennetwork', "Specify the password for authentication"], ] @@ -134,7 +142,7 @@ if __name__ == "__main__": config.parseOptions() if not config.subCommand: - print "Error! No Test Specified." + #print "Error! No Test Specified." config.opt_help() sys.exit(1) diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 13c06ed..a827093 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -58,7 +58,8 @@ class OONITest(object): return {} def __repr__(self): - return "<OONITest %s %s %s>" % (self.options, self.global_options, + return "<OONITest %s %s %s>" % (self.local_options, + self.global_options, self.assets) def end(self): @@ -132,6 +133,11 @@ class OONITest(object): @param args: the asset(s) lines that we are working on. """ self.start_time = date.now() - log.msg("Starting test %s" % self.__class__) + + if self.shortName: + log.msg("Starting test %s" % self.shortName) + else: + log.msg("Starting test %s" % self.__class__) + return self._do_experiment(args) diff --git a/ooni/utils/log.py b/ooni/utils/log.py index 3dc52dd..cf57186 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -1,43 +1,101 @@ """ OONI logging facility. """ -import sys -import logging -import warnings +from sys import stderr, stdout -from twisted.python import log - -# Logging levels -DEBUG = logging.DEBUG -INFO = logging.INFO -WARNING = logging.WARNING -ERROR = logging.ERROR -CRITICAL = logging.CRITICAL +from twisted.python import log, util +from twisted.python.failure import Failure def _get_log_level(level): + english = ['debug', 'info', 'warn', 'err', 'crit'] + + levels = dict(zip(range(len(english)), english)) + number = dict(zip(english, range(len(english)))) + if not level: - return INFO + return number['info'] else: - return level + ve = "Unknown log level: %s\n" % level + ve += "Allowed levels: %s\n" % [word for word in english] + + if type(level) is int: + if 0 <= level <= 4: + return level + elif type(level) is str: + if number.has_key(level.lower()): + return number[level] + else: + raise ValueError, ve + else: + raise ValueError, ve -def start(logfile=None, loglevel=None, logstdout=True): - if log.defaultObserver: - print "%s" % logstdout - loglevel = _get_log_level(loglevel) - file = open(logfile, 'a') if logfile else sys.stderr - observer = log.FileLogObserver(file) - if logstdout: - log.startLogging(sys.stdout) +class OONITestFailure(Failure): + """ + For handling Exceptions asynchronously. + + Can be given an Exception as an argument, else will use the + most recent Exception from the current stack frame. + """ + def __init__(self, exception=None, _type=None, + _traceback=None, _capture=False): + Failure.__init__(self, exc_value=exception, exc_type=_type, + exc_tb=_traceback, captureVars=_capture) + +class OONILogObserver(log.FileLogObserver): + """ + Supports logging level verbosity. + """ + def __init__(self, logfile, verb=None): + log.FileLogObserver.__init__(self, logfile) + self.level = _get_log_level(verb) if verb is not None else 1 + assert type(self.level) is int + + def emit(self, eventDict): + if 'logLevel' in eventDict: + msgLvl = _get_log_level(eventDict['logLevel']) + assert type(msgLvl) is int + ## only log our level and higher + if self.level <= msgLvl: + text = log.textFromEventDict(eventDict) + else: + text = None else: - log.startLogging() - log.addObserver(observer.emit) - msg("Started OONI") + text = log.textFromEventDict(eventDict) + + if text is None: + return + + timeStr = self.formatTime(eventDict['time']) + fmtDict = {'system': eventDict['system'], + 'text': text.replace('\n','\n\t')} + msgStr = log._safeFormat("[%(system)s] %(text)s\n", fmtDict) + + util.untilConcludes(self.write, timeStr + " " + msgStr) + util.untilConcludes(self.flush) + +def start(logstdout, logfile=None, verbosity=None): + if log.defaultObserver: + verbosity = _get_log_level(verbosity) + + ## Always log to file, keep level at info + file = open(logfile, 'a') if logfile else stderr + OONILogObserver(file, "info").start() + + if logstdout is True: + observerooni = OONILogObserver(stdout, verbosity) + log.addObserver(observerooni.emit) + + log.msg("Starting OONI...") + +def debug(message, level="debug", **kw): + log.msg(message, logLevel=level, **kw) -def msg(message, level=INFO, **kw): +def msg(message, level="info", **kw): log.msg(message, logLevel=level, **kw) -def err(message, **kw): - log.err(message, **kw) +## XXX fixme log.err messages get printed to stdout twice +def err(message, level="err", **kw): + log.err(message, logLevel=level, **kw) -def debug(message, **kw): - log.msg(message, logLevel=DEBUG, **kw) +def fail(message, exception=None, level="crit", **kw): + log.failure(message, OONITestFailure(exception, **kw), logLevel=level)

1 0

[ooni-probe/master] Fixed module import issues with directory structure.
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 2c7552e3da6e72b1d55b9599dc3e7281a5ee266e Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Thu May 31 12:36:50 2012 -0300 Fixed module import issues with directory structure. --- ooni/__init__.py | 2 +- ooni/example_plugins/skel.py | 4 ++-- ooni/ooniprobe.py | 7 +++---- ooni/plugins/blocking.py | 4 ++-- ooni/plugins/httphost.py | 4 ++-- ooni/plugoo/reports.py | 2 +- ooni/plugoo/tests.py | 8 +++----- ooni/plugoo/work.py | 2 ++ 8 files changed, 16 insertions(+), 17 deletions(-) diff --git a/ooni/__init__.py b/ooni/__init__.py index dbb3824..a03c856 100644 --- a/ooni/__init__.py +++ b/ooni/__init__.py @@ -1 +1 @@ -__all__ = ['plugoo', 'utils', 'config', 'logo', 'lib'] +__all__ = ['plugoo', 'utils', 'config', 'logo', 'lib', 'log'] diff --git a/ooni/example_plugins/skel.py b/ooni/example_plugins/skel.py index 52c7d4f..5f46620 100644 --- a/ooni/example_plugins/skel.py +++ b/ooni/example_plugins/skel.py @@ -1,8 +1,8 @@ from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin -from plugoo.tests import ITest, OONITest -from ooni.utils import log +from plugoo.tests import ITest, TwistedTest +import log class SkelArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index 4b5c086..3b48339 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -27,10 +27,9 @@ from zope.interface.verify import verifyObject from zope.interface.exceptions import BrokenImplementation from zope.interface.exceptions import BrokenMethodImplementation -from ooni.plugoo import tests, work, assets, reports -from ooni.utils.logo import getlogo -from ooni.utils import log -from ooni import plugins +from plugoo import tests, work, assets, reports +from logo import getlogo +import plugins, log __version__ = "0.0.1-prealpha" diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py index 6ddb5a4..72fd49f 100644 --- a/ooni/plugins/blocking.py +++ b/ooni/plugins/blocking.py @@ -2,8 +2,8 @@ from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin -from ooni.plugoo.assets import Asset -from ooni.plugoo.tests import ITest, OONITest +from plugoo.assets import Asset +from plugoo.tests import ITest, TwistedTest class BlockingArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py index b57a50d..e8808bd 100644 --- a/ooni/plugins/httphost.py +++ b/ooni/plugins/httphost.py @@ -20,8 +20,8 @@ from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin -from ooni.plugoo.assets import Asset -from ooni.plugoo.tests import ITest, OONITest +from plugoo.assets import Asset +from plugoo.tests import ITest, TwistedTest class HTTPHostArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index a543151..ef15a04 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -4,7 +4,7 @@ import os import yaml import itertools -from ooni.utils import log, date, net +import log class Report: """This is the ooni-probe reporting mechanism. It allows diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 7c190eb..544d82d 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -7,11 +7,9 @@ import itertools from twisted.internet import reactor, defer, threads from twisted.python import failure -from ooni.utils import log -from ooni.utils import date -from ooni.plugoo import assets, work -from ooni.plugoo.reports import Report -from ooni.plugoo.interface import ITest +import log +from plugoo import assets, work +from plugoo.reports import Report class OONITest(object): diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index db88fbf..e32d76d 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -19,6 +19,8 @@ from zope.interface import Interface, Attribute from twisted.python import failure from twisted.internet import reactor, defer +from plugoo.nodes import LocalNode + class Worker(object): """ This is the core of OONI. It takes as input Work Units and

1 0

[ooni-probe/master] Mostly done porting to twisted, still need rewrite of reverse DNS lookup method.
by isis＠torproject.org 13 Sep '12

13 Sep '12

commit 7150331304702d53e85a1b3123e768c551ac233d Author: Isis Lovecruft <isis(a)patternsinthevoid.net> Date: Wed Jun 13 20:41:17 2012 -0700 Mostly done porting to twisted, still need rewrite of reverse DNS lookup method. --- ooni/oonitests/dnstamper.py | 129 ++++++++++++++++++++++++++---------------- 1 files changed, 80 insertions(+), 49 deletions(-) diff --git a/ooni/oonitests/dnstamper.py b/ooni/oonitests/dnstamper.py index 498ba04..4ad7e7f 100644 --- a/ooni/oonitests/dnstamper.py +++ b/ooni/oonitests/dnstamper.py @@ -20,29 +20,26 @@ :copyright: (c) 2012 Arturo Filastò, Isis Lovecruft :license: see LICENSE for more details -""" -try: - from dns import resolver, reversename -except: - print "Error: dnspython is not installed! (http://www.dnspython.org/)" -try: - import gevent -except: - print "Error: gevent is not installed! (http://www.gevent.org/)" + TODO: + * Switch to using Twisted's DNS builtins instead of dnspython + * +""" import os -import plugoo -from plugoo.assets import Asset -from plugoo.tests import Test +from twisted.names import client +from twisted.python import usage +from twisted.plugin import IPlugin +from zope.interface import implements -__plugoo__ = "DNST" -__desc__ = "DNS censorship detection test" +from ooni.plugoo.assets import Asset +from ooni.plugoo.tests import ITest, OONITest +from ooni import log class Top1MAsset(Asset): """ - Class for parsing top-1m.txt as an asset. + Class for parsing the Alexa top-1m.txt as an asset. """ def __init__(self, file=None): self = Asset.__init__(self, file) @@ -51,22 +48,56 @@ class Top1MAsset(Asset): self = Asset.parse_line(self, line) return line.split(',')[1].replace('\n','') -class DNSTAsset(Asset): +class DNSTamperAsset(Asset): """ Creates DNS testing specific Assets. """ def __init__(self, file=None): self = Asset.__init__(self, file) -class DNST(Test): - def lookup(self, hostname, ns): +class DNSTamperArgs(usage.Options): + optParameters = [['asset', 'a', None, 'Asset file of hostnames to resolve'], + ['controlserver', 'c', '8.8.8.8', 'Known good DNS server'], + ['testservers', 't', None, 'Asset file of the DNS servers to test'], + ['resume', 'r', 0, 'Resume at this index in the asset file']] +''' + def control(self, experiment_result, args): + print "Experiment Result:", experiment_result + print "Args", args + return experiment_result + + def experiment(self, args): +''' + +class DNSTamperTest(OONITest): + implements(IPlugin, ITest) + + shortName = "DNSTamper" + description = "DNS censorship detection test" + requirements = None + options = DNSTamperArgs + blocking = False + + def load_asset(self): + if self.local_options: + if self.local_options['asset']: + assetf = self.local_options['asset'] + if assetf == 'top-1m.txt': + return {'asset': Top1MAsset(assetf)} + else: + return {'asset': DNSTamperAsset(assetf)} + else: + return {} + + def lookup(self, hostname, nameserver): """ - Resolves a hostname through a DNS nameserver, ns, to the corresponding - IP address(es). + Resolves a hostname through a DNS nameserver to the corresponding + IP addresses. """ - res = resolver.Resolver(configure=False) - res.nameservers = [ns] - answer = res.query(hostname) + res = client.createResolver(servers=[('nameserver', 53)]) + answer = res.getHostByName(hostname) + ## XXX MAY ALSO BE: + #answer = res.getAddress(servers=[('nameserver', 53)]) ret = [] @@ -75,14 +106,13 @@ class DNST(Test): return ret - def reverse_lookup(self, ip, ns): + def reverse_lookup(self, ip, nameserver): """ Attempt to do a reverse DNS lookup to determine if the control and exp sets from a positive result resolve to the same domain, in order to remove false positives due to GeoIP load balancing. """ - res = resolver.Resolver(configure=False) - res.nameservers = [ns] + res = client.createResolver(servers=nameserver) n = reversename.from_address(ip) revn = res.query(n, "PTR").__iter__().next().to_text()[:-1] @@ -131,26 +161,27 @@ class DNST(Test): print "\n" return result -def run(ooni): - """ - Run the test. - """ - config = ooni.config - urls = [] - - if (config.tests.dns_experiment == "top-1m.txt"): - dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, - config.tests.dns_experiment)) - else: - dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, - config.tests.dns_experiment)) - dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, - config.tests.dns_experiment_dns)) - - assets = [dns_experiment, dns_experiment_dns] - - dnstest = DNST(ooni) - ooni.logger.info("Beginning dnstamper test...") - dnstest.run(assets, {'index': 1}) - ooni.logger.info("Dnstamper test completed!") - +#def run(ooni): +# """ +# Run the test. +# """ +# config = ooni.config +# urls = [] +# +# if (config.tests.dns_experiment == "top-1m.txt"): +# dns_experiment = Top1MAsset(os.path.join(config.main.assetdir, +# config.tests.dns_experiment)) +# else: +# dns_experiment = DNSTAsset(os.path.join(config.main.assetdir, +# config.tests.dns_experiment)) +# dns_experiment_dns = DNSTAsset(os.path.join(config.main.assetdir, +# config.tests.dns_experiment_dns)) +# +# assets = [dns_experiment, dns_experiment_dns] +# +# dnstest = DNST(ooni) +# ooni.logger.info("Beginning dnstamper test...") +# dnstest.run(assets, {'index': 1}) +# ooni.logger.info("Dnstamper test completed!") + +dnstamper = DNSTamperTest(None, None, None)

1 0