[tor-commits] [ooni-probe/master] Remove dead code.
art at torproject.org
art at torproject.org
Fri Feb 28 14:32:21 UTC 2014
commit 2da8195f5677fe99d832a902a7497f39111eb5b0
Author: Arturo Filastò <art at fuffa.org>
Date: Thu Jan 30 17:19:32 2014 +0100
Remove dead code.
---
ooni/utils/onion.py | 704 ---------------------------------------------------
1 file changed, 704 deletions(-)
diff --git a/ooni/utils/onion.py b/ooni/utils/onion.py
deleted file mode 100644
index 58b4c02..0000000
--- a/ooni/utils/onion.py
+++ /dev/null
@@ -1,704 +0,0 @@
-#
-# onion.py
-# ----------
-# Utilities for working with Tor.
-#
-# This code is largely taken from txtorcon and its documentation, and as such
-# any and all credit should go to Meejah. Minor adjustments have been made to
-# use OONI's logging system, and to build custom circuits without actually
-# attaching streams.
-#
-# :author: Meejah, Isis Lovecruft
-# :license: see included LICENSE file
-# :copyright: copyright (c) 2012 The Tor Project, Inc.
-# :version: 0.1.0-alpha
-#
-# XXX TODO add report keys for onion methods
-
-import random
-import sys
-
-from twisted.internet import defer
-from zope.interface import implements
-
-from txtorcon import CircuitListenerMixin, IStreamAttacher
-from txtorcon import TorState, TorConfig
-from ooni.utils import log
-from ooni.utils.timer import deferred_timeout, TimeoutError
-
-# XXX This can be refactored to os.path.abspath
-# os.path.abspath(path)
-# Return a normalized absolutized version of the pathname path. On most
-# platforms, this is equivalent to normpath(join(os.getcwd(), path)).
-def parse_data_dir(data_dir):
- """
- Parse a string that a has been given as a DataDirectory and determine
- its absolute path on the filesystem.
-
- :param data_dir:
- A directory for Tor's DataDirectory, to be parsed.
- :return:
- The absolute path of :param:data_dir.
- """
- from os import path, getcwd
- import sys
-
- try:
- assert isinstance(data_dir, str), \
- "Parameter type(data_dir) must be str"
- except AssertionError, ae:
- log.err(ae)
-
- if data_dir.startswith('~'):
- data_dir = path.expanduser(data_dir)
- elif data_dir.startswith('/'):
- data_dir = path.join(getcwd(), data_dir)
- elif data_dir.startswith('./'):
- data_dir = path.abspath(data_dir)
- else:
- data_dir = path.join(getcwd(), data_dir)
-
- try:
- assert path.isdir(data_dir), "Could not find %s" % data_dir
- except AssertionError, ae:
- log.err(ae)
- sys.exit()
- else:
- return data_dir
-
-# XXX txtorcon handles this already.
-# Also this function is called write_torrc but it has hardcoded inside of it
-# bridget-tordata.
-def write_torrc(conf, data_dir=None):
- """
- Create a torrc in our data_dir. If we don't yet have a data_dir, create a
- temporary one. Any temporary files or folders are added to delete_list.
-
- :param conf:
- A :class:`ooni.lib.txtorcon.TorConfig` object, with all configuration
- values saved.
- :param data_dir:
- The Tor DataDirectory to use.
- :return: torrc, data_dir, delete_list
- """
- try:
- from os import write, close
- from tempfile import mkstemp, mkdtemp
- except ImportError, ie:
- log.err(ie)
-
- delete_list = []
-
- if data_dir is None:
- data_dir = mkdtemp(prefix='bridget-tordata')
- delete_list.append(data_dir)
- conf.DataDirectory = data_dir
-
- (fd, torrc) = mkstemp(dir=data_dir)
- delete_list.append(torrc)
- write(fd, conf.create_torrc())
- close(fd)
-
- return torrc, data_dir, delete_list
-
-def delete_files_or_dirs(delete_list):
- """
- Given a list of files or directories to delete, delete all and suppress
- all errors.
-
- :param delete_list:
- A list of files or directories to delete.
- """
- try:
- from os import unlink
- from shutil import rmtree
- except ImportError, ie:
- log.err(ie)
-
- for temp in delete_list:
- try:
- unlink(temp)
- except OSError:
- rmtree(temp, ignore_errors=True)
-
-def remove_node_from_list(node, list):
- for item in list: ## bridges don't match completely
- if item.startswith(node): ## due to the :<port>.
- try:
- log.msg("Removing %s because it is a public relay" % node)
- list.remove(item)
- except ValueError, ve:
- log.err(ve)
-
-def remove_public_relays(state, bridges):
- """
- Remove bridges from our bridge list which are also listed as public
- relays. This must be called after Tor has fully bootstrapped and we have a
- :class:`ooni.lib.txtorcon.TorState` with the
- :attr:`ooni.lib.txtorcon.TorState.routers` attribute assigned.
-
- XXX Does state.router.values() have all of the relays in the consensus, or
- just the ones we know about so far?
-
- XXX FIXME: There is a problem in that Tor needs a Bridge line to already be
- configured in order to bootstrap. However, after bootstrapping, we grab the
- microdescriptors of all the relays and check if any of our bridges are
- listed as public relays. Because of this, the first bridge does not get
- checked for being a relay.
- """
- IPs = map(lambda addr: addr.split(':',1)[0], bridges['all'])
- both = set(state.routers.values()).intersection(IPs)
-
- if len(both) > 0:
- try:
- updated = map(lambda node: remove_node_from_list(node), both)
- log.debug("Bridges in both: %s" % both)
- log.debug("Updated = %s" % updated)
- #if not updated:
- # defer.returnValue(state)
- #else:
- # defer.returnValue(state)
- return state
- except Exception, e:
- log.err("Removing public relays %s from bridge list failed:\n%s"
- % (both, e))
-
-# XXX It is unclear to me how all of these functions would be reused. Why must
-# hey be inside of a module?
-def setup_done(proto):
- log.msg("Setup Complete")
- state = TorState(proto.tor_protocol)
- state.post_bootstrap.addCallback(state_complete)
- state.post_bootstrap.addErrback(setup_fail)
-
-
-def setup_fail(proto):
- log.msg("Setup Failed:\n%s" % proto)
- return proto
- #reactor.stop()
-
-def state_complete(state):
- """Called when we've got a TorState."""
- log.msg("We've completely booted up a Tor version %s at PID %d"
- % (state.protocol.version, state.tor_pid))
- log.msg("This Tor has the following %d Circuits:"
- % len(state.circuits))
- for circ in state.circuits.values():
- log.msg("%s" % circ)
- return state
-
-def updates(_progress, _tag, _summary):
- """Log updates on the Tor bootstrapping process."""
- log.msg("%d%%: %s" % (_progress, _summary))
-
-def bootstrap(ctrl):
- """
- Bootstrap Tor from an instance of
- :class:`ooni.lib.txtorcon.TorControlProtocol`.
- """
- conf = TorConfig(ctrl)
- conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail)
- log.msg("Tor process connected, bootstrapping ...")
-
-# XXX txtorcon does this already for us.
-def start_tor(reactor, config, control_port, tor_binary, data_dir,
- report=None, progress=updates,
- process_cb=None, process_eb=None):
- """
- Use a txtorcon.TorConfig() instance, config, to write a torrc to a
- tempfile in our DataDirectory, data_dir. If data_dir is None, a temp
- directory will be created. Finally, create a TCP4ClientEndpoint at our
- control_port, and connect it to our reactor and a spawned Tor
- process. Compare with :meth:`txtorcon.launch_tor` for differences.
-
- :param reactor:
- An instance of class:`twisted.internet.reactor`.
- :param config:
- An instance of class:`txtorcon.TorConfig` with all torrc options
- already configured. ivar:`config.ControlPort`,
- ivar:`config.SocksPort`, ivar:`config.CookieAuthentication`, should
- already be set, as well as ivar:`config.UseBridges` and
- ivar:`config.Bridge` if bridges are to be used.
- ivar:`txtorcon.DataDirectory` does not need to be set.
- :param control_port:
- The port number to use for Tor's ControlPort.
- :param tor_binary:
- The full path to the Tor binary to use.
- :param data_dir:
- The directory to use as Tor's DataDirectory.
- :param report:
- The class:`ooni.plugoo.reports.Report` instance.
- :param progress:
- A non-blocking function to handle bootstrapping updates, which takes
- three parameters: _progress, _tag, and _summary.
- :param process_cb:
- The function to callback to after
- class:`ooni.lib.txtorcon.TorProcessProtocol` returns with the fully
- bootstrapped Tor process.
- :param process_eb:
- The function to errback to if
- class:`ooni.lib.txtorcon.TorProcessProtocol` fails.
- :return:
- The result of the callback of a
- class:`ooni.lib.txtorcon.TorProcessProtocol` which callbacks with a
- class:`txtorcon.TorControlProtocol` as .protocol.
- """
- try:
- from functools import partial
- from twisted.internet.endpoints import TCP4ClientEndpoint
- from ooni.lib.txtorcon import TorProtocolFactory
- from ooni.lib.txtorcon import TorProcessProtocol
- except ImportError, ie:
- log.err(ie)
-
- ## TODO: add option to specify an already existing torrc, which
- ## will require prior parsing to enforce necessary lines
- (torrc, data_dir, to_delete) = write_torrc(config, data_dir)
-
- log.msg("Starting Tor ...")
- log.msg("Using the following as our torrc:\n%s" % config.create_torrc())
- if report is None:
- report = {'torrc': config.create_torrc()}
- else:
- report.update({'torrc': config.create_torrc()})
-
- end_point = TCP4ClientEndpoint(reactor, 'localhost', control_port)
- connection_creator = partial(end_point.connect, TorProtocolFactory())
- process_protocol = TorProcessProtocol(connection_creator, progress)
- process_protocol.to_delete = to_delete
-
- if process_cb is not None and process_eb is not None:
- process_protocol.connected_cb.addCallbacks(process_cb, process_eb)
-
- reactor.addSystemEventTrigger('before', 'shutdown',
- partial(delete_files_or_dirs, to_delete))
- try:
- transport = reactor.spawnProcess(process_protocol,
- tor_binary,
- args=(tor_binary,'-f',torrc),
- env={'HOME': data_dir},
- path=data_dir)
- transport.closeStdin()
- except RuntimeError, e:
- log.err("Starting Tor failed:")
- process_protocol.connected_cb.errback(e)
- except NotImplementedError, e:
- url = "http://starship.python.net/crew/mhammond/win32/Downloads.html"
- log.msg("Running bridget on Windows requires pywin32: %s" % url)
- process_protocol.connected_cb.errback(e)
-
- return process_protocol.connected_cb
-
- at defer.inlineCallbacks
-def start_tor_filter_nodes(reactor, config, control_port, tor_binary,
- data_dir, bridges):
- """
- Bootstrap a Tor process and return a fully-setup
- :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges
- to test in the list of known public relays,
- :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges
- which are known public relays.
-
- :param reactor:
- The :class:`twisted.internet.reactor`.
- :param config:
- An instance of :class:`ooni.lib.txtorcon.TorConfig`.
- :param control_port:
- The port to use for Tor's ControlPort. If already configured in
- the TorConfig instance, this can be given as
- TorConfig.config.ControlPort.
- :param tor_binary:
- The full path to the Tor binary to execute.
- :param data_dir:
- The full path to the directory to use as Tor's DataDirectory.
- :param bridges:
- A dictionary which has a key 'all' which is a list of bridges to
- test connecting to, e.g.:
- bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001']
- :return:
- A fully initialized :class:`ooni.lib.txtorcon.TorState`.
- """
- setup = yield start_tor(reactor, config, control_port,
- tor_binary, data_dir,
- process_cb=setup_done, process_eb=setup_fail)
- filter_nodes = yield remove_public_relays(setup, bridges)
- defer.returnValue(filter_nodes)
-
-# XXX Why is this needed?
- at defer.inlineCallbacks
-def start_tor_with_timer(reactor, config, control_port, tor_binary, data_dir,
- bridges, timeout):
- """
- Start bootstrapping a Tor process wrapped with an instance of the class
- decorator :func:`ooni.utils.timer.deferred_timeout` and complete callbacks
- to either :func:`setup_done` or :func:`setup_fail`. Return a fully-setup
- :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges to test
- in the list of known public relays,
- :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges which
- are listed as known public relays.
-
- :param reactor:
- The :class:`twisted.internet.reactor`.
- :param config:
- An instance of :class:`ooni.lib.txtorcon.TorConfig`.
- :param control_port:
- The port to use for Tor's ControlPort. If already configured in
- the TorConfig instance, this can be given as
- TorConfig.config.ControlPort.
- :param tor_binary:
- The full path to the Tor binary to execute.
- :param data_dir:
- The full path to the directory to use as Tor's DataDirectory.
- :param bridges:
- A dictionary which has a key 'all' which is a list of bridges to
- test connecting to, e.g.:
- bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001']
- :param timeout:
- The number of seconds to attempt to bootstrap the Tor process before
- raising a :class:`ooni.utils.timer.TimeoutError`.
- :return:
- If the timeout limit is not exceeded, return a fully initialized
- :class:`ooni.lib.txtorcon.TorState`, else return None.
- """
- error_msg = "Bootstrapping has exceeded the timeout limit..."
- with_timeout = deferred_timeout(timeout, e=error_msg)(start_tor)
- try:
- setup = yield with_timeout(reactor, config, control_port, tor_binary,
- data_dir, process_cb=setup_done,
- process_eb=setup_fail)
- except TimeoutError, te:
- log.err(te)
- defer.returnValue(None)
- #except Exception, e:
- # log.err(e)
- # defer.returnValue(None)
- else:
- state = yield remove_public_relays(setup, bridges)
- defer.returnValue(state)
-
-# XXX This is a copy and paste of the above class with just an extra argument.
- at defer.inlineCallbacks
-def start_tor_filter_nodes_with_timer(reactor, config, control_port,
- tor_binary, data_dir, bridges, timeout):
- """
- Start bootstrapping a Tor process wrapped with an instance of the class
- decorator :func:`ooni.utils.timer.deferred_timeout` and complete callbacks
- to either :func:`setup_done` or :func:`setup_fail`. Then, filter our list
- of bridges to remove known public relays by calling back to
- :func:`remove_public_relays`. Return a fully-setup
- :class:`ooni.lib.txtorcon.TorState`. Then search for our bridges to test
- in the list of known public relays,
- :ivar:`ooni.lib.txtorcon.TorState.routers`, and remove any bridges which
- are listed as known public relays.
-
- :param reactor:
- The :class:`twisted.internet.reactor`.
- :param config:
- An instance of :class:`ooni.lib.txtorcon.TorConfig`.
- :param control_port:
- The port to use for Tor's ControlPort. If already configured in
- the TorConfig instance, this can be given as
- TorConfig.config.ControlPort.
- :param tor_binary:
- The full path to the Tor binary to execute.
- :param data_dir:
- The full path to the directory to use as Tor's DataDirectory.
- :param bridges:
- A dictionary which has a key 'all' which is a list of bridges to
- test connecting to, e.g.:
- bridges['all'] = ['1.1.1.1:443', '22.22.22.22:9001']
- :param timeout:
- The number of seconds to attempt to bootstrap the Tor process before
- raising a :class:`ooni.utils.timer.TimeoutError`.
- :return:
- If the timeout limit is not exceeded, return a fully initialized
- :class:`ooni.lib.txtorcon.TorState`, else return None.
- """
- error_msg = "Bootstrapping has exceeded the timeout limit..."
- with_timeout = deferred_timeout(timeout, e=error_msg)(start_tor_filter_nodes)
- try:
- state = yield with_timeout(reactor, config, control_port,
- tor_binary, data_dir, bridges)
- except TimeoutError, te:
- log.err(te)
- defer.returnValue(None)
- #except Exception, e:
- # log.err(e)
- # defer.returnValue(None)
- else:
- defer.returnValue(state)
-
-class CustomCircuit(CircuitListenerMixin):
- """
- Utility class for controlling circuit building. See
- 'attach_streams_by_country.py' in the txtorcon documentation.
-
- :param state:
- A fully bootstrapped instance of :class:`ooni.lib.txtorcon.TorState`.
- :param relays:
- A dictionary containing a key 'all', which is a list of relays to
- test connecting to.
- :ivar waiting_circuits:
- The list of circuits which we are waiting to attach to. You shouldn't
- need to touch this.
- """
- # XXX
- # 14:57 < meejah> to build a custom circuit (with no streams) in txtorcon,
- # call TorState.build_circuit -- the Deferred callbacks with the circid
-
- implements(IStreamAttacher)
-
- def __init__(self, state, relays=None):
- self.state = state
- self.waiting_circuits = []
- self.relays = relays
-
- def waiting_on(self, circuit):
- """
- Whether or not we are waiting on the given circuit before attaching to
- it.
-
- :param circuit:
- An item from :ivar:`ooni.lib.txtorcon.TorState.circuits`.
- :return:
- True if we are waiting on the circuit, False if not waiting.
- """
- for (circid, d) in self.waiting_circuits:
- if circuit.id == circid:
- return True
- return False
-
- def circuit_extend(self, circuit, router):
- "ICircuitListener"
- if circuit.purpose != 'GENERAL':
- return
- if self.waiting_on(circuit):
- log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex))
-
- def circuit_built(self, circuit):
- "ICircuitListener"
- if circuit.purpose != 'GENERAL':
- return
- log.msg("Circuit %s built ..." % circuit.id)
- log.msg("Full path of %s: %s" % (circuit.id, circuit.path))
- for (circid, d) in self.waiting_circuits:
- if circid == circuit.id:
- self.waiting_circuits.remove((circid, d))
- d.callback(circuit)
-
- def circuit_failed(self, circuit, reason):
- """
- If building a circuit has failed, try to remove it from our list of
- :ivar:`waiting_circuits`, else request to build it.
-
- :param circuit:
- An item from :ivar:`ooni.lib.txtorcon.TorState.circuits`.
- :param reason:
- A :class:`twisted.python.fail.Failure` instance.
- :return:
- None
- """
- if self.waiting_on(circuit):
- log.msg("Circuit %s failed for reason %s" % (circuit.id, reason))
- circid, d = None, None
- for c in self.waiting_circuits:
- if c[0] == circuit.id:
- circid, d = c
- if d is None:
- raise Exception("Expected to find circuit.")
-
- self.waiting_circuits.remove((circid, d))
- log.msg("Trying to build a circuit for %s" % circid)
- self.request_circuit_build(d)
-
- def check_circuit_route(self, router):
- """
- Check if a relay is a hop in one of our already built circuits.
-
- :param router:
- An item from the list
- :func:`ooni.lib.txtorcon.TorState.routers.values()`.
- """
- for circ in self.state.circuits.values():
- if router in circ.path:
- #router.update() ## XXX can i use without args? no.
- TorInfo.dump(self)
-
- def request_circuit_build(self, deferred, path=None):
- """
- Request a custom circuit.
-
- :param deferred:
- A :class:`twisted.internet.defer.Deferred` for this circuit.
- :param path:
- A list of router ids to build a circuit from. The length of this
- list must be at least three.
- """
- if path is None:
-
- pick = self.relays['all'].pop
- n = self.state.entry_guards.values()
- choose = random.choice
-
- first, middle, last = (None for i in range(3))
-
- if self.relays['remaining']() >= 3:
- first, middle, last = (pick() for i in range(3))
- elif self.relays['remaining']() < 3:
- first = choose(n)
- middle = pick()
- if self.relays['remaining'] == 2:
- middle, last = (pick() for i in range(2))
- elif self.relay['remaining'] == 1:
- middle = pick()
- last = choose(n)
- else:
- log.msg("Qu'est-que fuque?")
- else:
- middle, last = (random.choice(self.state.routers.values())
- for i in range(2))
-
- path = [first, middle, last]
-
- else:
- assert isinstance(path, list), \
- "Circuit path must be a list of relays!"
- assert len(path) >= 3, \
- "Circuit path must be at least three hops!"
-
- log.msg("Requesting a circuit: %s"
- % '->'.join(map(lambda node: node, path)))
-
- class AppendWaiting:
- def __init__(self, attacher, deferred):
- self.attacher = attacher
- self.d = deferred
- def __call__(self, circ):
- """
- Return from build_circuit is a Circuit, however,
- we want to wait until it is built before we can
- issue an attach on it and callback to the Deferred
- we issue here.
- """
- log.msg("Circuit %s is in progress ..." % circ.id)
- self.attacher.waiting_circuits.append((circ.id, self.d))
-
- return self.state.build_circuit(path).addCallback(
- AppendWaiting(self, deferred)).addErrback(
- log.err)
-
-class TxtorconImportError(ImportError):
- """
- Raised when ooni.lib.txtorcon cannot be imported from. Checks our current
- working directory and the path given to see if txtorcon has been
- initialized via /ooni/lib/Makefile.
- """
- from os import getcwd, path
-
- cwd, tx = getcwd(), 'lib/txtorcon/torconfig.py'
- try:
- log.msg("Unable to import from ooni.lib.txtorcon")
- if cwd.endswith('ooni'):
- check = path.join(cwd, tx)
- elif cwd.endswith('utils'):
- check = path.join(cwd, '../'+tx)
- else:
- check = path.join(cwd, 'ooni/'+tx)
- assert path.isfile(check)
- except:
- log.msg("Error: Some OONI libraries are missing!")
- log.msg("Please go to /ooni/lib/ and do \"make all\"")
-
-class PTNoBridgesException(Exception):
- """Raised when a pluggable transport is specified, but not bridges."""
- def __init__(self):
- log.msg("Pluggable transport requires the bridges option")
- return sys.exit()
-
-class PTNotFoundException(Exception):
- def __init__(self, transport_type):
- m = "Pluggable Transport type %s was unaccounted " % transport_type
- m += "for, please contact isis(at)torproject(dot)org and it will "
- m += "get included."
- log.msg("%s" % m)
- return sys.exit()
-
- at defer.inlineCallbacks
-def __start_tor_with_timer__(reactor, config, control_port, tor_binary,
- data_dir, bridges=None, relays=None, timeout=None,
- retry=None):
- """
- A wrapper for :func:`start_tor` which wraps the bootstrapping of a Tor
- process and its connection to a reactor with a
- :class:`twisted.internet.defer.Deferred` class decorator utility,
- :func:`ooni.utils.timer.deferred_timeout`, and a mechanism for resets.
-
- ## XXX fill me in
- """
- raise NotImplementedError
-
- class RetryException(Exception):
- pass
-
- import sys
- from ooni.utils.timer import deferred_timeout, TimeoutError
-
- def __make_var__(old, default, _type):
- if old is not None:
- assert isinstance(old, _type)
- new = old
- else:
- new = default
- return new
-
- reactor = reactor
- timeout = __make_var__(timeout, 120, int)
- retry = __make_var__(retry, 1, int)
-
- with_timeout = deferred_timeout(timeout)(start_tor)
-
- @defer.inlineCallbacks
- def __start_tor__(rc=reactor, cf=config, cp=control_port, tb=tor_binary,
- dd=data_dir, br=bridges, rl=relays, cb=setup_done,
- eb=setup_fail, af=remove_public_relays, retry=retry):
- try:
- setup = yield with_timeout(rc,cf,cp,tb,dd)
- except TimeoutError:
- retry -= 1
- defer.returnValue(retry)
- else:
- if setup.callback:
- setup = yield cb(setup)
- elif setup.errback:
- setup = yield eb(setup)
- else:
- setup = setup
-
- if br is not None:
- state = af(setup,br)
- else:
- state = setup
- defer.returnValue(state)
-
- @defer.inlineCallbacks
- def __try_until__(tries):
- result = yield __start_tor__()
- try:
- assert isinstance(result, int)
- except AssertionError:
- defer.returnValue(result)
- else:
- if result >= 0:
- tried = yield __try_until__(result)
- defer.returnValue(tried)
- else:
- raise RetryException
- try:
- tried = yield __try_until__(retry)
- except RetryException:
- log.msg("All retry attempts to bootstrap Tor have timed out.")
- log.msg("Exiting ...")
- defer.returnValue(sys.exit())
- else:
- defer.returnValue(tried)
-
More information about the tor-commits
mailing list