- tor-commits - lists.torproject.org

[ooni-probe/master] Merge branch 'nettest'
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 16c3cfedbd80d23337579d6d5fc3f9bff367e714 Merge: 2e49b73 d621155 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 12:06:54 2012 +0000 Merge branch 'nettest' Conflicts: ooni/plugins/new_bridget.py .gitignore | 1 + README.md | 10 + bin/ooniprobe | 16 +- docs/design.dia | Bin 1706 -> 1706 bytes docs/source/conf.py | 7 +- docs/source/index.rst | 100 ++---- docs/source/install.rst | 55 +++ docs/source/tutorial.rst | 1 - docs/source/writing_tests.rst | 88 ++++- docs/writing_tests.md | 17 - ooni/input.py | 65 ++++ ooni/nettest.py | 90 +++++ ooni/oonicli.py | 105 ++++++ ooni/oonitests/bridget.py | 373 -------------------- ooni/plugins/httpt.py | 9 +- ooni/plugins/new_bridget.py | 4 +- ooni/plugoo/tests.py | 5 +- ooni/protocols/http.py | 17 +- ooni/reporter.py | 199 +++++++++++ ooni/runner.py | 191 ++++++++++ 20 files changed, 852 insertions(+), 501 deletions(-) diff --cc README.md index 7d6e37a,123d837..23deb32 --- a/README.md +++ b/README.md @@@ -32,8 -26,18 +32,18 @@@ If you see some errors see INSTALL to i To list the help for a specific test: - python ooniprobe.py httpt --help + $ python ooniprobe.py httpt --help + ## Virtualenv way (Recommended) + + virtualenv2 ENV/ + source ENV/bin/activate + pip install twisted Scapy + + To install the most up to date scapy version (requires mercurial): + + pip install hg+http://hg.secdev.org/scapy + # More details diff --cc ooni/ooniprobe.py index 86f4e2d,f80669e..86f4e2d mode 100755,100644..100644 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py diff --cc ooni/plugins/new_bridget.py index 736b084,b26455e..673cacc --- a/ooni/plugins/new_bridget.py +++ b/ooni/plugins/new_bridget.py @@@ -1,421 -1,105 +1,421 @@@ -""" -This is a self genrated test created by scaffolding.py. -you will need to fill it up with all your necessities. -Safe hacking :). -""" -from exceptions import Exception -from datetime import datetime -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import reactor, task - -from ooni.utils import log -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset - -from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState -from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor - -class bridgetArgs(usage.Options): - optParameters = [['bridges', 'b', None, 'List of bridges to scan'], - ['relays', 'f', None, 'List of relays to scan'], - ['resume', 'r', 0, 'Resume at this index'], - ['timeout', 't', 5, 'Timeout in seconds after which to consider a bridge not working'] - ] - -class bridgetTest(OONITest): +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# +-----------+ +# | BRIDGET | +# | +----------------------------------------------+ +# +--------| Use a slave Tor process to test making a Tor | +# | connection to a list of bridges or relays. | +# +----------------------------------------------+ +# +# :authors: Arturo Filasto, Isis Lovecruft +# :licence: see included LICENSE +# :version: 0.1.0-alpha + +from __future__ import with_statement +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import defer, error, reactor + +import random +import sys + +try: + from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher +except: + print "BridgeT requires txtorcon: https://github.com/meejah/txtorcon.git" + print "Your copy of OONI should have it included, if you're seeing this" + print "message, please file a bug report." + log.msg ("Bridget: Unable to import from ooni.lib.txtorcon") + +from ooni.utils import log +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset + + +class BridgetArgs(usage.Options): + + def portCheck(number): + number = int(number) + if number not in range(1024, 65535): + raise ValueError("Port out of range") + portCheck.coerceDoc = "Ports must be between 1024 and 65535" + + optParameters = [ + ['bridges', 'b', None, + 'List of bridges to scan <IP>:<ORport>'], + ['relays', 'f', None, + 'List of relays to scan <IP>'], + ['socks', 's', 9049, portCheck, + 'Tor SocksPort to use'], + ['control', 'c', 9052, portCheck, + 'Tor ControlPort to use'], + ['tor-path', 'p', None, + 'Path to the Tor binary to use'], + ['data-dir', 'd', None, + 'Tor DataDirectory to use'], + ['transport', 't', None, + 'Tor ClientTransportPlugin'], + ['resume', 'r', 0, + 'Resume at this index']] + optFlags = [['random', 'x', 'Randomize control and socks ports']] + + def postOptions(self): + ## We can't test pluggable transports without bridges + if self['transport'] and not self['bridges']: + e = "Pluggable transport requires the bridges option" + raise usage.UsageError, e + ## We can't use random and static port simultaneously + if self['socks'] and self['control']: + if self['random']: + e = "Unable to use random and specific ports simultaneously" + raise usageError, e + +class CustomCircuit(CircuitListenerMixin): + implements(IStreamAttacher) + + from txtorcon.interface import IRouterContainer, ICircuitContainer + + def __init__(self, state): + self.state = state + self.waiting_circuits = [] + + def waiting_on(self, circuit): + for (circid, d) in self.waiting_circuits: + if circuit.id == circid: + return true + return False + + def circuit_extend(self, circuit, router): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + if self.waiting_on(circuit): + log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) + + def circuit_built(self, circuit): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + + log.msg("Circuit %s built ..." % circuit.id) + log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) + + for (circid, d) in self.waiting_circuits: + if circid == circuit.id: + self.waiting_circuits.remove(circid, d) + d.callback(circuit) + + def circuit_failed(self, circuit, reason): + if self.waiting_on(circuit): + log.msg("A circuit we requested %s failed for reason %s" % + (circuit.id, reason)) + circid, d = None, None + for x in self.waiting_circuits: + if x[0] == circuit.id: + circid, d, stream_cc = x + if d is None: + raise Exception("Expected to find circuit.") + + self.waiting_circuits.remove((circid, d)) + log.msg("Trying to build a circuit for %s" % circid) + self.request_circuit_build(d) + + def check_circuit_route(self, circuit, router): + if router in circuit.path: + #router.update() ## XXX can i use without args? no. + TorInfo.dump(self) + + def request_circuit_build(self, deferred): + entries = self.state.entry_guards.value() + relays = self.state.routers.values() + + log.msg("We have these nodes listed as entry guards:") + log.msg("%s" % entries) + log.msg("We have these nodes listed as relays:") + log.msg("%s" % relays) + + path = [random.choice(entries), + random.choice(relays), + random.choice(relays)] + + log.msg("Requesting a circuit: %s" + % '-->'.join(map(lambda x: x.location.countrycode, path))) + + class AppendWaiting: + def __init__(self, attacher, deferred): + self.attacher = attacher + self.d = deferred + + def __call__(self, circuit): + """ + Return from build_circuit is a Circuit, however, we want to + wait until it is built before we can issue an attach on it and + callback to the Deferred we issue here. + """ + log.msg("Circuit %s is in progress ..." % circuit.id) + self.attacher.waiting_circuits.append((circuit.id, self.d)) + + return self.state.build_circuit(path).addCallback(AppendWaiting(self, deferred_to_callback)).addErrback(log.err) + +class BridgetAsset(Asset): + """ + Class for parsing bridge assets so that they can be commented out. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + if line.startswith('#'): + return + else: + return line.replace('\n','') + +class BridgetTest(OONITest): + """ + XXX fill me in + + :ivar config: + An :class:`ooni.lib.txtorcon.TorConfig` instance. + :ivar relay_list: + A list of all provided relays to test. We have to do this because + txtorcon.TorState().entry_guards won't build a custom circuit if the + first hop isn't in the torrc's EntryNodes. + :ivar bridge_list: + A list of all provided bridges to test. + :ivar socks_port: + Integer for Tor's SocksPort. + :ivar control_port: + Integer for Tor's ControlPort. + :ivar plug_transport: + String defining the Tor's ClientTransportPlugin, for testing + a bridge's pluggable transport functionality. + :ivar tor_binary: + Path to the Tor binary to use, e.g. \'/usr/sbin/tor\' + """ implements(IPlugin, ITest) - shortName = "bridget" - description = "Use a Tor process to test connecting to bridges and relays" + shortName = "newbridget" + description = "bridget" requirements = None - options = bridgetArgs + options = BridgetArgs blocking = False - def experiment(self, args): - log.msg("Doing test") - last_update = datetime.now() - tor_log = [] - - def check_timeout(): - log.msg("Checking for timeout") - time_since_update = datetime.now() - last_update - if time_since_update.seconds > self.local_options['timeout']: - log.msg("Timed out when connecting to %s" % args) - l.stop() - self.result['reason'] = 'timeout' - d.errback(args) - return + def initialize(self): + """ + Extra initialization steps. We only want one child Tor process + running, so we need to deal with the creation of TorConfig() only + once, before the experiment runs. + """ + self.socks_port = 9049 + self.control_port = 9052 + self.tor_binary = '/usr/sbin/tor' + self.data_directory = None - def updates(prog, tag, summary): - tor_log.append((prog, tag, summary)) - last_update = datetime.now() - log.msg("%d%%: %s" % (prog, summary)) + if self.local_options: + try: + from ooni.lib.txtorcon import TorConfig + except: + e = "Could not import TorConfig class from txtorcon!" + raise ImportError, e - def setup_failed(failure): - log.msg("Setup Failed.") - if not self.result['reason']: - self.result['reason'] = 'unknown' - self.result['input'] = args - self.result['result'] = 'failed' - self.result['tor_log'] = tor_log - return + options = self.local_options + self.config = TorConfig() - def setup_complete(proto): - log.msg("Setup Complete.") - self.result['input'] = args - self.result['result'] = 'success' - return + ## Don't run the experiment if we don't have anything to test + if not options['bridges'] and not options['relays']: + self.suicide = True + + if options['bridges']: + self.config.UseBridges = 1 - config = TorConfig() - import random - config.SocksPort = random.randint(1024, 2**16) - config.ControlPort = random.randint(1024, 2**16) + if options['relays']: + ## Stupid hack for testing only relays: + ## Tor doesn't use EntryNodes when UseBridges is enabled, but + ## config.state.entry_guards needs to include the first hop to + ## build a custom circuit. + self.config.EntryNodes = ','.join(relay_list) - if 'bridge' in args: - config.UseBridges = 1 - config.Bridge = args['bridge'] + if options['socks']: + self.socks_port = options['socks'] - config.save() + if options['control']: + self.control_port = options['control'] - print config.config - self.result['tor_config'] = config.config - log.msg("Starting Tor connecting to %s" % args['bridge']) + if options['random']: + log.msg("Using randomized ControlPort and SocksPort ...") + self.socks_port = random.randint(1024, 2**16) + self.control_port = random.randint(1024, 2**16) - l = task.LoopingCall(check_timeout) - l.start(1.0) + if options['tor-path']: + self.tor_binary = options['tor-path'] - d = launch_tor(config, self.reactor, control_port=config.ControlPort, progress_updates=updates) - d.addCallback(setup_complete) - d.addErrback(setup_failed) - return d + if options['data-dir']: + self.config.DataDirectory = options['data-dir'] + + if options['transport']: + ## ClientTransportPlugin transport socks4|socks5 IP:PORT + ## ClientTransportPlugin transport exec path-to-binary [options] + if not options['bridges']: + e = "You must use the bridge option to test a transport." + raise usage.UsageError("%s" % e) + + log.msg("Using pluggable transport ...") + ## XXX fixme there's got to be a better way to check the exec + assert type(options['transport']) is str + self.config.ClientTransportPlugin = options['transport'] + + self.config.SocksPort = self.socks_port + self.config.ControlPort = self.control_port + self.config.save() def load_assets(self): - assets = {} + """ + Load bridges and/or relays from files given in user options. Bridges + should be given in the form IP:ORport. We don't want to load these as + assets, because it's inefficient to start a Tor process for each one. + """ + assets = {} + self.bridge_list = [] + self.relay_list = [] + + ## XXX fix me + ## we should probably find a more memory nice way to load addresses, + ## in case the files are really large if self.local_options: if self.local_options['bridges']: - assets.update({'bridge': Asset(self.local_options['bridges'])}) - elif self.local_options['relays']: - assets.update({'relay': Asset(self.local_options['relay'])}) + log.msg("Loading bridge information from %s ..." + % self.local_options['bridges']) + with open(self.local_options['bridges']) as bridge_file: + for line in bridge_file.readlines(): + if line.startswith('#'): + continue + else: + self.bridge_list.append(line.replace('\n','')) + assets.update({'bridges': self.bridge_list}) + + if self.local_options['relays']: + log.msg("Loading relay information from %s ..." + % self.local_options['relays']) + with open(options['relays']) as relay_file: + for line in relay_file.readlines(): + if line.startswith('#'): + continue + else: + self.relay_list.append(line.replace('\n','')) + assets.update({'relays': self.relay_list}) return assets -# We need to instantiate it otherwise getPlugins does not detect it -# XXX Find a way to load plugins without instantiating them. -bridget = bridgetTest(None, None, None) + def experiment(self, args): + """ + XXX fill me in + + :param args: + The :class:`ooni.plugoo.asset.Asset <Asset>` line currently being + used. + :meth launch_tor: + Returns a Deferred which callbacks with a + :class:`ooni.lib.txtorcon.torproto.TorProcessProtocol + <TorProcessProtocol>` connected to the fully-bootstrapped Tor; + this has a :class:`ooni.lib.txtorcon.torcontol.TorControlProtocol + <TorControlProtocol>` instance as .protocol. + """ + from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState + from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor + + def bootstrap(ctrl): + """ + Launch a Tor process with the TorConfig instance returned from + initialize(). + """ + conf = TorConfig(ctrl) + conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) + log.msg("Tor process connected, bootstrapping ...") + + def reconf_controller(conf, bridge): + ## if bridges and relays, use one bridge then build a circuit + ## from three relays + conf.Bridge = bridge + ## XXX do we need a SIGHUP to restart? + + ## XXX see txtorcon.TorControlProtocol.add_event_listener we + ## may not need full CustomCircuit class + + ## if bridges only, try one bridge at a time, but don't build + ## circuits, just return + ## if relays only, build circuits from relays + + def reconf_fail(args): + log.msg("Reconfiguring Tor config with args %s failed" % args) + reactor.stop() + + def setup_fail(args): + log.msg("Setup Failed.") + report.update({'failed': args}) + reactor.stop() + + def setup_done(proto): + log.msg("Setup Complete: %s" % proto) + state = TorState(proto.tor_protocol) + state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail) + report.update({'success': args}) + + def updates(prog, tag, summary): + log.msg("%d%%: %s" % (prog, summary)) + + if len(args) == 0: + log.msg("Bridget needs lists of bridges and/or relays to test!") + log.msg("Exiting ...") + d = sys.exit() + return d + + else: + if len(self.bridge_list) >= 1: + for bridge in self.bridge_list: + try: + print "BRIDGE IS %s" % bridge + reconf_controller(self.config, bridge) + except: + reconf_fail(bridge) + + log.msg("Bridget: initiating test ... ") + log.msg("Using the following as our torrc:\n%s" + % self.config.create_torrc()) + report = {'tor_config': self.config.config} + log.msg("Starting Tor ...") + + ## :return: a Deferred which callbacks with a TorProcessProtocol + ## connected to the fully-bootstrapped Tor; this has a + ## txtorcon.TorControlProtocol instance as .protocol. + d = launch_tor(self.config, + reactor, + progress_updates=updates, + tor_binary=self.tor_binary) + d.addCallback(bootstrap, self.config) + d.addErrback(setup_fail) + ## now build circuits + + #print "Tor process ID: %s" % d.transport.pid + return d + +## So that getPlugins() can register the Test: +bridget = BridgetTest(None, None, None) + +## ISIS' NOTES +## ----------- +## self.config.save() only needs to be called if Tor is already running. +## +## need to add transport type to torrc Bridge line: +## Bridge <transport> IP:ORPort <fingerprint> +## +## TODO: +## o add option for any kwarg=arg self.config setting +## o cleanup documentation +## o check if bridges are public relays +## o take bridge_desc file as input, also be able to give same +## format as output +## o change the stupid name +## +## FIX: +## data directory is not found, or permissions aren't right

1 0

[ooni-probe/master] Implement the first Test Template based on the new API.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit f358389ab248fdd4d6ced99e9e2b9f6265cdd38b Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 14:13:54 2012 +0000 Implement the first Test Template based on the new API. * Fix some bugs --- ooni/plugoo/reports.py | 2 +- ooni/plugoo/tests.py | 10 ++-- ooni/plugoo/work.py | 2 - ooni/runner.py | 8 ++- ooni/templates/http.py | 147 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 11 deletions(-) diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index b48ffdf..1bfdac0 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -4,7 +4,7 @@ import os import yaml import itertools -from utils import log, date, net +from ooni.utils import log, date, net class Report: """This is the ooni-probe reporting mechanism. It allows diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 75c23f7..21ac7bf 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -8,10 +8,10 @@ from twisted.internet import reactor, defer, threads ## XXX why is this imported and not used? from twisted.python import failure -from utils import log, date -from plugoo import assets, work -from plugoo.reports import Report -from plugoo.interface import ITest +from ooni.utils import log, date +from ooni.plugoo import assets, work +from ooni.plugoo.reports import Report +from ooni.plugoo.interface import ITest class OONITest(object): @@ -58,7 +58,7 @@ class OONITest(object): return {} def __repr__(self): - return "<OONITest %s %s %s>" % (self.local_options, + return "<OONITest %s %s %s>" % (self.local_options, self.global_options, self.assets) diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index e32d76d..db88fbf 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -19,8 +19,6 @@ from zope.interface import Interface, Attribute from twisted.python import failure from twisted.internet import reactor, defer -from plugoo.nodes import LocalNode - class Worker(object): """ This is the core of OONI. It takes as input Work Units and diff --git a/ooni/runner.py b/ooni/runner.py index ff6b47e..1a9b4ad 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -5,7 +5,8 @@ import time import inspect from twisted.internet import defer, reactor -from twisted.python import reflect, log, failure +from twisted.python import reflect, failure +from twisted.python import log from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader from twisted.trial.runner import isPackage, isTestCase, ErrorHolder @@ -15,7 +16,7 @@ from ooni.reporter import ReporterFactory from ooni.input import InputUnitFactory from ooni.nettest import InputTestSuite from ooni import nettest -from ooni.utils import log +#from ooni.utils import log from ooni.plugoo import tests as oonitests def isTestCase(thing): @@ -182,7 +183,8 @@ class ORunner(object): result.done() def run(self): - log.start() + log.startLogging(sys.stdout) + #log.start(True) self.reporterFactory.writeHeader() for inputUnit in InputUnitFactory(self.inputs): diff --git a/ooni/templates/__init__.py b/ooni/templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ooni/templates/http.py b/ooni/templates/http.py new file mode 100644 index 0000000..222e44c --- /dev/null +++ b/ooni/templates/http.py @@ -0,0 +1,147 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +import random + +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import protocol, defer +from twisted.web.http_headers import Headers + +from ooni.nettest import TestCase +from ooni.utils import log + +useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), + ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), + ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), + ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] + +class BodyReceiver(protocol.Protocol): + def __init__(self, finished): + self.finished = finished + self.data = "" + + def dataReceived(self, bytes): + self.data += bytes + + def connectionLost(self, reason): + self.finished.callback(self.data) + +class HTTPTest(TestCase): + """ + A utility class for dealing with HTTP based testing. It provides methods to + be overriden for dealing with HTTP based testing. + The main functions to look at are processResponseBody and + processResponseHeader that are invoked once the headers have been received + and once the request body has been received. + """ + randomizeUA = True + followRedirects = False + + def setUp(self): + from twisted.web.client import Agent + from twisted.internet import reactor + import yaml + self.agent = Agent(reactor) + if self.followRedirects: + from twisted.web.client import RedirectAgent + self.agent = RedirectAgent(self.agent) + self.request = {} + self.response = {} + + def _processResponseBody(self, data): + self.response['body'] = data + self.report['response'] = self.response + + self.processResponseBody(data) + + def processResponseBody(self, data): + """ + This should handle all the response body smushing for getting it ready + to be passed onto the control. + + @param data: The content of the body returned. + """ + pass + + def processResponseHeaders(self, headers): + """ + This should take care of dealing with the returned HTTP headers. + + @param headers: The content of the returned headers. + """ + pass + + def processRedirect(self, location): + """ + Handle a redirection via a 3XX HTTP status code. + + @param location: the url that is being redirected to. + """ + pass + + def doRequest(self, url): + d = self.build_request(url) + def finished(data): + return data + + d.addCallback(self._cbResponse) + d.addCallback(finished) + return d + + def test_http(self): + log.msg("Running experiment") + + if self.input: + url = self.input + else: + raise Exception("No input supplied") + + self.mainDefer = self.doRequest(url) + return self.mainDefer + + def _cbResponse(self, response): + self.response['headers'] = response.headers + self.response['code'] = response.code + self.response['length'] = response.length + self.response['version'] = response.length + + if str(self.response['code']).startswith('3'): + self.processRedirect(response.headers.getRawHeaders('Location')[0]) + + self.processResponseHeaders(self.response['headers']) + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished)) + finished.addCallback(self._processResponseBody) + + return finished + + def randomize_useragent(self): + user_agent = random.choice(useragents) + self.request['headers']['User-Agent'] = [user_agent] + + def build_request(self, url, method="GET", headers=None, body=None): + self.request['method'] = method + self.request['url'] = url + self.request['headers'] = headers if headers else {} + self.request['body'] = body + if self.randomizeUA: + self.randomize_useragent() + + self.report['request'] = self.request + self.report['url'] = url + return self.agent.request(self.request['method'], self.request['url'], + Headers(self.request['headers']), + self.request['body']) +

1 0

[ooni-probe/master] * Need to fix the Deferred class decorator timeout.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 365bd5a1a631191808a110ded51f520806ab342e Author: Isis Lovecruft <isis(a)torproject.org> Date: Fri Sep 28 05:37:49 2012 +0000 * Need to fix the Deferred class decorator timeout. --- ooni/plugins/bridget.py | 20 +++++++++++--------- ooni/utils/timer.py | 31 +++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 208b4c3..538e2f3 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -282,6 +282,7 @@ class BridgetTest(OONITest): from ooni.utils.onion import start_tor, remove_public_relays from ooni.utils.onion import setup_done, setup_fail from ooni.utils.onion import CustomCircuit + from ooni.utils.timer import timeout from ooni.lib.txtorcon import TorConfig, TorState except ImportError: raise TxtorconImportError @@ -365,19 +366,19 @@ class BridgetTest(OONITest): log.err("Attaching custom circuit builder failed: %s" % state) - log.msg("Bridget: initiating test ... ") ## Start the experiment - all_of_the_bridges = self.bridges['all'] - all_of_the_relays = self.relays['all'] ## Local copy of orginal lists + log.msg("Bridget: initiating test ... ") ## Start the experiment - if self.bridges['remaining']() >= 1 and not 'Bridge' in self.config.config: + if self.bridges['remaining']() >= 1 \ + and not 'Bridge' in self.config.config: self.bridges['current'] = self.bridges['all'][0] self.config.Bridge = self.bridges['current'] - ## avoid starting several - self.config.save() ## processes + ## avoid starting several + self.config.save() ## processes assert self.config.config.has_key('Bridge'), "NO BRIDGE" - state = start_tor(reactor, self.config, self.control_port, - self.tor_binary, self.data_directory) + state = timeout(self.circuit_timeout)(start_tor( + reactor, self.config, self.control_port, + self.tor_binary, self.data_directory)) state.addCallbacks(setup_done, setup_fail) state.addCallback(remove_public_relays, self.bridges) @@ -396,7 +397,8 @@ class BridgetTest(OONITest): all = [] for bridge in self.bridges['all'][1:]: self.bridges['current'] = bridge - new = defer.Deferred() + #new = defer.Deferred() + new = defer.waitForDeferred(state) new.addCallback(reconfigure_bridge, state, self.bridges) all.append(new) diff --git a/ooni/utils/timer.py b/ooni/utils/timer.py index 8ad9b79..24074f9 100644 --- a/ooni/utils/timer.py +++ b/ooni/utils/timer.py @@ -9,28 +9,39 @@ # :copyright: copyright (c) 2012, Isis Lovecruft, The Tor Project Inc. # - class TimeoutError(Exception): """Raised when a timer runs out.""" pass -def timeout(secs, e=None): +def timeout(seconds, e=None): """ - A decorator for blocking methods to cause them to timeout. + A decorator for blocking methods to cause them to timeout. Can be used + like this: + @timeout(30) + def foo(): + for x in xrange(1000000000): + print x + or like this: + ridiculous = timeout(30)(foo) + + :param seconds: + Number of seconds to wait before raising :class:`TimeoutError`. + :param e: + Error message to pass to :class:`TimeoutError`. Default None. """ - import signal - import functools.wraps + from signal import alarm, signal, SIGALRM + from functools import wraps + def decorator(func): def _timeout(signum, frame): raise TimeoutError, e - def wrapper(*args, **kwargs): - signal.signal(signal.SIGALRM, _timeout) - signal.alarm(secs) + signal(SIGALRM, _timeout) + alarm(seconds) try: res = func(*args, **kwargs) finally: - signal.alarm(0) + alarm(0) return res - return functools.wraps(func)(wrapper) + return wraps(func)(wrapper) return decorator

1 0

[ooni-probe/master] Fix bug in tests
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit d621155c27143c6fad8743d0bfbdc15727a3b0db Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 12:06:00 2012 +0000 Fix bug in tests --- ooni/plugoo/tests.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 37e0664..4e7e094 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -30,14 +30,14 @@ class OONITest(object): ended = False def __init__(self, local_options, global_options, report, ooninet=None, - my_reactor=reactor): + reactor=reactor): # These are the options that are read through the tests suboptions self.local_options = local_options # These are the options global to all of OONI self.global_options = global_options self.report = report #self.ooninet = ooninet - self.reactor = my_reactor + self.reactor = reactor self.result = {} self.initialize()

1 0

[ooni-probe/master] Add note about backward compatibility.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 7c645d40702c21c45059a01c67fe67150bd109d5 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 12:01:39 2012 +0000 Add note about backward compatibility. --- docs/source/writing_tests.rst | 8 ++++++++ 1 files changed, 8 insertions(+), 0 deletions(-) diff --git a/docs/source/writing_tests.rst b/docs/source/writing_tests.rst index a18b096..3d58dab 100644 --- a/docs/source/writing_tests.rst +++ b/docs/source/writing_tests.rst @@ -74,3 +74,11 @@ To access the current input you can use the ``input`` attribute, for example:: This will at each iteration over the list of inputs do something with the input. +Backward compatibility +---------------------- + +All ooni tests written using the experiment(), control() pattern are supported, +but all new tests should no longer be written using such pattern. + +Code in protocols should be refactored to follow the new API. +

1 0

[ooni-probe/master] Add an example of using the HTTP test template.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 8e8f3502944295b019f919e703c2f1396153b45e Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 14:14:41 2012 +0000 Add an example of using the HTTP test template. --- nettests/example_http.py | 28 ++++++++++++++++++++++++++++ 1 files changed, 28 insertions(+), 0 deletions(-) diff --git a/nettests/example_http.py b/nettests/example_http.py new file mode 100644 index 0000000..7528d48 --- /dev/null +++ b/nettests/example_http.py @@ -0,0 +1,28 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from ooni.templates import http +class Example(http.HTTPTest): + inputs = ['http://google.com/', 'http://wikileaks.org/', + 'http://torproject.org/'] + + def processResponseBody(self, body): + # XXX here shall go your logic + # for processing the body + if 'blocked' in body: + self.report['censored'] = True + else: + self.report['censored'] = False + + def processResponseHeaders(self, headers): + # XXX place in here all the logic for handling the processing of HTTP + # Headers. + if headers.hasHeader('location'): + self.report['redirect'] = True + + server = headers.getRawHeaders("Server") + if server: + self.report['http_server'] = str(server.pop()) +

1 0

[ooni-probe/master] Merge branch 'master' of https://git.torproject.org/ooni-probe
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit edfaff0c38c8ad5f8772ba23721413f0878a0d35 Merge: 16c3cfe 21505f8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 12:15:42 2012 +0000 Merge branch 'master' of https://git.torproject.org/ooni-probe Conflicts: .gitignore ooni/assets/bridgetests.txt ooni/plugins/new_bridget.py .gitignore | 4 + docs/bridget.md | 102 ++++++ ooni/assets/bridgetests.txt | 11 - ooni/lib/Makefile | 36 ++ ooni/lib/__init__.py | 43 +--- ooni/lib/txscapy | 1 - ooni/lib/txscapy.py | 348 ++++++++++++++++++++ ooni/lib/txtorcon | 1 - ooni/lib/txtraceroute | 1 - ooni/lib/txtraceroute.py | 752 +++++++++++++++++++++++++++++++++++++++++++ ooni/ooniprobe.py | 10 +- ooni/plugins/blocking.py | 2 +- ooni/plugins/dnstamper.py | 6 +- ooni/plugins/httphost.py | 2 +- ooni/plugins/new_bridget.py | 239 ++++++++++---- ooni/plugins/tcpconnect.py | 5 +- ooni/plugoo/nodes.py | 6 +- ooni/plugoo/reports.py | 2 +- ooni/plugoo/tests.py | 3 +- ooni/utils/log.py | 1 - 20 files changed, 1437 insertions(+), 138 deletions(-) diff --cc .gitignore index 6b1b9ce,686e319..0458e37 --- a/.gitignore +++ b/.gitignore @@@ -8,4 -8,4 +8,8 @@@ proxy-lists/italy-http-ips.tx private/* /ooni/plugins/dropin.cache oonib/oonibackend.conf -ooni/lib/txtorcon ++<<<<<<< HEAD +ooni/assets/* ++======= ++ooni/lib/txtorcon ++>>>>>>> 21505f84aef5d60c7e138590a1a40e3df773d680 diff --cc ooni/ooniprobe.py index 86f4e2d,539c2ac..ec529d6 mode 100644,100755..100644 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py diff --cc ooni/plugins/new_bridget.py index 673cacc,0000000..c85caeb mode 100644,000000..100644 --- a/ooni/plugins/new_bridget.py +++ b/ooni/plugins/new_bridget.py @@@ -1,421 -1,0 +1,528 @@@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# +-----------+ +# | BRIDGET | +# | +----------------------------------------------+ +# +--------| Use a slave Tor process to test making a Tor | +# | connection to a list of bridges or relays. | +# +----------------------------------------------+ +# +# :authors: Arturo Filasto, Isis Lovecruft +# :licence: see included LICENSE +# :version: 0.1.0-alpha + +from __future__ import with_statement - from zope.interface import implements ++from os import getcwd ++from os.path import isfile ++from os.path import join as pj +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import defer, error, reactor ++from zope.interface import implements + +import random +import sys + - try: - from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher - except: - print "BridgeT requires txtorcon: https://github.com/meejah/txtorcon.git" - print "Your copy of OONI should have it included, if you're seeing this" - print "message, please file a bug report." - log.msg ("Bridget: Unable to import from ooni.lib.txtorcon") - +from ooni.utils import log +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset + + - class BridgetArgs(usage.Options): ++def portCheck(number): ++ number = int(number) ++ if number not in range(1024, 65535): ++ raise ValueError("Port out of range") + - def portCheck(number): - number = int(number) - if number not in range(1024, 65535): - raise ValueError("Port out of range") - portCheck.coerceDoc = "Ports must be between 1024 and 65535" ++portCheckAllowed = "must be between 1024 and 65535." ++sockCheck, ctrlCheck = portCheck, portCheck ++sockCheck.coerceDoc = "Port to use for Tor's SocksPort, " + portCheckAllowed ++ctrlCheck.coerceDoc = "Port to use for Tor's ControlPort, " + portCheckAllowed + ++ ++class BridgetArgs(usage.Options): + optParameters = [ + ['bridges', 'b', None, - 'List of bridges to scan <IP>:<ORport>'], ++ 'File listing bridge IP:ORPorts to test'], + ['relays', 'f', None, - 'List of relays to scan <IP>'], - ['socks', 's', 9049, portCheck, - 'Tor SocksPort to use'], - ['control', 'c', 9052, portCheck, - 'Tor ControlPort to use'], - ['tor-path', 'p', None, ++ 'File listing relay IPs to test'], ++ ['socks', 's', 9049, None, portCheck], ++ ['control', 'c', 9052, None, portCheck], ++ ['torpath', 'p', None, + 'Path to the Tor binary to use'], - ['data-dir', 'd', None, ++ ['datadir', 'd', None, + 'Tor DataDirectory to use'], + ['transport', 't', None, + 'Tor ClientTransportPlugin'], + ['resume', 'r', 0, + 'Resume at this index']] - optFlags = [['random', 'x', 'Randomize control and socks ports']] ++ optFlags = [ ++ ['random', 'x', 'Use random ControlPort and SocksPort']] + + def postOptions(self): - ## We can't test pluggable transports without bridges + if self['transport'] and not self['bridges']: + e = "Pluggable transport requires the bridges option" + raise usage.UsageError, e - ## We can't use random and static port simultaneously + if self['socks'] and self['control']: + if self['random']: + e = "Unable to use random and specific ports simultaneously" + raise usageError, e + +class CustomCircuit(CircuitListenerMixin): + implements(IStreamAttacher) + + from txtorcon.interface import IRouterContainer, ICircuitContainer + + def __init__(self, state): + self.state = state + self.waiting_circuits = [] + + def waiting_on(self, circuit): + for (circid, d) in self.waiting_circuits: + if circuit.id == circid: + return true + return False + + def circuit_extend(self, circuit, router): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + if self.waiting_on(circuit): + log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) + + def circuit_built(self, circuit): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + + log.msg("Circuit %s built ..." % circuit.id) + log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) + + for (circid, d) in self.waiting_circuits: + if circid == circuit.id: + self.waiting_circuits.remove(circid, d) + d.callback(circuit) + + def circuit_failed(self, circuit, reason): + if self.waiting_on(circuit): + log.msg("A circuit we requested %s failed for reason %s" % + (circuit.id, reason)) + circid, d = None, None + for x in self.waiting_circuits: + if x[0] == circuit.id: + circid, d, stream_cc = x + if d is None: + raise Exception("Expected to find circuit.") + + self.waiting_circuits.remove((circid, d)) + log.msg("Trying to build a circuit for %s" % circid) + self.request_circuit_build(d) + + def check_circuit_route(self, circuit, router): + if router in circuit.path: + #router.update() ## XXX can i use without args? no. + TorInfo.dump(self) + + def request_circuit_build(self, deferred): + entries = self.state.entry_guards.value() + relays = self.state.routers.values() + + log.msg("We have these nodes listed as entry guards:") + log.msg("%s" % entries) + log.msg("We have these nodes listed as relays:") + log.msg("%s" % relays) + + path = [random.choice(entries), + random.choice(relays), + random.choice(relays)] + + log.msg("Requesting a circuit: %s" + % '-->'.join(map(lambda x: x.location.countrycode, path))) + + class AppendWaiting: + def __init__(self, attacher, deferred): + self.attacher = attacher + self.d = deferred + + def __call__(self, circuit): + """ + Return from build_circuit is a Circuit, however, we want to + wait until it is built before we can issue an attach on it and + callback to the Deferred we issue here. + """ + log.msg("Circuit %s is in progress ..." % circuit.id) + self.attacher.waiting_circuits.append((circuit.id, self.d)) + + return self.state.build_circuit(path).addCallback(AppendWaiting(self, deferred_to_callback)).addErrback(log.err) + +class BridgetAsset(Asset): + """ - Class for parsing bridge assets so that they can be commented out. ++ Class for parsing bridget Assets ignoring commented out lines. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + if line.startswith('#'): + return + else: + return line.replace('\n','') + +class BridgetTest(OONITest): + """ + XXX fill me in + + :ivar config: + An :class:`ooni.lib.txtorcon.TorConfig` instance. + :ivar relay_list: + A list of all provided relays to test. We have to do this because + txtorcon.TorState().entry_guards won't build a custom circuit if the + first hop isn't in the torrc's EntryNodes. + :ivar bridge_list: + A list of all provided bridges to test. + :ivar socks_port: + Integer for Tor's SocksPort. + :ivar control_port: + Integer for Tor's ControlPort. + :ivar plug_transport: + String defining the Tor's ClientTransportPlugin, for testing + a bridge's pluggable transport functionality. + :ivar tor_binary: + Path to the Tor binary to use, e.g. \'/usr/sbin/tor\' + """ + implements(IPlugin, ITest) + + shortName = "newbridget" + description = "bridget" + requirements = None + options = BridgetArgs + blocking = False + + def initialize(self): + """ + Extra initialization steps. We only want one child Tor process + running, so we need to deal with the creation of TorConfig() only + once, before the experiment runs. + """ + self.socks_port = 9049 + self.control_port = 9052 + self.tor_binary = '/usr/sbin/tor' + self.data_directory = None + + if self.local_options: - try: - from ooni.lib.txtorcon import TorConfig - except: - e = "Could not import TorConfig class from txtorcon!" - raise ImportError, e ++ options = self.local_options + - options = self.local_options - self.config = TorConfig() - - ## Don't run the experiment if we don't have anything to test + if not options['bridges'] and not options['relays']: + self.suicide = True ++ return ++ ++ try: ++ from ooni.lib.txtorcon import TorConfig ++ except ImportError: ++ log.msg ("Bridget: Unable to import from ooni.lib.txtorcon") ++ wd, tx = getcwd(), 'lib/txtorcon/torconfig.py' ++ chk = pj(wd,tx) if wd.endswith('ooni') else pj(wd,'ooni/'+tx) ++ try: ++ assert isfile(chk) ++ except: ++ log.msg("Error: Some OONI libraries are missing!") ++ log.msg("Please go to /ooni/lib/ and do \"make all\"") ++ ++ self.config = TorConfig() + + if options['bridges']: + self.config.UseBridges = 1 + + if options['relays']: + ## Stupid hack for testing only relays: + ## Tor doesn't use EntryNodes when UseBridges is enabled, but + ## config.state.entry_guards needs to include the first hop to + ## build a custom circuit. + self.config.EntryNodes = ','.join(relay_list) + + if options['socks']: + self.socks_port = options['socks'] + + if options['control']: + self.control_port = options['control'] + + if options['random']: + log.msg("Using randomized ControlPort and SocksPort ...") + self.socks_port = random.randint(1024, 2**16) + self.control_port = random.randint(1024, 2**16) + - if options['tor-path']: - self.tor_binary = options['tor-path'] ++ if options['torpath']: ++ self.tor_binary = options['torpath'] + - if options['data-dir']: - self.config.DataDirectory = options['data-dir'] ++ if options['datadir']: ++ self.config.DataDirectory = options['datadir'] + + if options['transport']: + ## ClientTransportPlugin transport socks4|socks5 IP:PORT + ## ClientTransportPlugin transport exec path-to-binary [options] + if not options['bridges']: + e = "You must use the bridge option to test a transport." + raise usage.UsageError("%s" % e) + + log.msg("Using pluggable transport ...") + ## XXX fixme there's got to be a better way to check the exec + assert type(options['transport']) is str + self.config.ClientTransportPlugin = options['transport'] + + self.config.SocksPort = self.socks_port + self.config.ControlPort = self.control_port + self.config.save() + + def load_assets(self): + """ + Load bridges and/or relays from files given in user options. Bridges + should be given in the form IP:ORport. We don't want to load these as + assets, because it's inefficient to start a Tor process for each one. + """ + assets = {} + self.bridge_list = [] + self.relay_list = [] + + ## XXX fix me + ## we should probably find a more memory nice way to load addresses, + ## in case the files are really large + if self.local_options: - if self.local_options['bridges']: - log.msg("Loading bridge information from %s ..." - % self.local_options['bridges']) - with open(self.local_options['bridges']) as bridge_file: - for line in bridge_file.readlines(): ++ def make_asset_list(opt, lst): ++ log.msg("Loading information from %s ..." % opt) ++ with open(opt) as opt_file: ++ for line in opt_file.readlines(): + if line.startswith('#'): + continue + else: - self.bridge_list.append(line.replace('\n','')) - assets.update({'bridges': self.bridge_list}) ++ lst.append(line.replace('\n','')) + ++ if self.local_options['bridges']: ++ make_asset_list(self.local_options['bridges'], ++ self.bridge_list) ++ assets.update({'bridges': self.bridge_list}) + if self.local_options['relays']: - log.msg("Loading relay information from %s ..." - % self.local_options['relays']) - with open(options['relays']) as relay_file: - for line in relay_file.readlines(): - if line.startswith('#'): - continue - else: - self.relay_list.append(line.replace('\n','')) ++ make_asset_list(self.local_options['relays'], ++ self.relay_list) + assets.update({'relays': self.relay_list}) + return assets + + def experiment(self, args): + """ + XXX fill me in + + :param args: + The :class:`ooni.plugoo.asset.Asset <Asset>` line currently being + used. + :meth launch_tor: + Returns a Deferred which callbacks with a + :class:`ooni.lib.txtorcon.torproto.TorProcessProtocol + <TorProcessProtocol>` connected to the fully-bootstrapped Tor; + this has a :class:`ooni.lib.txtorcon.torcontol.TorControlProtocol + <TorControlProtocol>` instance as .protocol. + """ - from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState - from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor ++ try: ++ from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher ++ from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState ++ from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor ++ except ImportError: ++ log.msg("Error: Unable to import from ooni.lib.txtorcon") ++ wd, tx = getcwd(), 'lib/txtorcon/torconfig.py' ++ chk = pj(wd,tx) if wd.endswith('ooni') else pj(wd,'ooni/'+tx) ++ try: ++ assert isfile(chk) ++ except: ++ log.msg("Error: Some OONI libraries are missing!") ++ log.msg(" Please go to /ooni/lib/ and do \"make all\"") ++ return sys.exit() + + def bootstrap(ctrl): + """ + Launch a Tor process with the TorConfig instance returned from + initialize(). + """ + conf = TorConfig(ctrl) + conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) + log.msg("Tor process connected, bootstrapping ...") + + def reconf_controller(conf, bridge): + ## if bridges and relays, use one bridge then build a circuit + ## from three relays + conf.Bridge = bridge + ## XXX do we need a SIGHUP to restart? + + ## XXX see txtorcon.TorControlProtocol.add_event_listener we + ## may not need full CustomCircuit class + + ## if bridges only, try one bridge at a time, but don't build + ## circuits, just return + ## if relays only, build circuits from relays + + def reconf_fail(args): + log.msg("Reconfiguring Tor config with args %s failed" % args) + reactor.stop() + + def setup_fail(args): + log.msg("Setup Failed.") + report.update({'failed': args}) + reactor.stop() + + def setup_done(proto): + log.msg("Setup Complete: %s" % proto) + state = TorState(proto.tor_protocol) + state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail) + report.update({'success': args}) + + def updates(prog, tag, summary): + log.msg("%d%%: %s" % (prog, summary)) + + if len(args) == 0: - log.msg("Bridget needs lists of bridges and/or relays to test!") ++ log.msg("Bridget can't run without bridges or relays to test!") + log.msg("Exiting ...") - d = sys.exit() - return d - ++ return sys.exit() + else: ++ ++ class CustomCircuit(CircuitListenerMixin): ++ implements(IStreamAttacher) ++ ++ from txtorcon.interface import IRouterContainer ++ from txtorcon.interface import ICircuitContainer ++ ++ def __init__(self, state): ++ self.state = state ++ self.waiting_circuits = [] ++ ++ def waiting_on(self, circuit): ++ for (circid, d) in self.waiting_circuits: ++ if circuit.id == circid: ++ return true ++ return False ++ ++ def circuit_extend(self, circuit, router): ++ "ICircuitListener" ++ if circuit.purpose != 'GENERAL': ++ return ++ if self.waiting_on(circuit): ++ log.msg("Circuit %d (%s)" ++ % (circuit.id, router.id_hex)) ++ ++ def circuit_built(self, circuit): ++ "ICircuitListener" ++ if circuit.purpose != 'GENERAL': ++ return ++ log.msg("Circuit %s built ..." % circuit.id) ++ log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) ++ for (circid, d) in self.waiting_circuits: ++ if circid == circuit.id: ++ self.waiting_circuits.remove(circid, d) ++ d.callback(circuit) ++ ++ def circuit_failed(self, circuit, reason): ++ if self.waiting_on(circuit): ++ log.msg("A circuit we requested %s failed for reason %s" ++ % (circuit.id, reason)) ++ circid, d = None, None ++ for x in self.waiting_circuits: ++ if x[0] == circuit.id: ++ circid, d, stream_cc = x ++ if d is None: ++ raise Exception("Expected to find circuit.") ++ ++ self.waiting_circuits.remove((circid, d)) ++ log.msg("Trying to build a circuit for %s" % circid) ++ self.request_circuit_build(d) ++ ++ def check_circuit_route(self, circuit, router): ++ if router in circuit.path: ++ #router.update() ## XXX can i use without args? no. ++ TorInfo.dump(self) ++ ++ def request_circuit_build(self, deferred): ++ entries = self.state.entry_guards.value() ++ relays = self.state.routers.values() ++ log.msg("We have these nodes listed as entry guards:") ++ log.msg("%s" % entries) ++ log.msg("We have these nodes listed as relays:") ++ log.msg("%s" % relays) ++ path = [random.choice(entries), ++ random.choice(relays), ++ random.choice(relays)] ++ log.msg("Requesting a circuit: %s" ++ % '-->'.join(map(lambda x: x.location.countrycode, ++ path))) ++ ++ class AppendWaiting: ++ def __init__(self, attacher, deferred): ++ self.attacher = attacher ++ self.d = deferred ++ ++ def __call__(self, circuit): ++ """ ++ Return from build_circuit is a Circuit, however, ++ we want to wait until it is built before we can ++ issue an attach on it and callback to the Deferred ++ we issue here. ++ """ ++ log.msg("Circuit %s is in progress ..." % circuit.id) ++ self.attacher.waiting_circuits.append((circuit.id, ++ self.d)) ++ ++ fin = self.state.build_circuit(path) ++ fin.addCallback(AppendWaiting(self, deferred_to_callback)) ++ fin.addErrback(log.err) ++ return fin ++ ++ + if len(self.bridge_list) >= 1: + for bridge in self.bridge_list: + try: - print "BRIDGE IS %s" % bridge ++ log.msg("Current Bridge: %s" % bridge) + reconf_controller(self.config, bridge) + except: + reconf_fail(bridge) + + log.msg("Bridget: initiating test ... ") + log.msg("Using the following as our torrc:\n%s" + % self.config.create_torrc()) + report = {'tor_config': self.config.config} + log.msg("Starting Tor ...") + + ## :return: a Deferred which callbacks with a TorProcessProtocol + ## connected to the fully-bootstrapped Tor; this has a + ## txtorcon.TorControlProtocol instance as .protocol. + d = launch_tor(self.config, + reactor, + progress_updates=updates, + tor_binary=self.tor_binary) + d.addCallback(bootstrap, self.config) + d.addErrback(setup_fail) + ## now build circuits + + #print "Tor process ID: %s" % d.transport.pid + return d + +## So that getPlugins() can register the Test: +bridget = BridgetTest(None, None, None) + +## ISIS' NOTES +## ----------- +## self.config.save() only needs to be called if Tor is already running. +## - ## need to add transport type to torrc Bridge line: - ## Bridge <transport> IP:ORPort <fingerprint> ++## to test gid, uid, and euid: ++## with open('/proc/self/state') as uidfile: ++## print uidfile.read(1000) +## +## TODO: +## o add option for any kwarg=arg self.config setting +## o cleanup documentation ++## x add DataDirectory option +## o check if bridges are public relays +## o take bridge_desc file as input, also be able to give same +## format as output - ## o change the stupid name ++## o Add assychronous timout for deferred, so that we don't wait ++## forever for bridges that don't work. ++## o Add mechanism for testing through another host +## +## FIX: - ## data directory is not found, or permissions aren't right ++## o DataDirectory is not found, or permissions aren't right ++## o Bridge line needs generation of transport properties ++## Bridge <transport> IP:ORPort <fingerprint>

1 0

[ooni-probe/master] Add documentation on the OONI API
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit d01d501efe93a9672d37b9aa0dae61e0d3065106 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 14:41:21 2012 +0000 Add documentation on the OONI API --- docs/source/api/ooni.rst | 60 ++++++++++++++++++++++++++++++++++++ docs/source/api/ooni.templates.rst | 11 ++++++ docs/source/api/ooni.utils.rst | 51 ++++++++++++++++++++++++++++++ docs/source/index.rst | 1 + 4 files changed, 123 insertions(+), 0 deletions(-) diff --git a/docs/source/api/ooni.rst b/docs/source/api/ooni.rst new file mode 100644 index 0000000..bcdd402 --- /dev/null +++ b/docs/source/api/ooni.rst @@ -0,0 +1,60 @@ +ooni Package +============ + +:mod:`ooni` Package +------------------- + +.. automodule:: ooni.__init__ + :members: + :undoc-members: + :show-inheritance: + +:mod:`input` Module +------------------- + +.. automodule:: ooni.input + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettest` Module +--------------------- + +.. automodule:: ooni.nettest + :members: + :undoc-members: + :show-inheritance: + +:mod:`oonicli` Module +--------------------- + +.. automodule:: ooni.oonicli + :members: + :undoc-members: + :show-inheritance: + +:mod:`reporter` Module +---------------------- + +.. automodule:: ooni.reporter + :members: + :undoc-members: + :show-inheritance: + +:mod:`runner` Module +-------------------- + +.. automodule:: ooni.runner + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + ooni.lib + ooni.templates + ooni.utils + diff --git a/docs/source/api/ooni.templates.rst b/docs/source/api/ooni.templates.rst new file mode 100644 index 0000000..662da1d --- /dev/null +++ b/docs/source/api/ooni.templates.rst @@ -0,0 +1,11 @@ +templates Package +================= + +:mod:`http` Module +------------------ + +.. automodule:: ooni.templates.http + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/source/api/ooni.utils.rst b/docs/source/api/ooni.utils.rst new file mode 100644 index 0000000..5361b43 --- /dev/null +++ b/docs/source/api/ooni.utils.rst @@ -0,0 +1,51 @@ +utils Package +============= + +:mod:`utils` Package +-------------------- + +.. automodule:: ooni.utils + :members: + :undoc-members: + :show-inheritance: + +:mod:`config` Module +-------------------- + +.. automodule:: ooni.utils.config + :members: + :undoc-members: + :show-inheritance: + +:mod:`date` Module +------------------ + +.. automodule:: ooni.utils.date + :members: + :undoc-members: + :show-inheritance: + +:mod:`log` Module +----------------- + +.. automodule:: ooni.utils.log + :members: + :undoc-members: + :show-inheritance: + +:mod:`logo` Module +------------------ + +.. automodule:: ooni.utils.logo + :members: + :undoc-members: + :show-inheritance: + +:mod:`net` Module +----------------- + +.. automodule:: ooni.utils.net + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/source/index.rst b/docs/source/index.rst index 34ca87c..e744f7f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -46,5 +46,6 @@ Contents install tutorial writing_tests + api/*

1 0

[ooni-probe/master] Deprecate scaffolding script
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit a957cd5f469d34bfda98848bca586a0c409181af Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 15:02:32 2012 +0000 Deprecate scaffolding script --- ooni/scaffolding.py | 10 ++++++++-- 1 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ooni/scaffolding.py b/ooni/scaffolding.py index 0a832aa..dffe342 100644 --- a/ooni/scaffolding.py +++ b/ooni/scaffolding.py @@ -3,10 +3,16 @@ """ This script should be used for creating the scaffolding for a test. """ +from __future__ import print_function import os import sys from ooni.utils import log +print("!!!! This test writing strategy is now deprecated !!!") +print("visit: https://ooni.readthedocs.org/en/latest/writing_tests.html " + "for new instructions") +sys.exit(1) + test_template = """\"\"\" This is a self genrated test created by scaffolding.py. you will need to fill it up with all your necessities. @@ -61,9 +67,9 @@ test_vars['testSNlower'] = test_vars['testShortname'].lower() fname = os.path.join('plugins', test_vars['testSNlower']+'.py') if os.path.exists(fname): - print 'WARNING! File named "%s" already exists.' % fname + print('WARNING! File named "%s" already exists.' % fname) if raw_input("Do you wish to continue (y/N)? ").lower() != 'y': - print "gotcha! Dying.." + print("gotcha! Dying..") sys.exit(0) fp = open(fname, 'w')

1 0

[ooni-probe/master] Add patch for bug in yaml.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 532e2e9853b4bdd61c951dc3073d8dc68d35f754 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 21:52:40 2012 +0000 Add patch for bug in yaml. Add assets to ignore file. --- .gitignore | 5 +- bin/ooniprobe | 11 + docs/source/conf.py | 7 +- ooni/assets/captive_portal_tests.txt | 4 - ooni/assets/example.txt | 2 - ooni/assets/greatfire.lst | 2100 ---------------------------------- ooni/assets/redirects.yaml | 17 - ooni/assets/tcpscan.txt | 2 - ooni/reporter.py | 19 +- ooni/utils/__init__.py | 52 + 10 files changed, 86 insertions(+), 2133 deletions(-) diff --git a/.gitignore b/.gitignore index 0458e37..dc569e9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +assets/* build/* package/* proxy-lists/ip-cc.txt @@ -8,8 +9,4 @@ proxy-lists/italy-http-ips.txt private/* /ooni/plugins/dropin.cache oonib/oonibackend.conf -<<<<<<< HEAD -ooni/assets/* -======= ooni/lib/txtorcon ->>>>>>> 21505f84aef5d60c7e138590a1a40e3df773d680 diff --git a/bin/ooniprobe b/bin/ooniprobe index 1f0c26d..f25be7a 100755 --- a/bin/ooniprobe +++ b/bin/ooniprobe @@ -1,8 +1,19 @@ #!/usr/bin/python2 +# -*- encoding: utf-8 -*- +# # startup script based on twisted trial # See http://twistedmatrix.com/ +# +# :authors: Arturo Filastò +# :licence: see LICENSE + + import os, sys +# This is a hack to overcome a bug in python +from ooni.utils import patched_reduce_ex +copy_reg._reduce_ex = patched_reduce_ex + sys.path[:] = map(os.path.abspath, sys.path) sys.path.insert(0, os.path.abspath(os.getcwd())) diff --git a/docs/source/conf.py b/docs/source/conf.py index e066613..1f61842 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -94,8 +94,13 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'haiku' +html_theme_options = {'textcolor': "#222222", + 'headingcolor': "#555", + 'linkcolor': "#4183C4", + 'visitedlinkcolor': "#888", + 'hoverlinkcolor': "#888"} # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. diff --git a/ooni/assets/captive_portal_tests.txt b/ooni/assets/captive_portal_tests.txt deleted file mode 100644 index 7007411..0000000 --- a/ooni/assets/captive_portal_tests.txt +++ /dev/null @@ -1,4 +0,0 @@ - -http://ooni.nu, Open Observatory of Network Interference, 200 -https://w2.eff.org/Censorship/Internet_censorship_bills/barlow_0296.declaration, let us now take our leave of them, 200 -https://www.torproject.org, Defend yourself against network surveillance and traffic analysis, 200 diff --git a/ooni/assets/example.txt b/ooni/assets/example.txt deleted file mode 100644 index 01ffcdf..0000000 --- a/ooni/assets/example.txt +++ /dev/null @@ -1,2 +0,0 @@ -127.0.0.1:9050 -127.0.0.1:9051 diff --git a/ooni/assets/greatfire.lst b/ooni/assets/greatfire.lst deleted file mode 100644 index 30c2a96..0000000 --- a/ooni/assets/greatfire.lst +++ /dev/null @@ -1,2100 +0,0 @@ -http://www.new.facebook.com -http://www.facebook.com -http://facebook.com -http://m.facebook.com -http://sites.google.com -http://login.facebook.com -http://spreadsheets.google.com -http://apps.facebook.com -http://facebook.com -http://picasaweb.google.com -http://www.facebook.com -http://api.facebook.com -http://zh-cn.facebook.com -http://plus.google.com -http://encrypted.google.com -http://plusone.google.com -http://plusone.google.com -http://google.com -http://sites.google.com -http://feedproxy.google.com -http://www.docs.google.com -http://profiles.google.com -http://m.facebook.com -http://encrypted.google.com -http://code.google.com -http://spreadsheets.google.com -http://drive.google.com -http://www.picasaweb.google.com -http://groups.google.com -http://spreadsheets4.google.com -http://fr-fr.facebook.com -http://talkgadget.google.com -http://docs.google.com -http://www.plus.google.com -http://plus.google.com -http://tw.youtube.com -http://youtube.com -http://www.youtube.com -http://www.youtube.com -http://hk.youtube.com -http://youtube.com -http://accounts.youtube.com -http://hk.rd.yahoo.com -http://meme.yahoo.com -http://tw.yahoo.com -http://hk.news.yahoo.com -http://tw.myblog.yahoo.com -http://tw.news.yahoo.com -http://video.yahoo.com -http://pulse.yahoo.com -http://es.video.yahoo.com -http://hk.yahoo.com -http://en.mobile.wikipedia.org -http://zh.m.wikipedia.org -http://twitter.com -http://eccomi.qzone.qq.com -http://twitter.com -http://www.twitter.com -http://www.twitter.com -http://dev.twitter.com -http://search.twitter.com -http://esbensportfolio.blogspot.com -http://tf2spreadshet.blogspot.com -http://downmerng.blogspot.com -http://historyanarchy.blogspot.com -http://charlesmok.blogspot.com -http://www.stratosphera-nrb.blogspot.com -http://veryverylong.blogspot.com -http://indeterminacy.blogspot.com -http://spritthevirus.blogspot.com -http://hric-newsbrief.blogspot.com -http://basketbawful.blogspot.com -http://www.fademi.blogspot.com -http://wuxiao0715.blogspot.com -http://dramatisinterruptus.blogspot.com -http://martinsmithdesign.blogspot.com -http://www.geezericious.blogspot.com -http://hungryballerina.blogspot.com -http://www.malaysianchiks.blogspot.com -http://transranyf.blogspot.com -http://spaceman100-thesuperblog.blogspot.com -http://unreadbits.blogspot.com -http://hqlenglish.blogspot.com -http://newsabeta.blogspot.com -http://www.google.blogspot.com -http://www.tibetnewsonline.blogspot.com -http://sopravivalitalia.blogspot.com -http://perfectpeacellc.blogspot.com -http://www.taikotime.blogspot.com -http://www.arielefraimashbel.blogspot.com -http://ukiahcoachbrown.blogspot.com -http://www.transranyf.blogspot.com -http://unreadnits.blogspot.com -http://adornoskies.blogspot.com -http://freedomforegyptians.blogspot.com -http://www.nickdael.blogspot.com -http://postcardsfromrichard.blogspot.com -http://pennedscripts.blogspot.com -http://helloloverr.blogspot.com -http://american-sleepingdragon.blogspot.com -http://postcardsfromrichard.blogspot.com -http://pennedscripts.blogspot.com -http://spritthevirus.blogspot.com -http://froggyflix.blogspot.com -http://www.varadhjain.blogspot.com -http://ukiahcoachbrown.blogspot.com -http://charlesmok.blogspot.com -http://amdoamonk.blogspot.com -http://historyanarchy.blogspot.com -http://ajithfanswing.blogspot.com -http://freedomforegyptians.blogspot.com -http://www.spaceman100-thesuperblog.blogspot.com -http://helloloverr.blogspot.com -http://amongstsaintsandsailors.blogspot.com -http://hric-newsbrief.blogspot.com -http://wuxiao0715.blogspot.com -http://taiwanmatters.blogspot.com -http://www.tibetnewsonline.blogspot.com -http://www.happening24X7.blogspot.com -http://dramatisinterruptus.blogspot.com -http://perfectpeacellc.blogspot.com -http://google.blogspot.com -http://cittadeglignomi.blogspot.com -http://arielefraimashbel.blogspot.com -http://evolutionguide.blogspot.com -http://unreadbits.blogspot.com -http://sopravivalitalia.blogspot.com -http://www.raceboy24.blogspot.com -http://pfge-pfge.blogspot.com -http://www.evolutionguide.blogspot.com -http://unreadnits.blogspot.com -http://www.cajuncardon.blogspot.com -http://owpandprojectdirect.blogspot.com -http://www.blogspot.com -http://codeslut.blogspot.com -http://www.panfletoliberal.blogspot.com -http://michaelturton.blogspot.com -http://raceboy24.blogspot.com -http://generatorblog.blogspot.com -http://gustavoguillenzulia.blogspot.com -http://manyouyinli.blogspot.com -http://sruv-pitbulls.blogspot.com -http://www.stratosphera-nrb.blogspot.com -http://www.ukiahcoachbrown.blogspot.com -http://www.ncbcr.blogspot.com -http://blogspot.com -http://trangridiculous.blogspot.com -http://le-rubrique.blogspot.com -http://roxy-angel.blogspot.com -http://bettermyths.blogspot.com -http://inexplorados.blogspot.com -http://thoughtthemes.blogspot.com -http://blakelychinatrip.blogspot.com -http://zhangxin1989.blogspot.com -http://kangdiy.blogspot.com -http://henacg.blogspot.com -http://oldvirginiablog.blogspot.com -http://imperturbable-music.blogspot.com -http://ang-newswire.blogspot.com -http://yuhaotech.blogspot.com -http://tokyobikepolo.blogspot.com -http://virtuasapiens.blogspot.com -http://hldpn.blogspot.com -http://blogs.yahoo.co.jp -http://picasaweb.google.com.hk -http://www.wordpress.com -http://abcdefg.wordpress.com -http://astimegoesbyinchina.wordpress.com -http://www.wordpress.com -http://thechinesedream.wordpress.com -http://xinchina.wordpress.com -http://theworldofguava.wordpress.com -http://wuxiao0715.wordpress.com -http://wordpress.com -http://www.annishan2011.wordpress.com -http://publichealthinchina.wordpress.com -http://teflandlouise.wordpress.com -http://uabinchina2011.wordpress.com -http://nfolamp.wordpress.com -http://smurfswillbefree.wordpress.com -http://storiesfromtheceiling.wordpress.com -http://culturalexposure.wordpress.com -http://bullrunnings.wordpress.com -http://radiofreechina.wordpress.com -http://stephenmooredesign.wordpress.com -http://fromcanadatofinland.wordpress.com -http://www.richardjangus.wordpress.com -http://underthejacaranda.wordpress.com -http://www.quhelan.wordpress.com -http://sinologica.wordpress.com -http://wuxiao0715.wordpress.com -http://indonesiastoriesdotcom.wordpress.com -http://martianchronicles.wordpress.com -http://livefromgaza.wordpress.com -http://foureverfamily.wordpress.com -http://abc.wordpress.com -http://fashiontech.wordpress.com -http://sample.wordpress.com -http://zh.wordpress.com -http://lemasabachthani.wordpress.com -http://hmetro.com.my -http://www.blogcatalog.com -http://cbs.ntu.edu.tw -http://thisav.com -http://blogcatalog.com -http://tapuz.co.il -http://www.identi.ca -http://shifeike.blog125.fc2blog.net -http://boysfood.com -http://kaufmich.com -http://elnuevoherald.com -http://sinembargo.mx -http://8591.com.tw -http://rapgenius.com -http://ticketmonster.co.kr -http://hotfrog.jp -http://oddschecker.com -http://danawa.com -http://www.stickam.com -http://identi.ca -http://vtc.vn -http://blog.co.uk -http://nsgalleries.com -http://tweetdeck.com -http://crutchfield.com -http://hotpornshow.com -http://jagranjosh.com -http://reviewcentre.com -http://stickam.com -http://tek-tips.com -http://www.hotpornshow.com -http://jumptags.com -http://wizaz.pl -http://hotel-urbano.com -http://destructoid.com -http://maturetube.com -http://5i01.com -http://he.net -http://bbs-tw.com -http://ray-ban.com -http://viddler.com -http://ccbs.ntu.edu.tw -http://cnsnews.com -http://transunion.com -http://lulzimg.com -http://foum.hkgolden.com -http://newspaperdirect.com -http://creaders.net -http://orgasmof.ru -http://potterybarnkids.com -http://hack371.com -http://virustotal.com -http://forum.hkgolden.com -http://mymodernmet.com -http://kmtlbb.com -http://siol.net -http://meiyingie.com -http://socialsecurity.gov -http://network54.com -http://gotovim-doma.ru -http://cheaperthandirt.com -http://hotgoo.com -http://www.worldjournal.com -http://lamoda.ru -http://information.com -http://savevid.com -http://tsekouratoi.gr -http://www.viddler.com -http://wowtv.co.kr -http://movies.com -http://www.dateinasia.com -http://pornaccess.com -http://www.creaders.net -http://hkgolden.com -http://forum5.hkgolden.com -http://24smile.org -http://www.he.net -http://savings.com -http://dreamamateurs.com -http://carscoop.blogspot.com -http://girlfriendvideos.com -http://worldjournal.com -http://anneahira.com -http://ftvgirls.com -http://artnudegalleries.com -http://mobawaba.com -http://gadaadmongol.com -http://busuk.org -http://lawyers.com -http://www.network54.com -http://kabeldeutschland.de -http://privet.ru -http://sitetrail.com -http://www.mingpao.com -http://hyundai.com -http://woopie.jp -http://jetpack.wordpress.com -http://westelm.com -http://oursogo.com -http://mysoftplace.com -http://shangqu.info -http://newplaynow.com -http://jaidefinichon.com -http://tommy.com -http://trendhunter.com -http://kansascity.com -http://tomshw.it -http://pacopacomama.com -http://newplaysite.com -http://meemessa.com -http://mlbtraderumors.com -http://el5antuario.org -http://shemale-porn-galls.com -http://bergfiles.com -http://ratedporntube.com -http://chanel.com -http://chirpme.com -http://zomobo.net -http://blogdeizquierda.com -http://sarkarinaukriblog.com -http://php.su -http://subscribe.wordpress.com -http://signup.wordpress.com -http://rfa.org -http://upworthy.com -http://klmty.net -http://cfun.ir -http://momovideo.net -http://videobox.com -http://ggsoku.com -http://shoppingdaisy.com -http://sannas.jp -http://pornstarnetwork.com -http://flashkhor.com -http://parsonline.net -http://huyandex.com -http://akhbarelyom.org.eg -http://cplusplus.com -http://passion.com -http://sbt.com.br -http://remax.com -http://korabia.com -http://levante-emv.com -http://mingpao.com -http://sexpixbox.com -http://jjgirls.com -http://anyap.info -http://gossip9.com -http://googlewebmastercentral.blogspot.com -http://kiwi.kz -http://xxxaporn.com -http://youtu.be -http://tubegals.com -http://preloved.co.uk -http://basicscan.com -http://kerala.gov.in -http://fin24.com -http://alrakoba.net -http://travelandleisure.com -http://hugoboss.com -http://google.com.pa -http://skybet.com -http://tah.jp -http://info.51.ca -http://blog.51.ca -http://saveyoutube.com -http://manatelugumovies.net -http://img.ly -http://www.1pondo.tv -http://gpil.org -http://fora.pl -http://homemade-voyeur.com -http://thisdaylive.com -http://fotopages.com -http://ekra2.com -http://youversion.com -http://1pondo.tv -http://funpic.de -http://eg.ru -http://jobkorea.co.kr -http://Www.youversion.com -http://google.is -http://wwitv.com -http://yidio.com -http://lifehacker.ru -http://lunarpages.com -http://bbs.51.ca -http://theoutnet.com -http://www.livestation.com -http://www.img.ly -http://yyv.co -http://chrispederick.com -http://sharenxs.com -http://www.wikileaks.org -http://wikileaks.org -http://dxlive.com -http://amateursexy.net -http://www.baidu.jp -http://vertvonlinegratis.org -http://taiwannation.50webs.com -http://photographyblog.com -http://fscstore.com -http://whoismind.com -http://www.jjgirls.com -http://dancingbear.com -http://ipicture.ru -http://livestation.com -http://articlerich.com -http://jossandmain.com -http://www.yidio.com -http://jp-sex.com -http://insidefacebook.com -http://bet.hkjc.com -http://foodandwine.com -http://wordle.net -http://scrippsnetworks.com -http://baidu.jp -http://bustnow.com -http://lefora.com -http://store.valvesoftware.com -http://kpo-faq.blogspot.com -http://www.sex141.com -http://wazzub.com -http://kingstone.com.tw -http://hostingbulk.com -http://smile-adult.com -http://zend2.com -http://hotfrog.in -http://errotica-archives.com -http://server8.kproxy.com -http://drupal.ru -http://www.kingstone.com.tw -http://twilightsex.com -http://bt.no -http://qualityhealth.com -http://ping.fm -http://cwer.ru -http://www.spike.com -http://secretchina.com -http://shanbara.jp -http://www.ugo.com -http://www.xxx.com -http://server15.kproxy.com -http://dotnetperls.com -http://sexandsubmission.com -http://hautetfort.com -http://www.sexandsubmission.com -http://tiavastube.com -http://git-scm.com -http://pagemodo.com -http://stadt-bremerhaven.de -http://blog.rockmelt.com -http://gagnons-argent.com -http://wealth2xtreme.com -http://forum.baby-kingdom.com -http://googleblog.blogspot.com -http://fleshbot.com -http://videobam.com -http://21sextury.com -http://sakuralive.com -http://asredas.com -http://www.18qt.com -http://expekt.com -http://hkbisi.com -http://www.nexttv.com.tw -http://porlaputa.com -http://jkforum.net -http://henhaoji.com -http://anobii.com -http://www.torproject.org -http://torproject.org -http://proxy.org -http://sexinsex.net -http://trendsmap.com -http://ntdtv.com -http://www.sexinsex.net -http://nexttv.com.tw -http://www.popyard.com -http://www.ntdtv.com -http://media.ntdtv.com -http://www.pussy.org -http://79.133.201.84.nyud.net -http://temasektimes.wordpress.com -http://chinanews.blog.ntdtv.com -http://tunnel.shellmix.com.nyud.net -http://powerapple.com -http://popyard.com -http://globalvoicesonline.org -http://codereview.chromium.org -http://feedbox.com -http://rthk.org.hk -http://blog.iphone-dev.org -http://www.saveyoutube.com -http://www.rthk.org.hk -http://zh.globalvoicesonline.org -http://www.globalvoicesonline.org -http://www.taipei.gov.tw -http://www.footytube.com -http://twttr.com -http://fr.globalvoicesonline.org -http://yorkbbs.ca -http://bbs.powerapple.com -http://www.steves-digicams.com -http://pleated-jeans.com -http://forum.yorkbbs.ca -http://lalulalu.com -http://www.lalulalu.com -http://page2rss.com -http://virb.com -http://iask.ca -http://www.fileflyer.com -http://tw.jiepang.com -http://youtube-nocookie.com -http://blog.sparrowmailapp.com -http://www.islamhouse.com -http://hellotxt.com -http://toronto.iask.ca -http://blog.path.com -http://www.youtube-nocookie.com -http://www.rthk.hk -http://www.boxun.com -http://www.wuala.com -http://dumblittleman.com -http://s20.youmaker.com -http://westca.com -http://www.huaren.us -http://mysinablog.com -http://rthk.hk -http://hk.jiepang.com -http://content.wuala.com -http://www.sevenload.com -http://news.boxun.com -http://moviefap.com -http://translate.twttr.com -http://www.uswebproxy.com -http://youmaker.com -http://boxun.com -http://sevenload.com -http://htmldog.com -http://en.favotter.net -http://aboluowang.com -http://www.cna.com.tw -http://cna.com.tw -http://www.aboluowang.com -http://www.cn.sevenload.com -http://im.tv -http://translate.twttr.com -http://www.t66y.com -http://doujin-moe.us -http://wuala.com -http://www.minghui.org -http://www.perfspot.com -http://www.singtao.com -http://t66y.com -http://aisex.com -http://webupd8.org -http://www.voy.com -http://www.megaporn.com -http://wahas.com -http://www.fxdd.com -http://transferbigfiles.com -http://megaporn.com -http://enmasse.com -http://www.openvpn.net -http://minghui.org -http://www.91porn.com -http://tor.eff.org -http://www.881903.com -http://tor.updatestar.com -http://www.wahas.com -http://67.220.91.30 -http://zh.wikinews.org -http://www.hrw.org -http://kwongwah.com.my -http://strongvpn.com -http://welide.com -http://www.strongvpn.com -http://twitgoo.com -http://marines.mil -http://vanpeople.com -http://scriptlance.com -http://www.somee.com -http://strongvpn.com -http://bbs.skykiwi.com -http://www.thepiratebay.org -http://hrw.org -http://www.theepochtimes.com -http://ntq.comlu.com -http://www.want-daily.com -http://www.jensonusa.com -http://faststone.org -http://thepiratebay.org -http://www.pcdvd.com.tw -http://thepiratebay.org -http://lesscss.org -http://uncyclopedia.info -http://www.fooooo.com -http://movieworldsite.com -http://db.stoptui.site88.net -http://www.e-sim.org -http://popyard.org -http://www.gopetition.com -http://gopetition.com -http://amnesty.org -http://www.amnesty.org -http://www.es.amnesty.org -http://kompozer.net -http://www.popyard.org -http://e-sim.org -http://aculo.us -http://m.tweete.net -http://www.1688.com.au -http://www.dopehood.com -http://rocksoccer.co.cc -http://www01.ktzhk.com -http://disp.cc -http://playpcesor.blogspot.com -http://sports.now.com -http://movie.now.com -http://tweete.net -http://www.oiktv.com -http://www.searchtruth.com -http://bbs.9gal.com -http://kok2.no-blog.jp -http://www.viewsonic.com -http://pornstarclub.com -http://nonstoptrafficformula.com -http://oiktv.com -http://apigee.com -http://www.doh.gov.tw -http://pioneer-worker.forums-free.com -http://fooooo.com -http://www.fring.com -http://www.fb.me -http://www.pornstarclub.com -http://fring.com -http://news.now.com -http://www.now.com -http://now.com -http://blog.kowalczyk.info -http://eic-av.com -http://www.coolaler.com -http://coolaler.com -http://www.onthesamehost.com -http://picasa-readme.blogspot.com -http://twbbs.org -http://twitiq.com -http://0to255.com -http://www.dongtaiwang.com -http://www.moi.gov.tw -http://dongtaiwang.com -http://myfreshnet.com -http://skimtube.com -http://www.c-spanvideo.org -http://www.surfagain.com -http://en.bitcoin.it -http://webconference.eadgear.com -http://forum.slime.com.tw -http://www.trtc.com.tw -http://marketingsurvivalskills.com -http://www.myfreshnet.com -http://nakido.com -http://wiki.oauth.net -http://www.cyberghostvpn.com -http://www.kenyanlist.com -http://www.voachinese.com -http://www.nakido.com -http://turbohide.com -http://www.sopcast.com -http://wearn.com -http://www.wearn.com -http://cn2.streetvoice.com -http://graphis.ne.jp -http://www.porn365.com -http://sopcast.com -http://cyberghostvpn.com -http://cyberghostvpn.com -http://ahalife.com -http://list.ly -http://danlambaovn.blogspot.com -http://getcloudapp.com -http://news.singtao.ca -http://web.pts.org.tw -http://www.fulltiltpoker.com -http://c-spanvideo.org -http://developers.box.net -http://dl.box.net -http://www.apigee.com -http://www.falundafa.org -http://news.pts.org.tw -http://twittergadget.com -http://android-x86.org -http://www.kaiyuan.de -http://www.juzp.net -http://ww.kaiyuan.de -http://Www.ruero.com -http://dropshots.com -http://www.android-x86.org -http://mingpaonews.com -http://talk.news.pts.org.tw -http://www.opendemocracy.net -http://www.lanecrawford.com -http://www.insidesearch.blogspot.com -http://stitcher.com -http://www.dw-world.de -http://dw-world.de -http://www.dotsub.com -http://scrapetorrent.com -http://www.anchorfree.com -http://www.twittergadget.com -http://tagxedo.com -http://anchorfree.com -http://kaiyuan.de -http://twitthat.com -http://newtalk.tw -http://www.xys.org -http://www.comefromchina.com -http://alwaysdata.net -http://np.cpami.gov.tw -http://bbs.kimy.com.tw -http://dabr.co.uk -http://www.wantchinatimes.com -http://www.moztw.org -http://www.dabr.co.uk -http://taiwankiss.com -http://www.cotweet.com -http://twbbs.tw -http://city9x.com -http://gutteruncensored.com -http://www.troyhunt.com -http://cotweet.com -http://htl.li -http://forosdz.info -http://retweetrank.com -http://www.englishdaily626.com -http://alwaysdata.com -http://www.myeasytv.com -http://junauza.com -http://comefromchina.com -http://www.jaiku.com -http://referer.us -http://blog.gowalla.com -http://www.newsblur.com -http://williamlong.jaiku.com -http://load.to -http://www.sinoquebec.com -http://www.rsf.org -http://666kb.com -http://getsmartlinks.com -http://www.bullogs.com -http://rsf.org -http://twimbow.com -http://zeutch.com -http://bbs.mychat.to -http://en.rsf.org -http://www.soundofhope.org -http://www.chinadigitaltimes.net -http://twitoaster.com -http://www.f3.proxymice.com -http://chinadigitaltimes.net -http://gtricks.com -http://anonymizer.com -http://dadi360.com -http://www.letscorp.net -http://www.wforum.com -http://www.dadi360.com -http://tvunetworks.com -http://mypopescu.com -http://littlebigdetails.com -http://sitebro.tw -http://echofon.com -http://wforum.com -http://tradus.in -http://nexton-net.jp -http://xys.org -http://guitarlist.net -http://www.anonymizer.com -http://www.palacemoon.com -http://zhengjian.org -http://www.made-in-japan.org -http://www.aolnews.com -http://www.peacehall.com -http://presentationzen.com -http://twibs.com -http://www.sitebro.tw -http://palacemoon.com -http://sharebee.com -http://www.sharebee.com -http://www.sex8.cc -http://tumblr.awflasher.com -http://www.pin6.com -http://j.mp -http://www.e-catworld.com -http://dvorak.org -http://renminbao.com -http://www.e-gold.com -http://peacehall.com -http://thechangelog.com -http://ranwen.com -http://www.bullogger.com -http://cocoawithlove.com -http://www.branch.com -http://www.pk.com -http://fsurf.com -http://www.fsurf.com -http://tgn.tv -http://overplay.net -http://claimfans.com -http://pk.com -http://ultrasurf.us -http://www.webnode.cn -http://webnode.cn -http://hkg.westkit.net -http://asianews.it -http://www.ultrasurf.us -http://www.moea.gov.tw -http://taiwan-sex.com -http://hwinfo.com -http://www.peopo.org -http://www.ironhidegames.com -http://soundofhope.org -http://atnext.com -http://bbs4.2djgame.net -http://www.iredmail.org -http://www.amnesty.org.uk -http://yuming.flnet.org -http://www.atnext.com -http://0rz.tw -http://chinahush.com -http://www.spotflux.com -http://www.privatetunnel.com -http://www.myaudiocast.com -http://www.eulam.com -http://welivetogether.com -http://bullogger.com -http://www.0rz.tw -http://democrats.org -http://www.chinahush.com -http://suprememastertv.com -http://www.democrats.org -http://www.iu45.com -http://myaudiocast.com -http://www.ezpeer.com -http://bookshelfporn.com -http://ezpeer.com -http://e-gold.com -http://pbxes.com -http://elgoog.im -http://www.elgoog.im -http://www.zideo.nl -http://lsforum.net -http://twtr2src.ogaoga.org -http://www.mingpaovan.com -http://shaunthesheep.com -http://www.laowaixiaohan.bloguje.cz -http://sex8.cc -http://blog.lightbox.com -http://holz.byethost8.com -http://jiehua.cz -http://centurys.net -http://www.centurys.net -http://cpj.org -http://nas.immigration.gov.tw -http://www.tunnelbear.com -http://www.shaunthesheep.com -http://www.oxid.it -http://www.sanmin.com.tw -http://www.gov.tw -http://freegateget.googlepages.com -http://www.appinonline.com -http://www.overplay.net -http://www.overplay.net -http://anti.anti.cnn.googlepages.com -http://myboooks.googlepages.com -http://bbs.ozchinese.com -http://www.hideipvpn.com -http://hideipvpn.com -http://cnd.org -http://www.cnd.org -http://blog.summify.com -http://mpinews.com -http://storagenewsletter.com -http://pacificpoker.com -http://www.mingpaotor.com -http://www.securitykiss.com -http://www.pacificpoker.com -http://appempire.com -http://laowaixiaohan.bloguje.cz -http://www.immigration.gov.tw -http://www.f2.chinoxy.com -http://i2p2.de -http://www.i2p2.de -http://art.twgg.org -http://chinoxy.com -http://www.hohosex.com -http://imageflea.com -http://freewallpaper4.me -http://noor.moe.sa -http://www.facebook.co -http://tornadoweb.org -http://www.your-freedom.net -http://db.tt -http://www.f1.chinoxy.com -http://hardtied.com -http://blog.aegisub.org -http://facebok.com -http://alasbarricadas.org -http://91porn.44.lt -http://www.choxe.net -http://www.facebok.com -http://www.chinoxy.com -http://blog.dayoneapp.com -http://www.jiehua.cz -http://yawningbread.wordpress.com -http://www.aceros-de-hispania.com -http://www.kony2012.com -http://e-info.org.tw -http://iu45.com -http://www.huayuworld.org -http://www.brandibelle.com -http://info.we.bs -http://bloodshed.net -http://www.revengebucks.com -http://devio.us -http://forum.my903.com -http://www.onbux.com -http://onbux.com -http://www.marxist.com -http://kurtmunger.com -http://www.freedomhouse.org -http://freedomhouse.org -http://www.dalailama.com -http://marxist.com -http://www.facebook.fr -http://goldwave.com -http://zhongguo.ca -http://yzzk.com -http://pin6.com -http://blog.pikchur.com -http://shenyunperformingarts.org -http://aniscartujo.com -http://www.privateinternetaccess.com -http://2-hand.info -http://app.heywire.com -http://privatepaste.com -http://your-freedom.net -http://paper-replika.com -http://hogbaysoftware.com -http://www.rileyguide.com -http://www.yzzk.com -http://sfileydy.com -http://wiki2.jp -http://nedproxy.com -http://www.nedproxy.com -http://dalailama.com -http://newcenturynews.com -http://www.jyzj.waqn.com -http://jyzj.waqn.com -http://www.ly.gov.tw -http://www.piratebay.org -http://www.rcinet.ca -http://wenku.com -http://www.ping.fm -http://tagwalk.com -http://singularitys.spaces.live.com -http://wangyi64.spaces.live.com -http://yezimary.spaces.live.com -http://littlefishblueprint.com -http://edoors.com -http://surfert.nl -http://isunaffairs.com -http://www.sfileydy.com -http://edmontonchina.cn -http://www.1kho.com -http://www.proxpn.com -http://proxpn.com -http://www.inmediahk.net -http://inmediahk.net -http://www.proxpn.com -http://twistar.cc -http://bbsland.com -http://www.bbsland.com -http://rileyguide.com -http://piratebay.org -http://zhllg.spaces.live.com -http://chinese.soifind.com -http://www.2-hand.info -http://china-green-party.spaces.live.com -http://hujiachina.spaces.live.com -http://program-think.spaces.live.com -http://wenyunchao.spaces.live.com -http://huajiadi.spaces.live.com -http://yanghengjun.spaces.live.com -http://ieemdai.spaces.live.com -http://www.wenku.com -http://siyi123123123.spaces.live.com -http://twitese.spaces.live.com -http://williamlong.spaces.live.com -http://bsd10.qnetau.com -http://www.myupload.dk -http://www.wisevid.com -http://portis21.spaces.live.com -http://zengjinyan.spaces.live.com -http://cc9007.spaces.live.com -http://flowerofhappiness.spaces.live.com -http://chenyehao.spaces.live.com -http://sublexical.spaces.live.com -http://gufeng521.spaces.live.com -http://honeonet.spaces.live.com -http://www.aiuonline.edu -http://chrome.blogspot.com -http://facebook.de -http://twip.me -http://1-apple.com.tw -http://anthonycalzadilla.com -http://wisevid.com -http://makemymood.com -http://embr.in -http://embr.in -http://hotpot.hk -http://linuxreviews.org -http://cacnw.com -http://www.tibet.com -http://www.uan.edu.mx -http://www.expatshield.com -http://www.freeproxy.ro -http://www.facbook.com -http://www.yacy.net -http://bfnn.org -http://www.tweetbackup.com -http://www.bfnn.org -http://www.porn2.com -http://www.anpopo.com -http://badassjs.com -http://waffle1999.com -http://jpopforum.net -http://www.kcna.co.jp -http://rcinet.ca -http://tsctv.net -http://twiffo.com -http://tweetbackup.com -http://twftp.org -http://etools.ncol.com -http://4bluestones.biz -http://geek-art.net -http://zlib.net -http://everyday-carry.com -http://www.sobees.com -http://www.mofa.gov.tw -http://www.freenetproject.org -http://www.bbs.tw789.net -http://www.greatfirewallofchina.org -http://koolsolutions.com -http://www.xbookcn.com -http://readingtimes.com.tw -http://www.readingtimes.com.tw -http://internationalrivers.org -http://67.220.91.23 -http://kyck.com -http://www.zqzj.net -http://tibet.com -http://www.facebook.de -http://www.livevideo.com -http://www.movements.org -http://www.witopia.net -http://www.witopia.net -http://www.president.gov.tw -http://www.ibvpn.com -http://stats.wordpress.com -http://program-think.blogspot.com -http://www.shareonall.com -http://www.sim-outhouse.com -http://ait.org.tw -http://fimotro.blogspot.com -http://www.now.com.hk -http://www.china-proxy.org -http://xbookcn.com -http://greatfirewallofchina.org -http://fileim.com -http://www.tw789.net -http://8teenporn.com -http://tweetwally.com -http://qkshare.com -http://bbs.tw789.net -http://nuexpo.com -http://www.tsemtulku.com -http://www.internationalrivers.org -http://snapbrowse.com -http://maiio.net -http://csuchen.de -http://jingpin.org -http://twbbs.net.tw -http://thefacebookproxy.com -http://bbg.gov -http://www.pelaut.go.id -http://witopia.net -http://www.maiio.net -http://igfw.net -http://youtube.com.br -http://www.chinageeks.org -http://www.boxpn.com -http://media.cottagecountrynow.ca -http://www.myca168.com -http://chinageeks.org -http://alestic.com -http://sorting-algorithms.com -http://status.twhirl.org -http://www.ginatv.com -http://www.bbg.gov -http://porn2.com -http://rocmp.org -http://mangareader.com -http://www.seeon.tv -http://photos.dailyme.com -http://coolloud.org.tw -http://sobees.com -http://farxian.com -http://tsemtulku.com -http://www.worldpress.com -http://torstatus.blutmagie.de -http://www.mnnit.ac.in -http://mirrorbooks.com -http://expatshield.com -http://qidian.ca -http://drugpolicy.org -http://www.anonasurf.com -http://pbwiki.com -http://boxitvn.blogspot.com -http://appempire1.com -http://canyu.org -http://www.googl.com -http://www.thepiratebay.com -http://twitter.jp -http://favorious.com -http://vpn-pay.itshidden.com -http://truveo.com -http://www.tsctv.net -http://kenengba.com -http://internetfreedom.org -http://naol.ca -http://www.farxian.com -http://www.fpmt.org -http://www.vpn4all.com -http://mk5000.com -http://www.phayul.com -http://phayul.com -http://ccthere.com -http://googlevideo.com -http://www.itshidden.com -http://www.wujieliulan.com -http://twiyia.com -http://www.taiwandaily.net -http://www.powerpointninja.com -http://www.twbbs.net.tw -http://www.brookemarks.com -http://www.techinterview.org -http://zonaeuropa.com -http://www.canyu.org -http://www.webrush.net -http://www.mckaywei.com -http://web2project.net -http://www.civicparty.hk -http://mademoisellerobot.com -http://tibet.net -http://xijie.wordpress.com -http://www.kenengba.com -http://x1949x.com -http://www.zonaeuropa.com -http://anonasurf.com -http://www.asiaharvest.org -http://www.naol.ca -http://wujieliulan.com -http://yx51.net -http://mobileways.de -http://www.ccthere.com -http://techlifeweb.com -http://cn.fmnnow.com -http://www.supervpn.net -http://www.googlevideo.com -http://www.expressvpn.com -http://www.autospec.co.za -http://www.truveo.com -http://areca-backup.org -http://www.vpnreactor.com -http://www.duping.net -http://zarias.com -http://www.ned.org -http://www.mk5000.com -http://mckaywei.com -http://www.boxpn.com -http://www.cryptocloud.com -http://civicparty.hk -http://kenengba.com -http://www.tibet.net -http://brandonhutchinson.com -http://www.expressvpn.com -http://duping.net -http://www.kenengba.com -http://triangulationblog.com -http://myca168.com -http://radiofreechina.wordpress.com -http://stephenmooredesign.wordpress.com -http://abc.wordpress.com -http://teflandlouise.wordpress.com -http://fromcanadatofinland.wordpress.com -http://fashiontech.wordpress.com -http://www.richardjangus.wordpress.com -http://beijingnaarhongkong.wordpress.com -http://sample.wordpress.com -http://caiting666.wordpress.com -http://anyname.wordpress.com -http://whotolduthat.wordpress.com -http://rfaunplugged.wordpress.com -http://reinesklareslicht.wordpress.com -http://afterthestroke.wordpress.com -http://skittlesinchina.wordpress.com -http://abcdefg.wordpress.com -http://zh-sg.wordpress.com -http://facebookdata.wordpress.com -http://lancasterleaflets.wordpress.com -http://markerhunter.wordpress.com -http://zhenxiang.wordpress.com -http://imsirius.wordpress.com -http://tibetnorway.wordpress.com -http://cenantua.wordpress.com -http://seotch.wordpress.com -http://djcadchina.wordpress.com -http://socialnetworkguru.wordpress.com -http://quhelan.wordpress.com -http://maoistrebelnews.wordpress.com -http://zh.wordpress.com -http://wordpress.com -http://mw2mk.wordpress.com -http://translate.googleusercontent.com -http://webcache.googleusercontent.com -http://lh4.googleusercontent.com -http://webcache.googleusercontent.com -http://winqual.microsoft.com -http://my.mail.ru -http://t.co -http://t.co -http://www.t.co -http://www.google.it -http://www.blogger.com -http://blogger.com -http://draft.blogger.com -http://xhamster.com -http://www.xhamster.com -http://imdb.com -http://www.imdb.com -http://www.fc2.com -http://fc2.com -http://toruyamada.web.fc2.com -http://kitchenremodelideas.blog138.fc2.com -http://ninionlytw.web.fc2.com -http://nijibug.blog131.fc2.com -http://www.xvideos.com -http://xvideos.com -http://paulsumiko.blog46.fc2.com -http://video.aol.com -http://aolchannels.aol.com -http://search.aol.com -http://geezericious.blogspot.in -http://evolutionguide.blogspot.in -http://hqlenglish.blogspot.in -http://ukiahcoachbrown.blogspot.in -http://www.thepiratebay.se -http://thepiratebay.se -http://thepiratebay.se -http://cart.godaddy.com -http://www.netflix.com -http://signup.netflix.com -http://www.redtube.com -http://www.netflix.com -http://netflix.com -http://redtube.com -http://3.bp.blogspot.com -http://youporn.com -http://www.youporn.com -http://dailymotion.com -http://www.dailymotion.com -http://4.bp.blogspot.com -http://2.bp.blogspot.com -http://xnxx.com -http://www.badoo.com -http://badoo.com -http://www.vimeo.com -http://tube8.com -http://vimeo.com -http://www.vimeo.com -http://www.tube8.com -http://email10.secureserver.net -http://login.secureserver.net -http://www.secureserver.net -http://default.secureserver.net -http://secureserver.net -http://ip-173-201-135-189.ip.secureserver.net -http://www.dropbox.com -http://dl-web.dropbox.com -http://d.dropbox.com -http://hk32168.com -http://www.hemispheregames.com -http://airvpn.org -http://www.gourmetgaming.co.uk -http://askakorean.blogspot.com -http://08charterbbs.blogspot.com -http://asiaharvest.org -http://www.ultravpn.fr -http://www.ivacy.com -http://tjholowaychuk.com -http://mychinamyhome.com -http://mixx.com -http://www.mixx.com -http://open.com.hk -http://danwei.org -http://www.hecaitou.net -http://evhead.com -http://christianstudy.com -http://ultravpn.fr -http://forum.mymaji.com -http://www.pekingduck.org -http://www.puretna.com -http://www.nogomi.com -http://hecaitou.net -http://dphk.org -http://www.hk32168.com -http://www.smarterdailydeal.com -http://taiwandaily.net -http://www.medical-explorer.com -http://yatsen.gov.tw -http://tweetymail.com -http://tvider.com -http://njuice.com -http://www.5pic.com -http://furl.net -http://dit-inc.us -http://www.inxian.com -http://www.youtube.de -http://airvpn.org -http://www.twitstat.com -http://twitstat.com -http://taiwanyes.ning.com -http://www.gpass1.com -http://www.snaptu.com -http://snaptu.com -http://tweetree.com -http://www.open.com.hk -http://gpass1.com -http://www.vpnaccounts.com -http://www.koornk.com -http://koornk.com -http://www.mingpaony.com -http://www.mychinamyhome.com -http://mash.to -http://www.proxeasy.com -http://chevronwp7.com -http://www.danwei.org -http://chinaaffairs.org -http://www.64tianwang.com -http://www.toodoc.com -http://tunnel.shellmix.com -http://savetibet.org -http://zh.greatfire.org -http://64tianwang.com -http://www.allinfa.com -http://www.cmule.com -http://tweetboard.com -http://vpnreactor.com -http://www.twitturly.com -http://chaos.e-spacy.com -http://www.my-private-network.co.uk -http://www.my-private-network.co.uk -http://twitturly.com -http://www.lastwordonnothing.com -http://www.swissvpn.net -http://iphonix.fr -http://zh.greatfire.org -http://sharecool.org -http://my-private-network.co.uk -http://www.greatfire.org -http://www.greatfire.org -http://www.online.hk -http://www.freexinwen.com -http://alkasir.com -http://www.chinagfw.org -http://www.swingasia.com -http://openleaks.org -http://www.openleaks.org -http://salvation.org.hk -http://blog.jtbworld.com -http://deibert.citizenlab.org -http://tsunamivpn.com -http://www.xxx -http://test.greatfire.org -http://allaboutalpha.com -http://facebook.it -http://uwants.net -http://www.superfreevpn.com -http://www.salvation.org.hk -http://pekingduck.org -http://dropbox.com -http://dl.dropbox.com -http://zh.uncyclopedia.wikia.com -http://downloads.sourceforge.net -http://ultrasurf.en.softonic.com -http://xwpm.sourceforge.net -http://sourceforge.net -http://sourceforge.net -http://audacity.sourceforge.net -http://cn.uncyclopedia.wikia.com -http://foxnews.com -http://secure.wikimedia.org -http://www.justbeenpaid.com -http://secure.wikimedia.org -http://skype.com -http://ui.skype.com -http://fr.myspace.com -http://www.fbcdn.net -http://www.myspace.com -http://fbcdn.net -http://www.skype.com -http://viewmorepics.myspace.com -http://blogs.myspace.com -http://www.blogs.myspace.com -http://myspace.com -http://justbeenpaid.com -http://adultfriendfinder.com -http://www.adultfriendfinder.com -http://slideshare.net -http://hootsuite.com -http://www.slideshare.net -http://www.files.wordpress.com -http://scribd.com -http://www.hootsuite.com -http://allonlinux.free.fr -http://www.scribd.com -http://kineox.free.fr -http://smile-pow13.nicovideo.jp -http://cam4.com -http://files.wordpress.com -http://www.nicovideo.jp -http://de.cam4.com -http://nicovideo.jp -http://www.cam4.com -http://google.com.ua -http://imageshack.us -http://live.nicovideo.jp -http://isohunt.com -http://www.isohunt.com -http://www.imageshack.us -http://www.xing.com -http://xing.com -http://www.hardsextube.com -http://www.mashable.com -http://mashable.com -http://adserverplus.com -http://hardsextube.com -http://rutracker.org -http://evolutionguide.blogspot.co.uk -http://www.cthulhupodcast.blogspot.co.uk -http://1234.weebly.com -http://blackheathbabysitting.weebly.com -http://easychinese.weebly.com -http://transferstudent.weebly.com -http://drtuber.com -http://www.drtuber.com -http://zimbio.com -http://transferstudentcn.weebly.com -http://www.zimbio.com -http://hulu.com -http://www.hulu.com -http://bet365.com -http://www.bet365.com -http://google.cz -http://www.battle.net -http://www.gutefrage.net -http://gutefrage.net -http://www.twitpic.com -http://wretch.cc -http://www.dayaarmongol.ning.com -http://www.wretch.cc -http://dayaarmongol.ning.com -http://twitpic.com -http://bloomberg.com -http://www.longlivehhdl.ning.com -http://www.linkbucks.com -http://www.bloomberg.com -http://tmz.com -http://linkbucks.com -http://www.multiply.com -http://miles6k.multiply.com -http://multiply.com -http://chinese.engadget.com -http://taiwanblogs.blogspot.it -http://www.spankwire.com -http://goodreads.com -http://www.goodreads.com -http://spankwire.com -http://blog.bitly.com -http://www.bitly.com -http://bitly.com -http://www.newegg.com -http://888.com -http://www.888.com -http://feeds.feedburner.com -http://123rf.com -http://nba.com -http://www.123rf.com -http://store.nokia.com -http://pornhublive.com -http://www.pornhublive.com -http://www.nba.com -http://www.macys.com -http://macys.com -http://www.pixnet.net -http://blekko.com -http://ustream.tv -http://www.ustream.tv -http://porntube.com -http://www.imagevenue.com -http://imagevenue.com -http://metacafe.com -http://www.blogimg.jp -http://www.metacafe.com -http://pixnet.net -http://freakshare.com -http://www.freakshare.com -http://posterous.com -http://www.pornerbros.com -http://alarabiya.net -http://www.alarabiya.net -http://www.justin.tv -http://www.geocities.jp -http://partypoker.com -http://justin.tv -http://www.partypoker.com -http://hidemyass.com -http://7.hidemyass.com -http://1.hidemyass.com -http://www.businessweek.com -http://businessweek.com -http://paper.li -http://arielefraimashbel.blogspot.com.ar -http://blogspot.com.ar -http://urbandictionary.com -http://vpn.hidemyass.com -http://pornerbros.com -http://blog.udn.com -http://www.udn.com -http://alnaddy.com -http://udn.com -http://www.yfrog.com -http://www.hidemyass.com -http://6.hidemyass.com -http://goo.gl -http://a.yfrog.com -http://kriz-cpec.posterous.com -http://nimingdepengyou.posterous.com -http://writer.zoho.com -http://arielefraimashbel.blogspot.com.au -http://biocouture.posterous.com -http://www.paper.li -http://www.posterous.com -http://mbc.net -http://eyny.com -http://www.empflix.com -http://www.eyny.com -http://yfrog.com -http://nfl.com -http://city.udn.com -http://video.udn.com -http://forum.udn.com -http://blog.disqus.com -http://networkedblogs.com -http://geezericious.blogspot.com.au -http://newsabeta.blogspot.com.au -http://freeones.com -http://blogspot.com.au -http://www.88db.com -http://www.delta.com -http://yobt.com -http://www.freeones.com -http://franklin-chow.blogspot.com.au -http://hqlenglish.blogspot.com.au -http://www.xvideoslive.com -http://lci.tf1.fr -http://extremetube.com -http://meetcheap.com -http://pornoxo.com -http://www.foursquare.com -http://foursquare.com -http://fapdu.com -http://www.lenovo.com -http://www.wn.com -http://forum.pchome.com.tw -http://mypaper.pchome.com.tw -http://blackhatteam.com -http://wn.com -http://www.demonoid.me -http://demonoid.me -http://www.networkedblogs.com -http://www.mobile01.com -http://mobile01.com -http://dtiblog.com -http://www.dtiblog.com -http://www.pchome.com.tw -http://blog.pchome.com.tw -http://news.pchome.com.tw -http://fandango.com -http://pchome.com.tw -http://n.yam.com -http://asg.to -http://diigo.com -http://mymedia.yam.com -http://screencast.com -http://news.yam.com -http://na.leagueoflegends.com -http://break.com -http://blog.yam.com -http://times.hinet.net -http://livestream.com -http://www.livestream.com -http://xa.yimg.com -http://autotrader.com -http://comcast.com -http://www.break.com -http://real.com -http://www.diigo.com -http://www.betfair.com -http://betfair.com -http://change.org -http://www.change.org -http://www.tunein.com -http://tunein.com -http://g.e-hentai.org -http://chn.chosun.com -http://www.asg.to -http://technorati.com -http://balagana.net -http://www.panoramio.com -http://google.com.om -http://appledaily.com.tw -http://tigerdirect.com -http://wwe.com -http://shopping.com -http://friendfeed.com -http://blog.dribbble.com -http://www.yousendit.com -http://google.co.ke -http://www.tigerdirect.com -http://forum.chinatimes.com -http://www.tweetmeme.com -http://yebhi.com -http://www.technorati.com -http://tweetmeme.com -http://blog.chinatimes.com -http://wikimapia.org -http://rk.com -http://tubewolf.com -http://6park.com -http://blog.nownews.com -http://news.chinatimes.com -http://camera.chinatimes.com -http://forum.nownews.com -http://plurk.com -http://cafepress.com -http://boardreader.com -http://torrenthound.com -http://tala.ir -http://yobt.tv -http://www.cafepress.com -http://www.2shared.com -http://realitykings.com -http://2shared.com -http://www.6park.com -http://ime.nu -http://wnd.com -http://gamestlbb.com -http://wenxuecity.com -http://hk.apple.nextmedia.com -http://www.plurk.com -http://jamnews.ir -http://bdnews24.com -http://forever21.com -http://elkhabar.com -http://www.wenxuecity.com -http://caribbeancom.com -http://togetter.com -http://rutube.ru -http://luntan.zaobao.com -http://tuite.googlecode.com -http://spartoo.com -http://tradus.com -http://sttlbb.com -http://naughtyamerica.com -http://segundamano.es -http://expressen.se -http://www.lusongsong.com -http://gfwinterceptor.googlecode.com -http://ibibo.com -http://www.ibibo.com -http://old.nabble.com -http://wiley.com -http://leilao.jp -http://v8.googlecode.com -http://www.rutube.ru -http://forums.wordpress.com -http://www.efukt.com -http://gtap.googlecode.com -http://xmatch.com -http://efukt.com -http://fotomac.com.tr -http://tv.com -http://boinc.berkeley.edu -http://binaryoptions24h.com -http://index.hr -http://cliphunter.com -http://en.wordpress.com -http://ow.ly -http://blog.xuite.net -http://www.ow.ly -http://www.sex.com -http://livetv.ru -http://torrentcrazy.com -http://topsy.com -http://tiao8.info -http://pornbb.org -http://shorouknews.com -http://sex.com -http://www.topsy.com -http://voanews.com -http://cams.com -http://www.dw.de -http://dw.de -http://myvideo.de -http://sbnation.com -http://www.erepublik.com -http://bbs.sina.com -http://home.sina.com -http://rconversation.blogs.com -http://sina.com -http://dailynews.sina.com -http://tweakers.net -http://www.sina.com -http://erepublik.com -http://tokyo-porn-tube.com -http://www.myvideo.de -http://veoh.com -http://discuss.com.hk -http://support.wordpress.com -http://babosas.com -http://googleapis.com -http://www.ck101.com -http://www.discuss.com.hk -http://www.cloudflare.com -http://uniqlo.com -http://twipple.jp -http://wiki.jqueryui.com -http://ptcbox.com -http://www.veoh.com -http://www.yuvutu.com -http://www.nrk.no -http://tokyo-tube.com -http://google.co.ug -http://eforosh.com -http://ck101.com -http://hotelurbano.com.br -http://yuvutu.com -http://anonymouse.org -http://findicons.com -http://apetube.com -http://blogspot.fi -http://gogvo.com -http://twitterfeed.com -http://www.mobypicture.com -http://aukro.cz -http://www.books.com.tw -http://nubiles.net -http://zh.pokerstrategy.com -http://cn.calameo.com -http://ebookbrowse.com -http://bsnl.co.in -http://avaaz.org -http://www.plus28.com -http://elespectador.com -http://nrk.no -http://www.avaaz.org -http://plus28.com -http://mobypicture.com -http://www.twitterfeed.com -http://homemademoviez.com -http://v-mire.com -http://www.geocities.co.jp -http://problogger.net -http://jagran.com -http://willhaben.at -http://tv2.dk -http://maps.ovi.com -http://reclameaqui.com.br -http://store.ovi.com -http://whatismyip.com -http://snagajob.com -http://books.com.tw -http://bwin.com -http://izlesene.com -http://corrieredellosport.it -http://pizzahut.com -http://xxxymovies.com -http://freshxxxtube.com -http://www.aftenposten.no -http://mouthshut.com -http://yammer.com -http://www5.kinghost.com -http://twitlonger.com -http://www.twitlonger.com -http://forum.tvb.com -http://pornsake.com -http://homemadesextube.com -http://wayfair.com -http://cuantocabron.com -http://devshed.com -http://the-sun.on.cc -http://gossiplankanews.com -http://blogspot.co.at -http://thehun.com -http://rotoworld.com -http://video.foxbusiness.com -http://petsmart.com -http://badjojo.com -http://www.hgtv.com -http://allanalpass.com -http://gammae.com -http://frankfinn.in -http://mysitecost.ru -http://hgtv.com -http://aaa.com -http://twitvid.com -http://kinghost.com -http://ht.ly -http://crackle.com -http://malaysiakini.com -http://punyu.com -http://ssa.gov -http://lists.debian.org -http://bloglovin.com -http://nps.gov -http://www.bebo.com -http://www.nps.gov -http://sportsdirect.com -http://china.dwnews.com -http://bravenet.com -http://perfectworld.com -http://bebo.com -http://dickssportinggoods.com -http://femjoy.com -http://rubias19.com -http://bloomingdales.com -http://mmaaxx.com -http://www.twitvid.com -http://indianpornvid.com -http://potterybarn.com -http://animecrazy.net -http://web.ics.purdue.edu -http://www.blog.de -http://webshots.com -http://urbanoutfitters.com -http://crackberry.com -http://rg.ru -http://kajabi.com -http://calottery.com -http://www.webshots.com -http://freeservers.com -http://commentshk.blogspot.hk -http://wqw2010.blogspot.hk -http://www.fakku.net -http://camfrog.com -http://royalty7.com -http://blog.de -http://blog.roodo.com -http://www.urbanoutfitters.com -http://wearehairy.com -http://fakku.net -http://libertytimes.com.tw -http://microworkers.com -http://supersport.com -http://www.hotspotshield.com -http://dwnews.com -http://teamliquid.net -http://wickedfire.com -http://tunisia-sat.com -http://miami.com -http://blog.sina.com.tw -http://bangbros1.com -http://pandora.tv -http://videosurf.com -http://vh1.com -http://pbase.com -http://akiba-online.com -http://politiken.dk -http://www.pandora.tv -http://news.sina.com.tw -http://startpage.com -http://occ.com.mx -http://www.camfrog.com -http://hotspotshield.com -http://www.libertytimes.com.tw -http://mitbbs.com -http://www.animecrazy.net -http://cosmiq.de -http://desidime.com -http://sacbee.com -http://mforos.com -http://www.twittercounter.com -http://www.mitbbs.com -http://cimbclicks.com.my -http://www.mefeedia.com -http://www.pbworks.com -http://magazines.sina.com.tw -http://kariyer.net -http://boots.com -http://pbworks.com -http://websiteforever.com -http://www.chinese.rfi.fr -http://wikiwiki.jp -http://idol-blog.com -http://amakings.com -http://amino.dk -http://causes.com -http://coches.net -http://rfi.fr -http://www.rfi.fr -http://imgsrc.ru -http://galeon.com -http://ripoffreport.com -http://freelotto.com -http://blip.tv -http://www.blip.tv -http://kobobooks.com -http://derekhsu.homeip.net -http://bookofsex.com -http://italia-film.com -http://pimpandhost.com -http://arte.tv -http://mefeedia.com -http://partycasino.com -http://governmentjobs.com -http://stltoday.com -http://yad2.co.il -http://www.freelotto.com -http://pagewash.com -http://guitarcenter.com -http://sexfinder.com -http://elifting.com -http://joomlaforum.ru -http://maomaotlbb.com -http://korben.info -http://self.com -http://hdporn.in -http://tinychat.com -http://wattpad.com -http://www.wattpad.com -http://boyfriendtv.com -http://via.me -http://bodisparking.com -http://www.paltalk.com -http://paltalk.com -http://simplyrecipes.com -http://epochtimes.com -http://khan.co.kr -http://www.simplyrecipes.com -http://goldline.pro -http://www.alabout.com -http://www.tinychat.com -http://gidonline.ru -http://lexilogos.com -http://mmtlsf.com -http://www.epochtimes.com -http://activerain.com -http://www.korben.info -http://moviehuts.com -http://myfreepaysite.com -http://mobli.com -http://gsmspain.com -http://www.17wtlbb.com -http://englishtown.com -http://cn.news.cnyes.com -http://gamebase.com.tw -http://www.gamebase.com.tw -http://ddfcash.com -http://cl.ly -http://alabout.com -http://flalottery.com -http://gohappy.com.tw -http://favstar.fm -http://swtor.com -http://immi.gov.au -http://heydouga.com -http://justunfollow.com -http://getiton.com -http://radioshack.com -http://topic.youthwant.com.tw -http://anchorfree.net -http://cakephp.org -http://forum.sina.com.hk -http://goibibo.com -http://www.sugarsync.com -http://fux.com -http://onekingslane.com -http://ghacks.net -http://definebabe.com -http://blog.youthwant.com.tw -http://sport24.co.za -http://twilog.org -http://eventful.com -http://gfrevenge.com -http://onlinewatchmovies.net -http://duga.jp -http://zh.wikisource.org -http://rghost.ru -http://hkreporter.com -http://bigtits.com -http://ladbrokes.com -http://mid-day.com -http://wiki.phonegap.com -http://internetslang.com -http://gnc.com -http://vuku.ru -http://kporno.com -http://tro-ma-ktiko.blogspot.com -http://tadawul.com.sa -http://mail-archive.com -http://www.mail-archive.com -http://xtc-modified.org -http://aviny.com -http://mybloggertricks.com -http://listentoyoutube.com -http://articlesnatch.com -http://aukro.ua -http://burningcamel.com -http://freepornofreeporn.com -http://www.zinio.com -http://weblagu.com -http://www.sugarsync.com -http://tvp.pl -http://1stcngame.com -http://xtlbb.com -http://powersearchingwithgoogle.com -http://www.hegre-art.com -http://imagecherry.com -http://assembla.com -http://career.co.kr -http://zinio.com -http://sytes.net -http://www.ptt.cc -http://theme.wordpress.com -http://www.rfa.org -http://lindaikeji.blogspot.com -http://wehkamp.nl -http://sugarsync.com -http://www.hkreporter.com -http://key.com -http://williams-sonoma.com -http://club.backchina.com -http://fastlink.sytes.net -http://www.sytes.net -http://hotfrog.com -http://www.hrnm.navy.mil -http://ch.shvoong.com -http://browsershots.org -http://www.backchina.com -http://codecademy.com -http://share.youthwant.com.tw -http://puritanas.com -http://blinkx.com -http://www.cam.ac.uk -http://www.tweetdeck.com -http://iran-eng.com -http://green-flowers.net -http://oszone.net -http://gq.com -http://lastfm.es -http://websurf.ru -http://tmi.me -http://dziennik.pl -http://pantheranetwork.com -http://newsbusters.org -http://xuk.ru -http://www.megavideo.com -http://allstate.com -http://megavideo.com -http://allfacebook.com -http://www.thisav.com -http://uberhumor.com -http://dallasnews.com -http://dotup.org -http://www.blinkx.com -http://evolutionguide.blogspot.co.nz -http://kjendis.no -http://backchina.com -http://hegre-art.com diff --git a/ooni/assets/redirects.yaml b/ooni/assets/redirects.yaml deleted file mode 100644 index 43582c4..0000000 --- a/ooni/assets/redirects.yaml +++ /dev/null @@ -1,17 +0,0 @@ -o2: - name: O2 Networks - patterns: - - {type: re, value: '(https://bango.net/).*'} - - {type: re, value: 'http://wap.o2.co.uk/(18plusaccess).*'} -orange: - name: Orange Telecom - patterns: {type: eq, value: 'http://orangeworld.co.uk/r/avblocked/'} -three: - name: Three - patterns: {type: re, value: '(http://mobile.three.co.uk/pc/Live/pcreator/live/).*'} -tmobile: - name: T-Mobile - patterns: {type: eq, value: 'http://www.t-mobile.co.uk/common/system_error_pages/outage_wnw.html'} -vodafone: - name: Vodafone UK - patterns: {type: eq, value: 'http://online.vodafone.co.uk/en_GB/assets/static/contentcontrol/unbranded/r…'} diff --git a/ooni/assets/tcpscan.txt b/ooni/assets/tcpscan.txt deleted file mode 100644 index 4204d77..0000000 --- a/ooni/assets/tcpscan.txt +++ /dev/null @@ -1,2 +0,0 @@ -127.0.0.1:9050 -127.0.0.1:6666 diff --git a/ooni/reporter.py b/ooni/reporter.py index b145e87..51a0166 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -8,6 +8,15 @@ from datetime import datetime from twisted.python.util import OrderedDict, untilConcludes from twisted.trial import unittest, reporter, runner +try: + from scapy.all import packet +except: + class FooClass: + pass + packet = object + packet.Packet = FooClass + + pyunit = __import__('unittest') class OReporter(pyunit.TestResult): @@ -44,8 +53,8 @@ class OReporter(pyunit.TestResult): self._write('\n') def writeYamlLine(self, line): - self._write(yaml.dump([line])) - + to_write = yaml.dump([line]) + self._write(to_write) class ReporterFactory(OReporter): @@ -122,7 +131,11 @@ class OONIReporter(OReporter): self._tests[idx] = {} self._tests[idx]['testStarted'] = self._getTime() - self._tests[idx]['input'] = test.input + if isinstance(test.input, packet.Packet): + test_input = repr(test.input) + else: + test_input = test.input + self._tests[idx]['input'] = test_input self._tests[idx]['idx'] = idx self._tests[idx]['name'] = test.name #self._tests[idx]['test'] = test diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index e9c55f1..5593027 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -143,4 +143,56 @@ class Log(): except: raise StopIteration +import copy_reg + +def patched_reduce_ex(self, proto): + """ + This is a hack to overcome a bug in one of pythons core functions. It is + located inside of copy_reg and is called _reduce_ex. + + Some background on the issue can be found here: + http://stackoverflow.com/questions/569754/how-to-tell-for-which-object-attr… + http://stackoverflow.com/questions/2049849/why-cant-i-pickle-this-object + + There was also an open bug on the pyyaml trac repo, but it got closed because + they could not reproduce. + http://pyyaml.org/ticket/190 + + It turned out to be easier to patch the python core library than to monkey + patch yaml. + + XXX see if there is a better way. sigh... + """ + _HEAPTYPE = 1<<9 + assert proto < 2 + for base in self.__class__.__mro__: + if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE: + break + else: + base = object # not really reachable + if base is object: + state = None + elif base is int: + state = None + else: + if base is self.__class__: + raise TypeError, "can't pickle %s objects" % base.__name__ + state = base(self) + args = (self.__class__, base, state) + try: + getstate = self.__getstate__ + except AttributeError: + if getattr(self, "__slots__", None): + raise TypeError("a class that defines __slots__ without " + "defining __getstate__ cannot be pickled") + try: + dict = self.__dict__ + except AttributeError: + dict = None + else: + dict = getstate() + if dict: + return copy_reg._reconstructor, args, dict + else: + return copy_reg._reconstructor, args

1 0