October 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Merge branch 'master' of https://git.torproject.org/ooni-probe
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit edfaff0c38c8ad5f8772ba23721413f0878a0d35 Merge: 16c3cfe 21505f8 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 12:15:42 2012 +0000 Merge branch 'master' of https://git.torproject.org/ooni-probe Conflicts: .gitignore ooni/assets/bridgetests.txt ooni/plugins/new_bridget.py .gitignore | 4 + docs/bridget.md | 102 ++++++ ooni/assets/bridgetests.txt | 11 - ooni/lib/Makefile | 36 ++ ooni/lib/__init__.py | 43 +--- ooni/lib/txscapy | 1 - ooni/lib/txscapy.py | 348 ++++++++++++++++++++ ooni/lib/txtorcon | 1 - ooni/lib/txtraceroute | 1 - ooni/lib/txtraceroute.py | 752 +++++++++++++++++++++++++++++++++++++++++++ ooni/ooniprobe.py | 10 +- ooni/plugins/blocking.py | 2 +- ooni/plugins/dnstamper.py | 6 +- ooni/plugins/httphost.py | 2 +- ooni/plugins/new_bridget.py | 239 ++++++++++---- ooni/plugins/tcpconnect.py | 5 +- ooni/plugoo/nodes.py | 6 +- ooni/plugoo/reports.py | 2 +- ooni/plugoo/tests.py | 3 +- ooni/utils/log.py | 1 - 20 files changed, 1437 insertions(+), 138 deletions(-) diff --cc .gitignore index 6b1b9ce,686e319..0458e37 --- a/.gitignore +++ b/.gitignore @@@ -8,4 -8,4 +8,8 @@@ proxy-lists/italy-http-ips.tx private/* /ooni/plugins/dropin.cache oonib/oonibackend.conf -ooni/lib/txtorcon ++<<<<<<< HEAD +ooni/assets/* ++======= ++ooni/lib/txtorcon ++>>>>>>> 21505f84aef5d60c7e138590a1a40e3df773d680 diff --cc ooni/ooniprobe.py index 86f4e2d,539c2ac..ec529d6 mode 100644,100755..100644 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py diff --cc ooni/plugins/new_bridget.py index 673cacc,0000000..c85caeb mode 100644,000000..100644 --- a/ooni/plugins/new_bridget.py +++ b/ooni/plugins/new_bridget.py @@@ -1,421 -1,0 +1,528 @@@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +# +# +-----------+ +# | BRIDGET | +# | +----------------------------------------------+ +# +--------| Use a slave Tor process to test making a Tor | +# | connection to a list of bridges or relays. | +# +----------------------------------------------+ +# +# :authors: Arturo Filasto, Isis Lovecruft +# :licence: see included LICENSE +# :version: 0.1.0-alpha + +from __future__ import with_statement - from zope.interface import implements ++from os import getcwd ++from os.path import isfile ++from os.path import join as pj +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import defer, error, reactor ++from zope.interface import implements + +import random +import sys + - try: - from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher - except: - print "BridgeT requires txtorcon: https://github.com/meejah/txtorcon.git" - print "Your copy of OONI should have it included, if you're seeing this" - print "message, please file a bug report." - log.msg ("Bridget: Unable to import from ooni.lib.txtorcon") - +from ooni.utils import log +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset + + - class BridgetArgs(usage.Options): ++def portCheck(number): ++ number = int(number) ++ if number not in range(1024, 65535): ++ raise ValueError("Port out of range") + - def portCheck(number): - number = int(number) - if number not in range(1024, 65535): - raise ValueError("Port out of range") - portCheck.coerceDoc = "Ports must be between 1024 and 65535" ++portCheckAllowed = "must be between 1024 and 65535." ++sockCheck, ctrlCheck = portCheck, portCheck ++sockCheck.coerceDoc = "Port to use for Tor's SocksPort, " + portCheckAllowed ++ctrlCheck.coerceDoc = "Port to use for Tor's ControlPort, " + portCheckAllowed + ++ ++class BridgetArgs(usage.Options): + optParameters = [ + ['bridges', 'b', None, - 'List of bridges to scan <IP>:<ORport>'], ++ 'File listing bridge IP:ORPorts to test'], + ['relays', 'f', None, - 'List of relays to scan <IP>'], - ['socks', 's', 9049, portCheck, - 'Tor SocksPort to use'], - ['control', 'c', 9052, portCheck, - 'Tor ControlPort to use'], - ['tor-path', 'p', None, ++ 'File listing relay IPs to test'], ++ ['socks', 's', 9049, None, portCheck], ++ ['control', 'c', 9052, None, portCheck], ++ ['torpath', 'p', None, + 'Path to the Tor binary to use'], - ['data-dir', 'd', None, ++ ['datadir', 'd', None, + 'Tor DataDirectory to use'], + ['transport', 't', None, + 'Tor ClientTransportPlugin'], + ['resume', 'r', 0, + 'Resume at this index']] - optFlags = [['random', 'x', 'Randomize control and socks ports']] ++ optFlags = [ ++ ['random', 'x', 'Use random ControlPort and SocksPort']] + + def postOptions(self): - ## We can't test pluggable transports without bridges + if self['transport'] and not self['bridges']: + e = "Pluggable transport requires the bridges option" + raise usage.UsageError, e - ## We can't use random and static port simultaneously + if self['socks'] and self['control']: + if self['random']: + e = "Unable to use random and specific ports simultaneously" + raise usageError, e + +class CustomCircuit(CircuitListenerMixin): + implements(IStreamAttacher) + + from txtorcon.interface import IRouterContainer, ICircuitContainer + + def __init__(self, state): + self.state = state + self.waiting_circuits = [] + + def waiting_on(self, circuit): + for (circid, d) in self.waiting_circuits: + if circuit.id == circid: + return true + return False + + def circuit_extend(self, circuit, router): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + if self.waiting_on(circuit): + log.msg("Circuit %d (%s)" % (circuit.id, router.id_hex)) + + def circuit_built(self, circuit): + "ICircuitListener" + if circuit.purpose != 'GENERAL': + return + + log.msg("Circuit %s built ..." % circuit.id) + log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) + + for (circid, d) in self.waiting_circuits: + if circid == circuit.id: + self.waiting_circuits.remove(circid, d) + d.callback(circuit) + + def circuit_failed(self, circuit, reason): + if self.waiting_on(circuit): + log.msg("A circuit we requested %s failed for reason %s" % + (circuit.id, reason)) + circid, d = None, None + for x in self.waiting_circuits: + if x[0] == circuit.id: + circid, d, stream_cc = x + if d is None: + raise Exception("Expected to find circuit.") + + self.waiting_circuits.remove((circid, d)) + log.msg("Trying to build a circuit for %s" % circid) + self.request_circuit_build(d) + + def check_circuit_route(self, circuit, router): + if router in circuit.path: + #router.update() ## XXX can i use without args? no. + TorInfo.dump(self) + + def request_circuit_build(self, deferred): + entries = self.state.entry_guards.value() + relays = self.state.routers.values() + + log.msg("We have these nodes listed as entry guards:") + log.msg("%s" % entries) + log.msg("We have these nodes listed as relays:") + log.msg("%s" % relays) + + path = [random.choice(entries), + random.choice(relays), + random.choice(relays)] + + log.msg("Requesting a circuit: %s" + % '-->'.join(map(lambda x: x.location.countrycode, path))) + + class AppendWaiting: + def __init__(self, attacher, deferred): + self.attacher = attacher + self.d = deferred + + def __call__(self, circuit): + """ + Return from build_circuit is a Circuit, however, we want to + wait until it is built before we can issue an attach on it and + callback to the Deferred we issue here. + """ + log.msg("Circuit %s is in progress ..." % circuit.id) + self.attacher.waiting_circuits.append((circuit.id, self.d)) + + return self.state.build_circuit(path).addCallback(AppendWaiting(self, deferred_to_callback)).addErrback(log.err) + +class BridgetAsset(Asset): + """ - Class for parsing bridge assets so that they can be commented out. ++ Class for parsing bridget Assets ignoring commented out lines. + """ + def __init__(self, file=None): + self = Asset.__init__(self, file) + + def parse_line(self, line): + if line.startswith('#'): + return + else: + return line.replace('\n','') + +class BridgetTest(OONITest): + """ + XXX fill me in + + :ivar config: + An :class:`ooni.lib.txtorcon.TorConfig` instance. + :ivar relay_list: + A list of all provided relays to test. We have to do this because + txtorcon.TorState().entry_guards won't build a custom circuit if the + first hop isn't in the torrc's EntryNodes. + :ivar bridge_list: + A list of all provided bridges to test. + :ivar socks_port: + Integer for Tor's SocksPort. + :ivar control_port: + Integer for Tor's ControlPort. + :ivar plug_transport: + String defining the Tor's ClientTransportPlugin, for testing + a bridge's pluggable transport functionality. + :ivar tor_binary: + Path to the Tor binary to use, e.g. \'/usr/sbin/tor\' + """ + implements(IPlugin, ITest) + + shortName = "newbridget" + description = "bridget" + requirements = None + options = BridgetArgs + blocking = False + + def initialize(self): + """ + Extra initialization steps. We only want one child Tor process + running, so we need to deal with the creation of TorConfig() only + once, before the experiment runs. + """ + self.socks_port = 9049 + self.control_port = 9052 + self.tor_binary = '/usr/sbin/tor' + self.data_directory = None + + if self.local_options: - try: - from ooni.lib.txtorcon import TorConfig - except: - e = "Could not import TorConfig class from txtorcon!" - raise ImportError, e ++ options = self.local_options + - options = self.local_options - self.config = TorConfig() - - ## Don't run the experiment if we don't have anything to test + if not options['bridges'] and not options['relays']: + self.suicide = True ++ return ++ ++ try: ++ from ooni.lib.txtorcon import TorConfig ++ except ImportError: ++ log.msg ("Bridget: Unable to import from ooni.lib.txtorcon") ++ wd, tx = getcwd(), 'lib/txtorcon/torconfig.py' ++ chk = pj(wd,tx) if wd.endswith('ooni') else pj(wd,'ooni/'+tx) ++ try: ++ assert isfile(chk) ++ except: ++ log.msg("Error: Some OONI libraries are missing!") ++ log.msg("Please go to /ooni/lib/ and do \"make all\"") ++ ++ self.config = TorConfig() + + if options['bridges']: + self.config.UseBridges = 1 + + if options['relays']: + ## Stupid hack for testing only relays: + ## Tor doesn't use EntryNodes when UseBridges is enabled, but + ## config.state.entry_guards needs to include the first hop to + ## build a custom circuit. + self.config.EntryNodes = ','.join(relay_list) + + if options['socks']: + self.socks_port = options['socks'] + + if options['control']: + self.control_port = options['control'] + + if options['random']: + log.msg("Using randomized ControlPort and SocksPort ...") + self.socks_port = random.randint(1024, 2**16) + self.control_port = random.randint(1024, 2**16) + - if options['tor-path']: - self.tor_binary = options['tor-path'] ++ if options['torpath']: ++ self.tor_binary = options['torpath'] + - if options['data-dir']: - self.config.DataDirectory = options['data-dir'] ++ if options['datadir']: ++ self.config.DataDirectory = options['datadir'] + + if options['transport']: + ## ClientTransportPlugin transport socks4|socks5 IP:PORT + ## ClientTransportPlugin transport exec path-to-binary [options] + if not options['bridges']: + e = "You must use the bridge option to test a transport." + raise usage.UsageError("%s" % e) + + log.msg("Using pluggable transport ...") + ## XXX fixme there's got to be a better way to check the exec + assert type(options['transport']) is str + self.config.ClientTransportPlugin = options['transport'] + + self.config.SocksPort = self.socks_port + self.config.ControlPort = self.control_port + self.config.save() + + def load_assets(self): + """ + Load bridges and/or relays from files given in user options. Bridges + should be given in the form IP:ORport. We don't want to load these as + assets, because it's inefficient to start a Tor process for each one. + """ + assets = {} + self.bridge_list = [] + self.relay_list = [] + + ## XXX fix me + ## we should probably find a more memory nice way to load addresses, + ## in case the files are really large + if self.local_options: - if self.local_options['bridges']: - log.msg("Loading bridge information from %s ..." - % self.local_options['bridges']) - with open(self.local_options['bridges']) as bridge_file: - for line in bridge_file.readlines(): ++ def make_asset_list(opt, lst): ++ log.msg("Loading information from %s ..." % opt) ++ with open(opt) as opt_file: ++ for line in opt_file.readlines(): + if line.startswith('#'): + continue + else: - self.bridge_list.append(line.replace('\n','')) - assets.update({'bridges': self.bridge_list}) ++ lst.append(line.replace('\n','')) + ++ if self.local_options['bridges']: ++ make_asset_list(self.local_options['bridges'], ++ self.bridge_list) ++ assets.update({'bridges': self.bridge_list}) + if self.local_options['relays']: - log.msg("Loading relay information from %s ..." - % self.local_options['relays']) - with open(options['relays']) as relay_file: - for line in relay_file.readlines(): - if line.startswith('#'): - continue - else: - self.relay_list.append(line.replace('\n','')) ++ make_asset_list(self.local_options['relays'], ++ self.relay_list) + assets.update({'relays': self.relay_list}) + return assets + + def experiment(self, args): + """ + XXX fill me in + + :param args: + The :class:`ooni.plugoo.asset.Asset <Asset>` line currently being + used. + :meth launch_tor: + Returns a Deferred which callbacks with a + :class:`ooni.lib.txtorcon.torproto.TorProcessProtocol + <TorProcessProtocol>` connected to the fully-bootstrapped Tor; + this has a :class:`ooni.lib.txtorcon.torcontol.TorControlProtocol + <TorControlProtocol>` instance as .protocol. + """ - from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState - from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor ++ try: ++ from ooni.lib.txtorcon import CircuitListenerMixin, IStreamAttacher ++ from ooni.lib.txtorcon import TorProtocolFactory, TorConfig, TorState ++ from ooni.lib.txtorcon import DEFAULT_VALUE, launch_tor ++ except ImportError: ++ log.msg("Error: Unable to import from ooni.lib.txtorcon") ++ wd, tx = getcwd(), 'lib/txtorcon/torconfig.py' ++ chk = pj(wd,tx) if wd.endswith('ooni') else pj(wd,'ooni/'+tx) ++ try: ++ assert isfile(chk) ++ except: ++ log.msg("Error: Some OONI libraries are missing!") ++ log.msg(" Please go to /ooni/lib/ and do \"make all\"") ++ return sys.exit() + + def bootstrap(ctrl): + """ + Launch a Tor process with the TorConfig instance returned from + initialize(). + """ + conf = TorConfig(ctrl) + conf.post_bootstrap.addCallback(setup_done).addErrback(setup_fail) + log.msg("Tor process connected, bootstrapping ...") + + def reconf_controller(conf, bridge): + ## if bridges and relays, use one bridge then build a circuit + ## from three relays + conf.Bridge = bridge + ## XXX do we need a SIGHUP to restart? + + ## XXX see txtorcon.TorControlProtocol.add_event_listener we + ## may not need full CustomCircuit class + + ## if bridges only, try one bridge at a time, but don't build + ## circuits, just return + ## if relays only, build circuits from relays + + def reconf_fail(args): + log.msg("Reconfiguring Tor config with args %s failed" % args) + reactor.stop() + + def setup_fail(args): + log.msg("Setup Failed.") + report.update({'failed': args}) + reactor.stop() + + def setup_done(proto): + log.msg("Setup Complete: %s" % proto) + state = TorState(proto.tor_protocol) + state.post_bootstrap.addCallback(state_complete).addErrback(setup_fail) + report.update({'success': args}) + + def updates(prog, tag, summary): + log.msg("%d%%: %s" % (prog, summary)) + + if len(args) == 0: - log.msg("Bridget needs lists of bridges and/or relays to test!") ++ log.msg("Bridget can't run without bridges or relays to test!") + log.msg("Exiting ...") - d = sys.exit() - return d - ++ return sys.exit() + else: ++ ++ class CustomCircuit(CircuitListenerMixin): ++ implements(IStreamAttacher) ++ ++ from txtorcon.interface import IRouterContainer ++ from txtorcon.interface import ICircuitContainer ++ ++ def __init__(self, state): ++ self.state = state ++ self.waiting_circuits = [] ++ ++ def waiting_on(self, circuit): ++ for (circid, d) in self.waiting_circuits: ++ if circuit.id == circid: ++ return true ++ return False ++ ++ def circuit_extend(self, circuit, router): ++ "ICircuitListener" ++ if circuit.purpose != 'GENERAL': ++ return ++ if self.waiting_on(circuit): ++ log.msg("Circuit %d (%s)" ++ % (circuit.id, router.id_hex)) ++ ++ def circuit_built(self, circuit): ++ "ICircuitListener" ++ if circuit.purpose != 'GENERAL': ++ return ++ log.msg("Circuit %s built ..." % circuit.id) ++ log.msg("Full path of %s: %s" % (circuit.id, circuit.path)) ++ for (circid, d) in self.waiting_circuits: ++ if circid == circuit.id: ++ self.waiting_circuits.remove(circid, d) ++ d.callback(circuit) ++ ++ def circuit_failed(self, circuit, reason): ++ if self.waiting_on(circuit): ++ log.msg("A circuit we requested %s failed for reason %s" ++ % (circuit.id, reason)) ++ circid, d = None, None ++ for x in self.waiting_circuits: ++ if x[0] == circuit.id: ++ circid, d, stream_cc = x ++ if d is None: ++ raise Exception("Expected to find circuit.") ++ ++ self.waiting_circuits.remove((circid, d)) ++ log.msg("Trying to build a circuit for %s" % circid) ++ self.request_circuit_build(d) ++ ++ def check_circuit_route(self, circuit, router): ++ if router in circuit.path: ++ #router.update() ## XXX can i use without args? no. ++ TorInfo.dump(self) ++ ++ def request_circuit_build(self, deferred): ++ entries = self.state.entry_guards.value() ++ relays = self.state.routers.values() ++ log.msg("We have these nodes listed as entry guards:") ++ log.msg("%s" % entries) ++ log.msg("We have these nodes listed as relays:") ++ log.msg("%s" % relays) ++ path = [random.choice(entries), ++ random.choice(relays), ++ random.choice(relays)] ++ log.msg("Requesting a circuit: %s" ++ % '-->'.join(map(lambda x: x.location.countrycode, ++ path))) ++ ++ class AppendWaiting: ++ def __init__(self, attacher, deferred): ++ self.attacher = attacher ++ self.d = deferred ++ ++ def __call__(self, circuit): ++ """ ++ Return from build_circuit is a Circuit, however, ++ we want to wait until it is built before we can ++ issue an attach on it and callback to the Deferred ++ we issue here. ++ """ ++ log.msg("Circuit %s is in progress ..." % circuit.id) ++ self.attacher.waiting_circuits.append((circuit.id, ++ self.d)) ++ ++ fin = self.state.build_circuit(path) ++ fin.addCallback(AppendWaiting(self, deferred_to_callback)) ++ fin.addErrback(log.err) ++ return fin ++ ++ + if len(self.bridge_list) >= 1: + for bridge in self.bridge_list: + try: - print "BRIDGE IS %s" % bridge ++ log.msg("Current Bridge: %s" % bridge) + reconf_controller(self.config, bridge) + except: + reconf_fail(bridge) + + log.msg("Bridget: initiating test ... ") + log.msg("Using the following as our torrc:\n%s" + % self.config.create_torrc()) + report = {'tor_config': self.config.config} + log.msg("Starting Tor ...") + + ## :return: a Deferred which callbacks with a TorProcessProtocol + ## connected to the fully-bootstrapped Tor; this has a + ## txtorcon.TorControlProtocol instance as .protocol. + d = launch_tor(self.config, + reactor, + progress_updates=updates, + tor_binary=self.tor_binary) + d.addCallback(bootstrap, self.config) + d.addErrback(setup_fail) + ## now build circuits + + #print "Tor process ID: %s" % d.transport.pid + return d + +## So that getPlugins() can register the Test: +bridget = BridgetTest(None, None, None) + +## ISIS' NOTES +## ----------- +## self.config.save() only needs to be called if Tor is already running. +## - ## need to add transport type to torrc Bridge line: - ## Bridge <transport> IP:ORPort <fingerprint> ++## to test gid, uid, and euid: ++## with open('/proc/self/state') as uidfile: ++## print uidfile.read(1000) +## +## TODO: +## o add option for any kwarg=arg self.config setting +## o cleanup documentation ++## x add DataDirectory option +## o check if bridges are public relays +## o take bridge_desc file as input, also be able to give same +## format as output - ## o change the stupid name ++## o Add assychronous timout for deferred, so that we don't wait ++## forever for bridges that don't work. ++## o Add mechanism for testing through another host +## +## FIX: - ## data directory is not found, or permissions aren't right ++## o DataDirectory is not found, or permissions aren't right ++## o Bridge line needs generation of transport properties ++## Bridge <transport> IP:ORPort <fingerprint>

1 0

[ooni-probe/master] Add documentation on the OONI API
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit d01d501efe93a9672d37b9aa0dae61e0d3065106 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 14:41:21 2012 +0000 Add documentation on the OONI API --- docs/source/api/ooni.rst | 60 ++++++++++++++++++++++++++++++++++++ docs/source/api/ooni.templates.rst | 11 ++++++ docs/source/api/ooni.utils.rst | 51 ++++++++++++++++++++++++++++++ docs/source/index.rst | 1 + 4 files changed, 123 insertions(+), 0 deletions(-) diff --git a/docs/source/api/ooni.rst b/docs/source/api/ooni.rst new file mode 100644 index 0000000..bcdd402 --- /dev/null +++ b/docs/source/api/ooni.rst @@ -0,0 +1,60 @@ +ooni Package +============ + +:mod:`ooni` Package +------------------- + +.. automodule:: ooni.__init__ + :members: + :undoc-members: + :show-inheritance: + +:mod:`input` Module +------------------- + +.. automodule:: ooni.input + :members: + :undoc-members: + :show-inheritance: + +:mod:`nettest` Module +--------------------- + +.. automodule:: ooni.nettest + :members: + :undoc-members: + :show-inheritance: + +:mod:`oonicli` Module +--------------------- + +.. automodule:: ooni.oonicli + :members: + :undoc-members: + :show-inheritance: + +:mod:`reporter` Module +---------------------- + +.. automodule:: ooni.reporter + :members: + :undoc-members: + :show-inheritance: + +:mod:`runner` Module +-------------------- + +.. automodule:: ooni.runner + :members: + :undoc-members: + :show-inheritance: + +Subpackages +----------- + +.. toctree:: + + ooni.lib + ooni.templates + ooni.utils + diff --git a/docs/source/api/ooni.templates.rst b/docs/source/api/ooni.templates.rst new file mode 100644 index 0000000..662da1d --- /dev/null +++ b/docs/source/api/ooni.templates.rst @@ -0,0 +1,11 @@ +templates Package +================= + +:mod:`http` Module +------------------ + +.. automodule:: ooni.templates.http + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/source/api/ooni.utils.rst b/docs/source/api/ooni.utils.rst new file mode 100644 index 0000000..5361b43 --- /dev/null +++ b/docs/source/api/ooni.utils.rst @@ -0,0 +1,51 @@ +utils Package +============= + +:mod:`utils` Package +-------------------- + +.. automodule:: ooni.utils + :members: + :undoc-members: + :show-inheritance: + +:mod:`config` Module +-------------------- + +.. automodule:: ooni.utils.config + :members: + :undoc-members: + :show-inheritance: + +:mod:`date` Module +------------------ + +.. automodule:: ooni.utils.date + :members: + :undoc-members: + :show-inheritance: + +:mod:`log` Module +----------------- + +.. automodule:: ooni.utils.log + :members: + :undoc-members: + :show-inheritance: + +:mod:`logo` Module +------------------ + +.. automodule:: ooni.utils.logo + :members: + :undoc-members: + :show-inheritance: + +:mod:`net` Module +----------------- + +.. automodule:: ooni.utils.net + :members: + :undoc-members: + :show-inheritance: + diff --git a/docs/source/index.rst b/docs/source/index.rst index 34ca87c..e744f7f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -46,5 +46,6 @@ Contents install tutorial writing_tests + api/*

1 0

[ooni-probe/master] Deprecate scaffolding script
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit a957cd5f469d34bfda98848bca586a0c409181af Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 15:02:32 2012 +0000 Deprecate scaffolding script --- ooni/scaffolding.py | 10 ++++++++-- 1 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ooni/scaffolding.py b/ooni/scaffolding.py index 0a832aa..dffe342 100644 --- a/ooni/scaffolding.py +++ b/ooni/scaffolding.py @@ -3,10 +3,16 @@ """ This script should be used for creating the scaffolding for a test. """ +from __future__ import print_function import os import sys from ooni.utils import log +print("!!!! This test writing strategy is now deprecated !!!") +print("visit: https://ooni.readthedocs.org/en/latest/writing_tests.html " + "for new instructions") +sys.exit(1) + test_template = """\"\"\" This is a self genrated test created by scaffolding.py. you will need to fill it up with all your necessities. @@ -61,9 +67,9 @@ test_vars['testSNlower'] = test_vars['testShortname'].lower() fname = os.path.join('plugins', test_vars['testSNlower']+'.py') if os.path.exists(fname): - print 'WARNING! File named "%s" already exists.' % fname + print('WARNING! File named "%s" already exists.' % fname) if raw_input("Do you wish to continue (y/N)? ").lower() != 'y': - print "gotcha! Dying.." + print("gotcha! Dying..") sys.exit(0) fp = open(fname, 'w')

1 0

[ooni-probe/master] Add patch for bug in yaml.
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 532e2e9853b4bdd61c951dc3073d8dc68d35f754 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 21:52:40 2012 +0000 Add patch for bug in yaml. Add assets to ignore file. --- .gitignore | 5 +- bin/ooniprobe | 11 + docs/source/conf.py | 7 +- ooni/assets/captive_portal_tests.txt | 4 - ooni/assets/example.txt | 2 - ooni/assets/greatfire.lst | 2100 ---------------------------------- ooni/assets/redirects.yaml | 17 - ooni/assets/tcpscan.txt | 2 - ooni/reporter.py | 19 +- ooni/utils/__init__.py | 52 + 10 files changed, 86 insertions(+), 2133 deletions(-) diff --git a/.gitignore b/.gitignore index 0458e37..dc569e9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +assets/* build/* package/* proxy-lists/ip-cc.txt @@ -8,8 +9,4 @@ proxy-lists/italy-http-ips.txt private/* /ooni/plugins/dropin.cache oonib/oonibackend.conf -<<<<<<< HEAD -ooni/assets/* -======= ooni/lib/txtorcon ->>>>>>> 21505f84aef5d60c7e138590a1a40e3df773d680 diff --git a/bin/ooniprobe b/bin/ooniprobe index 1f0c26d..f25be7a 100755 --- a/bin/ooniprobe +++ b/bin/ooniprobe @@ -1,8 +1,19 @@ #!/usr/bin/python2 +# -*- encoding: utf-8 -*- +# # startup script based on twisted trial # See http://twistedmatrix.com/ +# +# :authors: Arturo Filastò +# :licence: see LICENSE + + import os, sys +# This is a hack to overcome a bug in python +from ooni.utils import patched_reduce_ex +copy_reg._reduce_ex = patched_reduce_ex + sys.path[:] = map(os.path.abspath, sys.path) sys.path.insert(0, os.path.abspath(os.getcwd())) diff --git a/docs/source/conf.py b/docs/source/conf.py index e066613..1f61842 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -94,8 +94,13 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'haiku' +html_theme_options = {'textcolor': "#222222", + 'headingcolor': "#555", + 'linkcolor': "#4183C4", + 'visitedlinkcolor': "#888", + 'hoverlinkcolor': "#888"} # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. diff --git a/ooni/assets/captive_portal_tests.txt b/ooni/assets/captive_portal_tests.txt deleted file mode 100644 index 7007411..0000000 --- a/ooni/assets/captive_portal_tests.txt +++ /dev/null @@ -1,4 +0,0 @@ - -http://ooni.nu, Open Observatory of Network Interference, 200 -https://w2.eff.org/Censorship/Internet_censorship_bills/barlow_0296.declaration, let us now take our leave of them, 200 -https://www.torproject.org, Defend yourself against network surveillance and traffic analysis, 200 diff --git a/ooni/assets/example.txt b/ooni/assets/example.txt deleted file mode 100644 index 01ffcdf..0000000 --- a/ooni/assets/example.txt +++ /dev/null @@ -1,2 +0,0 @@ -127.0.0.1:9050 -127.0.0.1:9051 diff --git a/ooni/assets/greatfire.lst b/ooni/assets/greatfire.lst deleted file mode 100644 index 30c2a96..0000000 --- a/ooni/assets/greatfire.lst +++ /dev/null @@ -1,2100 +0,0 @@ -http://www.new.facebook.com -http://www.facebook.com -http://facebook.com -http://m.facebook.com -http://sites.google.com -http://login.facebook.com -http://spreadsheets.google.com -http://apps.facebook.com -http://facebook.com -http://picasaweb.google.com -http://www.facebook.com -http://api.facebook.com -http://zh-cn.facebook.com -http://plus.google.com -http://encrypted.google.com -http://plusone.google.com -http://plusone.google.com -http://google.com -http://sites.google.com -http://feedproxy.google.com -http://www.docs.google.com -http://profiles.google.com -http://m.facebook.com -http://encrypted.google.com -http://code.google.com -http://spreadsheets.google.com -http://drive.google.com -http://www.picasaweb.google.com -http://groups.google.com -http://spreadsheets4.google.com -http://fr-fr.facebook.com -http://talkgadget.google.com -http://docs.google.com -http://www.plus.google.com -http://plus.google.com -http://tw.youtube.com -http://youtube.com -http://www.youtube.com -http://www.youtube.com -http://hk.youtube.com -http://youtube.com -http://accounts.youtube.com -http://hk.rd.yahoo.com -http://meme.yahoo.com -http://tw.yahoo.com -http://hk.news.yahoo.com -http://tw.myblog.yahoo.com -http://tw.news.yahoo.com -http://video.yahoo.com -http://pulse.yahoo.com -http://es.video.yahoo.com -http://hk.yahoo.com -http://en.mobile.wikipedia.org -http://zh.m.wikipedia.org -http://twitter.com -http://eccomi.qzone.qq.com -http://twitter.com -http://www.twitter.com -http://www.twitter.com -http://dev.twitter.com -http://search.twitter.com -http://esbensportfolio.blogspot.com -http://tf2spreadshet.blogspot.com -http://downmerng.blogspot.com -http://historyanarchy.blogspot.com -http://charlesmok.blogspot.com -http://www.stratosphera-nrb.blogspot.com -http://veryverylong.blogspot.com -http://indeterminacy.blogspot.com -http://spritthevirus.blogspot.com -http://hric-newsbrief.blogspot.com -http://basketbawful.blogspot.com -http://www.fademi.blogspot.com -http://wuxiao0715.blogspot.com -http://dramatisinterruptus.blogspot.com -http://martinsmithdesign.blogspot.com -http://www.geezericious.blogspot.com -http://hungryballerina.blogspot.com -http://www.malaysianchiks.blogspot.com -http://transranyf.blogspot.com -http://spaceman100-thesuperblog.blogspot.com -http://unreadbits.blogspot.com -http://hqlenglish.blogspot.com -http://newsabeta.blogspot.com -http://www.google.blogspot.com -http://www.tibetnewsonline.blogspot.com -http://sopravivalitalia.blogspot.com -http://perfectpeacellc.blogspot.com -http://www.taikotime.blogspot.com -http://www.arielefraimashbel.blogspot.com -http://ukiahcoachbrown.blogspot.com -http://www.transranyf.blogspot.com -http://unreadnits.blogspot.com -http://adornoskies.blogspot.com -http://freedomforegyptians.blogspot.com -http://www.nickdael.blogspot.com -http://postcardsfromrichard.blogspot.com -http://pennedscripts.blogspot.com -http://helloloverr.blogspot.com -http://american-sleepingdragon.blogspot.com -http://postcardsfromrichard.blogspot.com -http://pennedscripts.blogspot.com -http://spritthevirus.blogspot.com -http://froggyflix.blogspot.com -http://www.varadhjain.blogspot.com -http://ukiahcoachbrown.blogspot.com -http://charlesmok.blogspot.com -http://amdoamonk.blogspot.com -http://historyanarchy.blogspot.com -http://ajithfanswing.blogspot.com -http://freedomforegyptians.blogspot.com -http://www.spaceman100-thesuperblog.blogspot.com -http://helloloverr.blogspot.com -http://amongstsaintsandsailors.blogspot.com -http://hric-newsbrief.blogspot.com -http://wuxiao0715.blogspot.com -http://taiwanmatters.blogspot.com -http://www.tibetnewsonline.blogspot.com -http://www.happening24X7.blogspot.com -http://dramatisinterruptus.blogspot.com -http://perfectpeacellc.blogspot.com -http://google.blogspot.com -http://cittadeglignomi.blogspot.com -http://arielefraimashbel.blogspot.com -http://evolutionguide.blogspot.com -http://unreadbits.blogspot.com -http://sopravivalitalia.blogspot.com -http://www.raceboy24.blogspot.com -http://pfge-pfge.blogspot.com -http://www.evolutionguide.blogspot.com -http://unreadnits.blogspot.com -http://www.cajuncardon.blogspot.com -http://owpandprojectdirect.blogspot.com -http://www.blogspot.com -http://codeslut.blogspot.com -http://www.panfletoliberal.blogspot.com -http://michaelturton.blogspot.com -http://raceboy24.blogspot.com -http://generatorblog.blogspot.com -http://gustavoguillenzulia.blogspot.com -http://manyouyinli.blogspot.com -http://sruv-pitbulls.blogspot.com -http://www.stratosphera-nrb.blogspot.com -http://www.ukiahcoachbrown.blogspot.com -http://www.ncbcr.blogspot.com -http://blogspot.com -http://trangridiculous.blogspot.com -http://le-rubrique.blogspot.com -http://roxy-angel.blogspot.com -http://bettermyths.blogspot.com -http://inexplorados.blogspot.com -http://thoughtthemes.blogspot.com -http://blakelychinatrip.blogspot.com -http://zhangxin1989.blogspot.com -http://kangdiy.blogspot.com -http://henacg.blogspot.com -http://oldvirginiablog.blogspot.com -http://imperturbable-music.blogspot.com -http://ang-newswire.blogspot.com -http://yuhaotech.blogspot.com -http://tokyobikepolo.blogspot.com -http://virtuasapiens.blogspot.com -http://hldpn.blogspot.com -http://blogs.yahoo.co.jp -http://picasaweb.google.com.hk -http://www.wordpress.com -http://abcdefg.wordpress.com -http://astimegoesbyinchina.wordpress.com -http://www.wordpress.com -http://thechinesedream.wordpress.com -http://xinchina.wordpress.com -http://theworldofguava.wordpress.com -http://wuxiao0715.wordpress.com -http://wordpress.com -http://www.annishan2011.wordpress.com -http://publichealthinchina.wordpress.com -http://teflandlouise.wordpress.com -http://uabinchina2011.wordpress.com -http://nfolamp.wordpress.com -http://smurfswillbefree.wordpress.com -http://storiesfromtheceiling.wordpress.com -http://culturalexposure.wordpress.com -http://bullrunnings.wordpress.com -http://radiofreechina.wordpress.com -http://stephenmooredesign.wordpress.com -http://fromcanadatofinland.wordpress.com -http://www.richardjangus.wordpress.com -http://underthejacaranda.wordpress.com -http://www.quhelan.wordpress.com -http://sinologica.wordpress.com -http://wuxiao0715.wordpress.com -http://indonesiastoriesdotcom.wordpress.com -http://martianchronicles.wordpress.com -http://livefromgaza.wordpress.com -http://foureverfamily.wordpress.com -http://abc.wordpress.com -http://fashiontech.wordpress.com -http://sample.wordpress.com -http://zh.wordpress.com -http://lemasabachthani.wordpress.com -http://hmetro.com.my -http://www.blogcatalog.com -http://cbs.ntu.edu.tw -http://thisav.com -http://blogcatalog.com -http://tapuz.co.il -http://www.identi.ca -http://shifeike.blog125.fc2blog.net -http://boysfood.com -http://kaufmich.com -http://elnuevoherald.com -http://sinembargo.mx -http://8591.com.tw -http://rapgenius.com -http://ticketmonster.co.kr -http://hotfrog.jp -http://oddschecker.com -http://danawa.com -http://www.stickam.com -http://identi.ca -http://vtc.vn -http://blog.co.uk -http://nsgalleries.com -http://tweetdeck.com -http://crutchfield.com -http://hotpornshow.com -http://jagranjosh.com -http://reviewcentre.com -http://stickam.com -http://tek-tips.com -http://www.hotpornshow.com -http://jumptags.com -http://wizaz.pl -http://hotel-urbano.com -http://destructoid.com -http://maturetube.com -http://5i01.com -http://he.net -http://bbs-tw.com -http://ray-ban.com -http://viddler.com -http://ccbs.ntu.edu.tw -http://cnsnews.com -http://transunion.com -http://lulzimg.com -http://foum.hkgolden.com -http://newspaperdirect.com -http://creaders.net -http://orgasmof.ru -http://potterybarnkids.com -http://hack371.com -http://virustotal.com -http://forum.hkgolden.com -http://mymodernmet.com -http://kmtlbb.com -http://siol.net -http://meiyingie.com -http://socialsecurity.gov -http://network54.com -http://gotovim-doma.ru -http://cheaperthandirt.com -http://hotgoo.com -http://www.worldjournal.com -http://lamoda.ru -http://information.com -http://savevid.com -http://tsekouratoi.gr -http://www.viddler.com -http://wowtv.co.kr -http://movies.com -http://www.dateinasia.com -http://pornaccess.com -http://www.creaders.net -http://hkgolden.com -http://forum5.hkgolden.com -http://24smile.org -http://www.he.net -http://savings.com -http://dreamamateurs.com -http://carscoop.blogspot.com -http://girlfriendvideos.com -http://worldjournal.com -http://anneahira.com -http://ftvgirls.com -http://artnudegalleries.com -http://mobawaba.com -http://gadaadmongol.com -http://busuk.org -http://lawyers.com -http://www.network54.com -http://kabeldeutschland.de -http://privet.ru -http://sitetrail.com -http://www.mingpao.com -http://hyundai.com -http://woopie.jp -http://jetpack.wordpress.com -http://westelm.com -http://oursogo.com -http://mysoftplace.com -http://shangqu.info -http://newplaynow.com -http://jaidefinichon.com -http://tommy.com -http://trendhunter.com -http://kansascity.com -http://tomshw.it -http://pacopacomama.com -http://newplaysite.com -http://meemessa.com -http://mlbtraderumors.com -http://el5antuario.org -http://shemale-porn-galls.com -http://bergfiles.com -http://ratedporntube.com -http://chanel.com -http://chirpme.com -http://zomobo.net -http://blogdeizquierda.com -http://sarkarinaukriblog.com -http://php.su -http://subscribe.wordpress.com -http://signup.wordpress.com -http://rfa.org -http://upworthy.com -http://klmty.net -http://cfun.ir -http://momovideo.net -http://videobox.com -http://ggsoku.com -http://shoppingdaisy.com -http://sannas.jp -http://pornstarnetwork.com -http://flashkhor.com -http://parsonline.net -http://huyandex.com -http://akhbarelyom.org.eg -http://cplusplus.com -http://passion.com -http://sbt.com.br -http://remax.com -http://korabia.com -http://levante-emv.com -http://mingpao.com -http://sexpixbox.com -http://jjgirls.com -http://anyap.info -http://gossip9.com -http://googlewebmastercentral.blogspot.com -http://kiwi.kz -http://xxxaporn.com -http://youtu.be -http://tubegals.com -http://preloved.co.uk -http://basicscan.com -http://kerala.gov.in -http://fin24.com -http://alrakoba.net -http://travelandleisure.com -http://hugoboss.com -http://google.com.pa -http://skybet.com -http://tah.jp -http://info.51.ca -http://blog.51.ca -http://saveyoutube.com -http://manatelugumovies.net -http://img.ly -http://www.1pondo.tv -http://gpil.org -http://fora.pl -http://homemade-voyeur.com -http://thisdaylive.com -http://fotopages.com -http://ekra2.com -http://youversion.com -http://1pondo.tv -http://funpic.de -http://eg.ru -http://jobkorea.co.kr -http://Www.youversion.com -http://google.is -http://wwitv.com -http://yidio.com -http://lifehacker.ru -http://lunarpages.com -http://bbs.51.ca -http://theoutnet.com -http://www.livestation.com -http://www.img.ly -http://yyv.co -http://chrispederick.com -http://sharenxs.com -http://www.wikileaks.org -http://wikileaks.org -http://dxlive.com -http://amateursexy.net -http://www.baidu.jp -http://vertvonlinegratis.org -http://taiwannation.50webs.com -http://photographyblog.com -http://fscstore.com -http://whoismind.com -http://www.jjgirls.com -http://dancingbear.com -http://ipicture.ru -http://livestation.com -http://articlerich.com -http://jossandmain.com -http://www.yidio.com -http://jp-sex.com -http://insidefacebook.com -http://bet.hkjc.com -http://foodandwine.com -http://wordle.net -http://scrippsnetworks.com -http://baidu.jp -http://bustnow.com -http://lefora.com -http://store.valvesoftware.com -http://kpo-faq.blogspot.com -http://www.sex141.com -http://wazzub.com -http://kingstone.com.tw -http://hostingbulk.com -http://smile-adult.com -http://zend2.com -http://hotfrog.in -http://errotica-archives.com -http://server8.kproxy.com -http://drupal.ru -http://www.kingstone.com.tw -http://twilightsex.com -http://bt.no -http://qualityhealth.com -http://ping.fm -http://cwer.ru -http://www.spike.com -http://secretchina.com -http://shanbara.jp -http://www.ugo.com -http://www.xxx.com -http://server15.kproxy.com -http://dotnetperls.com -http://sexandsubmission.com -http://hautetfort.com -http://www.sexandsubmission.com -http://tiavastube.com -http://git-scm.com -http://pagemodo.com -http://stadt-bremerhaven.de -http://blog.rockmelt.com -http://gagnons-argent.com -http://wealth2xtreme.com -http://forum.baby-kingdom.com -http://googleblog.blogspot.com -http://fleshbot.com -http://videobam.com -http://21sextury.com -http://sakuralive.com -http://asredas.com -http://www.18qt.com -http://expekt.com -http://hkbisi.com -http://www.nexttv.com.tw -http://porlaputa.com -http://jkforum.net -http://henhaoji.com -http://anobii.com -http://www.torproject.org -http://torproject.org -http://proxy.org -http://sexinsex.net -http://trendsmap.com -http://ntdtv.com -http://www.sexinsex.net -http://nexttv.com.tw -http://www.popyard.com -http://www.ntdtv.com -http://media.ntdtv.com -http://www.pussy.org -http://79.133.201.84.nyud.net -http://temasektimes.wordpress.com -http://chinanews.blog.ntdtv.com -http://tunnel.shellmix.com.nyud.net -http://powerapple.com -http://popyard.com -http://globalvoicesonline.org -http://codereview.chromium.org -http://feedbox.com -http://rthk.org.hk -http://blog.iphone-dev.org -http://www.saveyoutube.com -http://www.rthk.org.hk -http://zh.globalvoicesonline.org -http://www.globalvoicesonline.org -http://www.taipei.gov.tw -http://www.footytube.com -http://twttr.com -http://fr.globalvoicesonline.org -http://yorkbbs.ca -http://bbs.powerapple.com -http://www.steves-digicams.com -http://pleated-jeans.com -http://forum.yorkbbs.ca -http://lalulalu.com -http://www.lalulalu.com -http://page2rss.com -http://virb.com -http://iask.ca -http://www.fileflyer.com -http://tw.jiepang.com -http://youtube-nocookie.com -http://blog.sparrowmailapp.com -http://www.islamhouse.com -http://hellotxt.com -http://toronto.iask.ca -http://blog.path.com -http://www.youtube-nocookie.com -http://www.rthk.hk -http://www.boxun.com -http://www.wuala.com -http://dumblittleman.com -http://s20.youmaker.com -http://westca.com -http://www.huaren.us -http://mysinablog.com -http://rthk.hk -http://hk.jiepang.com -http://content.wuala.com -http://www.sevenload.com -http://news.boxun.com -http://moviefap.com -http://translate.twttr.com -http://www.uswebproxy.com -http://youmaker.com -http://boxun.com -http://sevenload.com -http://htmldog.com -http://en.favotter.net -http://aboluowang.com -http://www.cna.com.tw -http://cna.com.tw -http://www.aboluowang.com -http://www.cn.sevenload.com -http://im.tv -http://translate.twttr.com -http://www.t66y.com -http://doujin-moe.us -http://wuala.com -http://www.minghui.org -http://www.perfspot.com -http://www.singtao.com -http://t66y.com -http://aisex.com -http://webupd8.org -http://www.voy.com -http://www.megaporn.com -http://wahas.com -http://www.fxdd.com -http://transferbigfiles.com -http://megaporn.com -http://enmasse.com -http://www.openvpn.net -http://minghui.org -http://www.91porn.com -http://tor.eff.org -http://www.881903.com -http://tor.updatestar.com -http://www.wahas.com -http://67.220.91.30 -http://zh.wikinews.org -http://www.hrw.org -http://kwongwah.com.my -http://strongvpn.com -http://welide.com -http://www.strongvpn.com -http://twitgoo.com -http://marines.mil -http://vanpeople.com -http://scriptlance.com -http://www.somee.com -http://strongvpn.com -http://bbs.skykiwi.com -http://www.thepiratebay.org -http://hrw.org -http://www.theepochtimes.com -http://ntq.comlu.com -http://www.want-daily.com -http://www.jensonusa.com -http://faststone.org -http://thepiratebay.org -http://www.pcdvd.com.tw -http://thepiratebay.org -http://lesscss.org -http://uncyclopedia.info -http://www.fooooo.com -http://movieworldsite.com -http://db.stoptui.site88.net -http://www.e-sim.org -http://popyard.org -http://www.gopetition.com -http://gopetition.com -http://amnesty.org -http://www.amnesty.org -http://www.es.amnesty.org -http://kompozer.net -http://www.popyard.org -http://e-sim.org -http://aculo.us -http://m.tweete.net -http://www.1688.com.au -http://www.dopehood.com -http://rocksoccer.co.cc -http://www01.ktzhk.com -http://disp.cc -http://playpcesor.blogspot.com -http://sports.now.com -http://movie.now.com -http://tweete.net -http://www.oiktv.com -http://www.searchtruth.com -http://bbs.9gal.com -http://kok2.no-blog.jp -http://www.viewsonic.com -http://pornstarclub.com -http://nonstoptrafficformula.com -http://oiktv.com -http://apigee.com -http://www.doh.gov.tw -http://pioneer-worker.forums-free.com -http://fooooo.com -http://www.fring.com -http://www.fb.me -http://www.pornstarclub.com -http://fring.com -http://news.now.com -http://www.now.com -http://now.com -http://blog.kowalczyk.info -http://eic-av.com -http://www.coolaler.com -http://coolaler.com -http://www.onthesamehost.com -http://picasa-readme.blogspot.com -http://twbbs.org -http://twitiq.com -http://0to255.com -http://www.dongtaiwang.com -http://www.moi.gov.tw -http://dongtaiwang.com -http://myfreshnet.com -http://skimtube.com -http://www.c-spanvideo.org -http://www.surfagain.com -http://en.bitcoin.it -http://webconference.eadgear.com -http://forum.slime.com.tw -http://www.trtc.com.tw -http://marketingsurvivalskills.com -http://www.myfreshnet.com -http://nakido.com -http://wiki.oauth.net -http://www.cyberghostvpn.com -http://www.kenyanlist.com -http://www.voachinese.com -http://www.nakido.com -http://turbohide.com -http://www.sopcast.com -http://wearn.com -http://www.wearn.com -http://cn2.streetvoice.com -http://graphis.ne.jp -http://www.porn365.com -http://sopcast.com -http://cyberghostvpn.com -http://cyberghostvpn.com -http://ahalife.com -http://list.ly -http://danlambaovn.blogspot.com -http://getcloudapp.com -http://news.singtao.ca -http://web.pts.org.tw -http://www.fulltiltpoker.com -http://c-spanvideo.org -http://developers.box.net -http://dl.box.net -http://www.apigee.com -http://www.falundafa.org -http://news.pts.org.tw -http://twittergadget.com -http://android-x86.org -http://www.kaiyuan.de -http://www.juzp.net -http://ww.kaiyuan.de -http://Www.ruero.com -http://dropshots.com -http://www.android-x86.org -http://mingpaonews.com -http://talk.news.pts.org.tw -http://www.opendemocracy.net -http://www.lanecrawford.com -http://www.insidesearch.blogspot.com -http://stitcher.com -http://www.dw-world.de -http://dw-world.de -http://www.dotsub.com -http://scrapetorrent.com -http://www.anchorfree.com -http://www.twittergadget.com -http://tagxedo.com -http://anchorfree.com -http://kaiyuan.de -http://twitthat.com -http://newtalk.tw -http://www.xys.org -http://www.comefromchina.com -http://alwaysdata.net -http://np.cpami.gov.tw -http://bbs.kimy.com.tw -http://dabr.co.uk -http://www.wantchinatimes.com -http://www.moztw.org -http://www.dabr.co.uk -http://taiwankiss.com -http://www.cotweet.com -http://twbbs.tw -http://city9x.com -http://gutteruncensored.com -http://www.troyhunt.com -http://cotweet.com -http://htl.li -http://forosdz.info -http://retweetrank.com -http://www.englishdaily626.com -http://alwaysdata.com -http://www.myeasytv.com -http://junauza.com -http://comefromchina.com -http://www.jaiku.com -http://referer.us -http://blog.gowalla.com -http://www.newsblur.com -http://williamlong.jaiku.com -http://load.to -http://www.sinoquebec.com -http://www.rsf.org -http://666kb.com -http://getsmartlinks.com -http://www.bullogs.com -http://rsf.org -http://twimbow.com -http://zeutch.com -http://bbs.mychat.to -http://en.rsf.org -http://www.soundofhope.org -http://www.chinadigitaltimes.net -http://twitoaster.com -http://www.f3.proxymice.com -http://chinadigitaltimes.net -http://gtricks.com -http://anonymizer.com -http://dadi360.com -http://www.letscorp.net -http://www.wforum.com -http://www.dadi360.com -http://tvunetworks.com -http://mypopescu.com -http://littlebigdetails.com -http://sitebro.tw -http://echofon.com -http://wforum.com -http://tradus.in -http://nexton-net.jp -http://xys.org -http://guitarlist.net -http://www.anonymizer.com -http://www.palacemoon.com -http://zhengjian.org -http://www.made-in-japan.org -http://www.aolnews.com -http://www.peacehall.com -http://presentationzen.com -http://twibs.com -http://www.sitebro.tw -http://palacemoon.com -http://sharebee.com -http://www.sharebee.com -http://www.sex8.cc -http://tumblr.awflasher.com -http://www.pin6.com -http://j.mp -http://www.e-catworld.com -http://dvorak.org -http://renminbao.com -http://www.e-gold.com -http://peacehall.com -http://thechangelog.com -http://ranwen.com -http://www.bullogger.com -http://cocoawithlove.com -http://www.branch.com -http://www.pk.com -http://fsurf.com -http://www.fsurf.com -http://tgn.tv -http://overplay.net -http://claimfans.com -http://pk.com -http://ultrasurf.us -http://www.webnode.cn -http://webnode.cn -http://hkg.westkit.net -http://asianews.it -http://www.ultrasurf.us -http://www.moea.gov.tw -http://taiwan-sex.com -http://hwinfo.com -http://www.peopo.org -http://www.ironhidegames.com -http://soundofhope.org -http://atnext.com -http://bbs4.2djgame.net -http://www.iredmail.org -http://www.amnesty.org.uk -http://yuming.flnet.org -http://www.atnext.com -http://0rz.tw -http://chinahush.com -http://www.spotflux.com -http://www.privatetunnel.com -http://www.myaudiocast.com -http://www.eulam.com -http://welivetogether.com -http://bullogger.com -http://www.0rz.tw -http://democrats.org -http://www.chinahush.com -http://suprememastertv.com -http://www.democrats.org -http://www.iu45.com -http://myaudiocast.com -http://www.ezpeer.com -http://bookshelfporn.com -http://ezpeer.com -http://e-gold.com -http://pbxes.com -http://elgoog.im -http://www.elgoog.im -http://www.zideo.nl -http://lsforum.net -http://twtr2src.ogaoga.org -http://www.mingpaovan.com -http://shaunthesheep.com -http://www.laowaixiaohan.bloguje.cz -http://sex8.cc -http://blog.lightbox.com -http://holz.byethost8.com -http://jiehua.cz -http://centurys.net -http://www.centurys.net -http://cpj.org -http://nas.immigration.gov.tw -http://www.tunnelbear.com -http://www.shaunthesheep.com -http://www.oxid.it -http://www.sanmin.com.tw -http://www.gov.tw -http://freegateget.googlepages.com -http://www.appinonline.com -http://www.overplay.net -http://www.overplay.net -http://anti.anti.cnn.googlepages.com -http://myboooks.googlepages.com -http://bbs.ozchinese.com -http://www.hideipvpn.com -http://hideipvpn.com -http://cnd.org -http://www.cnd.org -http://blog.summify.com -http://mpinews.com -http://storagenewsletter.com -http://pacificpoker.com -http://www.mingpaotor.com -http://www.securitykiss.com -http://www.pacificpoker.com -http://appempire.com -http://laowaixiaohan.bloguje.cz -http://www.immigration.gov.tw -http://www.f2.chinoxy.com -http://i2p2.de -http://www.i2p2.de -http://art.twgg.org -http://chinoxy.com -http://www.hohosex.com -http://imageflea.com -http://freewallpaper4.me -http://noor.moe.sa -http://www.facebook.co -http://tornadoweb.org -http://www.your-freedom.net -http://db.tt -http://www.f1.chinoxy.com -http://hardtied.com -http://blog.aegisub.org -http://facebok.com -http://alasbarricadas.org -http://91porn.44.lt -http://www.choxe.net -http://www.facebok.com -http://www.chinoxy.com -http://blog.dayoneapp.com -http://www.jiehua.cz -http://yawningbread.wordpress.com -http://www.aceros-de-hispania.com -http://www.kony2012.com -http://e-info.org.tw -http://iu45.com -http://www.huayuworld.org -http://www.brandibelle.com -http://info.we.bs -http://bloodshed.net -http://www.revengebucks.com -http://devio.us -http://forum.my903.com -http://www.onbux.com -http://onbux.com -http://www.marxist.com -http://kurtmunger.com -http://www.freedomhouse.org -http://freedomhouse.org -http://www.dalailama.com -http://marxist.com -http://www.facebook.fr -http://goldwave.com -http://zhongguo.ca -http://yzzk.com -http://pin6.com -http://blog.pikchur.com -http://shenyunperformingarts.org -http://aniscartujo.com -http://www.privateinternetaccess.com -http://2-hand.info -http://app.heywire.com -http://privatepaste.com -http://your-freedom.net -http://paper-replika.com -http://hogbaysoftware.com -http://www.rileyguide.com -http://www.yzzk.com -http://sfileydy.com -http://wiki2.jp -http://nedproxy.com -http://www.nedproxy.com -http://dalailama.com -http://newcenturynews.com -http://www.jyzj.waqn.com -http://jyzj.waqn.com -http://www.ly.gov.tw -http://www.piratebay.org -http://www.rcinet.ca -http://wenku.com -http://www.ping.fm -http://tagwalk.com -http://singularitys.spaces.live.com -http://wangyi64.spaces.live.com -http://yezimary.spaces.live.com -http://littlefishblueprint.com -http://edoors.com -http://surfert.nl -http://isunaffairs.com -http://www.sfileydy.com -http://edmontonchina.cn -http://www.1kho.com -http://www.proxpn.com -http://proxpn.com -http://www.inmediahk.net -http://inmediahk.net -http://www.proxpn.com -http://twistar.cc -http://bbsland.com -http://www.bbsland.com -http://rileyguide.com -http://piratebay.org -http://zhllg.spaces.live.com -http://chinese.soifind.com -http://www.2-hand.info -http://china-green-party.spaces.live.com -http://hujiachina.spaces.live.com -http://program-think.spaces.live.com -http://wenyunchao.spaces.live.com -http://huajiadi.spaces.live.com -http://yanghengjun.spaces.live.com -http://ieemdai.spaces.live.com -http://www.wenku.com -http://siyi123123123.spaces.live.com -http://twitese.spaces.live.com -http://williamlong.spaces.live.com -http://bsd10.qnetau.com -http://www.myupload.dk -http://www.wisevid.com -http://portis21.spaces.live.com -http://zengjinyan.spaces.live.com -http://cc9007.spaces.live.com -http://flowerofhappiness.spaces.live.com -http://chenyehao.spaces.live.com -http://sublexical.spaces.live.com -http://gufeng521.spaces.live.com -http://honeonet.spaces.live.com -http://www.aiuonline.edu -http://chrome.blogspot.com -http://facebook.de -http://twip.me -http://1-apple.com.tw -http://anthonycalzadilla.com -http://wisevid.com -http://makemymood.com -http://embr.in -http://embr.in -http://hotpot.hk -http://linuxreviews.org -http://cacnw.com -http://www.tibet.com -http://www.uan.edu.mx -http://www.expatshield.com -http://www.freeproxy.ro -http://www.facbook.com -http://www.yacy.net -http://bfnn.org -http://www.tweetbackup.com -http://www.bfnn.org -http://www.porn2.com -http://www.anpopo.com -http://badassjs.com -http://waffle1999.com -http://jpopforum.net -http://www.kcna.co.jp -http://rcinet.ca -http://tsctv.net -http://twiffo.com -http://tweetbackup.com -http://twftp.org -http://etools.ncol.com -http://4bluestones.biz -http://geek-art.net -http://zlib.net -http://everyday-carry.com -http://www.sobees.com -http://www.mofa.gov.tw -http://www.freenetproject.org -http://www.bbs.tw789.net -http://www.greatfirewallofchina.org -http://koolsolutions.com -http://www.xbookcn.com -http://readingtimes.com.tw -http://www.readingtimes.com.tw -http://internationalrivers.org -http://67.220.91.23 -http://kyck.com -http://www.zqzj.net -http://tibet.com -http://www.facebook.de -http://www.livevideo.com -http://www.movements.org -http://www.witopia.net -http://www.witopia.net -http://www.president.gov.tw -http://www.ibvpn.com -http://stats.wordpress.com -http://program-think.blogspot.com -http://www.shareonall.com -http://www.sim-outhouse.com -http://ait.org.tw -http://fimotro.blogspot.com -http://www.now.com.hk -http://www.china-proxy.org -http://xbookcn.com -http://greatfirewallofchina.org -http://fileim.com -http://www.tw789.net -http://8teenporn.com -http://tweetwally.com -http://qkshare.com -http://bbs.tw789.net -http://nuexpo.com -http://www.tsemtulku.com -http://www.internationalrivers.org -http://snapbrowse.com -http://maiio.net -http://csuchen.de -http://jingpin.org -http://twbbs.net.tw -http://thefacebookproxy.com -http://bbg.gov -http://www.pelaut.go.id -http://witopia.net -http://www.maiio.net -http://igfw.net -http://youtube.com.br -http://www.chinageeks.org -http://www.boxpn.com -http://media.cottagecountrynow.ca -http://www.myca168.com -http://chinageeks.org -http://alestic.com -http://sorting-algorithms.com -http://status.twhirl.org -http://www.ginatv.com -http://www.bbg.gov -http://porn2.com -http://rocmp.org -http://mangareader.com -http://www.seeon.tv -http://photos.dailyme.com -http://coolloud.org.tw -http://sobees.com -http://farxian.com -http://tsemtulku.com -http://www.worldpress.com -http://torstatus.blutmagie.de -http://www.mnnit.ac.in -http://mirrorbooks.com -http://expatshield.com -http://qidian.ca -http://drugpolicy.org -http://www.anonasurf.com -http://pbwiki.com -http://boxitvn.blogspot.com -http://appempire1.com -http://canyu.org -http://www.googl.com -http://www.thepiratebay.com -http://twitter.jp -http://favorious.com -http://vpn-pay.itshidden.com -http://truveo.com -http://www.tsctv.net -http://kenengba.com -http://internetfreedom.org -http://naol.ca -http://www.farxian.com -http://www.fpmt.org -http://www.vpn4all.com -http://mk5000.com -http://www.phayul.com -http://phayul.com -http://ccthere.com -http://googlevideo.com -http://www.itshidden.com -http://www.wujieliulan.com -http://twiyia.com -http://www.taiwandaily.net -http://www.powerpointninja.com -http://www.twbbs.net.tw -http://www.brookemarks.com -http://www.techinterview.org -http://zonaeuropa.com -http://www.canyu.org -http://www.webrush.net -http://www.mckaywei.com -http://web2project.net -http://www.civicparty.hk -http://mademoisellerobot.com -http://tibet.net -http://xijie.wordpress.com -http://www.kenengba.com -http://x1949x.com -http://www.zonaeuropa.com -http://anonasurf.com -http://www.asiaharvest.org -http://www.naol.ca -http://wujieliulan.com -http://yx51.net -http://mobileways.de -http://www.ccthere.com -http://techlifeweb.com -http://cn.fmnnow.com -http://www.supervpn.net -http://www.googlevideo.com -http://www.expressvpn.com -http://www.autospec.co.za -http://www.truveo.com -http://areca-backup.org -http://www.vpnreactor.com -http://www.duping.net -http://zarias.com -http://www.ned.org -http://www.mk5000.com -http://mckaywei.com -http://www.boxpn.com -http://www.cryptocloud.com -http://civicparty.hk -http://kenengba.com -http://www.tibet.net -http://brandonhutchinson.com -http://www.expressvpn.com -http://duping.net -http://www.kenengba.com -http://triangulationblog.com -http://myca168.com -http://radiofreechina.wordpress.com -http://stephenmooredesign.wordpress.com -http://abc.wordpress.com -http://teflandlouise.wordpress.com -http://fromcanadatofinland.wordpress.com -http://fashiontech.wordpress.com -http://www.richardjangus.wordpress.com -http://beijingnaarhongkong.wordpress.com -http://sample.wordpress.com -http://caiting666.wordpress.com -http://anyname.wordpress.com -http://whotolduthat.wordpress.com -http://rfaunplugged.wordpress.com -http://reinesklareslicht.wordpress.com -http://afterthestroke.wordpress.com -http://skittlesinchina.wordpress.com -http://abcdefg.wordpress.com -http://zh-sg.wordpress.com -http://facebookdata.wordpress.com -http://lancasterleaflets.wordpress.com -http://markerhunter.wordpress.com -http://zhenxiang.wordpress.com -http://imsirius.wordpress.com -http://tibetnorway.wordpress.com -http://cenantua.wordpress.com -http://seotch.wordpress.com -http://djcadchina.wordpress.com -http://socialnetworkguru.wordpress.com -http://quhelan.wordpress.com -http://maoistrebelnews.wordpress.com -http://zh.wordpress.com -http://wordpress.com -http://mw2mk.wordpress.com -http://translate.googleusercontent.com -http://webcache.googleusercontent.com -http://lh4.googleusercontent.com -http://webcache.googleusercontent.com -http://winqual.microsoft.com -http://my.mail.ru -http://t.co -http://t.co -http://www.t.co -http://www.google.it -http://www.blogger.com -http://blogger.com -http://draft.blogger.com -http://xhamster.com -http://www.xhamster.com -http://imdb.com -http://www.imdb.com -http://www.fc2.com -http://fc2.com -http://toruyamada.web.fc2.com -http://kitchenremodelideas.blog138.fc2.com -http://ninionlytw.web.fc2.com -http://nijibug.blog131.fc2.com -http://www.xvideos.com -http://xvideos.com -http://paulsumiko.blog46.fc2.com -http://video.aol.com -http://aolchannels.aol.com -http://search.aol.com -http://geezericious.blogspot.in -http://evolutionguide.blogspot.in -http://hqlenglish.blogspot.in -http://ukiahcoachbrown.blogspot.in -http://www.thepiratebay.se -http://thepiratebay.se -http://thepiratebay.se -http://cart.godaddy.com -http://www.netflix.com -http://signup.netflix.com -http://www.redtube.com -http://www.netflix.com -http://netflix.com -http://redtube.com -http://3.bp.blogspot.com -http://youporn.com -http://www.youporn.com -http://dailymotion.com -http://www.dailymotion.com -http://4.bp.blogspot.com -http://2.bp.blogspot.com -http://xnxx.com -http://www.badoo.com -http://badoo.com -http://www.vimeo.com -http://tube8.com -http://vimeo.com -http://www.vimeo.com -http://www.tube8.com -http://email10.secureserver.net -http://login.secureserver.net -http://www.secureserver.net -http://default.secureserver.net -http://secureserver.net -http://ip-173-201-135-189.ip.secureserver.net -http://www.dropbox.com -http://dl-web.dropbox.com -http://d.dropbox.com -http://hk32168.com -http://www.hemispheregames.com -http://airvpn.org -http://www.gourmetgaming.co.uk -http://askakorean.blogspot.com -http://08charterbbs.blogspot.com -http://asiaharvest.org -http://www.ultravpn.fr -http://www.ivacy.com -http://tjholowaychuk.com -http://mychinamyhome.com -http://mixx.com -http://www.mixx.com -http://open.com.hk -http://danwei.org -http://www.hecaitou.net -http://evhead.com -http://christianstudy.com -http://ultravpn.fr -http://forum.mymaji.com -http://www.pekingduck.org -http://www.puretna.com -http://www.nogomi.com -http://hecaitou.net -http://dphk.org -http://www.hk32168.com -http://www.smarterdailydeal.com -http://taiwandaily.net -http://www.medical-explorer.com -http://yatsen.gov.tw -http://tweetymail.com -http://tvider.com -http://njuice.com -http://www.5pic.com -http://furl.net -http://dit-inc.us -http://www.inxian.com -http://www.youtube.de -http://airvpn.org -http://www.twitstat.com -http://twitstat.com -http://taiwanyes.ning.com -http://www.gpass1.com -http://www.snaptu.com -http://snaptu.com -http://tweetree.com -http://www.open.com.hk -http://gpass1.com -http://www.vpnaccounts.com -http://www.koornk.com -http://koornk.com -http://www.mingpaony.com -http://www.mychinamyhome.com -http://mash.to -http://www.proxeasy.com -http://chevronwp7.com -http://www.danwei.org -http://chinaaffairs.org -http://www.64tianwang.com -http://www.toodoc.com -http://tunnel.shellmix.com -http://savetibet.org -http://zh.greatfire.org -http://64tianwang.com -http://www.allinfa.com -http://www.cmule.com -http://tweetboard.com -http://vpnreactor.com -http://www.twitturly.com -http://chaos.e-spacy.com -http://www.my-private-network.co.uk -http://www.my-private-network.co.uk -http://twitturly.com -http://www.lastwordonnothing.com -http://www.swissvpn.net -http://iphonix.fr -http://zh.greatfire.org -http://sharecool.org -http://my-private-network.co.uk -http://www.greatfire.org -http://www.greatfire.org -http://www.online.hk -http://www.freexinwen.com -http://alkasir.com -http://www.chinagfw.org -http://www.swingasia.com -http://openleaks.org -http://www.openleaks.org -http://salvation.org.hk -http://blog.jtbworld.com -http://deibert.citizenlab.org -http://tsunamivpn.com -http://www.xxx -http://test.greatfire.org -http://allaboutalpha.com -http://facebook.it -http://uwants.net -http://www.superfreevpn.com -http://www.salvation.org.hk -http://pekingduck.org -http://dropbox.com -http://dl.dropbox.com -http://zh.uncyclopedia.wikia.com -http://downloads.sourceforge.net -http://ultrasurf.en.softonic.com -http://xwpm.sourceforge.net -http://sourceforge.net -http://sourceforge.net -http://audacity.sourceforge.net -http://cn.uncyclopedia.wikia.com -http://foxnews.com -http://secure.wikimedia.org -http://www.justbeenpaid.com -http://secure.wikimedia.org -http://skype.com -http://ui.skype.com -http://fr.myspace.com -http://www.fbcdn.net -http://www.myspace.com -http://fbcdn.net -http://www.skype.com -http://viewmorepics.myspace.com -http://blogs.myspace.com -http://www.blogs.myspace.com -http://myspace.com -http://justbeenpaid.com -http://adultfriendfinder.com -http://www.adultfriendfinder.com -http://slideshare.net -http://hootsuite.com -http://www.slideshare.net -http://www.files.wordpress.com -http://scribd.com -http://www.hootsuite.com -http://allonlinux.free.fr -http://www.scribd.com -http://kineox.free.fr -http://smile-pow13.nicovideo.jp -http://cam4.com -http://files.wordpress.com -http://www.nicovideo.jp -http://de.cam4.com -http://nicovideo.jp -http://www.cam4.com -http://google.com.ua -http://imageshack.us -http://live.nicovideo.jp -http://isohunt.com -http://www.isohunt.com -http://www.imageshack.us -http://www.xing.com -http://xing.com -http://www.hardsextube.com -http://www.mashable.com -http://mashable.com -http://adserverplus.com -http://hardsextube.com -http://rutracker.org -http://evolutionguide.blogspot.co.uk -http://www.cthulhupodcast.blogspot.co.uk -http://1234.weebly.com -http://blackheathbabysitting.weebly.com -http://easychinese.weebly.com -http://transferstudent.weebly.com -http://drtuber.com -http://www.drtuber.com -http://zimbio.com -http://transferstudentcn.weebly.com -http://www.zimbio.com -http://hulu.com -http://www.hulu.com -http://bet365.com -http://www.bet365.com -http://google.cz -http://www.battle.net -http://www.gutefrage.net -http://gutefrage.net -http://www.twitpic.com -http://wretch.cc -http://www.dayaarmongol.ning.com -http://www.wretch.cc -http://dayaarmongol.ning.com -http://twitpic.com -http://bloomberg.com -http://www.longlivehhdl.ning.com -http://www.linkbucks.com -http://www.bloomberg.com -http://tmz.com -http://linkbucks.com -http://www.multiply.com -http://miles6k.multiply.com -http://multiply.com -http://chinese.engadget.com -http://taiwanblogs.blogspot.it -http://www.spankwire.com -http://goodreads.com -http://www.goodreads.com -http://spankwire.com -http://blog.bitly.com -http://www.bitly.com -http://bitly.com -http://www.newegg.com -http://888.com -http://www.888.com -http://feeds.feedburner.com -http://123rf.com -http://nba.com -http://www.123rf.com -http://store.nokia.com -http://pornhublive.com -http://www.pornhublive.com -http://www.nba.com -http://www.macys.com -http://macys.com -http://www.pixnet.net -http://blekko.com -http://ustream.tv -http://www.ustream.tv -http://porntube.com -http://www.imagevenue.com -http://imagevenue.com -http://metacafe.com -http://www.blogimg.jp -http://www.metacafe.com -http://pixnet.net -http://freakshare.com -http://www.freakshare.com -http://posterous.com -http://www.pornerbros.com -http://alarabiya.net -http://www.alarabiya.net -http://www.justin.tv -http://www.geocities.jp -http://partypoker.com -http://justin.tv -http://www.partypoker.com -http://hidemyass.com -http://7.hidemyass.com -http://1.hidemyass.com -http://www.businessweek.com -http://businessweek.com -http://paper.li -http://arielefraimashbel.blogspot.com.ar -http://blogspot.com.ar -http://urbandictionary.com -http://vpn.hidemyass.com -http://pornerbros.com -http://blog.udn.com -http://www.udn.com -http://alnaddy.com -http://udn.com -http://www.yfrog.com -http://www.hidemyass.com -http://6.hidemyass.com -http://goo.gl -http://a.yfrog.com -http://kriz-cpec.posterous.com -http://nimingdepengyou.posterous.com -http://writer.zoho.com -http://arielefraimashbel.blogspot.com.au -http://biocouture.posterous.com -http://www.paper.li -http://www.posterous.com -http://mbc.net -http://eyny.com -http://www.empflix.com -http://www.eyny.com -http://yfrog.com -http://nfl.com -http://city.udn.com -http://video.udn.com -http://forum.udn.com -http://blog.disqus.com -http://networkedblogs.com -http://geezericious.blogspot.com.au -http://newsabeta.blogspot.com.au -http://freeones.com -http://blogspot.com.au -http://www.88db.com -http://www.delta.com -http://yobt.com -http://www.freeones.com -http://franklin-chow.blogspot.com.au -http://hqlenglish.blogspot.com.au -http://www.xvideoslive.com -http://lci.tf1.fr -http://extremetube.com -http://meetcheap.com -http://pornoxo.com -http://www.foursquare.com -http://foursquare.com -http://fapdu.com -http://www.lenovo.com -http://www.wn.com -http://forum.pchome.com.tw -http://mypaper.pchome.com.tw -http://blackhatteam.com -http://wn.com -http://www.demonoid.me -http://demonoid.me -http://www.networkedblogs.com -http://www.mobile01.com -http://mobile01.com -http://dtiblog.com -http://www.dtiblog.com -http://www.pchome.com.tw -http://blog.pchome.com.tw -http://news.pchome.com.tw -http://fandango.com -http://pchome.com.tw -http://n.yam.com -http://asg.to -http://diigo.com -http://mymedia.yam.com -http://screencast.com -http://news.yam.com -http://na.leagueoflegends.com -http://break.com -http://blog.yam.com -http://times.hinet.net -http://livestream.com -http://www.livestream.com -http://xa.yimg.com -http://autotrader.com -http://comcast.com -http://www.break.com -http://real.com -http://www.diigo.com -http://www.betfair.com -http://betfair.com -http://change.org -http://www.change.org -http://www.tunein.com -http://tunein.com -http://g.e-hentai.org -http://chn.chosun.com -http://www.asg.to -http://technorati.com -http://balagana.net -http://www.panoramio.com -http://google.com.om -http://appledaily.com.tw -http://tigerdirect.com -http://wwe.com -http://shopping.com -http://friendfeed.com -http://blog.dribbble.com -http://www.yousendit.com -http://google.co.ke -http://www.tigerdirect.com -http://forum.chinatimes.com -http://www.tweetmeme.com -http://yebhi.com -http://www.technorati.com -http://tweetmeme.com -http://blog.chinatimes.com -http://wikimapia.org -http://rk.com -http://tubewolf.com -http://6park.com -http://blog.nownews.com -http://news.chinatimes.com -http://camera.chinatimes.com -http://forum.nownews.com -http://plurk.com -http://cafepress.com -http://boardreader.com -http://torrenthound.com -http://tala.ir -http://yobt.tv -http://www.cafepress.com -http://www.2shared.com -http://realitykings.com -http://2shared.com -http://www.6park.com -http://ime.nu -http://wnd.com -http://gamestlbb.com -http://wenxuecity.com -http://hk.apple.nextmedia.com -http://www.plurk.com -http://jamnews.ir -http://bdnews24.com -http://forever21.com -http://elkhabar.com -http://www.wenxuecity.com -http://caribbeancom.com -http://togetter.com -http://rutube.ru -http://luntan.zaobao.com -http://tuite.googlecode.com -http://spartoo.com -http://tradus.com -http://sttlbb.com -http://naughtyamerica.com -http://segundamano.es -http://expressen.se -http://www.lusongsong.com -http://gfwinterceptor.googlecode.com -http://ibibo.com -http://www.ibibo.com -http://old.nabble.com -http://wiley.com -http://leilao.jp -http://v8.googlecode.com -http://www.rutube.ru -http://forums.wordpress.com -http://www.efukt.com -http://gtap.googlecode.com -http://xmatch.com -http://efukt.com -http://fotomac.com.tr -http://tv.com -http://boinc.berkeley.edu -http://binaryoptions24h.com -http://index.hr -http://cliphunter.com -http://en.wordpress.com -http://ow.ly -http://blog.xuite.net -http://www.ow.ly -http://www.sex.com -http://livetv.ru -http://torrentcrazy.com -http://topsy.com -http://tiao8.info -http://pornbb.org -http://shorouknews.com -http://sex.com -http://www.topsy.com -http://voanews.com -http://cams.com -http://www.dw.de -http://dw.de -http://myvideo.de -http://sbnation.com -http://www.erepublik.com -http://bbs.sina.com -http://home.sina.com -http://rconversation.blogs.com -http://sina.com -http://dailynews.sina.com -http://tweakers.net -http://www.sina.com -http://erepublik.com -http://tokyo-porn-tube.com -http://www.myvideo.de -http://veoh.com -http://discuss.com.hk -http://support.wordpress.com -http://babosas.com -http://googleapis.com -http://www.ck101.com -http://www.discuss.com.hk -http://www.cloudflare.com -http://uniqlo.com -http://twipple.jp -http://wiki.jqueryui.com -http://ptcbox.com -http://www.veoh.com -http://www.yuvutu.com -http://www.nrk.no -http://tokyo-tube.com -http://google.co.ug -http://eforosh.com -http://ck101.com -http://hotelurbano.com.br -http://yuvutu.com -http://anonymouse.org -http://findicons.com -http://apetube.com -http://blogspot.fi -http://gogvo.com -http://twitterfeed.com -http://www.mobypicture.com -http://aukro.cz -http://www.books.com.tw -http://nubiles.net -http://zh.pokerstrategy.com -http://cn.calameo.com -http://ebookbrowse.com -http://bsnl.co.in -http://avaaz.org -http://www.plus28.com -http://elespectador.com -http://nrk.no -http://www.avaaz.org -http://plus28.com -http://mobypicture.com -http://www.twitterfeed.com -http://homemademoviez.com -http://v-mire.com -http://www.geocities.co.jp -http://problogger.net -http://jagran.com -http://willhaben.at -http://tv2.dk -http://maps.ovi.com -http://reclameaqui.com.br -http://store.ovi.com -http://whatismyip.com -http://snagajob.com -http://books.com.tw -http://bwin.com -http://izlesene.com -http://corrieredellosport.it -http://pizzahut.com -http://xxxymovies.com -http://freshxxxtube.com -http://www.aftenposten.no -http://mouthshut.com -http://yammer.com -http://www5.kinghost.com -http://twitlonger.com -http://www.twitlonger.com -http://forum.tvb.com -http://pornsake.com -http://homemadesextube.com -http://wayfair.com -http://cuantocabron.com -http://devshed.com -http://the-sun.on.cc -http://gossiplankanews.com -http://blogspot.co.at -http://thehun.com -http://rotoworld.com -http://video.foxbusiness.com -http://petsmart.com -http://badjojo.com -http://www.hgtv.com -http://allanalpass.com -http://gammae.com -http://frankfinn.in -http://mysitecost.ru -http://hgtv.com -http://aaa.com -http://twitvid.com -http://kinghost.com -http://ht.ly -http://crackle.com -http://malaysiakini.com -http://punyu.com -http://ssa.gov -http://lists.debian.org -http://bloglovin.com -http://nps.gov -http://www.bebo.com -http://www.nps.gov -http://sportsdirect.com -http://china.dwnews.com -http://bravenet.com -http://perfectworld.com -http://bebo.com -http://dickssportinggoods.com -http://femjoy.com -http://rubias19.com -http://bloomingdales.com -http://mmaaxx.com -http://www.twitvid.com -http://indianpornvid.com -http://potterybarn.com -http://animecrazy.net -http://web.ics.purdue.edu -http://www.blog.de -http://webshots.com -http://urbanoutfitters.com -http://crackberry.com -http://rg.ru -http://kajabi.com -http://calottery.com -http://www.webshots.com -http://freeservers.com -http://commentshk.blogspot.hk -http://wqw2010.blogspot.hk -http://www.fakku.net -http://camfrog.com -http://royalty7.com -http://blog.de -http://blog.roodo.com -http://www.urbanoutfitters.com -http://wearehairy.com -http://fakku.net -http://libertytimes.com.tw -http://microworkers.com -http://supersport.com -http://www.hotspotshield.com -http://dwnews.com -http://teamliquid.net -http://wickedfire.com -http://tunisia-sat.com -http://miami.com -http://blog.sina.com.tw -http://bangbros1.com -http://pandora.tv -http://videosurf.com -http://vh1.com -http://pbase.com -http://akiba-online.com -http://politiken.dk -http://www.pandora.tv -http://news.sina.com.tw -http://startpage.com -http://occ.com.mx -http://www.camfrog.com -http://hotspotshield.com -http://www.libertytimes.com.tw -http://mitbbs.com -http://www.animecrazy.net -http://cosmiq.de -http://desidime.com -http://sacbee.com -http://mforos.com -http://www.twittercounter.com -http://www.mitbbs.com -http://cimbclicks.com.my -http://www.mefeedia.com -http://www.pbworks.com -http://magazines.sina.com.tw -http://kariyer.net -http://boots.com -http://pbworks.com -http://websiteforever.com -http://www.chinese.rfi.fr -http://wikiwiki.jp -http://idol-blog.com -http://amakings.com -http://amino.dk -http://causes.com -http://coches.net -http://rfi.fr -http://www.rfi.fr -http://imgsrc.ru -http://galeon.com -http://ripoffreport.com -http://freelotto.com -http://blip.tv -http://www.blip.tv -http://kobobooks.com -http://derekhsu.homeip.net -http://bookofsex.com -http://italia-film.com -http://pimpandhost.com -http://arte.tv -http://mefeedia.com -http://partycasino.com -http://governmentjobs.com -http://stltoday.com -http://yad2.co.il -http://www.freelotto.com -http://pagewash.com -http://guitarcenter.com -http://sexfinder.com -http://elifting.com -http://joomlaforum.ru -http://maomaotlbb.com -http://korben.info -http://self.com -http://hdporn.in -http://tinychat.com -http://wattpad.com -http://www.wattpad.com -http://boyfriendtv.com -http://via.me -http://bodisparking.com -http://www.paltalk.com -http://paltalk.com -http://simplyrecipes.com -http://epochtimes.com -http://khan.co.kr -http://www.simplyrecipes.com -http://goldline.pro -http://www.alabout.com -http://www.tinychat.com -http://gidonline.ru -http://lexilogos.com -http://mmtlsf.com -http://www.epochtimes.com -http://activerain.com -http://www.korben.info -http://moviehuts.com -http://myfreepaysite.com -http://mobli.com -http://gsmspain.com -http://www.17wtlbb.com -http://englishtown.com -http://cn.news.cnyes.com -http://gamebase.com.tw -http://www.gamebase.com.tw -http://ddfcash.com -http://cl.ly -http://alabout.com -http://flalottery.com -http://gohappy.com.tw -http://favstar.fm -http://swtor.com -http://immi.gov.au -http://heydouga.com -http://justunfollow.com -http://getiton.com -http://radioshack.com -http://topic.youthwant.com.tw -http://anchorfree.net -http://cakephp.org -http://forum.sina.com.hk -http://goibibo.com -http://www.sugarsync.com -http://fux.com -http://onekingslane.com -http://ghacks.net -http://definebabe.com -http://blog.youthwant.com.tw -http://sport24.co.za -http://twilog.org -http://eventful.com -http://gfrevenge.com -http://onlinewatchmovies.net -http://duga.jp -http://zh.wikisource.org -http://rghost.ru -http://hkreporter.com -http://bigtits.com -http://ladbrokes.com -http://mid-day.com -http://wiki.phonegap.com -http://internetslang.com -http://gnc.com -http://vuku.ru -http://kporno.com -http://tro-ma-ktiko.blogspot.com -http://tadawul.com.sa -http://mail-archive.com -http://www.mail-archive.com -http://xtc-modified.org -http://aviny.com -http://mybloggertricks.com -http://listentoyoutube.com -http://articlesnatch.com -http://aukro.ua -http://burningcamel.com -http://freepornofreeporn.com -http://www.zinio.com -http://weblagu.com -http://www.sugarsync.com -http://tvp.pl -http://1stcngame.com -http://xtlbb.com -http://powersearchingwithgoogle.com -http://www.hegre-art.com -http://imagecherry.com -http://assembla.com -http://career.co.kr -http://zinio.com -http://sytes.net -http://www.ptt.cc -http://theme.wordpress.com -http://www.rfa.org -http://lindaikeji.blogspot.com -http://wehkamp.nl -http://sugarsync.com -http://www.hkreporter.com -http://key.com -http://williams-sonoma.com -http://club.backchina.com -http://fastlink.sytes.net -http://www.sytes.net -http://hotfrog.com -http://www.hrnm.navy.mil -http://ch.shvoong.com -http://browsershots.org -http://www.backchina.com -http://codecademy.com -http://share.youthwant.com.tw -http://puritanas.com -http://blinkx.com -http://www.cam.ac.uk -http://www.tweetdeck.com -http://iran-eng.com -http://green-flowers.net -http://oszone.net -http://gq.com -http://lastfm.es -http://websurf.ru -http://tmi.me -http://dziennik.pl -http://pantheranetwork.com -http://newsbusters.org -http://xuk.ru -http://www.megavideo.com -http://allstate.com -http://megavideo.com -http://allfacebook.com -http://www.thisav.com -http://uberhumor.com -http://dallasnews.com -http://dotup.org -http://www.blinkx.com -http://evolutionguide.blogspot.co.nz -http://kjendis.no -http://backchina.com -http://hegre-art.com diff --git a/ooni/assets/redirects.yaml b/ooni/assets/redirects.yaml deleted file mode 100644 index 43582c4..0000000 --- a/ooni/assets/redirects.yaml +++ /dev/null @@ -1,17 +0,0 @@ -o2: - name: O2 Networks - patterns: - - {type: re, value: '(https://bango.net/).*'} - - {type: re, value: 'http://wap.o2.co.uk/(18plusaccess).*'} -orange: - name: Orange Telecom - patterns: {type: eq, value: 'http://orangeworld.co.uk/r/avblocked/'} -three: - name: Three - patterns: {type: re, value: '(http://mobile.three.co.uk/pc/Live/pcreator/live/).*'} -tmobile: - name: T-Mobile - patterns: {type: eq, value: 'http://www.t-mobile.co.uk/common/system_error_pages/outage_wnw.html'} -vodafone: - name: Vodafone UK - patterns: {type: eq, value: 'http://online.vodafone.co.uk/en_GB/assets/static/contentcontrol/unbranded/r…'} diff --git a/ooni/assets/tcpscan.txt b/ooni/assets/tcpscan.txt deleted file mode 100644 index 4204d77..0000000 --- a/ooni/assets/tcpscan.txt +++ /dev/null @@ -1,2 +0,0 @@ -127.0.0.1:9050 -127.0.0.1:6666 diff --git a/ooni/reporter.py b/ooni/reporter.py index b145e87..51a0166 100644 --- a/ooni/reporter.py +++ b/ooni/reporter.py @@ -8,6 +8,15 @@ from datetime import datetime from twisted.python.util import OrderedDict, untilConcludes from twisted.trial import unittest, reporter, runner +try: + from scapy.all import packet +except: + class FooClass: + pass + packet = object + packet.Packet = FooClass + + pyunit = __import__('unittest') class OReporter(pyunit.TestResult): @@ -44,8 +53,8 @@ class OReporter(pyunit.TestResult): self._write('\n') def writeYamlLine(self, line): - self._write(yaml.dump([line])) - + to_write = yaml.dump([line]) + self._write(to_write) class ReporterFactory(OReporter): @@ -122,7 +131,11 @@ class OONIReporter(OReporter): self._tests[idx] = {} self._tests[idx]['testStarted'] = self._getTime() - self._tests[idx]['input'] = test.input + if isinstance(test.input, packet.Packet): + test_input = repr(test.input) + else: + test_input = test.input + self._tests[idx]['input'] = test_input self._tests[idx]['idx'] = idx self._tests[idx]['name'] = test.name #self._tests[idx]['test'] = test diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index e9c55f1..5593027 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -143,4 +143,56 @@ class Log(): except: raise StopIteration +import copy_reg + +def patched_reduce_ex(self, proto): + """ + This is a hack to overcome a bug in one of pythons core functions. It is + located inside of copy_reg and is called _reduce_ex. + + Some background on the issue can be found here: + http://stackoverflow.com/questions/569754/how-to-tell-for-which-object-attr… + http://stackoverflow.com/questions/2049849/why-cant-i-pickle-this-object + + There was also an open bug on the pyyaml trac repo, but it got closed because + they could not reproduce. + http://pyyaml.org/ticket/190 + + It turned out to be easier to patch the python core library than to monkey + patch yaml. + + XXX see if there is a better way. sigh... + """ + _HEAPTYPE = 1<<9 + assert proto < 2 + for base in self.__class__.__mro__: + if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE: + break + else: + base = object # not really reachable + if base is object: + state = None + elif base is int: + state = None + else: + if base is self.__class__: + raise TypeError, "can't pickle %s objects" % base.__name__ + state = base(self) + args = (self.__class__, base, state) + try: + getstate = self.__getstate__ + except AttributeError: + if getattr(self, "__slots__", None): + raise TypeError("a class that defines __slots__ without " + "defining __getstate__ cannot be pickled") + try: + dict = self.__dict__ + except AttributeError: + dict = None + else: + dict = getstate() + if dict: + return copy_reg._reconstructor, args, dict + else: + return copy_reg._reconstructor, args

1 0

[ooni-probe/master] Implement port of ScapyTest to new API
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 36c8a8e149fa5d975fc0d39bd0e5bf72ab38086a Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 16:11:05 2012 +0000 Implement port of ScapyTest to new API * Rename templates to avoid namespace conflicts * XXX need to fix some bugs in how yaml serializes scapy objects. Basically the issue is that scapy Packet objects override getattr and setattr. This leads to pickle having some issues in serializing it. Some links describing the issue: http://stackoverflow.com/questions/2049849/why-cant-i-pickle-this-object http://stackoverflow.com/questions/569754/how-to-tell-for-which-object-attr… There was also an open bug on the pyyaml trac repo, but it got closed because they could not reproduce. http://pyyaml.org/ticket/190 --- nettests/example_http.py | 28 -------- nettests/example_httpt.py | 28 ++++++++ nettests/example_scapyt.py | 9 +++ ooni/templates/http.py | 147 -------------------------------------------- ooni/templates/httpt.py | 147 ++++++++++++++++++++++++++++++++++++++++++++ ooni/templates/scapyt.py | 65 +++++++++++++++++++ 6 files changed, 249 insertions(+), 175 deletions(-) diff --git a/nettests/example_http.py b/nettests/example_http.py deleted file mode 100644 index 7528d48..0000000 --- a/nettests/example_http.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -from ooni.templates import http -class Example(http.HTTPTest): - inputs = ['http://google.com/', 'http://wikileaks.org/', - 'http://torproject.org/'] - - def processResponseBody(self, body): - # XXX here shall go your logic - # for processing the body - if 'blocked' in body: - self.report['censored'] = True - else: - self.report['censored'] = False - - def processResponseHeaders(self, headers): - # XXX place in here all the logic for handling the processing of HTTP - # Headers. - if headers.hasHeader('location'): - self.report['redirect'] = True - - server = headers.getRawHeaders("Server") - if server: - self.report['http_server'] = str(server.pop()) - diff --git a/nettests/example_httpt.py b/nettests/example_httpt.py new file mode 100644 index 0000000..1814a90 --- /dev/null +++ b/nettests/example_httpt.py @@ -0,0 +1,28 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from ooni.templates import httpt +class Example(httpt.HTTPTest): + inputs = ['http://google.com/', 'http://wikileaks.org/', + 'http://torproject.org/'] + + def processResponseBody(self, body): + # XXX here shall go your logic + # for processing the body + if 'blocked' in body: + self.report['censored'] = True + else: + self.report['censored'] = False + + def processResponseHeaders(self, headers): + # XXX place in here all the logic for handling the processing of HTTP + # Headers. + if headers.hasHeader('location'): + self.report['redirect'] = True + + server = headers.getRawHeaders("Server") + if server: + self.report['http_server'] = str(server.pop()) + diff --git a/nettests/example_scapyt.py b/nettests/example_scapyt.py new file mode 100644 index 0000000..ef61f3d --- /dev/null +++ b/nettests/example_scapyt.py @@ -0,0 +1,9 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from ooni.templates import scapyt +from scapy.all import * +class ExampleScapy(scapyt.ScapyTest): + inputs = [IP()/TCP()] diff --git a/ooni/templates/http.py b/ooni/templates/http.py deleted file mode 100644 index 222e44c..0000000 --- a/ooni/templates/http.py +++ /dev/null @@ -1,147 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# :authors: Arturo Filastò -# :licence: see LICENSE - -import random - -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from twisted.internet import protocol, defer -from twisted.web.http_headers import Headers - -from ooni.nettest import TestCase -from ooni.utils import log - -useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), - ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), - ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), - ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), - ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), - ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] - -class BodyReceiver(protocol.Protocol): - def __init__(self, finished): - self.finished = finished - self.data = "" - - def dataReceived(self, bytes): - self.data += bytes - - def connectionLost(self, reason): - self.finished.callback(self.data) - -class HTTPTest(TestCase): - """ - A utility class for dealing with HTTP based testing. It provides methods to - be overriden for dealing with HTTP based testing. - The main functions to look at are processResponseBody and - processResponseHeader that are invoked once the headers have been received - and once the request body has been received. - """ - randomizeUA = True - followRedirects = False - - def setUp(self): - from twisted.web.client import Agent - from twisted.internet import reactor - import yaml - self.agent = Agent(reactor) - if self.followRedirects: - from twisted.web.client import RedirectAgent - self.agent = RedirectAgent(self.agent) - self.request = {} - self.response = {} - - def _processResponseBody(self, data): - self.response['body'] = data - self.report['response'] = self.response - - self.processResponseBody(data) - - def processResponseBody(self, data): - """ - This should handle all the response body smushing for getting it ready - to be passed onto the control. - - @param data: The content of the body returned. - """ - pass - - def processResponseHeaders(self, headers): - """ - This should take care of dealing with the returned HTTP headers. - - @param headers: The content of the returned headers. - """ - pass - - def processRedirect(self, location): - """ - Handle a redirection via a 3XX HTTP status code. - - @param location: the url that is being redirected to. - """ - pass - - def doRequest(self, url): - d = self.build_request(url) - def finished(data): - return data - - d.addCallback(self._cbResponse) - d.addCallback(finished) - return d - - def test_http(self): - log.msg("Running experiment") - - if self.input: - url = self.input - else: - raise Exception("No input supplied") - - self.mainDefer = self.doRequest(url) - return self.mainDefer - - def _cbResponse(self, response): - self.response['headers'] = response.headers - self.response['code'] = response.code - self.response['length'] = response.length - self.response['version'] = response.length - - if str(self.response['code']).startswith('3'): - self.processRedirect(response.headers.getRawHeaders('Location')[0]) - - self.processResponseHeaders(self.response['headers']) - - finished = defer.Deferred() - response.deliverBody(BodyReceiver(finished)) - finished.addCallback(self._processResponseBody) - - return finished - - def randomize_useragent(self): - user_agent = random.choice(useragents) - self.request['headers']['User-Agent'] = [user_agent] - - def build_request(self, url, method="GET", headers=None, body=None): - self.request['method'] = method - self.request['url'] = url - self.request['headers'] = headers if headers else {} - self.request['body'] = body - if self.randomizeUA: - self.randomize_useragent() - - self.report['request'] = self.request - self.report['url'] = url - return self.agent.request(self.request['method'], self.request['url'], - Headers(self.request['headers']), - self.request['body']) - diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py new file mode 100644 index 0000000..222e44c --- /dev/null +++ b/ooni/templates/httpt.py @@ -0,0 +1,147 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +import random + +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import protocol, defer +from twisted.web.http_headers import Headers + +from ooni.nettest import TestCase +from ooni.utils import log + +useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), + ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), + ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), + ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] + +class BodyReceiver(protocol.Protocol): + def __init__(self, finished): + self.finished = finished + self.data = "" + + def dataReceived(self, bytes): + self.data += bytes + + def connectionLost(self, reason): + self.finished.callback(self.data) + +class HTTPTest(TestCase): + """ + A utility class for dealing with HTTP based testing. It provides methods to + be overriden for dealing with HTTP based testing. + The main functions to look at are processResponseBody and + processResponseHeader that are invoked once the headers have been received + and once the request body has been received. + """ + randomizeUA = True + followRedirects = False + + def setUp(self): + from twisted.web.client import Agent + from twisted.internet import reactor + import yaml + self.agent = Agent(reactor) + if self.followRedirects: + from twisted.web.client import RedirectAgent + self.agent = RedirectAgent(self.agent) + self.request = {} + self.response = {} + + def _processResponseBody(self, data): + self.response['body'] = data + self.report['response'] = self.response + + self.processResponseBody(data) + + def processResponseBody(self, data): + """ + This should handle all the response body smushing for getting it ready + to be passed onto the control. + + @param data: The content of the body returned. + """ + pass + + def processResponseHeaders(self, headers): + """ + This should take care of dealing with the returned HTTP headers. + + @param headers: The content of the returned headers. + """ + pass + + def processRedirect(self, location): + """ + Handle a redirection via a 3XX HTTP status code. + + @param location: the url that is being redirected to. + """ + pass + + def doRequest(self, url): + d = self.build_request(url) + def finished(data): + return data + + d.addCallback(self._cbResponse) + d.addCallback(finished) + return d + + def test_http(self): + log.msg("Running experiment") + + if self.input: + url = self.input + else: + raise Exception("No input supplied") + + self.mainDefer = self.doRequest(url) + return self.mainDefer + + def _cbResponse(self, response): + self.response['headers'] = response.headers + self.response['code'] = response.code + self.response['length'] = response.length + self.response['version'] = response.length + + if str(self.response['code']).startswith('3'): + self.processRedirect(response.headers.getRawHeaders('Location')[0]) + + self.processResponseHeaders(self.response['headers']) + + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished)) + finished.addCallback(self._processResponseBody) + + return finished + + def randomize_useragent(self): + user_agent = random.choice(useragents) + self.request['headers']['User-Agent'] = [user_agent] + + def build_request(self, url, method="GET", headers=None, body=None): + self.request['method'] = method + self.request['url'] = url + self.request['headers'] = headers if headers else {} + self.request['body'] = body + if self.randomizeUA: + self.randomize_useragent() + + self.report['request'] = self.request + self.report['url'] = url + return self.agent.request(self.request['method'], self.request['url'], + Headers(self.request['headers']), + self.request['body']) + diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py new file mode 100644 index 0000000..894a476 --- /dev/null +++ b/ooni/templates/scapyt.py @@ -0,0 +1,65 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +import random +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import protocol, defer + +from ooni.nettest import TestCase +from ooni.utils import log + +from ooni.lib.txscapy import txsr, txsend + +from scapy.all import * +class ScapyTest(TestCase): + """ + A utility class for writing scapy driven OONI tests. + + * pcapfile: specify where to store the logged pcapfile + + * timeout: timeout in ms of when we should stop waiting to receive packets + + * receive: if we should also receive packets and not just send + """ + + receive = True + timeout = None + pcapfile = 'scapytest.pcap' + input = IP()/TCP() + def setUp(self): + + if not self.reactor: + from twisted.internet import reactor + self.reactor = reactor + + self.request = {} + self.response = {} + + def test_sendReceive(self): + log.msg("Running send receive") + if self.receive: + log.msg("Sending and receiving packets.") + d = txsr(self.buildPackets(), pcapfile=self.pcapfile, + timeout=self.timeout) + else: + log.msg("Sending packets.") + d = txsend(self.buildPackets()) + + def finished(data): + log.msg("Finished sending") + return data + + d.addCallback(finished) + return d + + def buildPackets(self): + """ + Override this method to build scapy packets. + """ + return self.input + +

1 0

[ooni-probe/master] Minor refactor to move hacks into separate module
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit fbd9ecdbbe0c8463716c3f0d7a70648e068c3bc7 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 22:03:42 2012 +0000 Minor refactor to move hacks into separate module --- bin/ooniprobe | 3 +- nettests/example_scapyt.py | 2 +- ooni/utils/__init__.py | 53 -------------------------------------------- 3 files changed, 3 insertions(+), 55 deletions(-) diff --git a/bin/ooniprobe b/bin/ooniprobe index f25be7a..62d60bb 100755 --- a/bin/ooniprobe +++ b/bin/ooniprobe @@ -9,9 +9,10 @@ import os, sys +import copy_reg # This is a hack to overcome a bug in python -from ooni.utils import patched_reduce_ex +from ooni.utils.hacks import patched_reduce_ex copy_reg._reduce_ex = patched_reduce_ex sys.path[:] = map(os.path.abspath, sys.path) diff --git a/nettests/example_scapyt.py b/nettests/example_scapyt.py index ef61f3d..4c31567 100644 --- a/nettests/example_scapyt.py +++ b/nettests/example_scapyt.py @@ -6,4 +6,4 @@ from ooni.templates import scapyt from scapy.all import * class ExampleScapy(scapyt.ScapyTest): - inputs = [IP()/TCP()] + inputs = [IP(dst='8.8.8.8')/UDP(), IP()/TCP()] diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py index 5593027..38239ba 100644 --- a/ooni/utils/__init__.py +++ b/ooni/utils/__init__.py @@ -143,56 +143,3 @@ class Log(): except: raise StopIteration -import copy_reg - -def patched_reduce_ex(self, proto): - """ - This is a hack to overcome a bug in one of pythons core functions. It is - located inside of copy_reg and is called _reduce_ex. - - Some background on the issue can be found here: - http://stackoverflow.com/questions/569754/how-to-tell-for-which-object-attr… - http://stackoverflow.com/questions/2049849/why-cant-i-pickle-this-object - - There was also an open bug on the pyyaml trac repo, but it got closed because - they could not reproduce. - http://pyyaml.org/ticket/190 - - It turned out to be easier to patch the python core library than to monkey - patch yaml. - - XXX see if there is a better way. sigh... - """ - _HEAPTYPE = 1<<9 - assert proto < 2 - for base in self.__class__.__mro__: - if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE: - break - else: - base = object # not really reachable - if base is object: - state = None - elif base is int: - state = None - else: - if base is self.__class__: - raise TypeError, "can't pickle %s objects" % base.__name__ - state = base(self) - args = (self.__class__, base, state) - try: - getstate = self.__getstate__ - except AttributeError: - if getattr(self, "__slots__", None): - raise TypeError("a class that defines __slots__ without " - "defining __getstate__ cannot be pickled") - try: - dict = self.__dict__ - except AttributeError: - dict = None - else: - dict = getstate() - if dict: - return copy_reg._reconstructor, args, dict - else: - return copy_reg._reconstructor, args -

1 0

[ooni-probe/master] Clean up some code
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit f7ea3343c0e148c2fd180b746cb651b9f664687a Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 22:28:44 2012 +0000 Clean up some code * Fix a bug in txscapy --- nettests/example_httpt.py | 6 +++++- nettests/example_scapyt.py | 4 ++++ ooni/lib/txscapy.py | 7 ++++--- ooni/nettest.py | 2 +- ooni/runner.py | 8 ++++---- ooni/templates/httpt.py | 3 +++ ooni/templates/scapyt.py | 18 +++++++++++------- ooni/utils/log.py | 11 ++++++----- 8 files changed, 38 insertions(+), 21 deletions(-) diff --git a/nettests/example_httpt.py b/nettests/example_httpt.py index 1814a90..6501ca7 100644 --- a/nettests/example_httpt.py +++ b/nettests/example_httpt.py @@ -4,7 +4,11 @@ # :licence: see LICENSE from ooni.templates import httpt -class Example(httpt.HTTPTest): +class ExampleHTTP(httpt.HTTPTest): + name = "Example HTTP Test" + author = "Arturo Filastò" + version = 0.1 + inputs = ['http://google.com/', 'http://wikileaks.org/', 'http://torproject.org/'] diff --git a/nettests/example_scapyt.py b/nettests/example_scapyt.py index 4c31567..54fce9a 100644 --- a/nettests/example_scapyt.py +++ b/nettests/example_scapyt.py @@ -6,4 +6,8 @@ from ooni.templates import scapyt from scapy.all import * class ExampleScapy(scapyt.ScapyTest): + name = "Example Scapy Test" + author = "Arturo Filastò" + version = 0.1 + inputs = [IP(dst='8.8.8.8')/UDP(), IP()/TCP()] diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py index 4d83dce..f67daa3 100644 --- a/ooni/lib/txscapy.py +++ b/ooni/lib/txscapy.py @@ -90,8 +90,7 @@ class Scapy(object): recv = False def __init__(self, pkts=None, maxPacketSize=8192, reactor=None, filter=None, - iface=None, nofilter=None, pcapfile=None): - + iface=None, nofilter=None, pcapfile=None, *arg, **kw): if self.debug: log.startLogging(sys.stdout) @@ -201,7 +200,7 @@ class Scapy(object): @param pkt: the packet that has been received. """ - #pkt.show() + pkt.show() def doRead(self): @@ -210,6 +209,8 @@ class Scapy(object): received packet. """ pkt = self.socket.recv() + if not pkt: + return if self.pcapwriter and not self.write_only_answers: self.pcapwriter.write(pkt) self.processPacket(pkt) diff --git a/ooni/nettest.py b/ooni/nettest.py index c0c7406..403a022 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -69,7 +69,7 @@ class TestCase(unittest.TestCase): * version: is the version string of the test. """ - name = "IDidNotChangeTheName" + name = "I Did Not Change The Name" author = "John Doe <foo(a)example.com>" version = "0" diff --git a/ooni/runner.py b/ooni/runner.py index 1a9b4ad..13fdda2 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -6,7 +6,7 @@ import inspect from twisted.internet import defer, reactor from twisted.python import reflect, failure -from twisted.python import log +#from twisted.python import log from twisted.trial import unittest from twisted.trial.runner import TrialRunner, TestLoader from twisted.trial.runner import isPackage, isTestCase, ErrorHolder @@ -16,7 +16,7 @@ from ooni.reporter import ReporterFactory from ooni.input import InputUnitFactory from ooni.nettest import InputTestSuite from ooni import nettest -#from ooni.utils import log +from ooni.utils import log from ooni.plugoo import tests as oonitests def isTestCase(thing): @@ -183,8 +183,8 @@ class ORunner(object): result.done() def run(self): - log.startLogging(sys.stdout) - #log.start(True) + #log.startLogging(sys.stdout) + log.start(True) self.reporterFactory.writeHeader() for inputUnit in InputUnitFactory(self.inputs): diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 222e44c..117a77a 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -45,6 +45,9 @@ class HTTPTest(TestCase): processResponseHeader that are invoked once the headers have been received and once the request body has been received. """ + name = "HTTP Test" + version = 0.1 + randomizeUA = True followRedirects = False diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index 894a476..f762ae5 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -9,12 +9,13 @@ from twisted.python import usage from twisted.plugin import IPlugin from twisted.internet import protocol, defer +from scapy.all import * + from ooni.nettest import TestCase from ooni.utils import log from ooni.lib.txscapy import txsr, txsend -from scapy.all import * class ScapyTest(TestCase): """ A utility class for writing scapy driven OONI tests. @@ -25,29 +26,33 @@ class ScapyTest(TestCase): * receive: if we should also receive packets and not just send """ + name = "Scapy Test" + version = 0.1 receive = True - timeout = None + timeout = 1 pcapfile = 'scapytest.pcap' input = IP()/TCP() + reactor = None def setUp(self): - if not self.reactor: from twisted.internet import reactor self.reactor = reactor - self.request = {} self.response = {} + def tearDown(self): + self.reactor.stop() + def test_sendReceive(self): log.msg("Running send receive") if self.receive: log.msg("Sending and receiving packets.") d = txsr(self.buildPackets(), pcapfile=self.pcapfile, - timeout=self.timeout) + timeout=self.timeout, reactor=self.reactor) else: log.msg("Sending packets.") - d = txsend(self.buildPackets()) + d = txsend(self.buildPackets(), reactor=self.reactor) def finished(data): log.msg("Finished sending") @@ -62,4 +67,3 @@ class ScapyTest(TestCase): """ return self.input - diff --git a/ooni/utils/log.py b/ooni/utils/log.py index dd5cf13..f79f4b6 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -17,7 +17,7 @@ def _get_log_level(level): else: ve = "Unknown log level: %s\n" % level ve += "Allowed levels: %s\n" % [word for word in english] - + if type(level) is int: if 0 <= level <= 4: return level @@ -31,16 +31,16 @@ def _get_log_level(level): class OONITestFailure(Failure): """ - For handling Exceptions asynchronously. + For handling Exceptions asynchronously. Can be given an Exception as an argument, else will use the most recent Exception from the current stack frame. """ - def __init__(self, exception=None, _type=None, + def __init__(self, exception=None, _type=None, _traceback=None, _capture=False): Failure.__init__(self, exc_value=exception, exc_type=_type, exc_tb=_traceback, captureVars=_capture) - + class OONILogObserver(log.FileLogObserver): """ Supports logging level verbosity. @@ -66,7 +66,7 @@ class OONILogObserver(log.FileLogObserver): return timeStr = self.formatTime(eventDict['time']) - fmtDict = {'system': eventDict['system'], + fmtDict = {'system': eventDict['system'], 'text': text.replace('\n','\n\t')} msgStr = log._safeFormat("[%(system)s] %(text)s\n", fmtDict) @@ -91,6 +91,7 @@ def debug(message, level="debug", **kw): log.msg(message, logLevel=level, **kw) def msg(message, level="info", **kw): + print "Msg %s" % message log.msg(message, logLevel=level, **kw) def err(message, level="err", **kw):

1 0

[ooni-probe/master] Make SSL work in HTTPT
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit ae87c2b74844372586d9c863ab63bb5f5e6bec38 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 23:16:44 2012 +0000 Make SSL work in HTTPT * Fix misc bugs --- README.md | 10 +++++++++- bin/ooniprobe | 8 ++++---- docs/source/index.rst | 2 +- nettests/example_scapyt.py | 1 + ooni/templates/httpt.py | 19 +++++++++++++++++-- ooni/utils/log.py | 1 - 6 files changed, 32 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 23deb32..4837275 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,15 @@ To list the help for a specific test: virtualenv2 ENV/ source ENV/bin/activate - pip install twisted Scapy + pip install twisted Scapy pyyaml + +The setup.py script of pyOpenSSL has some issues and it is therefore +recommended to install it system wide with: + + sudo pip install pyOpenSSL + +Note: be sure to do this over a secure network since pip does not by default +use SSL... To install the most up to date scapy version (requires mercurial): diff --git a/bin/ooniprobe b/bin/ooniprobe index 62d60bb..c653756 100755 --- a/bin/ooniprobe +++ b/bin/ooniprobe @@ -11,13 +11,13 @@ import os, sys import copy_reg +# Hack to set the proper sys.path. Overcomes the export PYTHONPATH pain. +sys.path[:] = map(os.path.abspath, sys.path) +sys.path.insert(0, os.path.abspath(os.getcwd())) + # This is a hack to overcome a bug in python from ooni.utils.hacks import patched_reduce_ex copy_reg._reduce_ex = patched_reduce_ex -sys.path[:] = map(os.path.abspath, sys.path) - -sys.path.insert(0, os.path.abspath(os.getcwd())) - from ooni.oonicli import run run() diff --git a/docs/source/index.rst b/docs/source/index.rst index e744f7f..9de0dba 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -20,7 +20,7 @@ need to do the following:: virtualenv ENV source ENV/bin/activate - pip install twisted Scapy pyyaml + pip install twisted Scapy pyyaml pyOpenSSL To get the latest version of scapy you will need mercurial. You can then install it with:: diff --git a/nettests/example_scapyt.py b/nettests/example_scapyt.py index 54fce9a..2e73150 100644 --- a/nettests/example_scapyt.py +++ b/nettests/example_scapyt.py @@ -11,3 +11,4 @@ class ExampleScapy(scapyt.ScapyTest): version = 0.1 inputs = [IP(dst='8.8.8.8')/UDP(), IP()/TCP()] + diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 117a77a..51775f0 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -9,6 +9,8 @@ from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin from twisted.internet import protocol, defer +from twisted.internet.ssl import ClientContextFactory + from twisted.web.http_headers import Headers from ooni.nettest import TestCase @@ -26,6 +28,11 @@ useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Geck ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] + +class WebClientContextFactory(ClientContextFactory): + def getContext(self, hostname, port): + return ClientContextFactory.getContext(self) + class BodyReceiver(protocol.Protocol): def __init__(self, finished): self.finished = finished @@ -52,10 +59,16 @@ class HTTPTest(TestCase): followRedirects = False def setUp(self): + try: + import OpenSSL + except: + log.err("Warning! pyOpenSSL is not installed. https websites will" + "not work") from twisted.web.client import Agent from twisted.internet import reactor - import yaml + self.agent = Agent(reactor) + if self.followRedirects: from twisted.web.client import RedirectAgent self.agent = RedirectAgent(self.agent) @@ -96,6 +109,7 @@ class HTTPTest(TestCase): def doRequest(self, url): d = self.build_request(url) def finished(data): + #self.mainDefer.callback() return data d.addCallback(self._cbResponse) @@ -144,7 +158,8 @@ class HTTPTest(TestCase): self.report['request'] = self.request self.report['url'] = url - return self.agent.request(self.request['method'], self.request['url'], + req = self.agent.request(self.request['method'], self.request['url'], Headers(self.request['headers']), self.request['body']) + return req diff --git a/ooni/utils/log.py b/ooni/utils/log.py index f79f4b6..6ff6bf9 100644 --- a/ooni/utils/log.py +++ b/ooni/utils/log.py @@ -91,7 +91,6 @@ def debug(message, level="debug", **kw): log.msg(message, logLevel=level, **kw) def msg(message, level="info", **kw): - print "Msg %s" % message log.msg(message, logLevel=level, **kw) def err(message, level="err", **kw):

1 0

[ooni-probe/master] Add test to obtain the clients IP address via check.tpo
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit 26213446116ffef0ee8d528a1a58aab918b16aa7 Author: Arturo Filastò <arturo(a)filasto.net> Date: Fri Sep 28 23:17:22 2012 +0000 Add test to obtain the clients IP address via check.tpo --- nettests/myip.py | 18 ++++++++++++++++++ 1 files changed, 18 insertions(+), 0 deletions(-) diff --git a/nettests/myip.py b/nettests/myip.py new file mode 100644 index 0000000..e7e96f2 --- /dev/null +++ b/nettests/myip.py @@ -0,0 +1,18 @@ +# -*- encoding: utf-8 -*- +# +# :authors: Arturo Filastò +# :licence: see LICENSE + +from ooni.templates import httpt +class MyIP(httpt.HTTPTest): + inputs = ['https://check.torproject.org'] + def processResponseBody(self, body): + print "FOOOO" + import re + regexp = "Your IP address appears to be: <b>(.+?)<\/b>" + match = re.search(regexp, body) + try: + self.report['myip'] = match.group(1) + except: + self.report['myip'] = None +

1 0

[ooni-probe/master] Properly implement timeout in txscapy
by isis＠torproject.org 04 Oct '12

04 Oct '12

commit b267f3bd882fb5da6aa20c9a106ff3a10630e4f9 Author: Arturo Filastò <arturo(a)filasto.net> Date: Sat Sep 29 11:39:02 2012 +0000 Properly implement timeout in txscapy * Fix the example scapy test and scapy template --- nettests/example_scapyt.py | 3 ++- ooni/lib/txscapy.py | 35 ++++++++++++++++++++++++++++------- ooni/templates/scapyt.py | 2 +- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/nettests/example_scapyt.py b/nettests/example_scapyt.py index 2e73150..c7f342d 100644 --- a/nettests/example_scapyt.py +++ b/nettests/example_scapyt.py @@ -10,5 +10,6 @@ class ExampleScapy(scapyt.ScapyTest): author = "Arturo Filastò" version = 0.1 - inputs = [IP(dst='8.8.8.8')/UDP(), IP()/TCP()] + inputs = [IP(dst="8.8.8.8")/TCP(dport=31337), + IP(dst="ooni.nu")/TCP(dport=31337)] diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py index f67daa3..ec605e5 100644 --- a/ooni/lib/txscapy.py +++ b/ooni/lib/txscapy.py @@ -36,8 +36,8 @@ from scapy.all import RawPcapWriter, MTU, BasePacketList, conf class PcapWriter(RawPcapWriter): def __init__(self, filename, linktype=None, gz=False, endianness="", append=False, sync=False): - RawPcapWriter.__init__(self, filename, linktype=None, gz=False, - endianness="", append=False, sync=False) + RawPcapWriter.__init__(self, filename, linktype=linktype, gz=gz, + endianness=endianness, append=append, sync=sync) fdesc.setNonBlocking(self.f) def _write_header(self, pkt): @@ -90,7 +90,7 @@ class Scapy(object): recv = False def __init__(self, pkts=None, maxPacketSize=8192, reactor=None, filter=None, - iface=None, nofilter=None, pcapfile=None, *arg, **kw): + iface=None, nofilter=None, pcapfile=None, timeout=None, *arg, **kw): if self.debug: log.startLogging(sys.stdout) @@ -102,7 +102,11 @@ class Scapy(object): if pkts: self._buildPacketQueues(pkts) - self._buildSocket() + try: + self._buildSocket() + except Exception, e: + log.err("Unable to build socket. Are you root?") + sys.exit() self.cthreads = 0 self.mthreads = 80 @@ -122,6 +126,9 @@ class Scapy(object): if pcapfile: self.pcapwriter = PcapWriter(pcapfile) + if timeout and self.recv: + pass + def _buildSocket(self, filter=None, iface=None, nofilter=None): self.socket = ScapySocket(filter, iface, nofilter) if self.recv: @@ -213,6 +220,7 @@ class Scapy(object): return if self.pcapwriter and not self.write_only_answers: self.pcapwriter.write(pkt) + self.processPacket(pkt) h = pkt.hashret() @@ -251,7 +259,13 @@ class Scapy(object): """ self.socket.send(pkt) - def sr(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw): + def timeout(self, *arg, **kw): + if not self.done: + self._reactor.callLater(self.timeoutSeconds, self.timeout, None) + else: + self.deferred.callback(None) + + def sr(self, pkts, filter=None, iface=None, nofilter=0, timeout=None, *args, **kw): """ Wraps the scapy sr function. @@ -271,6 +285,9 @@ class Scapy(object): @param iface: listen answers only on the given interface """ self.recv = True + if timeout: + self.timeoutSeconds = timeout + self._reactor.callLater(timeout, self.timeout, None) self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter) def send(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw): @@ -338,12 +355,16 @@ class Scapy(object): self.threadpool.stop() self.running = False +(a)defer.inlineCallbacks def txsr(*args, **kw): tr = Scapy(*args, **kw) tr.sr(*args, **kw) - return tr.deferred + yield tr.deferred + tr.finalClose() +(a)defer.inlineCallbacks def txsend(*arg, **kw): tr = Scapy(*arg, **kw) tr.send(*arg, **kw) - return tr.deferred + yield tr.deferred + tr.finalClose() diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py index f762ae5..ac32ad8 100644 --- a/ooni/templates/scapyt.py +++ b/ooni/templates/scapyt.py @@ -31,7 +31,7 @@ class ScapyTest(TestCase): receive = True timeout = 1 - pcapfile = 'scapytest.pcap' + pcapfile = '/home/x/code/networking/ooni-probe/scapytest.pcap' input = IP()/TCP() reactor = None def setUp(self):

1 0