commit bfd84aa39d4323e4ee2262b8fef4caeab2ec14a0 Author: Arturo Filastò hellais@gmail.com Date: Tue Feb 21 10:15:29 2012 -0800

Misc improvements

* Rename ooni-probe to ooniprobe * Improve error collection --- refactor/ooni-probe.conf | 1 + refactor/ooni-probe.py | 142 --------------------------------------------- refactor/ooniprobe.py | 142 +++++++++++++++++++++++++++++++++++++++++++++ refactor/plugoo.py | 9 +++- refactor/tests/bridget.py | 32 +++++++---- refactor/utils.py | 21 ++++++- 6 files changed, 191 insertions(+), 156 deletions(-)

diff --git a/refactor/ooni-probe.conf b/refactor/ooni-probe.conf index e74b241..8ea67e3 100644 --- a/refactor/ooni-probe.conf +++ b/refactor/ooni-probe.conf @@ -56,6 +56,7 @@ tor_bridges_timeout = 40

[report] file = report.log +timestamp = true #ssh = 127.0.0.1:22 #ssh_user = theusername #ssh_password = thepassword diff --git a/refactor/ooni-probe.py b/refactor/ooni-probe.py deleted file mode 100644 index 64d35f5..0000000 --- a/refactor/ooni-probe.py +++ /dev/null @@ -1,142 +0,0 @@ -# -*- coding: UTF-8 -#!/usr/bin/env python -""" - ooni-probe - ********** - - Open Observatory of Network Interference - - "The Net interprets censorship as damage and routes around it." - - John Gilmore; TIME magazine (6 December 1993) - - The goal of ooni-probe is to collect data about censorship around - the world. - - :copyright: (c) 2012 by Arturo Filastò - :license: see LICENSE for more details. -""" - -import imp -import os -import sys - -from pprint import pprint - -import plugoo -from utils import Storage, parse_asset, import_test, get_logger -from config import Config - -class ooni(object): - """ooni-probe is a suite designed to run tests on your - network to detect censorship. - This is the main class that is used to start ooni probe - select the assets and run tests. - """ - def __init__(self): - self.config = Storage() - self.config.main = Config("main") - self.config.tests = Config("tests") - self.config.report = Config("report") - - self.logger = get_logger(self.config.main) - - self.logger.info("Started ooni-probe") - - self.assets = [] - self.get_assets() - - self.tests = Storage() - self.load_tests() - - self.runtests = self.config.tests.run.split(",") - - - def get_assets(self): - """Parse all the assets in the asset directory. - Assets can optionaly contain the ooni-probe asset file - format: #:<something> <something_else>, that will then - be used to render the asset details to the user. - It is also possible to have an asset file link to multiple - other files. - """ - for root, dir, files in os.walk(self.config.main.assetdir): - for name in files: - asset = os.path.join(root, name) - self.assets.append(parse_asset(asset)) - - def list_assets(self): - """Enumerate all the assets in the directory specified - in the config file - """ - print "[-] There are a total of %s assets loaded" % len(self.assets) - for asset in self.assets: - print " name: %s" % asset.name - if asset.desc: - print " description: %s" % asset.desc - if asset.files: - print " files: %s" % asset.files - if asset.tests: - print " tests: %s" % asset.tests - print "" - - def load_tests(self): - """Iterate through the plugoos insite the folder specified by the - config file and instantiate them. - """ - pluginfiles = [fname[:-3] for fname in os.listdir(self.config.main.testdir)\ - if fname.endswith(".py")] - for fname in pluginfiles: - test = Storage() - test_name = fname - if not self.config.main.testdir in sys.path: - sys.path.insert(0, self.config.main.testdir) - - module = __import__(fname) - try: - test.name = module.__plugoo__ - test.desc = module.__desc__ - test.module = module - except Exception, e: - self.logger.warning("Soft fail %s", e) - test.name = test_name - test.desc = "" - test.module = module - - try: - self.tests[test_name] = test - except Exception, e: - print "Failed to load the test %s %s" % (name, e) - - def list_tests(self): - """Print the loaded plugoonis to screen - """ - print "[-] There are a total of %s tests available" % len(self.tests) - for name, test in self.tests.items(): - print " name: %s" % name - if test.name: - print " long name: %s" % test.name - if test.desc: - print " description: %s" % test.desc - print "" - - - def run_tests(self): - """Run all the tests that have been loaded - """ - for name in self.runtests: - print "running %s" % name - try: - self.tests[name].module.run(self) - except Exception, e: - print "ERR: %s" % e - - def run_test(self, test): - """Run a single test - """ - self.tests[test].module.run(self) - -if __name__ == "__main__": - o = ooni() - o.list_tests() - o.run_test("bridget") - diff --git a/refactor/ooniprobe.py b/refactor/ooniprobe.py new file mode 100644 index 0000000..64d35f5 --- /dev/null +++ b/refactor/ooniprobe.py @@ -0,0 +1,142 @@ +# -*- coding: UTF-8 +#!/usr/bin/env python +""" + ooni-probe + ********** + + Open Observatory of Network Interference + + "The Net interprets censorship as damage and routes around it." + - John Gilmore; TIME magazine (6 December 1993) + + The goal of ooni-probe is to collect data about censorship around + the world. + + :copyright: (c) 2012 by Arturo Filastò + :license: see LICENSE for more details. +""" + +import imp +import os +import sys + +from pprint import pprint + +import plugoo +from utils import Storage, parse_asset, import_test, get_logger +from config import Config + +class ooni(object): + """ooni-probe is a suite designed to run tests on your + network to detect censorship. + This is the main class that is used to start ooni probe + select the assets and run tests. + """ + def __init__(self): + self.config = Storage() + self.config.main = Config("main") + self.config.tests = Config("tests") + self.config.report = Config("report") + + self.logger = get_logger(self.config.main) + + self.logger.info("Started ooni-probe") + + self.assets = [] + self.get_assets() + + self.tests = Storage() + self.load_tests() + + self.runtests = self.config.tests.run.split(",") + + + def get_assets(self): + """Parse all the assets in the asset directory. + Assets can optionaly contain the ooni-probe asset file + format: #:<something> <something_else>, that will then + be used to render the asset details to the user. + It is also possible to have an asset file link to multiple + other files. + """ + for root, dir, files in os.walk(self.config.main.assetdir): + for name in files: + asset = os.path.join(root, name) + self.assets.append(parse_asset(asset)) + + def list_assets(self): + """Enumerate all the assets in the directory specified + in the config file + """ + print "[-] There are a total of %s assets loaded" % len(self.assets) + for asset in self.assets: + print " name: %s" % asset.name + if asset.desc: + print " description: %s" % asset.desc + if asset.files: + print " files: %s" % asset.files + if asset.tests: + print " tests: %s" % asset.tests + print "" + + def load_tests(self): + """Iterate through the plugoos insite the folder specified by the + config file and instantiate them. + """ + pluginfiles = [fname[:-3] for fname in os.listdir(self.config.main.testdir)\ + if fname.endswith(".py")] + for fname in pluginfiles: + test = Storage() + test_name = fname + if not self.config.main.testdir in sys.path: + sys.path.insert(0, self.config.main.testdir) + + module = __import__(fname) + try: + test.name = module.__plugoo__ + test.desc = module.__desc__ + test.module = module + except Exception, e: + self.logger.warning("Soft fail %s", e) + test.name = test_name + test.desc = "" + test.module = module + + try: + self.tests[test_name] = test + except Exception, e: + print "Failed to load the test %s %s" % (name, e) + + def list_tests(self): + """Print the loaded plugoonis to screen + """ + print "[-] There are a total of %s tests available" % len(self.tests) + for name, test in self.tests.items(): + print " name: %s" % name + if test.name: + print " long name: %s" % test.name + if test.desc: + print " description: %s" % test.desc + print "" + + + def run_tests(self): + """Run all the tests that have been loaded + """ + for name in self.runtests: + print "running %s" % name + try: + self.tests[name].module.run(self) + except Exception, e: + print "ERR: %s" % e + + def run_test(self, test): + """Run a single test + """ + self.tests[test].module.run(self) + +if __name__ == "__main__": + o = ooni() + o.list_tests() + o.run_test("bridget") + diff --git a/refactor/plugoo.py b/refactor/plugoo.py index 49b3e16..0ce1d28 100644 --- a/refactor/plugoo.py +++ b/refactor/plugoo.py @@ -12,7 +12,7 @@ """

import os - +from datetime import datetime import yaml

try: @@ -95,6 +95,13 @@ class Report: self.config = ooni.config.report self.logger = ooni.logger

+ if self.config.timestamp: + tmp = self.file.split('.') + self.file = '.'.join(tmp[:-1]) + "-" + \ + datetime.now().isoformat('-') + '.' + \ + tmp[-1] + print self.file + try: import paramiko except: diff --git a/refactor/tests/bridget.py b/refactor/tests/bridget.py index 7c1ca8b..1e18175 100644 --- a/refactor/tests/bridget.py +++ b/refactor/tests/bridget.py @@ -82,6 +82,8 @@ class BridgeT(Plugoo): datadir = "/tmp/" + randomname if bridge.startswith("obfs://"): obfsbridge = bridge.split("//")[1] + + self.logger.debug("Genearting torrc file for obfs bridge") torrc = """SocksPort %s UseBridges 1 Bridge obfs2 %s @@ -90,6 +92,7 @@ ClientTransportPlugin obfs2 exec /usr/local/bin/obfsproxy --managed ControlPort %s """ % (socksport, obfsbridge, datadir, controlport) else: + self.logger.debug("Generating torrc file for bridge") torrc = """SocksPort %s UseBridges 1 bridge %s @@ -97,7 +100,8 @@ DataDirectory %s usemicrodescriptors 0 ControlPort %s """ % (socksport, bridge, datadir, controlport) - print torrc + #print torrc +

with open(randomname, "wb") as f: f.write(torrc) @@ -117,17 +121,15 @@ ControlPort %s

#Can't use @torify as it doesn't support concurrency right now def download_file(self, socksport): - time_start=time.time() - opener = urllib2.build_opener(SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, '127.0.0.1', int(socksport))) - f = opener.open('http://check.torproject.org') - data= f.readlines() - print data - print len(data) + + time_start=time.time() + f = opener.open('http://38.229.72.16/bwauth.torproject.org/256k') + f.read() time_end = time.time() print (time_end-time_start) - return len(data)/(time_end-time_start) + return str(256/(time_end-time_start)) + " KB/s"

def connect(self, bridge, timeout=None): if not timeout: @@ -150,6 +152,7 @@ ControlPort %s except: self.logger.error("Unable to set file descriptor to non blocking")

+ self.logger.info("Testing bridge: %s" % bridge) while True: o = "" try: @@ -158,15 +161,17 @@ ControlPort %s self.logger.debug(str(o)) if re.search("100%", o): self.logger.info("Success in connecting to %s" % bridge) + print "%s bridge works" % bridge - print "%s controlport" % controlport + # print "%s controlport" % controlport try: c = TorCtl.connect('127.0.0.1', controlport) bridgeinfo = self.parsebridgeinfo(c.get_info('dir/server/all')['dir/server/all']) - bandwidth=self.download_file(socksport) + bandwidth = self.download_file(socksport) except: self.logger.error("Error in connecting to Tor Control port") - print bandwidth + + self.logger.info("Bandwidth: %s" % bandwidth) c.close() p.stdout.close() os.unlink(os.path.join(os.getcwd(), torrc)) @@ -182,6 +187,7 @@ ControlPort %s

if re.search("%", o): # Keep updating the timeout if there is progress + self.logger.debug("Updating time...") tupdate = time.time() #print o continue @@ -189,13 +195,17 @@ ControlPort %s except IOError: ex = sys.exc_info()[1] if ex[0] != errno.EAGAIN: + self.logger.error("Error IOError: EAGAIN") raise sys.exc_clear() + try: # Set the timeout for the socket wait ct = timeout-(time.time() - tupdate) socket.wait_read(p.stdout.fileno(), timeout=ct) + except: + self.logger.info("%s bridge does not work (%s s timeout)" % (bridge, timeout)) print "%s bridge does not work (%s s timeout)" % (bridge, timeout) self.failures.append(bridge) p.stdout.close() diff --git a/refactor/utils.py b/refactor/utils.py index 5c743ba..93bd2fd 100644 --- a/refactor/utils.py +++ b/refactor/utils.py @@ -63,7 +63,7 @@ def get_logger(config): console.setFormatter(formatter) logging.getLogger('').addHandler(console)

- return logging.getLogger('ooni-probe') + return logging.getLogger('ooniprobe')

def parse_asset(asset): parsed = Storage() @@ -80,7 +80,7 @@ def parse_asset(asset): parsed[n] = v.split(",") else: parsed[n] = v - + elif line.startswith("#"): continue else: @@ -113,3 +113,20 @@ def import_test(name, config):

return None, None

+class Log(): + def __init__(self, file=None): + if file: + self.fh = open(file) + + def __iter__(self): + return self + + def next(self): + try: + line = self.fh.readline() + while not line.startswith("---"): + line = self.fh.readline() + except: + raise StopIteration + +