[tor-commits] [ooni-probe/master] Misc improvements
art at torproject.org
art at torproject.org
Tue Feb 21 18:22:07 UTC 2012
commit bfd84aa39d4323e4ee2262b8fef4caeab2ec14a0
Author: Arturo Filastò <hellais at gmail.com>
Date: Tue Feb 21 10:15:29 2012 -0800
Misc improvements
* Rename ooni-probe to ooniprobe
* Improve error collection
---
refactor/ooni-probe.conf | 1 +
refactor/ooni-probe.py | 142 ---------------------------------------------
refactor/ooniprobe.py | 142 +++++++++++++++++++++++++++++++++++++++++++++
refactor/plugoo.py | 9 +++-
refactor/tests/bridget.py | 32 +++++++----
refactor/utils.py | 21 ++++++-
6 files changed, 191 insertions(+), 156 deletions(-)
diff --git a/refactor/ooni-probe.conf b/refactor/ooni-probe.conf
index e74b241..8ea67e3 100644
--- a/refactor/ooni-probe.conf
+++ b/refactor/ooni-probe.conf
@@ -56,6 +56,7 @@ tor_bridges_timeout = 40
[report]
file = report.log
+timestamp = true
#ssh = 127.0.0.1:22
#ssh_user = theusername
#ssh_password = thepassword
diff --git a/refactor/ooni-probe.py b/refactor/ooni-probe.py
deleted file mode 100644
index 64d35f5..0000000
--- a/refactor/ooni-probe.py
+++ /dev/null
@@ -1,142 +0,0 @@
-# -*- coding: UTF-8
-#!/usr/bin/env python
-"""
- ooni-probe
- **********
-
- Open Observatory of Network Interference
-
- "The Net interprets censorship as damage and routes around it."
- - John Gilmore; TIME magazine (6 December 1993)
-
- The goal of ooni-probe is to collect data about censorship around
- the world.
-
- :copyright: (c) 2012 by Arturo Filastò
- :license: see LICENSE for more details.
-"""
-
-import imp
-import os
-import sys
-
-from pprint import pprint
-
-import plugoo
-from utils import Storage, parse_asset, import_test, get_logger
-from config import Config
-
-class ooni(object):
- """ooni-probe is a suite designed to run tests on your
- network to detect censorship.
- This is the main class that is used to start ooni probe
- select the assets and run tests.
- """
- def __init__(self):
- self.config = Storage()
- self.config.main = Config("main")
- self.config.tests = Config("tests")
- self.config.report = Config("report")
-
- self.logger = get_logger(self.config.main)
-
- self.logger.info("Started ooni-probe")
-
- self.assets = []
- self.get_assets()
-
- self.tests = Storage()
- self.load_tests()
-
- self.runtests = self.config.tests.run.split(",")
-
-
- def get_assets(self):
- """Parse all the assets in the asset directory.
- Assets can optionaly contain the ooni-probe asset file
- format: #:<something> <something_else>, that will then
- be used to render the asset details to the user.
- It is also possible to have an asset file link to multiple
- other files.
- """
- for root, dir, files in os.walk(self.config.main.assetdir):
- for name in files:
- asset = os.path.join(root, name)
- self.assets.append(parse_asset(asset))
-
- def list_assets(self):
- """Enumerate all the assets in the directory specified
- in the config file
- """
- print "[-] There are a total of %s assets loaded" % len(self.assets)
- for asset in self.assets:
- print " name: %s" % asset.name
- if asset.desc:
- print " description: %s" % asset.desc
- if asset.files:
- print " files: %s" % asset.files
- if asset.tests:
- print " tests: %s" % asset.tests
- print ""
-
- def load_tests(self):
- """Iterate through the plugoos insite the folder specified by the
- config file and instantiate them.
- """
- pluginfiles = [fname[:-3] for fname in os.listdir(self.config.main.testdir)\
- if fname.endswith(".py")]
- for fname in pluginfiles:
- test = Storage()
- test_name = fname
- if not self.config.main.testdir in sys.path:
- sys.path.insert(0, self.config.main.testdir)
-
- module = __import__(fname)
- try:
- test.name = module.__plugoo__
- test.desc = module.__desc__
- test.module = module
- except Exception, e:
- self.logger.warning("Soft fail %s", e)
- test.name = test_name
- test.desc = ""
- test.module = module
-
- try:
- self.tests[test_name] = test
- except Exception, e:
- print "Failed to load the test %s %s" % (name, e)
-
- def list_tests(self):
- """Print the loaded plugoonis to screen
- """
- print "[-] There are a total of %s tests available" % len(self.tests)
- for name, test in self.tests.items():
- print " name: %s" % name
- if test.name:
- print " long name: %s" % test.name
- if test.desc:
- print " description: %s" % test.desc
- print ""
-
-
- def run_tests(self):
- """Run all the tests that have been loaded
- """
- for name in self.runtests:
- print "running %s" % name
- try:
- self.tests[name].module.run(self)
- except Exception, e:
- print "ERR: %s" % e
-
- def run_test(self, test):
- """Run a single test
- """
- self.tests[test].module.run(self)
-
-if __name__ == "__main__":
- o = ooni()
- o.list_tests()
- o.run_test("bridget")
-
diff --git a/refactor/ooniprobe.py b/refactor/ooniprobe.py
new file mode 100644
index 0000000..64d35f5
--- /dev/null
+++ b/refactor/ooniprobe.py
@@ -0,0 +1,142 @@
+# -*- coding: UTF-8
+#!/usr/bin/env python
+"""
+ ooni-probe
+ **********
+
+ Open Observatory of Network Interference
+
+ "The Net interprets censorship as damage and routes around it."
+ - John Gilmore; TIME magazine (6 December 1993)
+
+ The goal of ooni-probe is to collect data about censorship around
+ the world.
+
+ :copyright: (c) 2012 by Arturo Filastò
+ :license: see LICENSE for more details.
+"""
+
+import imp
+import os
+import sys
+
+from pprint import pprint
+
+import plugoo
+from utils import Storage, parse_asset, import_test, get_logger
+from config import Config
+
+class ooni(object):
+ """ooni-probe is a suite designed to run tests on your
+ network to detect censorship.
+ This is the main class that is used to start ooni probe
+ select the assets and run tests.
+ """
+ def __init__(self):
+ self.config = Storage()
+ self.config.main = Config("main")
+ self.config.tests = Config("tests")
+ self.config.report = Config("report")
+
+ self.logger = get_logger(self.config.main)
+
+ self.logger.info("Started ooni-probe")
+
+ self.assets = []
+ self.get_assets()
+
+ self.tests = Storage()
+ self.load_tests()
+
+ self.runtests = self.config.tests.run.split(",")
+
+
+ def get_assets(self):
+ """Parse all the assets in the asset directory.
+ Assets can optionaly contain the ooni-probe asset file
+ format: #:<something> <something_else>, that will then
+ be used to render the asset details to the user.
+ It is also possible to have an asset file link to multiple
+ other files.
+ """
+ for root, dir, files in os.walk(self.config.main.assetdir):
+ for name in files:
+ asset = os.path.join(root, name)
+ self.assets.append(parse_asset(asset))
+
+ def list_assets(self):
+ """Enumerate all the assets in the directory specified
+ in the config file
+ """
+ print "[-] There are a total of %s assets loaded" % len(self.assets)
+ for asset in self.assets:
+ print " name: %s" % asset.name
+ if asset.desc:
+ print " description: %s" % asset.desc
+ if asset.files:
+ print " files: %s" % asset.files
+ if asset.tests:
+ print " tests: %s" % asset.tests
+ print ""
+
+ def load_tests(self):
+ """Iterate through the plugoos insite the folder specified by the
+ config file and instantiate them.
+ """
+ pluginfiles = [fname[:-3] for fname in os.listdir(self.config.main.testdir)\
+ if fname.endswith(".py")]
+ for fname in pluginfiles:
+ test = Storage()
+ test_name = fname
+ if not self.config.main.testdir in sys.path:
+ sys.path.insert(0, self.config.main.testdir)
+
+ module = __import__(fname)
+ try:
+ test.name = module.__plugoo__
+ test.desc = module.__desc__
+ test.module = module
+ except Exception, e:
+ self.logger.warning("Soft fail %s", e)
+ test.name = test_name
+ test.desc = ""
+ test.module = module
+
+ try:
+ self.tests[test_name] = test
+ except Exception, e:
+ print "Failed to load the test %s %s" % (name, e)
+
+ def list_tests(self):
+ """Print the loaded plugoonis to screen
+ """
+ print "[-] There are a total of %s tests available" % len(self.tests)
+ for name, test in self.tests.items():
+ print " name: %s" % name
+ if test.name:
+ print " long name: %s" % test.name
+ if test.desc:
+ print " description: %s" % test.desc
+ print ""
+
+
+ def run_tests(self):
+ """Run all the tests that have been loaded
+ """
+ for name in self.runtests:
+ print "running %s" % name
+ try:
+ self.tests[name].module.run(self)
+ except Exception, e:
+ print "ERR: %s" % e
+
+ def run_test(self, test):
+ """Run a single test
+ """
+ self.tests[test].module.run(self)
+
+if __name__ == "__main__":
+ o = ooni()
+ o.list_tests()
+ o.run_test("bridget")
+
diff --git a/refactor/plugoo.py b/refactor/plugoo.py
index 49b3e16..0ce1d28 100644
--- a/refactor/plugoo.py
+++ b/refactor/plugoo.py
@@ -12,7 +12,7 @@
"""
import os
-
+from datetime import datetime
import yaml
try:
@@ -95,6 +95,13 @@ class Report:
self.config = ooni.config.report
self.logger = ooni.logger
+ if self.config.timestamp:
+ tmp = self.file.split('.')
+ self.file = '.'.join(tmp[:-1]) + "-" + \
+ datetime.now().isoformat('-') + '.' + \
+ tmp[-1]
+ print self.file
+
try:
import paramiko
except:
diff --git a/refactor/tests/bridget.py b/refactor/tests/bridget.py
index 7c1ca8b..1e18175 100644
--- a/refactor/tests/bridget.py
+++ b/refactor/tests/bridget.py
@@ -82,6 +82,8 @@ class BridgeT(Plugoo):
datadir = "/tmp/" + randomname
if bridge.startswith("obfs://"):
obfsbridge = bridge.split("//")[1]
+
+ self.logger.debug("Genearting torrc file for obfs bridge")
torrc = """SocksPort %s
UseBridges 1
Bridge obfs2 %s
@@ -90,6 +92,7 @@ ClientTransportPlugin obfs2 exec /usr/local/bin/obfsproxy --managed
ControlPort %s
""" % (socksport, obfsbridge, datadir, controlport)
else:
+ self.logger.debug("Generating torrc file for bridge")
torrc = """SocksPort %s
UseBridges 1
bridge %s
@@ -97,7 +100,8 @@ DataDirectory %s
usemicrodescriptors 0
ControlPort %s
""" % (socksport, bridge, datadir, controlport)
- print torrc
+ #print torrc
+
with open(randomname, "wb") as f:
f.write(torrc)
@@ -117,17 +121,15 @@ ControlPort %s
#Can't use @torify as it doesn't support concurrency right now
def download_file(self, socksport):
- time_start=time.time()
-
opener = urllib2.build_opener(SocksiPyHandler(socks.PROXY_TYPE_SOCKS5,
'127.0.0.1', int(socksport)))
- f = opener.open('http://check.torproject.org')
- data= f.readlines()
- print data
- print len(data)
+
+ time_start=time.time()
+ f = opener.open('http://38.229.72.16/bwauth.torproject.org/256k')
+ f.read()
time_end = time.time()
print (time_end-time_start)
- return len(data)/(time_end-time_start)
+ return str(256/(time_end-time_start)) + " KB/s"
def connect(self, bridge, timeout=None):
if not timeout:
@@ -150,6 +152,7 @@ ControlPort %s
except:
self.logger.error("Unable to set file descriptor to non blocking")
+ self.logger.info("Testing bridge: %s" % bridge)
while True:
o = ""
try:
@@ -158,15 +161,17 @@ ControlPort %s
self.logger.debug(str(o))
if re.search("100%", o):
self.logger.info("Success in connecting to %s" % bridge)
+
print "%s bridge works" % bridge
- print "%s controlport" % controlport
+ # print "%s controlport" % controlport
try:
c = TorCtl.connect('127.0.0.1', controlport)
bridgeinfo = self.parsebridgeinfo(c.get_info('dir/server/all')['dir/server/all'])
- bandwidth=self.download_file(socksport)
+ bandwidth = self.download_file(socksport)
except:
self.logger.error("Error in connecting to Tor Control port")
- print bandwidth
+
+ self.logger.info("Bandwidth: %s" % bandwidth)
c.close()
p.stdout.close()
os.unlink(os.path.join(os.getcwd(), torrc))
@@ -182,6 +187,7 @@ ControlPort %s
if re.search("%", o):
# Keep updating the timeout if there is progress
+ self.logger.debug("Updating time...")
tupdate = time.time()
#print o
continue
@@ -189,13 +195,17 @@ ControlPort %s
except IOError:
ex = sys.exc_info()[1]
if ex[0] != errno.EAGAIN:
+ self.logger.error("Error IOError: EAGAIN")
raise
sys.exc_clear()
+
try:
# Set the timeout for the socket wait
ct = timeout-(time.time() - tupdate)
socket.wait_read(p.stdout.fileno(), timeout=ct)
+
except:
+ self.logger.info("%s bridge does not work (%s s timeout)" % (bridge, timeout))
print "%s bridge does not work (%s s timeout)" % (bridge, timeout)
self.failures.append(bridge)
p.stdout.close()
diff --git a/refactor/utils.py b/refactor/utils.py
index 5c743ba..93bd2fd 100644
--- a/refactor/utils.py
+++ b/refactor/utils.py
@@ -63,7 +63,7 @@ def get_logger(config):
console.setFormatter(formatter)
logging.getLogger('').addHandler(console)
- return logging.getLogger('ooni-probe')
+ return logging.getLogger('ooniprobe')
def parse_asset(asset):
parsed = Storage()
@@ -80,7 +80,7 @@ def parse_asset(asset):
parsed[n] = v.split(",")
else:
parsed[n] = v
-
+
elif line.startswith("#"):
continue
else:
@@ -113,3 +113,20 @@ def import_test(name, config):
return None, None
+class Log():
+ def __init__(self, file=None):
+ if file:
+ self.fh = open(file)
+
+ def __iter__(self):
+ return self
+
+ def next(self):
+ try:
+ line = self.fh.readline()
+ while not line.startswith("---"):
+ line = self.fh.readline()
+ except:
+ raise StopIteration
+
+
More information about the tor-commits
mailing list