commit a129f69242588fdc679eb33cfcc38288b09d07e5 Author: Arturo Filastò <art@torproject.org> Date: Tue Jul 10 10:07:47 2012 +0200 Code refactoring, restructure directory tree. --- ooni/config.py | 53 --------- ooni/date.py | 22 ---- ooni/example_plugins/examplescapy.py | 2 +- ooni/example_plugins/skel.py | 2 +- ooni/log.py | 43 ------- ooni/logo.py | 214 ---------------------------------- ooni/ooniprobe.py | 11 +- ooni/plugins/bridget.py | 2 +- ooni/plugins/captiveportal.py | 2 +- ooni/plugins/chinatrigger.py | 2 +- ooni/plugins/httpt.py | 2 +- ooni/plugins/tcpconnect.py | 2 +- ooni/plugoo/nodes.py | 2 +- ooni/plugoo/reports.py | 2 +- ooni/plugoo/tests.py | 4 +- ooni/protocols/http.py | 2 +- ooni/protocols/scapy.py | 2 +- ooni/scaffolding.py | 4 +- ooni/utils.py | 146 ----------------------- ooni/utils/__init__.py | 146 +++++++++++++++++++++++ ooni/utils/config.py | 53 +++++++++ ooni/utils/date.py | 22 ++++ ooni/utils/log.py | 43 +++++++ ooni/utils/logo.py | 214 ++++++++++++++++++++++++++++++++++ 24 files changed, 499 insertions(+), 498 deletions(-) diff --git a/ooni/config.py b/ooni/config.py deleted file mode 100644 index 42a14f6..0000000 --- a/ooni/config.py +++ /dev/null @@ -1,53 +0,0 @@ -import ConfigParser -from utils import Storage - -class Config(Storage): - """ - A Storage-like class which loads and store each attribute into a portable - conf file. - """ - def __init__(self, section, cfgfile="ooni-probe.conf"): - super(Config, self).__init__() - - self._cfgfile = cfgfile - # setting up confgiparser - self._cfgparser = ConfigParser.ConfigParser() - self._cfgparser.read([self._cfgfile]) - self._section = section - - def __getattr__(self, name): - if name.startswith('_'): - return self.__dict__.get(name, None) - - try: - value = self._cfgparser.get(self._section, name) - if value.isdigit(): - return int(value) - elif value.lower() in ('true', 'false'): - return value.lower() == 'true' - else: - return value - except ConfigParser.NoOptionError: - return '' # if option doesn't exists return an empty string - - def __setattr__(self, name, value): - # keep an open port with private attributes - if name.startswith('_'): - self.__dict__[name] = value - return - - try: - # XXX: Automagically discover variable type - self._cfgparser.set(self._section, name, value) - except ConfigParser.NoOptionError: - raise NameError(name) - - def commit(self): - """ - Commit changes in config file. - """ - cfgfile = open(self._cfgfile, 'w') - try: - self._cfgparser.write(cfgfile) - finally: - cfgfile.close() diff --git a/ooni/date.py b/ooni/date.py deleted file mode 100644 index 59ec1f8..0000000 --- a/ooni/date.py +++ /dev/null @@ -1,22 +0,0 @@ -from ooni.lib.rfc3339 import rfc3339 -from datetime import datetime - -class odate(datetime): - def __str__(self): - return rfc3339(self) - - def __repr__(self): - return "'%s'" % rfc3339(self) - - def from_rfc(self, datestr): - pass - -def now(): - return odate.utcnow() - -def pretty_date(): - cur_time = datetime.utcnow() - d_format = "%d %B %Y %H:%M:%S" - pretty = cur_time.strftime(d_format) - return pretty - diff --git a/ooni/example_plugins/examplescapy.py b/ooni/example_plugins/examplescapy.py index aa6d81b..144afee 100644 --- a/ooni/example_plugins/examplescapy.py +++ b/ooni/example_plugins/examplescapy.py @@ -5,7 +5,7 @@ from twisted.plugin import IPlugin from twisted.internet import protocol, defer from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset -from ooni import log +from ooni.utils import log from ooni.protocols.scapy import ScapyTest from ooni.lib.txscapy import txsr, txsend diff --git a/ooni/example_plugins/skel.py b/ooni/example_plugins/skel.py index de95b48..52c7d4f 100644 --- a/ooni/example_plugins/skel.py +++ b/ooni/example_plugins/skel.py @@ -2,7 +2,7 @@ from zope.interface import implements from twisted.python import usage from twisted.plugin import IPlugin from plugoo.tests import ITest, OONITest -from ooni import log +from ooni.utils import log class SkelArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], diff --git a/ooni/log.py b/ooni/log.py deleted file mode 100644 index 3dc52dd..0000000 --- a/ooni/log.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -OONI logging facility. -""" -import sys -import logging -import warnings - -from twisted.python import log - -# Logging levels -DEBUG = logging.DEBUG -INFO = logging.INFO -WARNING = logging.WARNING -ERROR = logging.ERROR -CRITICAL = logging.CRITICAL - -def _get_log_level(level): - if not level: - return INFO - else: - return level - -def start(logfile=None, loglevel=None, logstdout=True): - if log.defaultObserver: - print "%s" % logstdout - loglevel = _get_log_level(loglevel) - file = open(logfile, 'a') if logfile else sys.stderr - observer = log.FileLogObserver(file) - if logstdout: - log.startLogging(sys.stdout) - else: - log.startLogging() - log.addObserver(observer.emit) - msg("Started OONI") - -def msg(message, level=INFO, **kw): - log.msg(message, logLevel=level, **kw) - -def err(message, **kw): - log.err(message, **kw) - -def debug(message, **kw): - log.msg(message, logLevel=DEBUG, **kw) diff --git a/ooni/logo.py b/ooni/logo.py deleted file mode 100644 index 8924462..0000000 --- a/ooni/logo.py +++ /dev/null @@ -1,214 +0,0 @@ -import random -def getlogo(): - logo = [] - logo.append(""" - _____ _____ __ _ _____ _____ ______ _____ ______ _______ - | | | | | \ | | |_____] |_____/ | | |_____] |______ - |_____| |_____| | \_| __|__ | | \_ |_____| |_____] |______""") - - logo.append(""" - _ _ - (_) | | - ___ ___ _ __ _ _ __ _ __ ___ | |__ ___ - / _ \ / _ \| '_ \| | '_ \| '__/ _ \| '_ \ / _ \\ -| (_) | (_) | | | | | |_) | | | (_) | |_) | __/ - \___/ \___/|_| |_|_| .__/|_| \___/|_.__/ \___| - | | - |_|""") - - - logo.append(""" - .__ ___. - ____ ____ ____ |__|_____________ ____\_ |__ ____ - / _ \ / _ \ / \| |\____ \_ __ \/ _ \| __ \_/ __ \\ -( <_> | <_> ) | \ || |_> > | \( <_> ) \_\ \ ___/ - \____/ \____/|___| /__|| __/|__| \____/|___ /\___ > - \/ |__| \/ \/""") - - - logo.append(""" - __ __ __ __ __ __ ___ -/ \ / \ |\ | | |__) |__) / \ |__) |__ -\__/ \__/ | \| | | | \ \__/ |__) |___ """) - - - logo.append(""" - _ _ ._ o ._ ._ _ |_ _ - (_) (_) | | | |_) | (_) |_) (/_ - | -""") - - logo.append(""" - _______ _______ __ _ ___ _______ ______ _______ _______ _______ -| || || | | || | | || _ | | || _ || | -| _ || _ || |_| || | | _ || | || | _ || |_| || ___| -| | | || | | || || | | |_| || |_|| | | | || || |___ -| |_| || |_| || _ || | | ___|| __ || |_| || _ | | ___| -| || || | | || | | | | | ||| || |_| || |___ -|_______||_______||_| |__||___| |___| |___| |||_______||_______||_______| -""") - - - logo.append(""" - _ _ - ___ ___ _ __ (_)_ __ _ __ ___ | |__ ___ - / _ \ / _ \| '_ \| | '_ \| '__/ _ \| '_ \ / _ \\ -| (_) | (_) | | | | | |_) | | | (_) | |_) | __/ - \___/ \___/|_| |_|_| .__/|_| \___/|_.__/ \___| - |_| -""") - - logo.append(""" - o - o O - O - o -.oOo. .oOo. 'OoOo. O .oOo. `OoOo. .oOo. OoOo. .oOo. -O o O o o O o O o o O o O o OooO' -o O o O O o O o O O o O o O O -`OoO' `OoO' o O o' oOoO' o `OoO' `OoO' `OoO' - O - o' -""") - - - logo.append(""" - _____ _____ ____ _ ____ _____ _____ _____ ______ ______ - / \/ \| \ | || || || | / \| >| ___| - | || || \| || || _|| \ | || < | ___| - \_____/\_____/|__/\____||____||___| |__|\__\\\\_____/|______>|______| -""") - - logo.append(""" - _ _ - ___ ___ ___|_|___ ___ ___| |_ ___ -| . | . | | | . | _| . | . | -_| -|___|___|_|_|_| _|_| |___|___|___| - |_| -""") - - logo.append(""" - _ _ - (_) | | - ___ ___ ____ _ ____ ____ ___ | |__ _____ - / _ \ / _ \| _ \| | _ \ / ___) _ \| _ \| ___ | -| |_| | |_| | | | | | |_| | | | |_| | |_) ) ____| - \___/ \___/|_| |_|_| __/|_| \___/|____/|_____) - |_| -""") - logo.append(""" - _ __ - ____ ____ ____ (_)____ _________ / /_ ___ - / __ \/ __ \/ __ \/ // __ \/ ___/ __ \/ __ \/ _ \\ -/ /_/ / /_/ / / / / // /_/ / / / /_/ / /_/ / __/ -\____/\____/_/ /_/_// .___/_/ \____/_.___/\___/ - /_/ -""") - logo.append(""" - _) | - _ \ _ \ \ | _ \ _| _ \ _ \ -_) - \___/ \___/ _| _| _| .__/ _| \___/ _.__/ \___| - _| -""") - logo.append(""" - _ _ - ___ ___ _ __ (_)_ __ _ __ ___ | |__ ___ - / _ \ / _ \| '_ \| | '_ \| '__/ _ \| '_ \ / _ \\ - | (_) | (_) | | | | | |_) | | | (_) | |_) | __/ - \___/ \___/|_| |_|_| .__/|_| \___/|_.__/ \___| - |_| -""") - logo.append(""" - .-. ( ) - .--. .--. ___ .-. ( __) .-.. ___ .-. .--. | |.-. .--. - / \ / \ ( ) \ (''") / \ ( ) \ / \ | / \ / \\ -| .-. ; | .-. ; | .-. . | | ' .-, ; | ' .-. ; | .-. ; | .-. | | .-. ; -| | | | | | | | | | | | | | | | . | | / (___) | | | | | | | | | | | | -| | | | | | | | | | | | | | | | | | | | | | | | | | | | | |/ | -| | | | | | | | | | | | | | | | | | | | | | | | | | | | | ' _.' -| ' | | | ' | | | | | | | | | | ' | | | | ' | | | ' | | | .'.-. -' `-' / ' `-' / | | | | | | | `-' ' | | ' `-' / ' `-' ; ' `-' / - `.__.' `.__.' (___)(___) (___) | \__.' (___) `.__.' `.__. `.__.' - | | - (___) -""") - logo.append(""" - o | -,---.,---.,---..,---.,---.,---.|---.,---. -| || || ||| || | || ||---' -`---'`---'` '`|---'` `---'`---'`---' - | -""") - logo.append(""" - ________________________________________ -/ OONI-PROBE -- WE FIGHT CENSORSHIP LIKE \\ -\ DRAGONS EAT KNIGHTS! / - ---------------------------------------- - \ ^ /^ - \ / \ // \\ - \ |\___/| / \// .\\ - \ /O O \__ / // | \ \ *----* - / / \/_/ // | \ \ \ | - @___@` \/_ // | \ \ \/\ \\ - 0/0/| \/_ // | \ \ \ \\ - 0/0/0/0/| \/// | \ \ | | - 0/0/0/0/0/_|_ / ( // | \ _\ | / - 0/0/0/0/0/0/`/,_ _ _/ ) ; -. | _ _\.-~ / / - ,-} _ *-.|.-~-. .~ ~ - \ \__/ `/\ / ~-. _ .-~ / - \____(oo) *. } { / - ( (--) .----~-.\ \-` .~ - //__\\\\ \__ Ack! ///.----..< \ _ -~ - // \\\\ ///-._ _ _ _ _ _ _{^ - - - - ~ -""") - logo.append(""" - _________________ -| OONI-PROBE | -| DON'T CENSOR ME | -|_________________| -\ . . - \ / `. .' " - \ .---. < > < > .---. - \ | \ \ - ~ ~ - / / | - _____ ..-~ ~-..-~ - | (A)| \~~~\.' `./~~~/ - --------- \__/ \__/ - .' O \ / / \ " - (_____, `._.' | } \/~~~/ - `----. / } | / \__/ - `-. | / | / `. ,~~| - ~-.__| /_ - ~ ^| /- _ `..-' - | / | / ~-. `-. _ _ _ - |_____| |_____| ~ - . _ _ _ _ _> -""") - logo.append(""" - _________________ - ,/~~, | | - /`` \ /| OONI-PROBE... | - `( O^O ) / | | - `\_-_/` | ...DOES THIS | - ~~~ | EXCITE YOU? | - __| |__ |_________________| - | | - | af | - | | | | - | | | \\ - | | |\ \\ - | | | \ \\ - | | | \ \\ - | |___| ((( - (((|/*\| - | | | - | | | - | | | - | | | - | | | - | | | - | | | - | | | - | | | - | | | - _| | |_ - (___|___) -""") - return random.choice(logo) diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index 22299ac..261eb74 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -17,19 +17,20 @@ # import sys +from pprint import pprint from twisted.python import usage -from twisted.plugin import getPlugins from twisted.internet import reactor +from twisted.plugin import getPlugins +from zope.interface.verify import verifyObject from zope.interface.exceptions import BrokenImplementation from zope.interface.exceptions import BrokenMethodImplementation -from zope.interface.verify import verifyObject -from pprint import pprint from ooni.plugoo import tests, work, assets, reports -from ooni.logo import getlogo -from ooni import plugins, log +from ooni.utils.logo import getlogo +from ooni.utils import log +from ooni import plugins __version__ = "0.0.1-prealpha" diff --git a/ooni/plugins/bridget.py b/ooni/plugins/bridget.py index 0614d5d..fa3cb67 100644 --- a/ooni/plugins/bridget.py +++ b/ooni/plugins/bridget.py @@ -8,7 +8,7 @@ from twisted.python import usage from twisted.plugin import IPlugin from twisted.internet import reactor -from ooni import log +from ooni.utils import log from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset diff --git a/ooni/plugins/captiveportal.py b/ooni/plugins/captiveportal.py index 4ed00b3..b48f70b 100644 --- a/ooni/plugins/captiveportal.py +++ b/ooni/plugins/captiveportal.py @@ -25,7 +25,7 @@ from twisted.plugin import IPlugin from ooni.plugoo.assets import Asset from ooni.plugoo.tests import ITest, OONITest from ooni.protocols import http -from ooni import log +from ooni.utils import log try: from dns import resolver diff --git a/ooni/plugins/chinatrigger.py b/ooni/plugins/chinatrigger.py index 4f2dc8c..538ef0b 100644 --- a/ooni/plugins/chinatrigger.py +++ b/ooni/plugins/chinatrigger.py @@ -9,7 +9,7 @@ from twisted.plugin import IPlugin from twisted.internet import protocol, defer from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset -from ooni import log +from ooni.utils import log from ooni.protocols.scapy import ScapyTest from ooni.lib.txscapy import txsr, txsend diff --git a/ooni/plugins/httpt.py b/ooni/plugins/httpt.py index d0ede0f..4113aef 100644 --- a/ooni/plugins/httpt.py +++ b/ooni/plugins/httpt.py @@ -9,7 +9,7 @@ from twisted.plugin import IPlugin from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset from ooni.protocols import http -from ooni import log +from ooni.utils import log class httptArgs(usage.Options): optParameters = [['urls', 'f', None, 'Urls file'], diff --git a/ooni/plugins/tcpconnect.py b/ooni/plugins/tcpconnect.py index 7c04994..db3d969 100644 --- a/ooni/plugins/tcpconnect.py +++ b/ooni/plugins/tcpconnect.py @@ -11,7 +11,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset -from ooni import log +from ooni.utils import log class tcpconnectArgs(usage.Options): optParameters = [['asset', 'a', None, 'File containing IP:PORT combinations, one per line.'], diff --git a/ooni/plugoo/nodes.py b/ooni/plugoo/nodes.py index 6cdba65..0d01348 100644 --- a/ooni/plugoo/nodes.py +++ b/ooni/plugoo/nodes.py @@ -99,7 +99,7 @@ class PlanetLab(CodeExecNode): def __init__(self, address, auth_creds, ooni): self.auth_creds = auth_creds - self.config = ooni.config + self.config = ooni.utils.config self.logger = ooni.logger self.name = "PlanetLab" diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py index a963f1c..3a9c5d2 100644 --- a/ooni/plugoo/reports.py +++ b/ooni/plugoo/reports.py @@ -4,7 +4,7 @@ import os import yaml import itertools -from ooni import log, date +from ooni.utils import log, date class Report: """This is the ooni-probe reporting mechanism. It allows diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 42294d6..a99a144 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -7,8 +7,8 @@ import itertools from twisted.internet import reactor, defer, threads from twisted.python import failure -from ooni import log -from ooni import date +from ooni.utils import log +from ooni.utils import date from ooni.plugoo import assets, work from ooni.plugoo.reports import Report from ooni.plugoo.interface import ITest diff --git a/ooni/protocols/http.py b/ooni/protocols/http.py index 1dd2261..100db88 100644 --- a/ooni/protocols/http.py +++ b/ooni/protocols/http.py @@ -5,7 +5,7 @@ from twisted.plugin import IPlugin from twisted.internet import protocol, defer from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset -from ooni import log +from ooni.utils import log useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), diff --git a/ooni/protocols/scapy.py b/ooni/protocols/scapy.py index bacc163..4166146 100644 --- a/ooni/protocols/scapy.py +++ b/ooni/protocols/scapy.py @@ -5,7 +5,7 @@ from twisted.plugin import IPlugin from twisted.internet import protocol, defer from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset -from ooni import log +from ooni.utils import log from ooni.lib.txscapy import txsr, txsend diff --git a/ooni/scaffolding.py b/ooni/scaffolding.py index b4c5645..0a832aa 100755 --- a/ooni/scaffolding.py +++ b/ooni/scaffolding.py @@ -5,7 +5,7 @@ This script should be used for creating the scaffolding for a test. """ import os import sys -from ooni import log +from ooni.utils import log test_template = """\"\"\" This is a self genrated test created by scaffolding.py. @@ -17,7 +17,7 @@ from twisted.python import usage from twisted.plugin import IPlugin from ooni.plugoo.tests import ITest, OONITest from ooni.plugoo.assets import Asset -from ooni import log +from ooni.utils import log class %(testShortname)sArgs(usage.Options): optParameters = [['asset', 'a', None, 'Asset file'], diff --git a/ooni/utils.py b/ooni/utils.py deleted file mode 100644 index e9c55f1..0000000 --- a/ooni/utils.py +++ /dev/null @@ -1,146 +0,0 @@ -""" - -""" - -import imp -import logging -try: - import yaml -except: - print "Error in importing YAML" - -class Storage(dict): - """ - A Storage object is like a dictionary except `obj.foo` can be used - in addition to `obj['foo']`. - - >>> o = Storage(a=1) - >>> o.a - 1 - >>> o['a'] - 1 - >>> o.a = 2 - >>> o['a'] - 2 - >>> del o.a - >>> o.a - None - """ - - def __getattr__(self, key): - try: - return self[key] - except KeyError, k: - return None - - def __setattr__(self, key, value): - self[key] = value - - def __delattr__(self, key): - try: - del self[key] - except KeyError, k: - raise AttributeError, k - - def __repr__(self): - return '<Storage ' + dict.__repr__(self) + '>' - - def __getstate__(self): - return dict(self) - - def __setstate__(self, value): - for (k, v) in value.items(): - self[k] = v - - -def get_logger(config): - loglevel = getattr(logging, config.loglevel.upper()) - logging.basicConfig(level=loglevel, - format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', - filename=config.logfile, - filemode='w') - - console = logging.StreamHandler() - console.setLevel(getattr(logging, config.consoleloglevel.upper())) - # Set the console logger to a different format - formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') - console.setFormatter(formatter) - logging.getLogger('').addHandler(console) - - return logging.getLogger('ooniprobe') - -def parse_asset(asset): - parsed = Storage() - try: - with open(asset, 'r') as f: - for line in f.readlines(): - # XXX This should be rewritten, if the values contain - # #: they will be rewritten with blank. - # should not be an issue but this is not a very good parser - if line.startswith("#:"): - n = line.split(' ')[0].replace('#:','') - v = line.replace('#:'+n+' ', '').strip() - if n in ('tests', 'files'): - parsed[n] = v.split(",") - else: - parsed[n] = v - - elif line.startswith("#"): - continue - else: - break - finally: - if not parsed.name: - parsed.name = asset - if not parsed.files: - parsed.files = asset - return parsed - -def import_test(name, config): - if name.endswith(".py"): - test = Storage() - test_name = name.split(".")[0] - fp, pathname, description = imp.find_module(test_name, - [config.main.testdir]) - module = imp.load_module(name, fp, pathname, description) - - try: - test.name = module.__name__ - test.desc = module.__desc__ - test.module = module - except: - test.name = test_name - test.desc = "" - test.module = module - - return test_name, test - - return None, None - -class Log(): - """ - This is a class necessary for parsing YAML log files. - It is required because pyYaml has a bug in parsing - log format YAML files. - """ - def __init__(self, file=None): - if file: - self.fh = open(file) - - def __iter__(self): - return self - - def next(self): - lines = [] - try: - line = self.fh.readline() - if not line: - raise StopIteration - while not line.startswith("---"): - lines.append(line) - line = self.fh.readline() - return lines - except: - raise StopIteration - - diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py new file mode 100644 index 0000000..e9c55f1 --- /dev/null +++ b/ooni/utils/__init__.py @@ -0,0 +1,146 @@ +""" + +""" + +import imp +import logging +try: + import yaml +except: + print "Error in importing YAML" + +class Storage(dict): + """ + A Storage object is like a dictionary except `obj.foo` can be used + in addition to `obj['foo']`. + + >>> o = Storage(a=1) + >>> o.a + 1 + >>> o['a'] + 1 + >>> o.a = 2 + >>> o['a'] + 2 + >>> del o.a + >>> o.a + None + """ + + def __getattr__(self, key): + try: + return self[key] + except KeyError, k: + return None + + def __setattr__(self, key, value): + self[key] = value + + def __delattr__(self, key): + try: + del self[key] + except KeyError, k: + raise AttributeError, k + + def __repr__(self): + return '<Storage ' + dict.__repr__(self) + '>' + + def __getstate__(self): + return dict(self) + + def __setstate__(self, value): + for (k, v) in value.items(): + self[k] = v + + +def get_logger(config): + loglevel = getattr(logging, config.loglevel.upper()) + logging.basicConfig(level=loglevel, + format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', + filename=config.logfile, + filemode='w') + + console = logging.StreamHandler() + console.setLevel(getattr(logging, config.consoleloglevel.upper())) + # Set the console logger to a different format + formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') + console.setFormatter(formatter) + logging.getLogger('').addHandler(console) + + return logging.getLogger('ooniprobe') + +def parse_asset(asset): + parsed = Storage() + try: + with open(asset, 'r') as f: + for line in f.readlines(): + # XXX This should be rewritten, if the values contain + # #: they will be rewritten with blank. + # should not be an issue but this is not a very good parser + if line.startswith("#:"): + n = line.split(' ')[0].replace('#:','') + v = line.replace('#:'+n+' ', '').strip() + if n in ('tests', 'files'): + parsed[n] = v.split(",") + else: + parsed[n] = v + + elif line.startswith("#"): + continue + else: + break + finally: + if not parsed.name: + parsed.name = asset + if not parsed.files: + parsed.files = asset + return parsed + +def import_test(name, config): + if name.endswith(".py"): + test = Storage() + test_name = name.split(".")[0] + fp, pathname, description = imp.find_module(test_name, + [config.main.testdir]) + module = imp.load_module(name, fp, pathname, description) + + try: + test.name = module.__name__ + test.desc = module.__desc__ + test.module = module + except: + test.name = test_name + test.desc = "" + test.module = module + + return test_name, test + + return None, None + +class Log(): + """ + This is a class necessary for parsing YAML log files. + It is required because pyYaml has a bug in parsing + log format YAML files. + """ + def __init__(self, file=None): + if file: + self.fh = open(file) + + def __iter__(self): + return self + + def next(self): + lines = [] + try: + line = self.fh.readline() + if not line: + raise StopIteration + while not line.startswith("---"): + lines.append(line) + line = self.fh.readline() + return lines + except: + raise StopIteration + + diff --git a/ooni/utils/config.py b/ooni/utils/config.py new file mode 100644 index 0000000..42a14f6 --- /dev/null +++ b/ooni/utils/config.py @@ -0,0 +1,53 @@ +import ConfigParser +from utils import Storage + +class Config(Storage): + """ + A Storage-like class which loads and store each attribute into a portable + conf file. + """ + def __init__(self, section, cfgfile="ooni-probe.conf"): + super(Config, self).__init__() + + self._cfgfile = cfgfile + # setting up confgiparser + self._cfgparser = ConfigParser.ConfigParser() + self._cfgparser.read([self._cfgfile]) + self._section = section + + def __getattr__(self, name): + if name.startswith('_'): + return self.__dict__.get(name, None) + + try: + value = self._cfgparser.get(self._section, name) + if value.isdigit(): + return int(value) + elif value.lower() in ('true', 'false'): + return value.lower() == 'true' + else: + return value + except ConfigParser.NoOptionError: + return '' # if option doesn't exists return an empty string + + def __setattr__(self, name, value): + # keep an open port with private attributes + if name.startswith('_'): + self.__dict__[name] = value + return + + try: + # XXX: Automagically discover variable type + self._cfgparser.set(self._section, name, value) + except ConfigParser.NoOptionError: + raise NameError(name) + + def commit(self): + """ + Commit changes in config file. + """ + cfgfile = open(self._cfgfile, 'w') + try: + self._cfgparser.write(cfgfile) + finally: + cfgfile.close() diff --git a/ooni/utils/date.py b/ooni/utils/date.py new file mode 100644 index 0000000..59ec1f8 --- /dev/null +++ b/ooni/utils/date.py @@ -0,0 +1,22 @@ +from ooni.lib.rfc3339 import rfc3339 +from datetime import datetime + +class odate(datetime): + def __str__(self): + return rfc3339(self) + + def __repr__(self): + return "'%s'" % rfc3339(self) + + def from_rfc(self, datestr): + pass + +def now(): + return odate.utcnow() + +def pretty_date(): + cur_time = datetime.utcnow() + d_format = "%d %B %Y %H:%M:%S" + pretty = cur_time.strftime(d_format) + return pretty + diff --git a/ooni/utils/log.py b/ooni/utils/log.py new file mode 100644 index 0000000..3dc52dd --- /dev/null +++ b/ooni/utils/log.py @@ -0,0 +1,43 @@ +""" +OONI logging facility. +""" +import sys +import logging +import warnings + +from twisted.python import log + +# Logging levels +DEBUG = logging.DEBUG +INFO = logging.INFO +WARNING = logging.WARNING +ERROR = logging.ERROR +CRITICAL = logging.CRITICAL + +def _get_log_level(level): + if not level: + return INFO + else: + return level + +def start(logfile=None, loglevel=None, logstdout=True): + if log.defaultObserver: + print "%s" % logstdout + loglevel = _get_log_level(loglevel) + file = open(logfile, 'a') if logfile else sys.stderr + observer = log.FileLogObserver(file) + if logstdout: + log.startLogging(sys.stdout) + else: + log.startLogging() + log.addObserver(observer.emit) + msg("Started OONI") + +def msg(message, level=INFO, **kw): + log.msg(message, logLevel=level, **kw) + +def err(message, **kw): + log.err(message, **kw) + +def debug(message, **kw): + log.msg(message, logLevel=DEBUG, **kw) diff --git a/ooni/utils/logo.py b/ooni/utils/logo.py new file mode 100644 index 0000000..8924462 --- /dev/null +++ b/ooni/utils/logo.py @@ -0,0 +1,214 @@ +import random +def getlogo(): + logo = [] + logo.append(""" + _____ _____ __ _ _____ _____ ______ _____ ______ _______ + | | | | | \ | | |_____] |_____/ | | |_____] |______ + |_____| |_____| | \_| __|__ | | \_ |_____| |_____] |______""") + + logo.append(""" + _ _ + (_) | | + ___ ___ _ __ _ _ __ _ __ ___ | |__ ___ + / _ \ / _ \| '_ \| | '_ \| '__/ _ \| '_ \ / _ \\ +| (_) | (_) | | | | | |_) | | | (_) | |_) | __/ + \___/ \___/|_| |_|_| .__/|_| \___/|_.__/ \___| + | | + |_|""") + + + logo.append(""" + .__ ___. + ____ ____ ____ |__|_____________ ____\_ |__ ____ + / _ \ / _ \ / \| |\____ \_ __ \/ _ \| __ \_/ __ \\ +( <_> | <_> ) | \ || |_> > | \( <_> ) \_\ \ ___/ + \____/ \____/|___| /__|| __/|__| \____/|___ /\___ > + \/ |__| \/ \/""") + + + logo.append(""" + __ __ __ __ __ __ ___ +/ \ / \ |\ | | |__) |__) / \ |__) |__ +\__/ \__/ | \| | | | \ \__/ |__) |___ """) + + + logo.append(""" + _ _ ._ o ._ ._ _ |_ _ + (_) (_) | | | |_) | (_) |_) (/_ + | +""") + + logo.append(""" + _______ _______ __ _ ___ _______ ______ _______ _______ _______ +| || || | | || | | || _ | | || _ || | +| _ || _ || |_| || | | _ || | || | _ || |_| || ___| +| | | || | | || || | | |_| || |_|| | | | || || |___ +| |_| || |_| || _ || | | ___|| __ || |_| || _ | | ___| +| || || | | || | | | | | ||| || |_| || |___ +|_______||_______||_| |__||___| |___| |___| |||_______||_______||_______| +""") + + + logo.append(""" + _ _ + ___ ___ _ __ (_)_ __ _ __ ___ | |__ ___ + / _ \ / _ \| '_ \| | '_ \| '__/ _ \| '_ \ / _ \\ +| (_) | (_) | | | | | |_) | | | (_) | |_) | __/ + \___/ \___/|_| |_|_| .__/|_| \___/|_.__/ \___| + |_| +""") + + logo.append(""" + o + o O + O + o +.oOo. .oOo. 'OoOo. O .oOo. `OoOo. .oOo. OoOo. .oOo. +O o O o o O o O o o O o O o OooO' +o O o O O o O o O O o O o O O +`OoO' `OoO' o O o' oOoO' o `OoO' `OoO' `OoO' + O + o' +""") + + + logo.append(""" + _____ _____ ____ _ ____ _____ _____ _____ ______ ______ + / \/ \| \ | || || || | / \| >| ___| + | || || \| || || _|| \ | || < | ___| + \_____/\_____/|__/\____||____||___| |__|\__\\\\_____/|______>|______| +""") + + logo.append(""" + _ _ + ___ ___ ___|_|___ ___ ___| |_ ___ +| . | . | | | . | _| . | . | -_| +|___|___|_|_|_| _|_| |___|___|___| + |_| +""") + + logo.append(""" + _ _ + (_) | | + ___ ___ ____ _ ____ ____ ___ | |__ _____ + / _ \ / _ \| _ \| | _ \ / ___) _ \| _ \| ___ | +| |_| | |_| | | | | | |_| | | | |_| | |_) ) ____| + \___/ \___/|_| |_|_| __/|_| \___/|____/|_____) + |_| +""") + logo.append(""" + _ __ + ____ ____ ____ (_)____ _________ / /_ ___ + / __ \/ __ \/ __ \/ // __ \/ ___/ __ \/ __ \/ _ \\ +/ /_/ / /_/ / / / / // /_/ / / / /_/ / /_/ / __/ +\____/\____/_/ /_/_// .___/_/ \____/_.___/\___/ + /_/ +""") + logo.append(""" + _) | + _ \ _ \ \ | _ \ _| _ \ _ \ -_) + \___/ \___/ _| _| _| .__/ _| \___/ _.__/ \___| + _| +""") + logo.append(""" + _ _ + ___ ___ _ __ (_)_ __ _ __ ___ | |__ ___ + / _ \ / _ \| '_ \| | '_ \| '__/ _ \| '_ \ / _ \\ + | (_) | (_) | | | | | |_) | | | (_) | |_) | __/ + \___/ \___/|_| |_|_| .__/|_| \___/|_.__/ \___| + |_| +""") + logo.append(""" + .-. ( ) + .--. .--. ___ .-. ( __) .-.. ___ .-. .--. | |.-. .--. + / \ / \ ( ) \ (''") / \ ( ) \ / \ | / \ / \\ +| .-. ; | .-. ; | .-. . | | ' .-, ; | ' .-. ; | .-. ; | .-. | | .-. ; +| | | | | | | | | | | | | | | | . | | / (___) | | | | | | | | | | | | +| | | | | | | | | | | | | | | | | | | | | | | | | | | | | |/ | +| | | | | | | | | | | | | | | | | | | | | | | | | | | | | ' _.' +| ' | | | ' | | | | | | | | | | ' | | | | ' | | | ' | | | .'.-. +' `-' / ' `-' / | | | | | | | `-' ' | | ' `-' / ' `-' ; ' `-' / + `.__.' `.__.' (___)(___) (___) | \__.' (___) `.__.' `.__. `.__.' + | | + (___) +""") + logo.append(""" + o | +,---.,---.,---..,---.,---.,---.|---.,---. +| || || ||| || | || ||---' +`---'`---'` '`|---'` `---'`---'`---' + | +""") + logo.append(""" + ________________________________________ +/ OONI-PROBE -- WE FIGHT CENSORSHIP LIKE \\ +\ DRAGONS EAT KNIGHTS! / + ---------------------------------------- + \ ^ /^ + \ / \ // \\ + \ |\___/| / \// .\\ + \ /O O \__ / // | \ \ *----* + / / \/_/ // | \ \ \ | + @___@` \/_ // | \ \ \/\ \\ + 0/0/| \/_ // | \ \ \ \\ + 0/0/0/0/| \/// | \ \ | | + 0/0/0/0/0/_|_ / ( // | \ _\ | / + 0/0/0/0/0/0/`/,_ _ _/ ) ; -. | _ _\.-~ / / + ,-} _ *-.|.-~-. .~ ~ + \ \__/ `/\ / ~-. _ .-~ / + \____(oo) *. } { / + ( (--) .----~-.\ \-` .~ + //__\\\\ \__ Ack! ///.----..< \ _ -~ + // \\\\ ///-._ _ _ _ _ _ _{^ - - - - ~ +""") + logo.append(""" + _________________ +| OONI-PROBE | +| DON'T CENSOR ME | +|_________________| +\ . . + \ / `. .' " + \ .---. < > < > .---. + \ | \ \ - ~ ~ - / / | + _____ ..-~ ~-..-~ + | (A)| \~~~\.' `./~~~/ + --------- \__/ \__/ + .' O \ / / \ " + (_____, `._.' | } \/~~~/ + `----. / } | / \__/ + `-. | / | / `. ,~~| + ~-.__| /_ - ~ ^| /- _ `..-' + | / | / ~-. `-. _ _ _ + |_____| |_____| ~ - . _ _ _ _ _> +""") + logo.append(""" + _________________ + ,/~~, | | + /`` \ /| OONI-PROBE... | + `( O^O ) / | | + `\_-_/` | ...DOES THIS | + ~~~ | EXCITE YOU? | + __| |__ |_________________| + | | + | af | + | | | | + | | | \\ + | | |\ \\ + | | | \ \\ + | | | \ \\ + | |___| ((( + (((|/*\| + | | | + | | | + | | | + | | | + | | | + | | | + | | | + | | | + | | | + | | | + _| | |_ + (___|___) +""") + return random.choice(logo)