November 2012 - tor-commits - lists.torproject.org

[ooni-probe/master] Loosen up the oonib regexp to support the timestamp report format
by art＠torproject.org 21 Nov '12

21 Nov '12

commit 3af0b5a12a5af750155cd06750d5a911f5c1900e Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 21 16:00:02 2012 +0100 Loosen up the oonib regexp to support the timestamp report format --- oonib/README.md | 1 + oonib/models.py | 9 +++++++-- oonib/report/api.py | 9 +++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/oonib/README.md b/oonib/README.md index 8136b06..f70c6db 100644 --- a/oonib/README.md +++ b/oonib/README.md @@ -15,3 +15,4 @@ The extra dependencies necessary to run OONIB are: openssl x509 -req -days 365 -in server.csr -signkey private.key -out certificate.crt rm private.key.org + diff --git a/oonib/models.py b/oonib/models.py index 5784673..22567ad 100644 --- a/oonib/models.py +++ b/oonib/models.py @@ -2,11 +2,16 @@ __all__ = ['Report', 'TestHelperTMP'] from storm.twisted.transact import transact from storm.locals import * +from ooni.utils import randomStr from oonib import transactor def generateReportID(): - size = 100 - report_id = ''.join(random.choice(string.ascii_letters) for x in range(size)) + """ + Generates a report ID for usage in the database backed oonib collector. + + XXX note how this function is different from the one in report/api.py + """ + report_id = randomStr(100) return report_id class OModel(object): diff --git a/oonib/report/api.py b/oonib/report/api.py index 708de37..5668857 100644 --- a/oonib/report/api.py +++ b/oonib/report/api.py @@ -59,7 +59,10 @@ def parseUpdateReportRequest(request): # XXX this and the function above can probably be refactored into something # more compact. There is quite a bit of code duplication going on here. - report_id_regexp = re.compile("[a-zA-Z0-9]+$") + #db_report_id_regexp = re.compile("[a-zA-Z0-9]+$") + + # this is the regexp for the reports that include the timestamp + report_id_regexp = re.compile("[a-zA-Z0-9_-]+$") # XXX here we are actually parsing a json object that could be quite big. # If we want this to scale properly we only want to look at the test_id @@ -121,12 +124,14 @@ class NewReportHandlerFile(web.RequestHandler): report_id = generateReportID() - #report_filename = '_'.join((report_id, + #report_filename = '_'.join((report_id, # report_data['software_name'], # report_data['software_version'], # report_data['test_name'], # report_data['test_version'])) + # The report filename contains the timestamp of the report plus a + # random nonce report_filename = os.path.join(config.main.report_dir, report_id) report_filename += '.yamloo'

1 0

[ooni-probe/master] Remove references to optFlags in runner
by art＠torproject.org 21 Nov '12

21 Nov '12

commit 6f0af75eb0e5cae52a03932c40a4c0562d64accc Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Nov 20 21:21:02 2012 +0100 Remove references to optFlags in runner --- ooni/nettest.py | 2 ++ ooni/runner.py | 3 --- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 0394564..4529c14 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -80,6 +80,8 @@ class NetTestCase(object): usageOptions = None + optParameters = None + requiredOptions = [] requiresRoot = False diff --git a/ooni/runner.py b/ooni/runner.py index 40aa40c..5e6f35e 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -66,9 +66,6 @@ def processTest(obj, cmd_line_options): log.debug("Got optParameters") class Options(usage.Options): optParameters = obj.optParameters - if obj.optFlags: - log.debug("Got optFlags") - optFlags = obj.optFlags options = Options() if options:

1 0

[ooni-probe/master] Refactoring of runner
by art＠torproject.org 21 Nov '12

21 Nov '12

commit 3f97ad5d4099bd7ed3b6c903d9cc39e17b519572 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Nov 20 22:03:06 2012 +0100 Refactoring of runner * Add support for baseParameters that allow the specification of a set of default parameters to be be used in all the test templates. --- ooni/nettest.py | 6 ++++-- ooni/runner.py | 50 ++++++++++++++++++-------------------------------- 2 files changed, 22 insertions(+), 34 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index 4529c14..ebb58ea 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -74,6 +74,7 @@ class NetTestCase(object): inputs = [None] inputFile = None + inputFilename = None report = {} report['errors'] = [] @@ -81,6 +82,7 @@ class NetTestCase(object): usageOptions = None optParameters = None + baseParameters = None requiredOptions = [] requiresRoot = False @@ -135,8 +137,8 @@ class NetTestCase(object): raise usage.UsageError("%s not specified!" % required_option) def _processOptions(self, options=None): - if self.inputFile: - self.inputs = self.inputProcessor(self.inputFile) + if self.inputFilename: + self.inputs = self.inputProcessor(self.inputFilename) self._checkRequiredOptions() diff --git a/ooni/runner.py b/ooni/runner.py index 5e6f35e..536ecfe 100644 --- a/ooni/runner.py +++ b/ooni/runner.py @@ -39,8 +39,7 @@ def processTest(obj, cmd_line_options): A configured and instantiated :class:`twisted.python.usage.Options` class. """ - - input_file = obj.inputFile + options = None if obj.requiresRoot: try: checkForRoot() @@ -48,33 +47,22 @@ def processTest(obj, cmd_line_options): log.err("%s requires root to run" % obj.name) sys.exit(1) + if obj.usageOptions and obj.inputFile: + obj.usageOptions.optParameters.append(obj.inputFile) - if obj.optParameters or input_file \ - or obj.usageOptions or obj.optFlags: - - if not obj.optParameters: - obj.optParameters = [] + if obj.usageOptions and obj.baseParameters: + for parameter in obj.baseParameters: + obj.usageOptions.optParameters.append(parameter) - if input_file: - obj.optParameters.append(input_file) + if obj.usageOptions: + options = obj.usageOptions() - if obj.usageOptions: - if input_file: - obj.usageOptions.optParameters.append(input_file) - options = obj.usageOptions() - elif obj.optParameters: - log.debug("Got optParameters") - class Options(usage.Options): - optParameters = obj.optParameters - options = Options() + if options: + options.parseOptions(cmd_line_options['subArgs']) + obj.localOptions = options - if options: - options.parseOptions(cmd_line_options['subArgs']) - obj.localOptions = options - - if input_file and options: - log.debug("Got input file") - obj.inputFile = options[input_file[0]] + if obj.inputFile: + obj.inputFilename = options[obj.inputFile[0]] try: log.debug("processing options") @@ -175,9 +163,8 @@ def runTestWithInput(test_class, test_method, test_input, oreporter): log.debug("returning %s input" % test_method) return d -def runTestWithInputUnit(test_class, - test_method, input_unit, - oreporter): +def runTestWithInputUnit(test_class, + test_method, input_unit, oreporter): """ test_class: the uninstantiated class of the test to be run @@ -194,13 +181,13 @@ def runTestWithInputUnit(test_class, log.debug("input unit %s" % input_unit) for test_input in input_unit: log.debug("running with input: %s" % test_input) - d = runTestWithInput(test_class, + d = runTestWithInput(test_class, test_method, test_input, oreporter) dl.append(d) return defer.DeferredList(dl) @defer.inlineCallbacks -def runTestCases(test_cases, options, +def runTestCases(test_cases, options, cmd_line_options, yamloo_filename): try: assert len(options) != 0, "Length of options is zero!" @@ -222,7 +209,6 @@ def runTestCases(test_cases, options, reportFile = open(yamloo_filename, 'w+') - if cmd_line_options['collector']: oreporter = reporter.OONIBReporter(cmd_line_options['collector']) else: @@ -235,7 +221,7 @@ def runTestCases(test_cases, options, yield oreporter.createReport(options) # This deferred list is a deferred list of deferred lists - # it is used to store all the deferreds of the tests that + # it is used to store all the deferreds of the tests that # are run try: for input_unit in input_unit_factory:

1 0

[ooni-probe/master] Add socks proxy support to HTTP Test
by art＠torproject.org 21 Nov '12

21 Nov '12

commit 213fcf08a4dee4776590c3321796bc7d9f7eb09c Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Nov 20 22:26:51 2012 +0100 Add socks proxy support to HTTP Test * Use required options in http keyword filtering test * Fix before_i_commit script --- before_i_commit.sh | 2 - nettests/core/http_keyword_filtering.py | 23 ++++++++++++--------- ooni/templates/httpt.py | 32 +++++++++++++++++++++++++++--- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/before_i_commit.sh b/before_i_commit.sh index 96117b3..3e01366 100755 --- a/before_i_commit.sh +++ b/before_i_commit.sh @@ -24,8 +24,6 @@ find . -type f -name "*.py[co]" -delete ./bin/ooniprobe -l before_i_commit.log -o http_keyword_filtering.yamloo nettests/core/http_keyword_filtering.py -b http://ooni.nu/test/ -f test_inputs/keyword_filtering_file.txt -./bin/ooniprobe -l before_i_commit.log -o url_lists.yamloo nettests/core/url_list.py -f test_inputs/url_lists_file.txt - echo "Below you should not see anything" echo "---------------------------------" grep "Error: " before_i_commit.log diff --git a/nettests/core/http_keyword_filtering.py b/nettests/core/http_keyword_filtering.py index fabdba6..0ae9c52 100644 --- a/nettests/core/http_keyword_filtering.py +++ b/nettests/core/http_keyword_filtering.py @@ -3,7 +3,14 @@ # :authors: Arturo Filastò # :licence: see LICENSE +from twisted.python import usage + from ooni.templates import httpt + +class UsageOptions(usage.Options): + optParameters = [['backend', 'b', 'http://127.0.0.1:57001', + 'URL of the test backend to use']] + class HTTPKeywordFiltering(httpt.HTTPTest): """ This test involves performing HTTP requests containing to be tested for @@ -14,29 +21,25 @@ class HTTPKeywordFiltering(httpt.HTTPTest): """ name = "HTTP Keyword Filtering" author = "Arturo Filastò" - version = 0.1 - - optParameters = [['backend', 'b', None, 'URL of the backend system to use for testing']] + version = "0.1.1" inputFile = ['file', 'f', None, 'List of keywords to use for censorship testing'] - def processInputs(self): - if 'backend' in self.localOptions: - self.url = self.localOptions['backend'] - else: - raise Exception("No backend specified") + usageOptions = UsageOptions + + requiredOptions = ['backend'] def test_get(self): """ Perform a HTTP GET request to the backend containing the keyword to be tested inside of the request body. """ - return self.doRequest(self.url, method="GET", body=self.input) + return self.doRequest(self.localOptions['backend'], method="GET", body=self.input) def test_post(self): """ Perform a HTTP POST request to the backend containing the keyword to be tested inside of the request body. """ - return self.doRequest(self.url, method="POST", body=self.input) + return self.doRequest(self.localOptions['backend'], method="POST", body=self.input) diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py index 85f8586..7caab39 100644 --- a/ooni/templates/httpt.py +++ b/ooni/templates/httpt.py @@ -44,6 +44,10 @@ class HTTPTest(NetTestCase): randomizeUA = True followRedirects = False + + baseParameters = [['socksproxy', 's', None, + 'Specify a socks proxy to use for requests (ip:port)']] + request = {} response = {} @@ -55,13 +59,21 @@ class HTTPTest(NetTestCase): log.err("Warning! pyOpenSSL is not installed. https websites will" "not work") - self.agent = Agent(reactor, - sockhost="127.0.0.1", + self.control_agent = Agent(reactor, sockhost="127.0.0.1", sockport=config.advanced.tor_socksport) + sockshost, socksport = (None, None) + if self.localOptions['socksproxy']: + sockshost, socksport = self.localOptions['socksproxy'].split(':') + socksport = int(socksport) + + self.agent = Agent(reactor, sockhost=sockshost, + sockport=socksport) + if self.followRedirects: try: from twisted.web.client import RedirectAgent + self.control_agent = RedirectAgent(self.control_agent) self.agent = RedirectAgent(self.agent) except: log.err("Warning! You are running an old version of twisted"\ @@ -150,6 +162,13 @@ class HTTPTest(NetTestCase): if use_tor: log.debug("Using tor for the request") url = 's'+url + agent = self.tor_agent + else: + agent = self.agent + + if self.localOptions['socksproxy']: + log.debug("Using SOCKS proxy %s for request" % (self.localOptions['socksproxy'])) + url = 's'+url log.debug("Performing request %s %s %s" % (url, method, headers)) @@ -188,7 +207,7 @@ class HTTPTest(NetTestCase): def finished(data): return - d = self.agent.request(request['method'], request['url'], headers, + d = agent.request(request['method'], request['url'], headers, body_producer) d.addErrback(errback) @@ -198,7 +217,12 @@ class HTTPTest(NetTestCase): def _cbResponse(self, response, request, headers_processor, body_processor): - log.debug("Got response %s" % response) + + if not response: + log.err("Got no response") + return + else: + log.debug("Got response %s" % response) if str(response.code).startswith('3'): self.processRedirect(response.headers.getRawHeaders('Location')[0])

1 0

[ooni-probe/master] Remove unused/unsupported functionality from NetTest
by art＠torproject.org 21 Nov '12

21 Nov '12

commit 5516cc6b1662fe3feeb4b0a849f4cf4f23f70f31 Author: Arturo Filastò <art(a)fuffa.org> Date: Tue Nov 20 21:10:22 2012 +0100 Remove unused/unsupported functionality from NetTest --- ooni/nettest.py | 27 ++++----------------------- 1 files changed, 4 insertions(+), 23 deletions(-) diff --git a/ooni/nettest.py b/ooni/nettest.py index a2f98c4..0394564 100644 --- a/ooni/nettest.py +++ b/ooni/nettest.py @@ -46,8 +46,8 @@ class NetTestCase(object): ``ooniprobe mytest.py -c path/to/file.txt`` - * inputProcessor: should be set to a function that takes as argument an - open file descriptor and it will return the input to be passed to the test + * inputProcessor: should be set to a function that takes as argument a + filename and it will return the input to be passed to the test instance. * name: should be set to the name of the test. @@ -61,24 +61,7 @@ class NetTestCase(object): * requiresRoot: set to True if the test must be run as root. - * optFlags: is assigned a list of lists. Each list represents a flag parameter, as so: - - optFlags = [['verbose', 'v', 'Makes it tell you what it doing.'], | ['quiet', 'q', 'Be vewy vewy quiet.']] - - As you can see, the first item is the long option name (prefixed with - '--' on the command line), followed by the short option name (prefixed with - '-'), and the description. The description is used for the built-in handling of - the --help switch, which prints a usage summary. - - - * optParameters: is much the same, except the list also contains a default value: - - | optParameters = [['outfile', 'O', 'outfile.log', 'Description...']] - - * usageOptions: a subclass of twisted.python.usage.Options for more advanced command line arguments fun. - - * requiredOptions: a list containing the name of the options that are - required for proper running of a test. + * usageOptions: a subclass of twisted.python.usage.Options for processing of command line arguments * localOptions: contains the parsed command line arguments. @@ -95,10 +78,8 @@ class NetTestCase(object): report = {} report['errors'] = [] - optFlags = None - optParameters = None - usageOptions = None + requiredOptions = [] requiresRoot = False

1 0

[ooni-probe/master] Remove dead code
by art＠torproject.org 21 Nov '12

21 Nov '12

commit a194337696e30a9efd0328d9a4943cfcd26ca571 Author: Arturo FilastÃ² <art(a)fuffa.org> Date: Tue Nov 20 21:04:32 2012 +0100 Remove dead code --- to-be-ported/old-api/.ropeproject/config.py | 85 --------- to-be-ported/old-api/.ropeproject/globalnames | Bin 108 -> 0 bytes to-be-ported/old-api/.ropeproject/history | 1 - to-be-ported/old-api/.ropeproject/objectdb | Bin 741 -> 0 bytes to-be-ported/old-api/TESTS_ARE_MOVING.txt | 8 - to-be-ported/old-api/domclass.py | 216 ---------------------- to-be-ported/old-api/dropin.cache | 243 ------------------------- to-be-ported/old-api/tcpscan.py | 84 --------- 8 files changed, 0 insertions(+), 637 deletions(-) diff --git a/to-be-ported/old-api/.ropeproject/config.py b/to-be-ported/old-api/.ropeproject/config.py deleted file mode 100644 index ffebcd4..0000000 --- a/to-be-ported/old-api/.ropeproject/config.py +++ /dev/null @@ -1,85 +0,0 @@ -# The default ``config.py`` - - -def set_prefs(prefs): - """This function is called before opening the project""" - - # Specify which files and folders to ignore in the project. - # Changes to ignored resources are not added to the history and - # VCSs. Also they are not returned in `Project.get_files()`. - # Note that ``?`` and ``*`` match all characters but slashes. - # '*.pyc': matches 'test.pyc' and 'pkg/test.pyc' - # 'mod*.pyc': matches 'test/mod1.pyc' but not 'mod/1.pyc' - # '.svn': matches 'pkg/.svn' and all of its children - # 'build/*.o': matches 'build/lib.o' but not 'build/sub/lib.o' - # 'build//*.o': matches 'build/lib.o' and 'build/sub/lib.o' - prefs['ignored_resources'] = ['*.pyc', '*~', '.ropeproject', - '.hg', '.svn', '_svn', '.git'] - - # Specifies which files should be considered python files. It is - # useful when you have scripts inside your project. Only files - # ending with ``.py`` are considered to be python files by - # default. - #prefs['python_files'] = ['*.py'] - - # Custom source folders: By default rope searches the project - # for finding source folders (folders that should be searched - # for finding modules). You can add paths to that list. Note - # that rope guesses project source folders correctly most of the - # time; use this if you have any problems. - # The folders should be relative to project root and use '/' for - # separating folders regardless of the platform rope is running on. - # 'src/my_source_folder' for instance. - #prefs.add('source_folders', 'src') - - # You can extend python path for looking up modules - #prefs.add('python_path', '~/python/') - - # Should rope save object information or not. - prefs['save_objectdb'] = True - prefs['compress_objectdb'] = False - - # If `True`, rope analyzes each module when it is being saved. - prefs['automatic_soa'] = True - # The depth of calls to follow in static object analysis - prefs['soa_followed_calls'] = 0 - - # If `False` when running modules or unit tests "dynamic object - # analysis" is turned off. This makes them much faster. - prefs['perform_doa'] = True - - # Rope can check the validity of its object DB when running. - prefs['validate_objectdb'] = True - - # How many undos to hold? - prefs['max_history_items'] = 32 - - # Shows whether to save history across sessions. - prefs['save_history'] = True - prefs['compress_history'] = False - - # Set the number spaces used for indenting. According to - # :PEP:`8`, it is best to use 4 spaces. Since most of rope's - # unit-tests use 4 spaces it is more reliable, too. - prefs['indent_size'] = 4 - - # Builtin and c-extension modules that are allowed to be imported - # and inspected by rope. - prefs['extension_modules'] = [] - - # Add all standard c-extensions to extension_modules list. - prefs['import_dynload_stdmods'] = True - - # If `True` modules with syntax errors are considered to be empty. - # The default value is `False`; When `False` syntax errors raise - # `rope.base.exceptions.ModuleSyntaxError` exception. - prefs['ignore_syntax_errors'] = False - - # If `True`, rope ignores unresolvable imports. Otherwise, they - # appear in the importing namespace. - prefs['ignore_bad_imports'] = False - - -def project_opened(project): - """This function is called after opening the project""" - # Do whatever you like here! diff --git a/to-be-ported/old-api/.ropeproject/globalnames b/to-be-ported/old-api/.ropeproject/globalnames deleted file mode 100644 index 2877ef5..0000000 Binary files a/to-be-ported/old-api/.ropeproject/globalnames and /dev/null differ diff --git a/to-be-ported/old-api/.ropeproject/history b/to-be-ported/old-api/.ropeproject/history deleted file mode 100644 index fcd9c96..0000000 --- a/to-be-ported/old-api/.ropeproject/history +++ /dev/null @@ -1 +0,0 @@ -]q(]q]qe. \ No newline at end of file diff --git a/to-be-ported/old-api/.ropeproject/objectdb b/to-be-ported/old-api/.ropeproject/objectdb deleted file mode 100644 index f276839..0000000 Binary files a/to-be-ported/old-api/.ropeproject/objectdb and /dev/null differ diff --git a/to-be-ported/old-api/TESTS_ARE_MOVING.txt b/to-be-ported/old-api/TESTS_ARE_MOVING.txt deleted file mode 100644 index f4c0084..0000000 --- a/to-be-ported/old-api/TESTS_ARE_MOVING.txt +++ /dev/null @@ -1,8 +0,0 @@ -7/10/2012 - -All new tests will be moved to the directory /nettests/. - -Tests that are in this directory are either here for historical reasons or have -not yet been properly tested and fully supporting the new API. - -A. diff --git a/to-be-ported/old-api/domclass.py b/to-be-ported/old-api/domclass.py deleted file mode 100644 index 3080c40..0000000 --- a/to-be-ported/old-api/domclass.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env python -#-*- encoding: utf-8 -*- -# -# domclass -# ******** -# -# :copyright: (c) 2012 by Arturo FilastÃ² -# :license: see LICENSE for more details. -# -# how this works -# -------------- -# -# This classifier uses the DOM structure of a website to determine how similar -# the two sites are. -# The procedure we use is the following: -# * First we parse all the DOM tree of the web page and we build a list of -# TAG parent child relationships (ex. <html><a><b></b></a><c></c></html> => -# (html, a), (a, b), (html, c)). -# -# * We then use this information to build a matrix (M) where m[i][j] = P(of -# transitioning from tag[i] to tag[j]). If tag[i] does not exists P() = 0. -# Note: M is a square matrix that is number_of_tags wide. -# -# * We then calculate the eigenvectors (v_i) and eigenvalues (e) of M. -# -# * The corelation between page A and B is given via this formula: -# correlation = dot_product(e_A, e_B), where e_A and e_B are -# resepectively the eigenvalues for the probability matrix A and the -# probability matrix B. -# - -try: - import numpy -except: - print "Error numpy not installed!" - -import yaml -from zope.interface import implements -from twisted.python import usage -from twisted.plugin import IPlugin -from ooni.plugoo.tests import ITest, OONITest -from ooni.plugoo.assets import Asset -from ooni.utils import log -from ooni.protocols.http import HTTPTest - -class domclassArgs(usage.Options): - optParameters = [['output', 'o', None, 'Output to write'], - ['file', 'f', None, 'Corpus file'], - ['fileb', 'b', None, 'Corpus file'], - ['urls', 'u', None, 'URL List'], - ['resume', 'r', 0, 'Resume at this index']] - -# All HTML4 tags -# XXX add link to W3C page where these came from -alltags = ['A', 'ABBR', 'ACRONYM', 'ADDRESS', 'APPLET', 'AREA', 'B', 'BASE', - 'BASEFONT', 'BD', 'BIG', 'BLOCKQUOTE', 'BODY', 'BR', 'BUTTON', 'CAPTION', - 'CENTER', 'CITE', 'CODE', 'COL', 'COLGROUP', 'DD', 'DEL', 'DFN', 'DIR', 'DIV', - 'DL', 'DT', 'E M', 'FIELDSET', 'FONT', 'FORM', 'FRAME', 'FRAMESET', 'H1', 'H2', - 'H3', 'H4', 'H5', 'H6', 'HEAD', 'HR', 'HTML', 'I', 'IFRAME ', 'IMG', - 'INPUT', 'INS', 'ISINDEX', 'KBD', 'LABEL', 'LEGEND', 'LI', 'LINK', 'MAP', - 'MENU', 'META', 'NOFRAMES', 'NOSCRIPT', 'OBJECT', 'OL', 'OPTGROUP', 'OPTION', - 'P', 'PARAM', 'PRE', 'Q', 'S', 'SAMP', 'SCRIPT', 'SELECT', 'SMALL', 'SPAN', - 'STRIKE', 'STRONG', 'STYLE', 'SUB', 'SUP', 'TABLE', 'TBODY', 'TD', - 'TEXTAREA', 'TFOOT', 'TH', 'THEAD', 'TITLE', 'TR', 'TT', 'U', 'UL', 'VAR'] - -# Reduced subset of only the most common tags -commontags = ['A', 'B', 'BLOCKQUOTE', 'BODY', 'BR', 'BUTTON', 'CAPTION', - 'CENTER', 'CITE', 'CODE', 'COL', 'DD', 'DIV', - 'DL', 'DT', 'EM', 'FIELDSET', 'FONT', 'FORM', 'FRAME', 'FRAMESET', 'H1', 'H2', - 'H3', 'H4', 'H5', 'H6', 'HEAD', 'HR', 'HTML', 'IFRAME ', 'IMG', - 'INPUT', 'INS', 'LABEL', 'LEGEND', 'LI', 'LINK', 'MAP', - 'MENU', 'META', 'NOFRAMES', 'NOSCRIPT', 'OBJECT', 'OL', 'OPTION', - 'P', 'PRE', 'SCRIPT', 'SELECT', 'SMALL', 'SPAN', - 'STRIKE', 'STRONG', 'STYLE', 'SUB', 'SUP', 'TABLE', 'TBODY', 'TD', - 'TEXTAREA', 'TFOOT', 'TH', 'THEAD', 'TITLE', 'TR', 'TT', 'U', 'UL'] - -# The tags we are intested in using for our analysis -thetags = ['A', 'DIV', 'FRAME', 'H1', 'H2', - 'H3', 'H4', 'IFRAME ', 'INPUT', - 'LABEL','LI', 'P', 'SCRIPT', 'SPAN', - 'STYLE', 'TR'] - -def compute_probability_matrix(dataset): - """ - Compute the probability matrix based on the input dataset. - - :dataset: an array of pairs representing the parent child relationships. - """ - import itertools - ret = {} - matrix = numpy.zeros((len(thetags) + 1, len(thetags) + 1)) - - for data in dataset: - x = data[0].upper() - y = data[1].upper() - try: - x = thetags.index(x) - except: - x = len(thetags) - - try: - y = thetags.index(y) - except: - y = len(thetags) - - matrix[x,y] += 1 - - for x in xrange(len(thetags) + 1): - possibilities = 0 - for y in matrix[x]: - possibilities += y - - for i in xrange(len(matrix[x])): - if possibilities != 0: - matrix[x][i] = matrix[x][i]/possibilities - - return matrix - -def compute_eigenvalues(matrix): - """ - Returns the eigenvalues of the supplied square matrix. - - :matrix: must be a square matrix and diagonalizable. - """ - return numpy.linalg.eigvals(matrix) - -def readDOM(content=None, filename=None): - """ - Parses the DOM of the HTML page and returns an array of parent, child - pairs. - - :content: the content of the HTML page to be read. - - :filename: the filename to be read from for getting the content of the - page. - """ - from bs4 import BeautifulSoup - - if filename: - f = open(filename) - content = ''.join(f.readlines()) - f.close() - - dom = BeautifulSoup(content) - couples = [] - for x in dom.findAll(): - couples.append((str(x.parent.name), str(x.name))) - - return couples - -class domclassTest(HTTPTest): - implements(IPlugin, ITest) - - shortName = "domclass" - description = "domclass" - requirements = None - options = domclassArgs - blocking = False - - follow_redirects = True - #tool = True - - def runTool(self): - site_a = readDOM(filename=self.local_options['file']) - site_b = readDOM(filename=self.local_options['fileb']) - a = {} - a['matrix'] = compute_probability_matrix(site_a) - a['eigen'] = compute_eigenvalues(a['matrix']) - - self.result['eigenvalues'] = a['eigen'] - b = {} - b['matrix'] = compute_probability_matrix(site_b) - b['eigen'] = compute_eigenvalues(b['matrix']) - - #print "A: %s" % a - #print "B: %s" % b - correlation = numpy.vdot(a['eigen'],b['eigen']) - correlation /= numpy.linalg.norm(a['eigen'])*numpy.linalg.norm(b['eigen']) - correlation = (correlation + 1)/2 - print "Corelation: %s" % correlation - self.end() - return a - - def processResponseBody(self, data): - site_a = readDOM(data) - #site_b = readDOM(self.local_options['fileb']) - a = {} - a['matrix'] = compute_probability_matrix(site_a) - a['eigen'] = compute_eigenvalues(a['matrix']) - - - if len(data) == 0: - self.result['eigenvalues'] = None - self.result['matrix'] = None - else: - self.result['eigenvalues'] = a['eigen'] - #self.result['matrix'] = a['matrix'] - #self.result['content'] = data[:200] - #b = compute_matrix(site_b) - print "A: %s" % a - return a['eigen'] - - def load_assets(self): - if self.local_options: - if self.local_options['file']: - self.tool = True - return {} - elif self.local_options['urls']: - return {'url': Asset(self.local_options['urls'])} - else: - self.end() - return {} - else: - return {} - -#domclass = domclassTest(None, None, None) diff --git a/to-be-ported/old-api/dropin.cache b/to-be-ported/old-api/dropin.cache deleted file mode 100755 index 65c2187..0000000 --- a/to-be-ported/old-api/dropin.cache +++ /dev/null @@ -1,243 +0,0 @@ -(dp1 -S'tcpconnect' -p2 -ccopy_reg -_reconstructor -p3 -(ctwisted.plugin -CachedDropin -p4 -c__builtin__ -object -p5 -NtRp6 -(dp7 -S'moduleName' -p8 -S'ooni.plugins.tcpconnect' -p9 -sS'description' -p10 -S'\nThis is a self genrated test created by scaffolding.py.\nyou will need to fill it up with all your necessities.\nSafe hacking :).\n' -p11 -sS'plugins' -p12 -(lp13 -g3 -(ctwisted.plugin -CachedPlugin -p14 -g5 -NtRp15 -(dp16 -S'provided' -p17 -(lp18 -ctwisted.plugin -IPlugin -p19 -acooni.plugoo.interface -ITest -p20 -asS'dropin' -p21 -g6 -sS'name' -p22 -S'tcpconnect' -p23 -sg10 -NsbasbsS'domclass' -p24 -g3 -(g4 -g5 -NtRp25 -(dp26 -g8 -S'ooni.plugins.domclass' -p27 -sg10 -Nsg12 -(lp28 -g3 -(g14 -g5 -NtRp29 -(dp30 -g17 -(lp31 -g19 -ag20 -asg21 -g25 -sg22 -S'domclass' -p32 -sg10 -NsbasbsS'bridget' -p33 -g3 -(g4 -g5 -NtRp34 -(dp35 -g8 -S'ooni.plugins.bridget' -p36 -sg10 -Nsg12 -(lp37 -g3 -(g14 -g5 -NtRp38 -(dp39 -g17 -(lp40 -g19 -ag20 -asg21 -g34 -sg22 -S'bridget' -p41 -sg10 -S"\n XXX fill me in\n\n :ivar config:\n An :class:`ooni.lib.txtorcon.TorConfig` instance.\n :ivar relays:\n A list of all provided relays to test.\n :ivar bridges:\n A list of all provided bridges to test.\n :ivar socks_port:\n Integer for Tor's SocksPort.\n :ivar control_port:\n Integer for Tor's ControlPort.\n :ivar transport:\n String defining the Tor's ClientTransportPlugin, for testing \n a bridge's pluggable transport functionality.\n :ivar tor_binary:\n Path to the Tor binary to use, e.g. '/usr/sbin/tor'\n " -p42 -sbasbsS'daphn3' -p43 -g3 -(g4 -g5 -NtRp44 -(dp45 -g8 -S'plugins.daphn3' -p46 -sg10 -S'\nThis is a self genrated test created by scaffolding.py.\nyou will need to fill it up with all your necessities.\nSafe hacking :).\n' -p47 -sg12 -(lp48 -g3 -(g14 -g5 -NtRp49 -(dp50 -g17 -(lp51 -g19 -ag20 -asg21 -g44 -sg22 -S'daphn3test' -p52 -sg10 -NsbasbsS'httpt' -p53 -g3 -(g4 -g5 -NtRp54 -(dp55 -g8 -S'ooni.plugins.httpt' -p56 -sg10 -S'\nThis is a self genrated test created by scaffolding.py.\nyou will need to fill it up with all your necessities.\nSafe hacking :).\n' -p57 -sg12 -(lp58 -sbsS'chinatrigger' -p59 -g3 -(g4 -g5 -NtRp60 -(dp61 -g8 -S'plugins.chinatrigger' -p62 -sg10 -Nsg12 -(lp63 -g3 -(g14 -g5 -NtRp64 -(dp65 -g17 -(lp66 -g19 -ag20 -asg21 -g60 -sg22 -S'chinatrigger' -p67 -sg10 -S'\n This test is a OONI based implementation of the C tool written\n by Philipp Winter to engage chinese probes in active scanning.\n\n Example of running it:\n ./ooni/ooniprobe.py chinatrigger -d 127.0.0.1 -p 8080 -f bla.pcap\n ' -p68 -sbasbsS'dnstamper' -p69 -g3 -(g4 -g5 -NtRp70 -(dp71 -g8 -S'ooni.plugins.dnstamper' -p72 -sg10 -S'\n dnstamper\n *********\n\n This test resolves DNS for a list of domain names, one per line, in the\n file specified in the ooni-config under the setting "dns_experiment". If\n the file is top-1m.txt, the test will be run using Amazon\'s list of top\n one million domains. The experimental dns servers to query should\n be specified one per line in assets/dns_servers.txt.\n\n The test reports censorship if the cardinality of the intersection of\n the query result set from the control server and the query result set\n from the experimental server is zero, which is to say, if the two sets\n have no matching results whatsoever.\n\n NOTE: This test frequently results in false positives due to GeoIP-based\n load balancing on major global sites such as google, facebook, and\n youtube, etc.\n\n :author: Isis Lovecruft, Arturo Filast\xc3\xb2\n :license: see LICENSE for more details\n\n TODO:\n * Finish porting to twisted\n * Finish the client.Resolver() subclass and test it\n * Use the DNS tests from captiveportal\n * Use plugoo/reports.py for final data\n' -p73 -sg12 -(lp74 -g3 -(g14 -g5 -NtRp75 -(dp76 -g17 -(lp77 -g19 -ag20 -asg21 -g70 -sg22 -S'dnstamper' -p78 -sg10 -S'\n XXX fill me in\n ' -p79 -sbasbsS'blocking' -p80 -g3 -(g4 -g5 -NtRp81 -(dp82 -g8 -S'plugins.blocking' -p83 -sg10 -Nsg12 -(lp84 -g3 -(g14 -g5 -NtRp85 -(dp86 -g17 -(lp87 -g19 -ag20 -asg21 -g81 -sg22 -S'blocking' -p88 -sg10 -Nsbasbs. \ No newline at end of file diff --git a/to-be-ported/old-api/tcpscan.py b/to-be-ported/old-api/tcpscan.py deleted file mode 100644 index b371c88..0000000 --- a/to-be-ported/old-api/tcpscan.py +++ /dev/null @@ -1,84 +0,0 @@ -""" - TCP Port Scanner - **************** - - Does a TCP connect scan on the IP:port pairs. - -""" -import os -from gevent import socket -from datetime import datetime -import socks - -from plugoo.assets import Asset -from plugoo.tests import Test - -__plugoo__ = "TCP Port Scanner" -__desc__ = "This a test template to be used to build your own tests" - -class TCPScanAsset(Asset): - """ - This is the asset that should be used by the Test. It will - contain all the code responsible for parsing the asset file - and should be passed on instantiation to the test. - """ - def __init__(self, file=None): - self = Asset.__init__(self, file) - - -class TCPScan(Test): - """ - The main Test class - """ - - def experiment(self, *a, **kw): - """ - Fill this up with the tasks that should be performed - on the "dirty" network and should be compared with the - control. - """ - addr = kw['data'] - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - res = False - try: - self.logger.debug('Doing a connection to %s' % addr) - s.connect((addr.split(':')[0], int(addr.split(':')[1]))) - res = True - except socket.error, msg: - self.logger.debug('Connection failed to %s: %s' % (addr, msg)) - - finally: - s.close() - - return {'Time': datetime.now(), - 'Address': addr, - 'Status': res} - - def control(self): - """ - Fill this up with the control related code. - """ - return True - -def run(ooni, asset=None): - """ - This is the function that will be called by OONI - and it is responsible for instantiating and passing - the arguments to the Test class. - """ - config = ooni.config - - # This the assets array to be passed to the run function of - # the test - if asset: - assets = [TCPScanAsset(asset)] - else: - assets = [TCPScanAsset(os.path.join(config.main.assetdir, \ - "tcpscan.txt"))] - - # Instantiate the Test - thetest = TCPScan(ooni) - ooni.logger.info("starting TCP Scan...") - # Run the test with argument assets - thetest.run(assets) - ooni.logger.info("finished.")

1 0

[translation/whisperback_completed] Update translations for whisperback_completed
by translation＠torproject.org 21 Nov '12

21 Nov '12

commit 2560a4006c8550664c5db4a7bb1ac2d5e4ac02a9 Author: Translation commit bot <translation(a)torproject.org> Date: Wed Nov 21 13:45:45 2012 +0000 Update translations for whisperback_completed --- eu/eu.po | 16 ++++++++-------- 1 files changed, 8 insertions(+), 8 deletions(-) diff --git a/eu/eu.po b/eu/eu.po index f983794..260a170 100644 --- a/eu/eu.po +++ b/eu/eu.po @@ -9,14 +9,14 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-02-16 22:33+0100\n" -"PO-Revision-Date: 2012-03-08 17:55+0000\n" +"PO-Revision-Date: 2012-11-21 13:45+0000\n" "Last-Translator: Antxon Baldarra <baldarra(a)euskalerria.org>\n" "Language-Team: LANGUAGE <LL(a)li.org>\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Language: eu\n" -"Plural-Forms: nplurals=2; plural=(n != 1)\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" #. XXX use a better exception #: ../whisperBack/whisperback.py:63 @@ -157,8 +157,8 @@ msgid "Send feedback in an encrypted mail." msgstr "Atzeraelikadura enkriptatutako email batean bidali." #: ../whisperBack/gui.py:405 -msgid "Copyright © 2009-2011 Tails developpers (tails(a)boum.org)" -msgstr "Copyright © 2009-2011 Tails garatzaileak (tails(a)boum.org)" +msgid "Copyright © 2009-2012 Tails developpers (tails(a)boum.org)" +msgstr "Copyright © 2009-2012 Tails garatzaileak (tails(a)boum.org)" #: ../whisperBack/gui.py:406 msgid "Tails developers <tails(a)boum.org>" @@ -182,8 +182,8 @@ msgid "Bug description" msgstr "Akatsaren deskribapena" #: ../data/whisperback.ui.h:3 -msgid "Copyright © 2009-2011 tails(a)boum.org" -msgstr "Copyright © 2009-2011 tails(a)boum.org" +msgid "Copyright © 2009-2012 tails(a)boum.org" +msgstr "Copyright © 2009-2012 tails(a)boum.org" #: ../data/whisperback.ui.h:4 msgid "Help" @@ -214,7 +214,7 @@ msgstr "Sartzeko xehetasun teknikoak" #: ../data/whisperback.ui.h:12 msgid "" "WhisperBack - Send feedback in an encrypted mail\n" -"Copyright (C) 2009-2011 Tails developers <tails(a)boum.org>\n" +"Copyright (C) 2009-2012 Tails developers <tails(a)boum.org>\n" "\n" "This program is free software; you can redistribute it and/or modify\n" "it under the terms of the GNU General Public License as published by\n" @@ -228,7 +228,7 @@ msgid "" "\n" "You should have received a copy of the GNU General Public License\n" "along with this program. If not, see <http://www.gnu.org/licenses/>.\n" -msgstr "WhisperBack - Bidali atzeraelikadura enkriptatutako postan\nCopyright (C) 2009-2011 Tails garatzaileak <tails(a)boum.org>\n\nPrograma hau software librea da; birbana eta/edo alda dezakezu \nSoftware Foundation-ek argitaratutako GNU Lizentzia Publiko \nOrokorraren 3. bertsioan, edo (nahiago baduzu) beste berriago \nbatean, jasotako baldintzak betez gero.\n\nPrograma hau erabilgarria izango delakoan banatzen da, baina, \nINOLAKO BERMERIK GABE; era berean, ez da bermatzen beraren \nEGOKITASUNA MERKATURATZEKO edo HELBURU \nPARTIKULARRETARAKO ERABILTZEKO. Argibide gehiago nahi izanez \ngero, ikus GNU Lizentzia Publiko Orokorra.\n\nPrograma honekin batera GNU Lizentzia Publiko Orokorraren kopia \nbat jasoko zenuen; hala ez bada, ikusi: <http://www.gnu.org/licenses/>.\n" +msgstr "WhisperBack - Bidali berrelikadura enkriptatutako mail batean\nCopyright (C) 2009-2012 Tails garatzaileak <tails(a)boum.org>\n\nPrograma hau software askea da; birbana eta/edo alda dezakezu Software \nFoundation-ek argitaratutako GNU Lizentzia Publiko Orokorraren 3. \nbertsioan, edo (nahiago baduzu) beste berriago batean.\n\nPrograma hau erabilgarria izango delakoan banatzen da, baina, INOLAKO \nBERMERIK GABE; era berean, ez da bermatzen beraren EGOKITASUNA \nMERKATURATZEKO edo HELBURU PARTIKULARRETARAKO \nERABILTZEKO. Argibide gehiago nahi izanez gero, ikus GNU Lizentzia \nPubliko Orokorra.\n\nPorgrama honekin batera GNU Lizentzia Publiko Orokorraren kopia bat \njasoko zenuen; hala ez bada, begiratu <http://www.gnu.org/licenses/>\n" #: ../data/whisperback.ui.h:28 msgid "debugging info"

1 0

[translation/whisperback] Update translations for whisperback
by translation＠torproject.org 21 Nov '12

21 Nov '12

commit 9b3fcfc9b2d95a1681b22d68f2a3b6b6dd886456 Author: Translation commit bot <translation(a)torproject.org> Date: Wed Nov 21 13:45:42 2012 +0000 Update translations for whisperback --- eu/eu.po | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eu/eu.po b/eu/eu.po index 39d4403..260a170 100644 --- a/eu/eu.po +++ b/eu/eu.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2012-02-16 22:33+0100\n" -"PO-Revision-Date: 2012-10-09 12:41+0000\n" +"PO-Revision-Date: 2012-11-21 13:45+0000\n" "Last-Translator: Antxon Baldarra <baldarra(a)euskalerria.org>\n" "Language-Team: LANGUAGE <LL(a)li.org>\n" "MIME-Version: 1.0\n" @@ -228,7 +228,7 @@ msgid "" "\n" "You should have received a copy of the GNU General Public License\n" "along with this program. If not, see <http://www.gnu.org/licenses/>.\n" -msgstr "" +msgstr "WhisperBack - Bidali berrelikadura enkriptatutako mail batean\nCopyright (C) 2009-2012 Tails garatzaileak <tails(a)boum.org>\n\nPrograma hau software askea da; birbana eta/edo alda dezakezu Software \nFoundation-ek argitaratutako GNU Lizentzia Publiko Orokorraren 3. \nbertsioan, edo (nahiago baduzu) beste berriago batean.\n\nPrograma hau erabilgarria izango delakoan banatzen da, baina, INOLAKO \nBERMERIK GABE; era berean, ez da bermatzen beraren EGOKITASUNA \nMERKATURATZEKO edo HELBURU PARTIKULARRETARAKO \nERABILTZEKO. Argibide gehiago nahi izanez gero, ikus GNU Lizentzia \nPubliko Orokorra.\n\nPorgrama honekin batera GNU Lizentzia Publiko Orokorraren kopia bat \njasoko zenuen; hala ez bada, begiratu <http://www.gnu.org/licenses/>\n" #: ../data/whisperback.ui.h:28 msgid "debugging info"

1 0

[apaf/master] Redirect to index.html when visiting / closes #52
by art＠torproject.org 21 Nov '12

21 Nov '12

commit a44888dc60f8dda2f2c2027ae3d9ea86207cc072 Author: Arturo Filastò <art(a)fuffa.org> Date: Wed Nov 21 13:03:00 2012 +0100 Redirect to index.html when visiting / closes #52 --- apaf/panel/handlers/base.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/apaf/panel/handlers/base.py b/apaf/panel/handlers/base.py index e21e578..966fd49 100644 --- a/apaf/panel/handlers/base.py +++ b/apaf/panel/handlers/base.py @@ -67,4 +67,4 @@ class IndexHandler(PanelHandler): depending on the browser type """ self.set_header('Content-Type', 'text/plain') - self.finish('Hello world') + self.redirect('/index.html')

1 0

[apaf/master] Fix typo in requirements.txt file parsing
by art＠torproject.org 21 Nov '12

21 Nov '12

commit 4fa06913a09a7aafb64d6c369c2c1ad43bcf5f5f Author: Arturo Filastò <hellais(a)gmail.com> Date: Wed Nov 14 21:33:58 2012 +0100 Fix typo in requirements.txt file parsing Thanks to @estevopaz for reporting it. Closes #49 --- setup.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/setup.py b/setup.py index 5f1ef10..d7a21fd 100644 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ if os.path.exists(join(mandir, 'apaf.1')): ## Dependencies ## requirements = None with open('requirements.txt') as dependencies: - requirements = [dependency.strip() for dependency in depencies] + requirements = [dependency.strip() for dependency in dependencies] setup( name=config.appname,

1 0