- tor-commits - lists.torproject.org

[ooni-probe/master] Implement utility class for dealing with HTTP based testing
by art＠torproject.org 13 Jun '12

13 Jun '12

commit b1e8d865f121854e9f96ca70b54eb4a656679b66 Author: Arturo Filastò <art(a)baculo.org> Date: Wed Jun 13 19:25:55 2012 +0200 Implement utility class for dealing with HTTP based testing --- ooni/ooniprobe.py | 3 +- ooni/plugoo/tests.py | 13 ++++- ooni/plugoo/work.py | 4 +- ooni/protocols/http.py | 118 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+), 6 deletions(-) diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py index 7248e70..a997041 100755 --- a/ooni/ooniprobe.py +++ b/ooni/ooniprobe.py @@ -64,7 +64,8 @@ def runTest(test, options, global_options): log.start(global_options['log'], 1) - wgen = work.WorkGenerator(test_class(options, global_options, report), + wgen = work.WorkGenerator(test_class(options, global_options, report, + reactor=reactor), dict(options), start=options['resume']) diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py index 0632670..608cee3 100644 --- a/ooni/plugoo/tests.py +++ b/ooni/plugoo/tests.py @@ -19,12 +19,19 @@ from ooni.plugoo.interface import ITest class OONITest(object): blocking = False - def __init__(self, local_options, global_options, report, ooninet=None): + def __init__(self, local_options, global_options, report, ooninet=None, + reactor=None): self.local_options = local_options self.global_options = global_options self.assets = self.load_assets() self.report = report #self.ooninet = ooninet + self.reactor = reactor + self.initialize() + self.result = {} + + def initialize(self): + pass def load_assets(self): """ @@ -40,11 +47,11 @@ class OONITest(object): def finished(self, control): #self.ooninet.report(result) self.end_time = datetime.now() - result = {} + result = self.result result['start_time'] = self.start_time result['end_time'] = self.end_time result['run_time'] = self.end_time - self.start_time - result['result'] = control + result['control'] = control log.msg("FINISHED %s" % result) self.report(result) return result diff --git a/ooni/plugoo/work.py b/ooni/plugoo/work.py index 16a6e2c..7e9be0c 100644 --- a/ooni/plugoo/work.py +++ b/ooni/plugoo/work.py @@ -54,8 +54,8 @@ class Worker(object): if self._running < self.maxconcurrent: asset, test, idx = workunit self._running += 1 - test.startTest(asset).addBoth(self._run) - return + return test.startTest(asset).addBoth(self._run) + d = defer.Deferred() self._queued.append((workunit, d)) return d diff --git a/ooni/protocols/http.py b/ooni/protocols/http.py new file mode 100644 index 0000000..068ec0d --- /dev/null +++ b/ooni/protocols/http.py @@ -0,0 +1,118 @@ +import random +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from twisted.internet import protocol, defer +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset +from ooni.protocols.useragents import useragents +from ooni import log + +useragents = [("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-GB; rv:1.8.1.6) Gecko/20070725 Firefox/2.0.0.6", "Firefox 2.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)", "Internet Explorer 7, Windows Vista"), + ("Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30)", "Internet Explorer 7, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 6, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 5.0; Windows NT 5.1; .NET CLR 1.1.4322)", "Internet Explorer 5, Windows XP"), + ("Opera/9.20 (Windows NT 6.0; U; en)", "Opera 9.2, Windows Vista"), + ("Opera/9.00 (Windows NT 5.1; U; en)", "Opera 9.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.50", "Opera 8.5, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera 8.0", "Opera 8.0, Windows XP"), + ("Mozilla/4.0 (compatible; MSIE 6.0; MSIE 5.5; Windows NT 5.1) Opera 7.02 [en]", "Opera 7.02, Windows XP"), + ("Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.7.5) Gecko/20060127 Netscape/8.1", "Netscape 8.1, Windows XP")] + +class BodyReceiver(protocol.Protocol): + def __init__(self, finished): + self.finished = finished + self.data = "" + + def dataReceived(self, bytes): + self.data += bytes + + def connectionLost(self, reason): + self.finished.callback(self.data) + + +from twisted.web.http_headers import Headers +class HTTPTest(OONITest): + """ + A utility class for dealing with HTTP based testing. It provides methods to + be overriden for dealing with HTTP based testing. + The main functions to look at are processResponseBody and + processResponseHeader that are invoked once the headers have been received + and once the request body has been received. + """ + randomize_ua = True + + def initialize(self, reactor=None): + from twisted.web.client import Agent + import yaml + + if not self.reactor: + from twisted.internet import reactor + self.reactor = reactor + self.agent = Agent(self.reactor) + self.request = {} + self.response = {} + + def _processResponseBody(self, data): + self.response['body'] = data + self.result['response'] = self.response + self.processResponseBody(data) + + def processResponseBody(self, data): + """ + This should handle all the response body smushing for getting it ready + to be passed onto the control. + + @param data: The content of the body returned. + """ + pass + + def processResponseHeaders(self, headers): + """ + This should take care of dealing with the returned HTTP headers. + + @param headers: The content of the returned headers. + """ + pass + + def experiment(self, args): + log.msg("Running experiment") + d = self.build_request(args['url']) + def finished(data): + return data + + d.addCallback(self._cbResponse) + d.addCallback(finished) + return d + + def _cbResponse(self, response): + self.response['headers'] = list(response.headers.getAllRawHeaders()) + self.processResponseHeaders(self.response['headers']) + finished = defer.Deferred() + response.deliverBody(BodyReceiver(finished)) + finished.addCallback(self._processResponseBody) + + def randomize_useragent(self): + user_agent = random.choice(useragents) + self.request['headers']['User-Agent'] = [user_agent] + + def build_request(self, url, method="GET", headers=None, body=None): + self.request['method'] = method + self.request['url'] = url + self.request['headers'] = headers if headers else {} + self.request['body'] = body + if self.randomize_ua: + self.randomize_useragent() + + self.result['request'] = self.request + return self.agent.request(self.request['method'], self.request['url'], + Headers(self.request['headers']), + self.request['body']) + + def load_assets(self): + if self.local_options: + return {'url': Asset(self.local_options['asset'])} + else: + return {} +

1 0

[ooni-probe/master] Show usage of utilty library with simple HTTP test
by art＠torproject.org 13 Jun '12

13 Jun '12

commit 76767e09cba5dcf3cb19263032331d41046665a0 Author: Arturo Filastò <art(a)baculo.org> Date: Wed Jun 13 19:26:17 2012 +0200 Show usage of utilty library with simple HTTP test --- ooni/plugins/httpt.py | 40 ++++++++++++++++++++++++++++++++++++++++ 1 files changed, 40 insertions(+), 0 deletions(-) diff --git a/ooni/plugins/httpt.py b/ooni/plugins/httpt.py new file mode 100644 index 0000000..3f92019 --- /dev/null +++ b/ooni/plugins/httpt.py @@ -0,0 +1,40 @@ +""" +This is a self genrated test created by scaffolding.py. +you will need to fill it up with all your necessities. +Safe hacking :). +""" +from zope.interface import implements +from twisted.python import usage +from twisted.plugin import IPlugin +from ooni.plugoo.tests import ITest, OONITest +from ooni.plugoo.assets import Asset +from ooni.protocols import http +from ooni import log + +class httptArgs(usage.Options): + optParameters = [['urls', 'u', None, 'Urls file'], + ['resume', 'r', 0, 'Resume at this index']] + +class httptTest(http.HTTPTest): + implements(IPlugin, ITest) + + shortName = "httpt" + description = "httpt" + requirements = None + options = httptArgs + blocking = False + + def control(self, experiment_result, args): + # What you return here ends up inside of the report. + log.msg("Running control") + return {} + + def load_assets(self): + if self.local_options: + return {'url': Asset(self.local_options['urls'])} + else: + return {} + +# We need to instantiate it otherwise getPlugins does not detect it +# XXX Find a way to load plugins without instantiating them. +httpt = httptTest(None, None, None)

1 0

[stem/master] Tuples lack an index method in python 2.5
by atagar＠torproject.org 13 Jun '12

13 Jun '12

commit 122d0a6aa57f21fe2ef7d41c6fd14736074345c6 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jun 13 09:33:20 2012 -0700 Tuples lack an index method in python 2.5 Yet another python 2.5 compatability hack. --- stem/util/enum.py | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diff --git a/stem/util/enum.py b/stem/util/enum.py index af9ac27..8b21100 100644 --- a/stem/util/enum.py +++ b/stem/util/enum.py @@ -134,7 +134,9 @@ class Enum: if not value in self._values: raise ValueError("No such enumeration exists: %s (options: %s)" % (value, ", ".join(self._values))) - next_index = (self._values.index(value) + 1) % len(self._values) + # TODO: python 2.5 lacks an index method on tuples, when we drop support + # we can drop this hack + next_index = (list(self._values).index(value) + 1) % len(self._values) return self._values[next_index] def previous(self, value): @@ -151,7 +153,9 @@ class Enum: if not value in self._values: raise ValueError("No such enumeration exists: %s (options: %s)" % (value, ", ".join(self._values))) - prev_index = (self._values.index(value) - 1) % len(self._values) + # TODO: python 2.5 lacks an index method on tuples, when we drop support + # we can drop this hack + prev_index = (list(self._values).index(value) - 1) % len(self._values) return self._values[prev_index] def __getitem__(self, item):

1 0

[stem/master] Dumbing down syntax for python 2.5
by atagar＠torproject.org 13 Jun '12

13 Jun '12

commit 98b3a8dfb10944ff3ec7b41a3b1f39297173bf26 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jun 13 08:39:57 2012 -0700 Dumbing down syntax for python 2.5 Evidently even with the 'with' keyword's future import it still chokes on comma delimited entries (instead you need to do nested 'with' blocks). Ick. Oh well, if that's all we need to maintain 2.5 compatability then I'll be happy. It's a pity we can't tell python 'run in 2.5 compatability mode' to check for these things. I doubt that we'll remember to run against a 2.5 instance to keep stem working there. Oh well, guess this is a problem that'll need to be addressed when figuring out a release procedure. --- stem/control.py | 33 +++++++++++++++++---------------- stem/util/conf.py | 15 ++++++++------- test/check_whitespace.py | 2 ++ 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/stem/control.py b/stem/control.py index f20451b..09c61e4 100644 --- a/stem/control.py +++ b/stem/control.py @@ -309,22 +309,23 @@ class BaseController: # Any changes to our is_alive() state happen under the send lock, so we # need to have it to ensure it doesn't change beneath us. - with self._socket._get_send_lock(), self._status_listeners_lock: - change_timestamp = time.time() - - if expect_alive != None and expect_alive != self.is_alive(): - return - - for listener, spawn in self._status_listeners: - if spawn: - name = "%s notification" % state - args = (self, state, change_timestamp) - - notice_thread = threading.Thread(target = listener, args = args, name = name) - notice_thread.setDaemon(True) - notice_thread.start() - else: - listener(self, state, change_timestamp) + with self._socket._get_send_lock(): + with self._status_listeners_lock: + change_timestamp = time.time() + + if expect_alive != None and expect_alive != self.is_alive(): + return + + for listener, spawn in self._status_listeners: + if spawn: + name = "%s notification" % state + args = (self, state, change_timestamp) + + notice_thread = threading.Thread(target = listener, args = args, name = name) + notice_thread.setDaemon(True) + notice_thread.start() + else: + listener(self, state, change_timestamp) def _launch_threads(self): """ diff --git a/stem/util/conf.py b/stem/util/conf.py index 70287c2..3f1865a 100644 --- a/stem/util/conf.py +++ b/stem/util/conf.py @@ -280,13 +280,14 @@ class Config(): elif not self._path: raise ValueError("Unable to save configuration: no path provided") - with self._contents_lock, open(self._path, 'w') as output_file: - for entry_key in sorted(self.keys()): - for entry_value in self.get_value(entry_key, multiple = True): - # check for multi line entries - if "\n" in entry_value: entry_value = "\n|" + entry_value.replace("\n", "\n|") - - output_file.write('%s %s\n' % (entry_key, entry_value)) + with self._contents_lock: + with open(self._path, 'w') as output_file: + for entry_key in sorted(self.keys()): + for entry_value in self.get_value(entry_key, multiple = True): + # check for multi line entries + if "\n" in entry_value: entry_value = "\n|" + entry_value.replace("\n", "\n|") + + output_file.write('%s %s\n' % (entry_key, entry_value)) def clear(self): """ diff --git a/test/check_whitespace.py b/test/check_whitespace.py index 6c47cab..23ca1af 100644 --- a/test/check_whitespace.py +++ b/test/check_whitespace.py @@ -14,6 +14,8 @@ it's so much easier to do here...): from __future__ import with_statement """ +from __future__ import with_statement + import re import os

1 0

[stem/master] Tor executable shouldn't be needed for unit tests
by atagar＠torproject.org 13 Jun '12

13 Jun '12

commit e09e6b00a4b603db6342d27a11e3073acf8c74c6 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jun 13 08:46:21 2012 -0700 Tor executable shouldn't be needed for unit tests We only need a tor variable when running the integ tests. Dropping the check that it exists in our path if we're just running the unit tests. --- run_tests.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/run_tests.py b/run_tests.py index cd7c688..a6d604c 100755 --- a/run_tests.py +++ b/run_tests.py @@ -213,7 +213,7 @@ def load_user_configuration(test_config): sys.exit(1) tor_config = CONFIG["argument.tor"] - if not os.path.exists(tor_config) and not stem.util.system.is_available(tor_config): + if CONFIG["argument.integ"] and not os.path.exists(tor_config) and not stem.util.system.is_available(tor_config): print "Unable to start tor, '%s' does not exists." % tor_config sys.exit(1)

1 0

[stem/master] Adding module to check for requirements
by atagar＠torproject.org 13 Jun '12

13 Jun '12

commit b84a75638942e61a58b2af8edd97448ea72857d7 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jun 13 09:20:34 2012 -0700 Adding module to check for requirements Stem requires at least python 2.5, and beyond that our python version and modules does impact what functionality we have available (yay). Adding a module for tracking what we have available. --- run_tests.py | 9 +++ stem/__init__.py | 2 +- stem/descriptor/server_descriptor.py | 11 +--- stem/prereq.py | 86 +++++++++++++++++++++++++++++ test/unit/descriptor/server_descriptor.py | 5 +- 5 files changed, 102 insertions(+), 11 deletions(-) diff --git a/run_tests.py b/run_tests.py index a6d604c..5c41f40 100755 --- a/run_tests.py +++ b/run_tests.py @@ -45,6 +45,7 @@ import test.integ.util.system import test.integ.process import test.integ.version +import stem.prereq import stem.util.conf import stem.util.enum import stem.util.log as log @@ -218,6 +219,14 @@ def load_user_configuration(test_config): sys.exit(1) if __name__ == '__main__': + try: + stem.prereq.check_requriements() + except ImportError, exc: + print exc + print + + sys.exit(1) + start_time = time.time() # override flag to indicate at the end that testing failed somewhere diff --git a/stem/__init__.py b/stem/__init__.py index 61374a6..8f5add1 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -8,5 +8,5 @@ __contact__ = 'atagar(a)torproject.org' __url__ = 'http://www.atagar.com/stem/' __license__ = 'LGPLv3' -__all__ = ["descriptor", "response", "util", "connection", "control", "process", "socket", "version"] +__all__ = ["descriptor", "response", "util", "connection", "control", "prereq", "process", "socket", "version"] diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 99080ab..765a5ba 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -32,6 +32,7 @@ import base64 import hashlib import datetime +import stem.prereq import stem.descriptor import stem.descriptor.extrainfo_descriptor import stem.version @@ -39,13 +40,6 @@ import stem.util.log as log import stem.util.connection import stem.util.tor_tools -try: - import rsa - IS_RSA_AVAILABLE = True -except ImportError: - log.info("Unable to import the rsa module. Because of this we'll be unable to verify descriptor signature integrity.") - IS_RSA_AVAILABLE = False - # relay descriptors must have exactly one of the following REQUIRED_FIELDS = ( "router", @@ -572,7 +566,8 @@ class RelayDescriptor(ServerDescriptor): # if we have a fingerprint then checks that our fingerprint is a hash of # our signing key - if IS_RSA_AVAILABLE and validate and self.fingerprint: + if validate and self.fingerprint and stem.prereq.is_rsa_available(): + import rsa pubkey = rsa.PublicKey.load_pkcs1(self.signing_key) der_encoded = pubkey.save_pkcs1(format = "DER") key_hash = hashlib.sha1(der_encoded).hexdigest() diff --git a/stem/prereq.py b/stem/prereq.py new file mode 100644 index 0000000..e443078 --- /dev/null +++ b/stem/prereq.py @@ -0,0 +1,86 @@ +""" +Checks for stem dependencies. We require python 2.5 or greater (in the 2.x +series). Other requirements for complete functionality are... + +* Python 2.6 + + * os.walk's followlinks argument + +* rsa module + + * validating descriptor signagure integrity + +:: + + check_requriements - checks for minimum requirements for running stem + + is_python_26 - checks if python 2.6 or later is available + is_python_27 - checks if python 2.7 or later is available + + is_rsa_available - checks if the rsa module is available +""" + +import sys + +import stem.util.log as log + +IS_RSA_AVAILABLE = None + +def check_requriements(): + """ + Checks that we meet the minimum requirements to run stem. If we don't then + this raises an ImportError with the issue. + + :raises: ImportError with the problem if we don't meet stem's requirements + """ + + major_version, minor_version = sys.version_info[0:2] + + if major_version > 2: + raise ImportError("stem isn't compatible beyond the python 2.x series") + elif major_version < 2 or minor_version < 5: + raise ImportError("stem requires python version 2.5 or greater") + +def is_python_26(): + """ + Checks if we're in the 2.6 - 2.x range. + + :returns: bool that is True if we meet this requirement and False otherwise + """ + + return _check_version(6) + +def is_python_27(): + """ + Checks if we're in the 2.7 - 2.x range. + + :returns: bool that is True if we meet this requirement and False otherwise + """ + + return _check_version(7) + +def is_rsa_available(): + global IS_RSA_AVAILABLE + + if IS_RSA_AVAILABLE == None: + try: + import rsa + IS_RSA_AVAILABLE = True + except ImportError: + IS_RSA_AVAILABLE = False + + msg = "Unable to import the rsa module. Because of this we'll be unable to verify descriptor signature integrity." + log.log_once("stem.prereq.is_rsa_availabe", log.INFO, msg) + + return IS_RSA_AVAILABLE + +def _check_version(minor_req): + major_version, minor_version = sys.version_info[0:2] + + if major_version > 2: + return False + elif major_version < 2 or minor_version < minor_req: + return False + + return True + diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 3bb2c7a..8502b56 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -6,6 +6,7 @@ import datetime import StringIO import unittest +import stem.prereq import stem.descriptor.server_descriptor from stem.descriptor.server_descriptor import RelayDescriptor, BridgeDescriptor @@ -312,7 +313,7 @@ class TestServerDescriptor(unittest.TestCase): Checks that a fingerprint matching the hash of our signing key will validate. """ - if not stem.descriptor.server_descriptor.IS_RSA_AVAILABLE: + if not stem.prereq.is_rsa_available(): self.skipTest("(rsa module unavailable)") fingerprint = "4F0C 867D F0EF 6816 0568 C826 838F 482C EA7C FE44" @@ -326,7 +327,7 @@ class TestServerDescriptor(unittest.TestCase): it doesn't match the hash of our signing key. """ - if not stem.descriptor.server_descriptor.IS_RSA_AVAILABLE: + if not stem.prereq.is_rsa_available(): self.skipTest("(rsa module unavailable)") fingerprint = "4F0C 867D F0EF 6816 0568 C826 838F 482C EA7C FE45"

1 0

[stem/master] Adding todo notes around the stupid concessions to 2.5
by atagar＠torproject.org 13 Jun '12

13 Jun '12

commit 4581c355d64f3241d649256d4e14d839aedc9ea5 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jun 13 09:27:47 2012 -0700 Adding todo notes around the stupid concessions to 2.5 I hate dirtying our code for ye old python 2.5. It's ancient and that fact isn't getting rosier very fast. However, a lot of people still use it so we should still support it for now. This won't always be the case so including TODO notes to remove these icky bits when we drop compatability for it. --- stem/control.py | 1 + stem/util/conf.py | 1 + 2 files changed, 2 insertions(+), 0 deletions(-) diff --git a/stem/control.py b/stem/control.py index 09c61e4..47b08c9 100644 --- a/stem/control.py +++ b/stem/control.py @@ -309,6 +309,7 @@ class BaseController: # Any changes to our is_alive() state happen under the send lock, so we # need to have it to ensure it doesn't change beneath us. + # TODO: when we drop python 2.5 compatability we can simplify this with self._socket._get_send_lock(): with self._status_listeners_lock: change_timestamp = time.time() diff --git a/stem/util/conf.py b/stem/util/conf.py index 3f1865a..79291c0 100644 --- a/stem/util/conf.py +++ b/stem/util/conf.py @@ -280,6 +280,7 @@ class Config(): elif not self._path: raise ValueError("Unable to save configuration: no path provided") + # TODO: when we drop python 2.5 compatability we can simplify this with self._contents_lock: with open(self._path, 'w') as output_file: for entry_key in sorted(self.keys()):

1 0

[stem/master] The followlinks arg of os.walk needs python 2.6
by atagar＠torproject.org 13 Jun '12

13 Jun '12

commit d2eef839544eaf1ed55a85df2e4fd466a175fc40 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jun 13 09:24:34 2012 -0700 The followlinks arg of os.walk needs python 2.6 We were using the 'followlinks' argument two places: in the descriptor reader and whitespace checker. For the former I'm noting that 2.6 is a requirement for the argument and coding around it to work on 2.5. For the later I don't really care if the whitespace checker follows links so I'm simply dropping the arg. --- stem/descriptor/reader.py | 10 ++++++++-- test/check_whitespace.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index 01f77df..676676f 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -79,6 +79,7 @@ import threading import mimetypes import Queue +import stem.prereq import stem.descriptor # flag to indicate when the reader thread is out of descriptor files to read @@ -204,7 +205,7 @@ class DescriptorReader: instead. :param str,list target: path or list of paths for files or directories to be read from - :param bool follow_links: determines if we'll follow symlinks when traversing directories + :param bool follow_links: determines if we'll follow symlinks when traversing directories (requires python 2.6) :param int buffer_size: descriptors we'll buffer before waiting for some to be read, this is unbounded if zero :param str persistence_path: if set we will load and save processed file listings from this path, errors are ignored """ @@ -343,8 +344,13 @@ class DescriptorReader: continue if os.path.isdir(target): + if stem.prereq.is_python_26(): + walker = os.walk(target, followlinks = self._follow_links) + else: + walker = os.walk(target) + # adds all of the files that it contains - for root, _, files in os.walk(target, followlinks = self._follow_links): + for root, _, files in walker: for filename in files: self._handle_file(os.path.join(root, filename), new_processed_files) diff --git a/test/check_whitespace.py b/test/check_whitespace.py index 23ca1af..a567d49 100644 --- a/test/check_whitespace.py +++ b/test/check_whitespace.py @@ -102,7 +102,7 @@ def _get_python_files(base_path): :returns: iterator that yields the absolute path for python source code """ - for root, _, files in os.walk(base_path, followlinks = True): + for root, _, files in os.walk(base_path): for filename in files: if filename.endswith(".py"): yield os.path.join(root, filename)

1 0

[tor/master] Script to generate ciphers.inc by Arturo
by nickm＠torproject.org 13 Jun '12

13 Jun '12

commit db07aaf45f116a7c221931f822715f25bc1484fd Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Mar 14 16:01:15 2012 -0400 Script to generate ciphers.inc by Arturo --- src/common/get_mozilla_ciphers.py | 117 +++++++++++++++++++++++++++++++++++++ 1 files changed, 117 insertions(+), 0 deletions(-) diff --git a/src/common/get_mozilla_ciphers.py b/src/common/get_mozilla_ciphers.py new file mode 100644 index 0000000..59f8b59 --- /dev/null +++ b/src/common/get_mozilla_ciphers.py @@ -0,0 +1,117 @@ +#!/usr/bin/python +# Copyright 2011, The Tor Project, Inc +# original version by Arturo Filastò + +# This script parses Firefox and OpenSSL sources, and uses this information +# to generate a ciphers.inc file. + +# Read the cpp file to understand what Ciphers map to what name +fileA = open('security/manager/ssl/src/nsNSSComponent.cpp','r') + +start = None +lines = fileA.readlines() +for i, line in enumerate(lines): + if line.strip().startswith('static CipherPref CipherPrefs'): + # Get the starting boundary of the Cipher Preferences + start = i + + if start and line.strip().startswith('{NULL, 0}'): + # Get the ending boundary of the Cipher Prefs + end = i + break +fileA.close() + +# Parse the lines and put them into a dict +ciphers = {} +for x in lines[start:end]: + line = x.strip() + if line.startswith('{'): + for i, y in enumerate(line): + if y == '}': + parsed = line[1:i] + key, value = parsed.split(',') + ciphers[key.replace("\"","")] = value.strip() + +# Read the JS file to understand what ciphers are enabled +fileB = open('netwerk/base/public/security-prefs.js', 'r') + +enabled_ciphers = {} +for x in fileB.readlines(): + start = None + line = x.strip() + for i, y in enumerate(line): + if y == "(": + start = i + if start and y == ")": + end = i + if start: + inner = line[start+1:end] + key, value = inner.replace("\"","").split(",") + if key.startswith('security.ssl3'): + enabled_ciphers[key.strip()] = value.strip() +fileB.close() + +used_ciphers = [] +for k, v in enabled_ciphers.items(): + if v == "true": + used_ciphers.append(ciphers[k]) + + +oSSLinclude = ('/usr/include/openssl/ssl3.h', '/usr/include/openssl/ssl.h', + '/usr/include/openssl/ssl2.h', '/usr/include/openssl/ssl23.h', + '/usr/include/openssl/tls1.h') + +sslProto = open('security/nss/lib/ssl/sslproto.h', 'r') +sslProtoD = {} + +# This reads the HEX code for the ciphers that are used +# by firefox. +for x in sslProto.readlines(): + line = x.strip() + if line.lower().startswith("#define"): + # If I ever see somebody putting mixed tab + # and spaces I will cut their fingers :) + line = line.replace('\t', ' ') + key = line.split(' ')[1] + value = line.split(' ')[-1] + key = key.strip() + value = value.strip() + print "%s %s\n\n" % (key, value) + sslProtoD[key] = value +sslProto.close() + +cipher_codes = [] +for x in used_ciphers: + cipher_codes.append(sslProtoD[x].lower()) + +cipher_hex = {} +for fl in oSSLinclude: + fp = open(fl, 'r') + for x in fp.readlines(): + line = x.strip() + if line.lower().startswith("#define"): + line = line.replace('\t', ' ') + value = line.split(' ')[1] + key = line.split(' ')[-1] + key = key.strip() + value = value.strip() + if key.startswith('0x'): + key = key.replace('0x0300','0x').lower() + #print "%s %s" % (key, value) + cipher_hex[key] = value + fp.close() + +for x in cipher_codes: + try: + res = """#ifdef %s + CIPHER(%s, %s) + #else + XCIPHER(%s, %s) + #endif""" % (cipher_hex[x], x, cipher_hex[x], x, cipher_hex[x]) + print res + except: + print "Not found %s" % x + + +#print enabled_ciphers +#print ciphers

1 0

[tor/master] Make get_mozilla_ciphers script a little more regexy and readable
by nickm＠torproject.org 13 Jun '12

13 Jun '12

commit a6a905cc11da1f13db08e43b17fb32ed5295a23e Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Mar 14 16:45:38 2012 -0400 Make get_mozilla_ciphers script a little more regexy and readable --- src/common/get_mozilla_ciphers.py | 123 +++++++++++++++++++------------------ 1 files changed, 62 insertions(+), 61 deletions(-) diff --git a/src/common/get_mozilla_ciphers.py b/src/common/get_mozilla_ciphers.py index 59f8b59..378ce01 100644 --- a/src/common/get_mozilla_ciphers.py +++ b/src/common/get_mozilla_ciphers.py @@ -1,54 +1,64 @@ #!/usr/bin/python +# coding=utf-8 # Copyright 2011, The Tor Project, Inc -# original version by Arturo Filastò +# original version by Arturo FilastÃ² # This script parses Firefox and OpenSSL sources, and uses this information # to generate a ciphers.inc file. -# Read the cpp file to understand what Ciphers map to what name +import re + +##### +# Read the cpp file to understand what Ciphers map to what name : +# Make "ciphers" a map from name used in the javascript to a cipher macro name fileA = open('security/manager/ssl/src/nsNSSComponent.cpp','r') -start = None -lines = fileA.readlines() -for i, line in enumerate(lines): - if line.strip().startswith('static CipherPref CipherPrefs'): +# The input format is a file containing exactly one section of the form: +# static CipherPref CipherPrefs[] = { +# {"name", MACRO_NAME}, // comment +# ... +# {NULL, 0} +# } + +inCipherSection = False +cipherLines = [] +for line in fileA: + if line.startswith('static CipherPref CipherPrefs'): # Get the starting boundary of the Cipher Preferences - start = i - - if start and line.strip().startswith('{NULL, 0}'): - # Get the ending boundary of the Cipher Prefs - end = i - break + inCipherSection = True + elif inCipherSection: + line = line.strip() + if line.startswith('{NULL, 0}'): + # At the ending boundary of the Cipher Prefs + break + else: + cipherLines.append(line) fileA.close() # Parse the lines and put them into a dict ciphers = {} -for x in lines[start:end]: - line = x.strip() - if line.startswith('{'): - for i, y in enumerate(line): - if y == '}': - parsed = line[1:i] - key, value = parsed.split(',') - ciphers[key.replace("\"","")] = value.strip() - -# Read the JS file to understand what ciphers are enabled +for line in cipherLines: + m = re.search(r'^{\s*\"([^\"]+)\",\s*(\S*)\s*}', line) + if m: + key,value = m.groups() + ciphers[key] = value + +##### +# Read the JS file to understand what ciphers are enabled. The format is +# pref("name", true/false); +# Build a map enabled_ciphers from javascript name to "true" or "false", +# and an (unordered!) list of the macro names for those ciphers that are +# enabled. fileB = open('netwerk/base/public/security-prefs.js', 'r') enabled_ciphers = {} -for x in fileB.readlines(): - start = None - line = x.strip() - for i, y in enumerate(line): - if y == "(": - start = i - if start and y == ")": - end = i - if start: - inner = line[start+1:end] - key, value = inner.replace("\"","").split(",") - if key.startswith('security.ssl3'): - enabled_ciphers[key.strip()] = value.strip() +for line in fileB: + m = re.match(r'pref\(\"([^\"]+)\"\s*,\s*(\S*)\s*\)', line) + if not m: + continue + key, val = m.groups() + if key.startswith("security.ssl3"): + enabled_ciphers[key] = val fileB.close() used_ciphers = [] @@ -56,27 +66,21 @@ for k, v in enabled_ciphers.items(): if v == "true": used_ciphers.append(ciphers[k]) - oSSLinclude = ('/usr/include/openssl/ssl3.h', '/usr/include/openssl/ssl.h', '/usr/include/openssl/ssl2.h', '/usr/include/openssl/ssl23.h', '/usr/include/openssl/tls1.h') +##### +# This reads the hex code for the ciphers that are used by firefox. +# sslProtoD is set to a map from macro name to macro value in sslproto.h; +# cipher_codes is set to an (unordered!) list of these hex values. sslProto = open('security/nss/lib/ssl/sslproto.h', 'r') sslProtoD = {} -# This reads the HEX code for the ciphers that are used -# by firefox. -for x in sslProto.readlines(): - line = x.strip() - if line.lower().startswith("#define"): - # If I ever see somebody putting mixed tab - # and spaces I will cut their fingers :) - line = line.replace('\t', ' ') - key = line.split(' ')[1] - value = line.split(' ')[-1] - key = key.strip() - value = value.strip() - print "%s %s\n\n" % (key, value) +for line in sslProto: + m = re.match('#define\s+(\S+)\s+(\S+)', line) + if m: + key, value = m.groups() sslProtoD[key] = value sslProto.close() @@ -84,23 +88,23 @@ cipher_codes = [] for x in used_ciphers: cipher_codes.append(sslProtoD[x].lower()) +#### +# Now read through all the openssl include files, and try to find the openssl +# macro names for those files. cipher_hex = {} for fl in oSSLinclude: fp = open(fl, 'r') - for x in fp.readlines(): - line = x.strip() - if line.lower().startswith("#define"): - line = line.replace('\t', ' ') - value = line.split(' ')[1] - key = line.split(' ')[-1] - key = key.strip() - value = value.strip() + for line in fp.readlines(): + m = re.match('#define\s+(\S+)\s+(\S+)', line) + if m: + value,key = m.groups() if key.startswith('0x'): key = key.replace('0x0300','0x').lower() #print "%s %s" % (key, value) cipher_hex[key] = value fp.close() +# Now generate the output. for x in cipher_codes: try: res = """#ifdef %s @@ -109,9 +113,6 @@ for x in cipher_codes: XCIPHER(%s, %s) #endif""" % (cipher_hex[x], x, cipher_hex[x], x, cipher_hex[x]) print res - except: + except KeyError: print "Not found %s" % x - -#print enabled_ciphers -#print ciphers

1 0