- tor-commits - lists.torproject.org

[stem/master] Revisions for GETCONF handling
by atagar＠torproject.org 04 Jul '12

04 Jul '12

commit c09b80ef912bb590a37e0d4533a34bbba14cabf9 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jul 4 14:25:32 2012 -0700 Revisions for GETCONF handling These changes have gone through several review iterations so there wasn't much to tweak with it. Only substantial twiddling on my part was... * The super() method is new to me, and looks like it's the preferred method (especially in Python 3.x when it's syntactically much nicer). Swapped all of our constructors to use super() rather than Parent.__init__(self, args). This also forced me to fix some base classes that weren't inheriting from object. * Moved _case_insensitive_lookup() helper into controller module rather than having a separate util. * Removed the "GETCONF reply doesn't match the parameters that we requested." check since this'll always produce an error if tor adds new context sensitive options (my bad). * The get_conf_map() method wouldn't accept context sensitive options like HiddenServiceDir. * The get_conf_map()'s default value is what we should return if the lookup fails, not the mapped values (ie, get_conf_map("blarg", {}) should give an empty map). * Integ test was missing a requirement check that we could make a control socket, causing the RUN_NONE target to fail. --- stem/connection.py | 12 ++-- stem/control.py | 147 ++++++++++++++++++------------ stem/descriptor/__init__.py | 2 +- stem/descriptor/extrainfo_descriptor.py | 6 +- stem/descriptor/reader.py | 10 +- stem/descriptor/server_descriptor.py | 7 +- stem/response/__init__.py | 9 +- stem/response/getconf.py | 5 +- stem/socket.py | 6 +- stem/util/__init__.py | 2 +- stem/util/control.py | 24 ----- stem/util/log.py | 2 +- test/integ/control/controller.py | 13 ++-- 13 files changed, 127 insertions(+), 118 deletions(-) diff --git a/stem/connection.py b/stem/connection.py index e22a2d0..52037a3 100644 --- a/stem/connection.py +++ b/stem/connection.py @@ -799,7 +799,7 @@ class AuthenticationFailure(Exception): """ def __init__(self, message, auth_response = None): - Exception.__init__(self, message) + super(AuthenticationFailure, self).__init__(message) self.auth_response = auth_response class UnrecognizedAuthMethods(AuthenticationFailure): @@ -810,7 +810,7 @@ class UnrecognizedAuthMethods(AuthenticationFailure): """ def __init__(self, message, unknown_auth_methods): - AuthenticationFailure.__init__(self, message) + super(UnrecognizedAuthMethods, self).__init__(message) self.unknown_auth_methods = unknown_auth_methods class IncorrectSocketType(AuthenticationFailure): @@ -844,7 +844,7 @@ class CookieAuthFailed(AuthenticationFailure): """ def __init__(self, message, cookie_path, is_safecookie, auth_response = None): - AuthenticationFailure.__init__(self, message, auth_response) + super(CookieAuthFailed, self).__init__(message, auth_response) self.is_safecookie = is_safecookie self.cookie_path = cookie_path @@ -866,7 +866,7 @@ class AuthChallengeFailed(CookieAuthFailed): """ def __init__(self, message, cookie_path): - CookieAuthFailed.__init__(self, message, cookie_path, True) + super(AuthChallengeFailed, self).__init__(message, cookie_path, True) class AuthChallengeUnsupported(AuthChallengeFailed): """ @@ -881,7 +881,7 @@ class UnrecognizedAuthChallengeMethod(AuthChallengeFailed): """ def __init__(self, message, cookie_path, authchallenge_method): - AuthChallengeFailed.__init__(self, message, cookie_path) + super(UnrecognizedAuthChallengeMethod, self).__init__(message, cookie_path) self.authchallenge_method = authchallenge_method class AuthSecurityFailure(AuthChallengeFailed): @@ -907,7 +907,7 @@ class NoAuthCookie(MissingAuthInfo): """ def __init__(self, message, is_safecookie): - AuthenticationFailure.__init__(self, message) + super(NoAuthCookie, self).__init__(message) self.is_safecookie = is_safecookie # authentication exceptions ordered as per the authenticate function's pydocs diff --git a/stem/control.py b/stem/control.py index 61dd453..1b830a5 100644 --- a/stem/control.py +++ b/stem/control.py @@ -31,7 +31,6 @@ interacting at a higher level. from __future__ import with_statement -import re import time import Queue import threading @@ -56,6 +55,17 @@ State = stem.util.enum.Enum("INIT", "RESET", "CLOSED") UNDEFINED = "<Undefined_ >" +# Configuration options that are fetched by a special key. The keys are +# lowercase to make case insensetive lookups easier. + +MAPPED_CONFIG_KEYS = { + "hiddenservicedir": "HiddenServiceOptions", + "hiddenserviceport": "HiddenServiceOptions", + "hiddenserviceversion": "HiddenServiceOptions", + "hiddenserviceauthorizeclient": "HiddenServiceOptions", + "hiddenserviceoptions": "HiddenServiceOptions" +} + # TODO: The Thread's isAlive() method and theading's currentThread() was # changed to the more conventional is_alive() and current_thread() in python # 2.6 and above. We should use that when dropping python 2.5 compatability. @@ -406,14 +416,6 @@ class Controller(BaseController): BaseController and provides a more user friendly API for library users. """ - _mapped_config_keys = { - "hiddenservicedir": "hiddenserviceoptions", - "hiddenserviceport": "hiddenserviceoptions", - "hiddenserviceversion": "hiddenserviceoptions", - "hiddenserviceauthorizeclient": "hiddenserviceoptions", - "hiddenserviceoptions": "hiddenserviceoptions" - } - def from_port(control_addr = "127.0.0.1", control_port = 9051): """ Constructs a ControlPort based Controller. @@ -464,7 +466,7 @@ class Controller(BaseController): * default if one was provided and our call failed :raises: - :class:`stem.socket.ControllerError` if the call fails, and we weren't provided a default response + :class:`stem.socket.ControllerError` if the call fails and we weren't provided a default response :class:`stem.socket.InvalidArguments` if the 'param' requested was invalid """ @@ -549,12 +551,11 @@ class Controller(BaseController): provided a default then that's returned as if the GETCONF option is undefined or if the call fails for any reason (invalid configuration option, error response, control port closed, initiated, etc). If the configuration key - consists of whitespace only, None is returned, unless a default value is given. + consists of whitespace only, None is returned unless a default value is given. - :param str param: GETCONF option to be queried + :param str param: configuration option to be queried :param object default: response if the query fails - :param bool multiple: if True, the value(s) provided are lists of all returned values, - otherwise this just provides the first value + :param bool multiple: if True, the value(s) provided are lists of all returned values, otherwise this just provides the first value :returns: Response depends upon how we were called as follows... @@ -568,25 +569,27 @@ class Controller(BaseController): :class:`stem.socket.InvalidArguments` if the configuration option requested was invalid """ - if re.match("^\s*$", param): - if default != UNDEFINED: return default - else: return - # automagically change the requested parameter if it's context sensitive - # and cannot be returned on it's own. - param = param.lower() - entries = self.get_conf_map(self._mapped_config_keys.get(param, param), default, multiple) - if param in self._mapped_config_keys: - return entries[stem.util.control.case_insensitive_lookup(entries, param)] - else: return entries[param] + # Config options are case insensitive and don't contain whitespace. Using + # strip so the following check will catch whitespace-only params. + + param = param.lower().strip() + + if not param: + return default if default != UNDEFINED else None + + entries = self.get_conf_map(param, default, multiple) + + if entries == default: return default + else: return _case_insensitive_lookup(entries, param) def get_conf_map(self, param, default = UNDEFINED, multiple = True): """ - Queries the control socket for the values of given configuration options and - provides a mapping of the keys to the values. If provided a default then - that's returned as if the GETCONF option is undefined or if the call fails - for any reason (invalid configuration option, error response, control port - closed, initiated, etc). Configuration keys that contain only whitespace are - ignored. + Queries the control socket for the values of given configuration options + and provides a mapping of the keys to the values. If provided a default + then that's returned if the GETCONF option is undefined or if the call + fails for any reason (invalid configuration option, error response, control + port closed, initiated, etc). Configuration keys that are empty or contain + only whitespace are ignored. There's three use cases for GETCONF: - a single value is provided @@ -603,17 +606,18 @@ class Controller(BaseController): The get_conf() and get_conf_map() functions both try to account for these special mappings, so queried like get_conf("HiddenServicePort") should - behave as you'd expect. + behave as you'd expect. This method, however, simply returns whatever Tor + provides so get_conf_map("HiddenServicePort") will give the same response + as get_conf_map("HiddenServiceOptions"). - :param str,list param: GETCONF option(s) to be queried + :param str,list param: configuration option(s) to be queried :param object default: response if the query fails - :param bool multiple: if True, the value(s) provided are lists of all returned values, - otherwise this just provides the first value + :param bool multiple: if True, the value(s) provided are lists of all returned values,otherwise this just provides the first value :returns: Response depends upon how we were called as follows... - * dict of 'config key => value' mappings, the value is a list if 'multiple' is True and otherwise is a str of just the first value + * dict of 'config key => value' mappings, the value is a list if 'multiple' is True and a str of just the first value otherwise * default if one was provided and our call failed :raises: @@ -626,37 +630,62 @@ class Controller(BaseController): try: # remove strings which contain only whitespace - param = filter(lambda x: not re.match("^\s*$", x), param) + param = filter(lambda entry: entry.strip(), param) if param == []: return {} - response = self.msg("GETCONF %s" % ' '.join(param)) - stem.response.convert("GETCONF", response) + # translate context sensitive options + lookup_param = set([MAPPED_CONFIG_KEYS.get(entry, entry) for entry in param]) - lookup = stem.util.control.case_insensitive_lookup - # map the entries back to the parameters given so the capitalization - # matches. Note: when the same configuration key is provided multiple - # times, this uses the first one and ignores the rest. - for key in response.entries: - if not key.lower() in self._mapped_config_keys: - response.entries[lookup(param, key)] = response.entries[key] - if key != lookup(param, key): del response.entries[key] + response = self.msg("GETCONF %s" % ' '.join(lookup_param)) + stem.response.convert("GETCONF", response) - requested_params = set(param) - reply_params = set(response.entries.keys()) + # Maps the entries back to the parameters that the user requested so the + # capitalization matches (ie, if they request "exitpolicy" then that + # should be the key rather than "ExitPolicy"). When the same + # configuration key is provided multiple times this determines the case + # based on the first and ignores the rest. + # + # This retains the tor provided camel casing of MAPPED_CONFIG_KEYS + # entries since the user didn't request those by their key, so we can't + # be sure what they wanted. - # if none of the requested parameters are context sensitive and if the - # parameters received don't match the parameters requested - if not set(self._mapped_config_keys.values()) & requested_params and requested_params != reply_params: - requested_label = ", ".join(requested_params) - reply_label = ", ".join(reply_params) - - raise stem.socket.ProtocolError("GETCONF reply doesn't match the parameters that we requested. Queried '%s' but got '%s'." % (requested_label, reply_label)) + for key in response.entries: + if not key.lower() in MAPPED_CONFIG_KEYS.values(): + user_expected_key = _case_insensitive_lookup(param, key) + + if key != user_expected_key: + response.entries[user_expected_key] = response.entries[key] + del response.entries[key] - if not multiple: + if multiple: + return response.entries + else: return dict([(entry[0], entry[1][0]) for entry in response.entries.items()]) - return response.entries - except stem.socket.ControllerError, exc: - if default != UNDEFINED: return dict([(p, default) for p in param]) + if default != UNDEFINED: return default else: raise exc +def _case_insensitive_lookup(entries, key): + """ + Makes a case insensitive lookup within a list or dictionary, providing the + first matching entry that we come across. + + :param list,dict entries: list or dictionary to be searched + :param str key: entry or key value to look up + + :returns: case insensitive match + + :raises: ValueError if no such value exists + """ + + if isinstance(entries, dict): + for k, v in entries.items(): + if k.lower() == key.lower(): + return v + else: + for entry in entries: + if entry.lower() == key.lower(): + return entry + + raise ValueError("key '%s' doesn't exist in dict: %s" % (key, entries)) + diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 21dad37..4bddbc5 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -95,7 +95,7 @@ def parse_file(path, descriptor_file): raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line)) -class Descriptor: +class Descriptor(object): """ Common parent for all types of descriptors. """ diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 979c679..b1c7b09 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -281,7 +281,7 @@ class ExtraInfoDescriptor(stem.descriptor.Descriptor): :raises: ValueError if the contents is malformed and validate is True """ - stem.descriptor.Descriptor.__init__(self, raw_contents) + super(ExtraInfoDescriptor, self).__init__(raw_contents) self.nickname = None self.fingerprint = None @@ -709,7 +709,7 @@ class RelayExtraInfoDescriptor(ExtraInfoDescriptor): self.signature = None self._digest = None - ExtraInfoDescriptor.__init__(self, raw_contents, validate) + super(RelayExtraInfoDescriptor, self).__init__(raw_contents, validate) def digest(self): if self._digest is None: @@ -750,7 +750,7 @@ class BridgeExtraInfoDescriptor(ExtraInfoDescriptor): self._digest = None self.transport = None - ExtraInfoDescriptor.__init__(self, raw_contents, validate) + super(BridgeExtraInfoDescriptor, self).__init__(raw_contents, validate) def digest(self): return self._digest diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index 8d4a462..ed07c88 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -97,7 +97,7 @@ class AlreadyRead(FileSkipped): "Already read a file with this 'last modified' timestamp or later." def __init__(self, last_modified, last_modified_when_read): - FileSkipped.__init__(self) + super(AlreadyRead, self).__init__() self.last_modified = last_modified self.last_modified_when_read = last_modified_when_read @@ -105,7 +105,7 @@ class ParsingFailure(FileSkipped): "File contents could not be parsed as descriptor data." def __init__(self, parsing_exception): - FileSkipped.__init__(self) + super(ParsingFailure, self).__init__() self.exception = parsing_exception class UnrecognizedType(FileSkipped): @@ -115,21 +115,21 @@ class UnrecognizedType(FileSkipped): """ def __init__(self, mime_type): - FileSkipped.__init__(self) + super(UnrecognizedType, self).__init__() self.mime_type = mime_type class ReadFailed(FileSkipped): "An IOError occured while trying to read the file." def __init__(self, read_exception): - FileSkipped.__init__(self) + super(ReadFailed, self).__init__() self.exception = read_exception class FileMissing(ReadFailed): "File does not exist." def __init__(self): - ReadFailed.__init__(self, None) + super(FileMissing, self).__init__(None) def load_processed_files(path): """ diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 765a5ba..fc15a3d 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -188,7 +188,7 @@ class ServerDescriptor(stem.descriptor.Descriptor): :raises: ValueError if the contents is malformed and validate is True """ - stem.descriptor.Descriptor.__init__(self, raw_contents) + super(ServerDescriptor, self).__init__(raw_contents) self.nickname = None self.fingerprint = None @@ -561,7 +561,7 @@ class RelayDescriptor(ServerDescriptor): self.signature = None self._digest = None - ServerDescriptor.__init__(self, raw_contents, validate, annotations) + super(RelayDescriptor, self).__init__(raw_contents, validate, annotations) # if we have a fingerprint then checks that our fingerprint is a hash of # our signing key @@ -648,7 +648,8 @@ class BridgeDescriptor(ServerDescriptor): self.address_alt = [] self._digest = None self._scrubbing_issues = None - ServerDescriptor.__init__(self, raw_contents, validate, annotations) + + super(BridgeDescriptor, self).__init__(raw_contents, validate, annotations) def digest(self): return self._digest diff --git a/stem/response/__init__.py b/stem/response/__init__.py index 192e313..155f262 100644 --- a/stem/response/__init__.py +++ b/stem/response/__init__.py @@ -48,12 +48,12 @@ def convert(response_type, message, **kwargs): an in-place conversion of the message from being a ControlMessage to a subclass for its response type. Recognized types include... - * GETINFO - * GETCONF + * **\*** GETINFO + * **\*** GETCONF * PROTOCOLINFO * AUTHCHALLENGE - If the response_type isn't recognized then this is leaves it alone. + **\*** can raise a :class:`stem.socket.InvalidArguments` exception :param str response_type: type of tor response to convert to :param stem.response.ControlMessage message: message to be converted @@ -61,8 +61,7 @@ def convert(response_type, message, **kwargs): :raises: * :class:`stem.socket.ProtocolError` the message isn't a proper response of that type - * :class:`stem.socket.InvalidArguments` the arguments given as input are - invalid. Raised when converting GETINFO or GETCONF requests + * :class:`stem.socket.InvalidArguments` the arguments given as input are invalid * TypeError if argument isn't a :class:`stem.response.ControlMessage` or response_type isn't supported """ diff --git a/stem/response/getconf.py b/stem/response/getconf.py index 30b7f68..627e7bc 100644 --- a/stem/response/getconf.py +++ b/stem/response/getconf.py @@ -8,7 +8,7 @@ class GetConfResponse(stem.response.ControlMessage): Note that configuration parameters won't match what we queried for if it's one of the special mapping options (ex. "HiddenServiceOptions"). - :var dict entries: mapping between the config parameter (string) and their values (list of strings) + :var dict entries: mapping between the config parameter (str) and their values (list of str) """ def _parse_message(self): @@ -41,6 +41,9 @@ class GetConfResponse(stem.response.ControlMessage): if line.is_next_mapping(quoted = False): key, value = line.split("=", 1) # TODO: make this part of the ControlLine? elif line.is_next_mapping(quoted = True): + # TODO: doesn't seem to occure yet in practice... + # https://trac.torproject.org/6172 + key, value = line.pop_mapping(True).items()[0] else: key, value = (line.pop(), None) diff --git a/stem/socket.py b/stem/socket.py index 62312a8..7d40056 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -45,7 +45,7 @@ import stem.response import stem.util.enum import stem.util.log as log -class ControlSocket: +class ControlSocket(object): """ Wrapper for a socket connection that speaks the Tor control protocol. To the better part this transparently handles the formatting for sending and @@ -278,7 +278,7 @@ class ControlPort(ControlSocket): :raises: :class:`stem.socket.SocketError` if connect is True and we're unable to establish a connection """ - ControlSocket.__init__(self) + super(ControlPort, self).__init__() self._control_addr = control_addr self._control_port = control_port @@ -326,7 +326,7 @@ class ControlSocketFile(ControlSocket): :raises: :class:`stem.socket.SocketError` if connect is True and we're unable to establish a connection """ - ControlSocket.__init__(self) + super(ControlSocketFile, self).__init__() self._socket_path = socket_path if connect: self.connect() diff --git a/stem/util/__init__.py b/stem/util/__init__.py index 75a2da4..5f6bb24 100644 --- a/stem/util/__init__.py +++ b/stem/util/__init__.py @@ -2,5 +2,5 @@ Utility functions used by the stem library. """ -__all__ = ["conf", "connection", "control", "enum", "log", "proc", "system", "term", "tor_tools"] +__all__ = ["conf", "connection", "enum", "log", "proc", "system", "term", "tor_tools"] diff --git a/stem/util/control.py b/stem/util/control.py deleted file mode 100644 index 5ea2efe..0000000 --- a/stem/util/control.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Helper functions utilized by the controller classes. - -**Module Overview:** - -:: - case_insensitive_lookup - does case insensitive lookups on python dictionaries -""" - -def case_insensitive_lookup(lst, key): - """ - Returns the first value equal to key in lst while ignoring case. - - :param list lst: list of strings - :param str key: string to be looked up - - :returns: case-insensitive equivalent of key in lst - - :raises: ValueError if such a key doesn't exist - """ - - for i in lst: - if i.lower() == key.lower(): - return i diff --git a/stem/util/log.py b/stem/util/log.py index c2eb817..38815ba 100644 --- a/stem/util/log.py +++ b/stem/util/log.py @@ -152,7 +152,7 @@ class LogBuffer(logging.Handler): """ def __init__(self, runlevel): - logging.Handler.__init__(self, level = logging_level(runlevel)) + super(LogBuffer, self).__init__(level = logging_level(runlevel)) self.formatter = logging.Formatter( fmt = '%(asctime)s [%(levelname)s] %(message)s', datefmt = '%D %H:%M:%S') diff --git a/test/integ/control/controller.py b/test/integ/control/controller.py index a9eee1b..55f7210 100644 --- a/test/integ/control/controller.py +++ b/test/integ/control/controller.py @@ -140,6 +140,8 @@ class TestController(unittest.TestCase): Exercises GETCONF with valid and invalid queries. """ + if test.runner.require_control(self): return + runner = test.runner.get_runner() with runner.get_tor_controller() as controller: @@ -160,19 +162,18 @@ class TestController(unittest.TestCase): self.assertEqual(expected, controller.get_conf_map([config_key])) self.assertEqual(expected, controller.get_conf_map([config_key], "la-di-dah")) - getconf_params = set(["ControlPORT", "dirport", "datadirectory"]) - self.assertEqual(getconf_params, set(controller.get_conf_map(["ControlPORT", - "dirport", "datadirectory"], multiple=False))) + request_params = ["ControlPORT", "dirport", "datadirectory"] + reply_params = controller.get_conf_map(request_params, multiple=False).keys() + self.assertEqual(set(request_params), set(reply_params)) # non-existant option(s) self.assertRaises(stem.socket.InvalidArguments, controller.get_conf, "blarg") self.assertEqual("la-di-dah", controller.get_conf("blarg", "la-di-dah")) self.assertRaises(stem.socket.InvalidArguments, controller.get_conf_map, "blarg") - self.assertEqual({"blarg": "la-di-dah"}, controller.get_conf_map("blarg", "la-di-dah")) + self.assertEqual("la-di-dah", controller.get_conf_map("blarg", "la-di-dah")) self.assertRaises(stem.socket.InvalidRequest, controller.get_conf_map, ["blarg", "huadf"], multiple = True) - self.assertEqual({"erfusdj": "la-di-dah", "afiafj": "la-di-dah"}, - controller.get_conf_map(["erfusdj", "afiafj"], "la-di-dah", multiple = True)) + self.assertEqual("la-di-dah", controller.get_conf_map(["erfusdj", "afiafj"], "la-di-dah", multiple = True)) # multivalue configuration keys nodefamilies = [("abc", "xyz", "pqrs"), ("mno", "tuv", "wxyz")]

1 0

[stem/master] GETCONF handling
by atagar＠torproject.org 04 Jul '12

04 Jul '12

commit c9e2f7be38e7dede69a7898c7627294f857ea395 Merge: af27984 c09b80e Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jul 4 14:32:25 2012 -0700 GETCONF handling run_tests.py | 2 + stem/connection.py | 12 +- stem/control.py | 160 ++++++++++++++++++++++++++++++- stem/descriptor/__init__.py | 2 +- stem/descriptor/extrainfo_descriptor.py | 6 +- stem/descriptor/reader.py | 10 +- stem/descriptor/server_descriptor.py | 7 +- stem/response/__init__.py | 16 ++- stem/response/getconf.py | 52 ++++++++++ stem/response/getinfo.py | 11 ++- stem/socket.py | 53 ++++++++++- stem/util/log.py | 2 +- test/integ/control/controller.py | 57 +++++++++++ test/unit/response/__init__.py | 2 +- test/unit/response/getconf.py | 115 ++++++++++++++++++++++ test/unit/response/getinfo.py | 17 ++++ 16 files changed, 494 insertions(+), 30 deletions(-)

1 0

[metrics-tasks/master] Add #6232 code written by gsathya and others.
by karsten＠torproject.org 04 Jul '12

04 Jul '12

commit e456caea6134de661d24be8a253394e323ac025a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jul 4 23:11:06 2012 +0200 Add #6232 code written by gsathya and others. Extract consensus weights from consensuses and calculate the Shannon Entropy for them. Most of this code was written by gsathya and later refined by asn and phw. --- task-6232/.gitignore | 3 ++ task-6232/plot-entropy.R | 10 +++++ task-6232/pyentropy.py | 79 ++++++++++++++++++++++++++++++++++++++++++++ task-6232/run-pyentropy.py | 5 +++ 4 files changed, 97 insertions(+), 0 deletions(-) diff --git a/task-6232/.gitignore b/task-6232/.gitignore new file mode 100644 index 0000000..8a0c627 --- /dev/null +++ b/task-6232/.gitignore @@ -0,0 +1,3 @@ +in/ +entropy.csv + diff --git a/task-6232/plot-entropy.R b/task-6232/plot-entropy.R new file mode 100644 index 0000000..1334b88 --- /dev/null +++ b/task-6232/plot-entropy.R @@ -0,0 +1,10 @@ +library(ggplot2) +d <- read.csv("entropy.csv", header = FALSE, + col.names = c("validafter", "entropy")) +ggplot(d, aes(x = as.POSIXct(validafter), y = entropy)) + +geom_line() + +scale_x_datetime(name = "\nDate") + +scale_y_continuous(name = "Entropy\n") +ggsave("entropy.png", width = 8, height = 6, dpi = 100) + + diff --git a/task-6232/pyentropy.py b/task-6232/pyentropy.py new file mode 100644 index 0000000..f13f709 --- /dev/null +++ b/task-6232/pyentropy.py @@ -0,0 +1,79 @@ +""" +Usage - python pyentropy.py <consensus-dir> <output-file> +Output - A CSV file of the format <valid-after>,<entropy> +rsync -arz --delete metrics.torproject.org::metrics-recent/relay-descriptors/consensuses in +""" + +import sys +import math +import os +from decimal import * + +RESULTS = [] +KEYS = ['r','s','v','w','p','m'] + + +class Router: + def __init__(self): + self.lines = [] + self.nick = None + self.bandwidth = None + self.flags = None + self.probability = None + + def add(self, key, values): + if key == 'r': + self.nick = values[0] + if key == 'w': + self.bandwidth = int(values[0].split('=')[1]) + if key == 's': + self.flags = values + + +def run(file_name): + routers = [] + # parse consensus + with open(file_name, 'r') as f: + for line in f.readlines(): + key = line.split()[0] + values = line.split()[1:] + if key =='r': + router = Router() + router.add(key, values) + elif key == 'p': + router.add(key, values) + routers.append(router) + elif key == 'valid-after': + valid_after = ' '.join(values) + elif key in KEYS: + router.add(key, values) + + # build hash table with freq. distribution + # key: bandwidth + # value: number of bandwidth's observations + bw_dist = {} + for router in routers: + if bw_dist.has_key(router.bandwidth): + bw_dist[router.bandwidth] += 1 + else: + bw_dist[router.bandwidth] = 1 + + if len(routers) <= 0: + print "Error: amount of routers must be > 0." + return; + + print "calculating entropy" + entropy = 0.0 + for bw in bw_dist.iterkeys(): + # p = probability of one particular bandwidth + p = float(bw_dist[bw]) / len(routers) + entropy += -(p * math.log(p, 2)) + + return ",".join([valid_after, str(entropy)]) + + +if __name__ == "__main__": + with open(sys.argv[2], 'w') as f: + for file_name in os.listdir(sys.argv[1]): + string = run(os.path.join(sys.argv[1], file_name)) + f.write("%s\n" % (string)) diff --git a/task-6232/run-pyentropy.py b/task-6232/run-pyentropy.py new file mode 100755 index 0000000..a94a7d6 --- /dev/null +++ b/task-6232/run-pyentropy.py @@ -0,0 +1,5 @@ +#!/bin/bash +#### Uncomment to use most recent data instead of extracted tarballs +###rsync -arz --delete metrics.torproject.org::metrics-recent/relay-descriptors/consensuses in +python pyentropy.py in/consensuses/ entropy.csv +

1 0

[flashproxy/master] Copyediting.
by dcf＠torproject.org 04 Jul '12

04 Jul '12

commit 2725010dd0ea0411cfedcccd5b4c3b5e352a57f4 Author: David Fifield <david(a)bamsoftware.com> Date: Wed Jul 4 14:01:43 2012 -0700 Copyediting. --- README | 7 +++---- 1 files changed, 3 insertions(+), 4 deletions(-) diff --git a/README b/README index 5fbd572..cb66dde 100644 --- a/README +++ b/README @@ -9,10 +9,9 @@ But as a user you only need these files: https://gitweb.torproject.org/flashproxy.git/blob_plain/HEAD:/connector.py https://gitweb.torproject.org/flashproxy.git/blob_plain/HEAD:/torrc -You must be able to receive TCP connections in order -The only communication your computer makes with a fixed IP address is -registration with the facilitator: everything else happens over -ephemeral proxies. +You must be able to receive TCP connections; unfortunately means that +you cannot be behind NAT. See the section "Using a public connector" +below to try out the system even behind NAT. 1. Run the connector. $ python connector.py --register

1 0

[flashproxy/master] Remove outdated SSH tunnel note.
by dcf＠torproject.org 04 Jul '12

04 Jul '12

commit b561f2c49e2708ba09ccc1a84fa6c8386dba325e Author: David Fifield <david(a)bamsoftware.com> Date: Wed Jul 4 08:09:42 2012 -0700 Remove outdated SSH tunnel note. --- notes.txt | 10 ---------- 1 files changed, 0 insertions(+), 10 deletions(-) diff --git a/notes.txt b/notes.txt index b521bef..8e5d2a6 100644 --- a/notes.txt +++ b/notes.txt @@ -8,13 +8,3 @@ We need to solve the problem of potential changing fingerprints when reconnecting through a new proxy. Tor remembers the host fingerprint from the first connection (to the bridge it thinks is at 127.0.0.1) and balks at connecting to the same address with a different fingerprint. - -== How to use with an SSH proxy - -Start swfcat, giving it a relay parameter pointing to your SSH server. - http://example.com/swfcat.swf?facilitator=<facilitator_addr>:<facilitator_port>&relay=<ssh_addr>:<ssh_port> -Start the connector. - python connector.py -f <facilitator_addr:facilitator_port> -Start SSH, giving it a proxy command to connect through the connector. - ssh -v -N -D 1080 <ssh_addr> -p <ssh_port> \ - -o 'ProxyCommand=socat - SOCKS4A:localhost:dummy:0,socksport=9001'

1 0

[flashproxy/master] Remove some more references to Adobe Flash.
by dcf＠torproject.org 04 Jul '12

04 Jul '12

commit 32c5609ab34f00e6ef92b75e37c68fe1d4b6386e Author: David Fifield <david(a)bamsoftware.com> Date: Wed Jul 4 08:16:52 2012 -0700 Remove some more references to Adobe Flash. --- README | 8 +++----- doc/design.txt | 18 ++++++++---------- facilitator.py | 3 --- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/README b/README index 71fbffe..8b7e9b5 100644 --- a/README +++ b/README @@ -2,8 +2,7 @@ This is a set of tools that make it possible to connect Tor through an browser-based proxy running on another computer. The flash proxy can be -run just by opening a web page in a computer that has Flash Player -installed. +run just by opening a web page in a browser. There are five main parts. Our terminology for each part is in quotes. 1. The Tor "client," running on someone's localhost. @@ -46,9 +45,8 @@ if you are able to configure port forwarding or receive direct connections from the Internet. If you don't get a connection within 30 seconds or so, make sure someone -with Flash Player is viewing http://crypto.stanford.edu/flashproxy/. -There aren't yet enough operational proxies that one is available all -the time. +is viewing http://crypto.stanford.edu/flashproxy/. There aren't yet +enough operational proxies that one is available all the time. === Direct TCP connection diff --git a/doc/design.txt b/doc/design.txt index ecee83c..c8141dd 100644 --- a/doc/design.txt +++ b/doc/design.txt @@ -19,16 +19,14 @@ Design of flash proxies temporary and short-lived, but their number will be great enough that they can't all be blocked effectively. - The implementation of a browser-based proxy using Adobe Flash or other - networking technologies is complicated by restrictions that prevent it - being a straightforward proxy. Chief among these is the lack of - listening sockets. Flash and, for example, WebSockets, can only - initiate outgoing connections, not receive incoming ones. The flash - proxy can only connect to external hosts by connecting directly to - them. The other significant restriction is that the proxy cannot - connect to just any destination. A third, but less important, - restriction is that browser-based networking does not provide - low-level socket access such as control of source address. + The implementation of a browser-based proxy using WebSocket is + complicated by restrictions that prevent it being a straightforward + proxy. Chief among these is the lack of listening sockets. WebSocket + can only initiate outgoing connections, not receive incoming ones. The + flash proxy can only connect to external hosts by connecting directly + to them. Another, but less important, restriction is that + browser-based networking does not provide low-level socket access such + as control of source address. 2. Components diff --git a/facilitator.py b/facilitator.py index 64de7a6..2a04e71 100755 --- a/facilitator.py +++ b/facilitator.py @@ -253,9 +253,6 @@ class Handler(BaseHTTPServer.BaseHTTPRequestHandler): if reg: client_str = str(reg) else: - # Send an empty string rather than a 404 or similar because Flash - # Player's URLLoader can't always distinguish a 404 from, say, - # "server not found." client_str = "" self.send_response(200) self.send_header("Content-Type", "application/x-www-form-urlencoded")

1 0

[flashproxy/master] README copyediting and rearrangement.
by dcf＠torproject.org 04 Jul '12

04 Jul '12

commit 3e733e46e6e11a042732e08860e19c83328682a1 Author: David Fifield <david(a)bamsoftware.com> Date: Wed Jul 4 13:53:58 2012 -0700 README copyediting and rearrangement. --- README | 88 +++++++++++++++++++++++++++++++-------------------------------- 1 files changed, 43 insertions(+), 45 deletions(-) diff --git a/README b/README index 8b7e9b5..5fbd572 100644 --- a/README +++ b/README @@ -1,33 +1,3 @@ -== Introduction - -This is a set of tools that make it possible to connect Tor through an -browser-based proxy running on another computer. The flash proxy can be -run just by opening a web page in a browser. - -There are five main parts. Our terminology for each part is in quotes. -1. The Tor "client," running on someone's localhost. -2. A "connector," which waits for connections from the flash proxy and - the Tor client, and joins them together. -3. A "flash proxy," an application running in someone's web browser. -4. A "facilitator," a pseudo-HTTP server that keeps a list of clients - that want a connection, and hands them out to proxies. -5. A Tor "relay," which is just a normal Tor relay. - -The purpose of this project is to create many, generally ephemeral -bridge IP addresses, with the goal of outpacing a censor's ability to -block them. Rather than increasing the number of bridges at static -addresses, we aim to make existing bridges reachable by a larger and -changing pool of addresses. - - -== Demonstration page - -This page has a description of the project; viewing it also turns your -computer into a flash proxy as long as the page is open. - -http://crypto.stanford.edu/flashproxy/ - - == Quick start for users You must have a version of Tor that supports pluggable transports. This @@ -39,19 +9,8 @@ But as a user you only need these files: https://gitweb.torproject.org/flashproxy.git/blob_plain/HEAD:/connector.py https://gitweb.torproject.org/flashproxy.git/blob_plain/HEAD:/torrc -There are different ways to get connected, depending on whether you are -able to receive direct TCP connections. The first method is preferred, -if you are able to configure port forwarding or receive direct -connections from the Internet. - -If you don't get a connection within 30 seconds or so, make sure someone -is viewing http://crypto.stanford.edu/flashproxy/. There aren't yet -enough operational proxies that one is available all the time. - -=== Direct TCP connection - -This is the preferred way to use a flash proxy when it's possible. The -only communication your computer makes with a fixed IP address is +You must be able to receive TCP connections in order +The only communication your computer makes with a fixed IP address is registration with the facilitator: everything else happens over ephemeral proxies. @@ -74,7 +33,45 @@ From tor you are looking for: [notice] Tor has successfully opened a circuit. Looks like client functionality is working. [notice] Bootstrapped 100%: Done. -=== Using a public connector +If you don't get a connection within 30 seconds or so, make sure someone +is viewing http://crypto.stanford.edu/flashproxy/. There aren't yet +enough operational proxies that one is available all the time. + + +== Overview + +This is a set of tools that make it possible to connect Tor through an +browser-based proxy running on another computer. The flash proxy can be +run just by opening a web page in a browser. Flash proxies are one of +several pluggable transports for Tor. + +There are five main parts. +1. The Tor client, running on someone's localhost. +2. A client transport plugin, which is a program that waits for + connections from a flash proxy and connects them to the Tor client. +3. A flash proxy, which is a JavaScript program running in someone's web + browser. +4. A facilitator, which is a server that keeps a list of clients that + want a connection and assigns those addresses to proxies. +5. A Tor relay running a server transport plugin capable of receiving + WebSocket connections. + +The purpose of this project is to create many ephemeral bridge IP +addresses, with the goal of outpacing a censor's ability to block them. +Rather than increasing the number of bridges at static addresses, we aim +to make existing bridges reachable by a larger and changing pool of +addresses. + + +== Demonstration page + +This page has a description of the project; viewing it also turns your +computer into a flash proxy as long as the page is open. + +http://crypto.stanford.edu/flashproxy/ + + +== Using a public connector Rather than running connector.py on your computer, you can use a public connector. This way is not as realistic because all your Tor traffic @@ -83,7 +80,8 @@ be easily blocked. However this is an easy way to try out the system without having to do port forwarding. $ tor ClientTransportPlugin "websocket socks4 tor-facilitator.bamsoftware.com:9999" UseBridges 1 Bridge "websocket 127.0.0.1:9001" -=== Troubleshooting + +== Troubleshooting Make sure someone is viewing http://crypto.stanford.edu/flashproxy/, or another web page with a flash proxy badge on it.

1 0

[flashproxy/master] Add tests for parse_socks_request.
by dcf＠torproject.org 04 Jul '12

04 Jul '12

commit b71898a781345d57c9ba7dbe1c4f02e6a7bbbba3 Author: David Fifield <david(a)bamsoftware.com> Date: Wed Jul 4 07:09:30 2012 -0700 Add tests for parse_socks_request. --- connector-test.py | 20 +++++++++++++++++++- 1 files changed, 19 insertions(+), 1 deletions(-) diff --git a/connector-test.py b/connector-test.py index b19fcc6..374b173 100755 --- a/connector-test.py +++ b/connector-test.py @@ -4,11 +4,29 @@ import socket import subprocess import unittest -from connector import WebSocketDecoder, WebSocketEncoder +from connector import parse_socks_request, WebSocketDecoder, WebSocketEncoder LOCAL_ADDRESS = ("127.0.0.1", 40000) REMOTE_ADDRESS = ("127.0.0.1", 40001) +class TestSocks(unittest.TestCase): + def test_parse_socks_request_empty(self): + self.assertRaises(ValueError, parse_socks_request, "") + def test_parse_socks_request_short(self): + self.assertRaises(ValueError, parse_socks_request, "\x04\x01\x99\x99\x01\x02\x03\x04") + def test_parse_socks_request_ip_userid_missing(self): + dest, port = parse_socks_request("\x04\x01\x99\x99\x01\x02\x03\x04\x00") + dest, port = parse_socks_request("\x04\x01\x99\x99\x01\x02\x03\x04\x00userid") + self.assertEqual((dest, port), ("1.2.3.4", 0x9999)) + def test_parse_socks_request_ip(self): + dest, port = parse_socks_request("\x04\x01\x99\x99\x01\x02\x03\x04userid\x00") + self.assertEqual((dest, port), ("1.2.3.4", 0x9999)) + def test_parse_socks_request_hostname_missing(self): + self.assertRaises(ValueError, parse_socks_request, "\x04\x01\x99\x99\x00\x00\x00\x01userid\x00") + self.assertRaises(ValueError, parse_socks_request, "\x04\x01\x99\x99\x00\x00\x00\x01userid\x00abc") + def test_parse_socks_request_hostname(self): + dest, port = parse_socks_request("\x04\x01\x99\x99\x00\x00\x00\x01userid\x00abc\x00") + def read_frames(dec): frames = [] while True:

1 0

[flashproxy/master] Remove some debugging logs logging function entry.
by dcf＠torproject.org 04 Jul '12

04 Jul '12

commit bc27dbbe195a4fe85d3e83ea4f9b3ec2d46bbf65 Author: David Fifield <david(a)bamsoftware.com> Date: Wed Jul 4 06:37:18 2012 -0700 Remove some debugging logs logging function entry. --- connector.py | 2 -- 1 files changed, 0 insertions(+), 2 deletions(-) diff --git a/connector.py b/connector.py index 8a2b7e1..ed4f02c 100755 --- a/connector.py +++ b/connector.py @@ -445,7 +445,6 @@ class WebSocketRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): MAGIC_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" def handle_websocket_request(fd): - log(u"handle_websocket_request") request_text = fd.recv(10 * 1024) handler = WebSocketRequestHandler(request_text, fd) if handler.error or not hasattr(handler, "path"): @@ -580,7 +579,6 @@ def parse_socks_request(data): return dest, dport def handle_socks_request(fd): - log(u"handle_socks_request") try: addr = fd.getpeername() data = fd.recv(100)

1 0

[flashproxy/master] Use an exception in parse_socks_header instead of an error return.
by dcf＠torproject.org 04 Jul '12

04 Jul '12

commit a04414d0518930fbfcdb21cfb6486e1efc9ed8b4 Author: David Fifield <david(a)bamsoftware.com> Date: Wed Jul 4 06:42:32 2012 -0700 Use an exception in parse_socks_header instead of an error return. --- connector.py | 23 +++++++++++------------ 1 files changed, 11 insertions(+), 12 deletions(-) diff --git a/connector.py b/connector.py index ed4f02c..353d6b5 100755 --- a/connector.py +++ b/connector.py @@ -554,26 +554,23 @@ def grab_string(s, pos): return pos, None def parse_socks_request(data): + """Parse the 8-byte SOCKS header at the beginning of data. Returns a + (dest, port) tuple. Raises ValueError on error.""" try: ver, cmd, dport, o1, o2, o3, o4 = struct.unpack(">BBHBBBB", data[:8]) except struct.error: - log(u"Couldn't unpack SOCKS4 header.") - return None + raise ValueError("Couldn't unpack SOCKS4 header") if ver != 4: - log(u"SOCKS header has wrong version (%d)." % ver) - return None + raise ValueError("SOCKS header has wrong version (%d)" % ver) if cmd != 1: - log(u"SOCKS header had wrong command (%d)." % cmd) - return None + raise ValueError("SOCKS header had wrong command (%d)" % cmd) pos, userid = grab_string(data, 8) if userid is None: - log(u"Couldn't read userid from SOCKS header.") - return None + raise ValueError("Couldn't read userid from SOCKS header") if o1 == 0 and o2 == 0 and o3 == 0 and o4 != 0: pos, dest = grab_string(data, pos) if dest is None: - log(u"Couldn't read destination from SOCKS4a header.") - return None + raise ValueError("Couldn't read destination from SOCKS4a header") else: dest = "%d.%d.%d.%d" % (o1, o2, o3, o4) return dest, dport @@ -585,8 +582,10 @@ def handle_socks_request(fd): except socket.error, e: log(u"Socket error from SOCKS-pending: %s" % repr(str(e))) return False - dest_addr = parse_socks_request(data) - if dest_addr is None: + try: + dest_addr = parse_socks_request(data) + except ValueError, e: + log(u"Error parsing SOCKS header: %s." % str(e)) # Error reply. fd.sendall(struct.pack(">BBHBBBB", 0, 91, 0, 0, 0, 0, 0)) return False

1 0