December 2015 - tor-commits - lists.torproject.org

[gettor/develop] Script and sample htaccess for creating GetTor API static mirror
by ilv＠torproject.org 31 Dec '15

31 Dec '15

commit 7735f5b69383ce26da39dc0dcdfd82f364ec10b8 Author: ilv <ilv(a)users.noreply.github.com> Date: Thu Dec 31 18:46:29 2015 -0300 Script and sample htaccess for creating GetTor API static mirror --- scripts/create_api_mirror.py | 305 ++++++++++++++++++++++++++++++++++++ scripts/sample-htaccess-api-mirror | 21 +++ 2 files changed, 326 insertions(+) diff --git a/scripts/create_api_mirror.py b/scripts/create_api_mirror.py new file mode 100644 index 0000000..e80ec4a --- /… [View More]dev/null +++ b/scripts/create_api_mirror.py @@ -0,0 +1,305 @@ +# -*- coding: utf-8 -*- +# +# This file is part of GetTor. +# +# :authors: Israel Leiva <ilv(a)torproject.org> +# see also AUTHORS file +# +# :copyright: (c) 2008-2015, The Tor Project, Inc. +# (c) 2015, Israel Leiva +# +# :license: This is Free Software. See LICENSE for license information. + +import os +import re +import json +import codecs +import urllib2 +import ConfigParser + +from time import gmtime, strftime + + +"""Script to build a static mirror of GetTor RESTful API""" + +# currently supported locales for Tor Browser +LC = ['ar', 'de', 'en-US', 'es-ES', 'fa', 'fr', 'it', 'ko', 'nl', 'pl', + 'pt-PT', 'ru', 'tr', 'vi', 'zh-CN'] + +# https://gitweb.tpo/tor-browser-spec.git/tree/processes/VersionNumbers +# does not say anything about operating systems, so it's possible the +# notation might change in the future. We should always use the same three +# strings though: linux, windows, osx. +OS = { + 'Linux': 'linux', + 'Windows': 'windows', + 'MacOS': 'osx' +} + +# based on +# https://gitweb.tpo.org/tor-browser-spec.git/tree/processes/VersionNumbers +# except for the first one, which is based on current RecommendedTBBVersions +RE = { + 'os': '(.*)-(\w+)', + 'alpha': '\d\.\d(\.\d)*a\d+', + 'beta': '\d\.\d(\.\d)*b\d+', + 'stable': '\d\.\d(\.\d)*' +} + +# strings to build names of packages depending on OS. +PKG = { + 'windows': 'torbrowser-install-%s_%s.exe', + 'linux': 'tor-browser-linux%s-%s_%s.tar.xz', + 'osx': 'TorBrowser-%s-osx64_%s.dmg' +} + +# bin and asc are used to build the download links for each version, os and lc +URL = { + 'version': 'https://gettor.torproject.org/api/latest.json', + 'mirrors': 'https://gettor.torproject.org/api/mirrors.json', + 'providers': 'https://gettor.torproject.org/api/providers.json' +} + +GETTOR_URL = 'http://gettor.torproject.org' +SERVER_URL = 'https://yourprivateserver.com' +API_TREE = '/home/ilv/Proyectos/tor/static_api/' + +class ConfigError(Exception): + pass + + +class InternalError(Exception): + pass + + +class APIMirror(object): + """ Provide useful resources via RESTful API. """ + def __init__(self, cfg=None): + self.url = SERVER_URL + self.tree = API_TREE + + def _is_json(self, my_json): + """ Check if json generated is valid. + + :param: my_json (string) data to ve verified. + + :return: (bool) true if data is json-valid, false otherwise. + + """ + try: + json_object = json.loads(my_json) + except ValueError, e: + return False + return True + + def _write_json(self, path, content): + """ + """ + try: + with codecs.open( + path, + "w", + "utf-8" + ) as jsonfile: + # Make pretty json + json.dump( + content, + jsonfile, + sort_keys=True, + indent=4, + separators=(',', ': '), + encoding="utf-8", + ) + except IOError as e: + #logging.error("Couldn't write json: %s" % str(e)) + print "Error building %s: %s" % (path, str(e)) + print "%s built" % path + + def _get_provider_name(self, p): + """ Return simplified version of provider's name. + + :param: p (string) provider's name. + + :return: (string) provider's name in lowercase and without spaces. + + """ + p = p.replace(' ', '-') + return p.lower() + + + def _load_latest_version(self): + """ Load latest version data from GetTor API. """ + response = urllib2.urlopen(URL['version']) + json_response = json.load(response) + self.lv = json_response + + def _load_links(self): + """ Load links and providers data. """ + + response = urllib2.urlopen(URL['providers']) + json_response = json.load(response) + + links = {} + + for provider in json_response: + if provider != 'updated_at': + provider_url = json_response[provider] + + provider_response = urllib2.urlopen("%s.json" % provider_url) + provider_content = json.load(provider_response) + + json_response[provider] = provider_url.replace( + GETTOR_URL, + self.url + ) + + + links[provider] = provider_content + + self.providers = json_response + self.links = links + + def _load_mirrors(self): + """ Load mirrors data from GetTor API. """ + response = urllib2.urlopen(URL['mirrors']) + json_response = json.load(response) + self.mirrors = json_response + + def _load_resources(self): + """ Load available resources data. """ + + self.resources = { + 'providers': '%s/providers' % self.url, + 'mirrors': '%s/mirrors' % self.url, + 'latest_version': '%s/latest' % self.url, + 'updated_at': strftime("%Y-%m-%d %H:%M:%S", gmtime()) + } + + def load_data(self): + """ Load all data.""" + self._load_links() + self._load_mirrors() + self._load_resources() + self._load_latest_version() + + def build(self): + """ Build RESTful API. """ + + print "Building API mirror" + + # resources + self._write_json( + os.path.join(self.tree, 'api'), + self.resources + ) + + api_path = os.path.join(self.tree, 'api-content') + if not os.path.isdir(api_path): + os.mkdir(api_path) + + # providers + self._write_json( + os.path.join(api_path, 'providers'), + self.providers + ) + + providers_path = os.path.join(api_path, 'providers-content') + if not os.path.isdir(providers_path): + os.mkdir(providers_path) + + for provider in self.links: + if provider == 'updated_at': + continue + + self._write_json( + os.path.join(providers_path, provider), + self.links[provider] + ) + + provider_path = os.path.join( + providers_path, + "%s-content" % provider + ) + + if not os.path.isdir(provider_path): + os.mkdir(provider_path) + + for osys in self.links[provider]: + self._write_json( + os.path.join(provider_path, osys), + self.links[provider][osys] + ) + + provider_os_path = os.path.join( + provider_path, "%s-content" % osys + ) + + if not os.path.isdir(provider_os_path): + os.mkdir(provider_os_path) + + for lc in self.links[provider][osys]: + self._write_json( + os.path.join(provider_os_path, lc), + self.links[provider][osys][lc] + ) + + # latest version + self._write_json( + os.path.join(api_path, 'latest'), + self.lv + ) + + lv_path = os.path.join(api_path, 'latest-content') + if not os.path.isdir(lv_path): + os.mkdir(lv_path) + + for release in self.lv: + if release == 'updated_at': + continue + + self._write_json( + os.path.join(lv_path, release), + self.lv[release] + ) + + release_path = os.path.join( + lv_path, + "%s-content" % release + ) + + if not os.path.isdir(release_path): + os.mkdir(release_path) + + for osys in self.lv[release]['downloads']: + self._write_json( + os.path.join(release_path, osys), + self.lv[release]['downloads'][osys] + ) + + release_os_path = os.path.join( + release_path, + "%s-content" % osys + ) + + if not os.path.isdir(release_os_path): + os.mkdir(release_os_path) + + for lc in self.lv[release]['downloads'][osys]: + self._write_json( + os.path.join(release_os_path, lc), + self.lv[release]['downloads'][osys][lc] + ) + + # mirrors + self._write_json( + os.path.join(api_path, 'mirrors'), + self.mirrors + ) + +def main(): + api = APIMirror() + api.load_data() + api.build() + +if __name__ == '__main__': + main() diff --git a/scripts/sample-htaccess-api-mirror b/scripts/sample-htaccess-api-mirror new file mode 100644 index 0000000..41594db --- /dev/null +++ b/scripts/sample-htaccess-api-mirror @@ -0,0 +1,21 @@ +Options +FollowSymLinks +RewriteEngine On + +# Operating System and Locale filters for providers +RewriteCond %{QUERY_STRING} os=([a-z]+)&lc=([a-z]+) +RewriteRule ^api/providers/([a-z\-]+)$ api-content/providers-content/$1-content/%1-content/%2.json +RewriteCond %{QUERY_STRING} os=([a-z]+) +RewriteRule ^api/providers/([a-z\-]+)$ api-content/providers-content/$1-content/%1 + +# Operating System and Locale filters for TPO downloads +RewriteCond %{QUERY_STRING} os=([a-z]+)&lc=([a-zA-Z\-]+) +RewriteRule ^api/latest/([a-z]+)$ api-content/latest-content/$1-content/%1-content/%2.json +RewriteCond %{QUERY_STRING} os=([a-z]+) +RewriteRule ^api/latest/([a-z]+)$ api-content/latest-content/$1-content/%1 + +# General rules +RewriteRule ^api/providers/([a-z\-]+)$ api-content/providers-content/$1 [NC] +RewriteRule ^api/latest/([a-z]+)$ api-content/latest-content/$1 [NC] +RewriteRule ^api/([a-z]+)$ api-content/$1 [NC] + +ErrorDocument 404 404-file [View Less]

1 0

[gettor/develop] Change from flask to build an static API
by ilv＠torproject.org 31 Dec '15

31 Dec '15

commit 3cc92b1b881ffdc2b6c441b44455560c11a6f907 Author: ilv <ilv(a)users.noreply.github.com> Date: Thu Dec 31 17:16:36 2015 -0300 Change from flask to build an static API --- gettor/http.py | 249 +++++++++++++++++++++++++++++-------------------------- process_http.py | 7 +- 2 files changed, 135 insertions(+), 121 deletions(-) diff --git a/gettor/http.py b/gettor/http.py index c1441c6..2a63502 100644 --- a/gettor/http.py +++ b/gettor/http.py @@ -19,9 +19,6 @@ import … [View More]ConfigParser from time import gmtime, strftime -from flask import Flask -from flask_restful import Api, Resource, reqparse - import core import utils @@ -65,10 +62,6 @@ URL = { 'asc': 'https://www.torproject.org/dist/torbrowser/%s/%s.asc' } -# filters for API resources -parser = reqparse.RequestParser() -parser.add_argument('os', type=str, help='Operating System') -parser.add_argument('lc', type=str, help='Locale') class ConfigError(Exception): @@ -100,8 +93,10 @@ class HTTP(object): raise ConfigError("File %s not found!" % cfg) try: + # path to static tree of API + self.tree = config.get('general', 'tree') # server that provides the RESTful API - self.server = config.get('general', 'server') + self.server = config.get('general', 'url') # path to the links files self.links_path = config.get('general', 'links') # path to mirrors in json @@ -129,6 +124,29 @@ class HTTP(object): except ValueError, e: return False return True + + def _write_json(self, path, content): + """ + """ + try: + with codecs.open( + path, + "w", + "utf-8" + ) as jsonfile: + # Make pretty json + json.dump( + content, + jsonfile, + sort_keys=True, + indent=4, + separators=(',', ': '), + encoding="utf-8", + ) + except IOError as e: + #logging.error("Couldn't write json: %s" % str(e)) + print "Error building %s: %s" % (path, str(e)) + print "%s built" % path def _get_provider_name(self, p): """ Return simplified version of provider's name. @@ -373,121 +391,116 @@ class HTTP(object): self._load_resources() self._load_latest_version() - def run(self): - """ Run RESTful API. """ - app = Flask(__name__) - api = Api(app) - - api.add_resource( - AvailableResources, - '/', - resource_class_kwargs={ - 'resources': self.resources - } + def build(self): + """ Build RESTful API. """ + + print "Building API" + + # resources + self._write_json( + os.path.join(self.tree, 'api'), + self.resources ) - api.add_resource( - Providers, - '/providers', - '/providers/<string:provider>', - resource_class_kwargs={ - 'links': self.links, - 'providers': self.providers - } + api_path = os.path.join(self.tree, 'api-content') + if not os.path.isdir(api_path): + os.mkdir(api_path) + + # providers + self._write_json( + os.path.join(api_path, 'providers'), + self.providers ) - api.add_resource( - LatestVersion, - '/latest', - '/latest/<string:release>', - resource_class_kwargs={ - 'latest_version': self.lv, - 'releases': self.releases - } - ) + providers_path = os.path.join(api_path, 'providers-content') + if not os.path.isdir(providers_path): + os.mkdir(providers_path) + + for provider in self.links: + if provider == 'updated_at': + continue + + self._write_json( + os.path.join(providers_path, provider), + self.links[provider] + ) + + provider_path = os.path.join( + providers_path, + "%s-content" % provider + ) + + if not os.path.isdir(provider_path): + os.mkdir(provider_path) + + for osys in self.links[provider]: + self._write_json( + os.path.join(provider_path, osys), + self.links[provider][osys] + ) - api.add_resource( - Mirrors, - '/mirrors', - resource_class_kwargs={ - 'mirrors': self.mirrors - } + provider_os_path = os.path.join( + provider_path, "%s-content" % osys + ) + + if not os.path.isdir(provider_os_path): + os.mkdir(provider_os_path) + + for lc in self.links[provider][osys]: + self._write_json( + os.path.join(provider_os_path, lc), + self.links[provider][osys][lc] + ) + + # latest version + self._write_json( + os.path.join(api_path, 'latest'), + self.lv ) + + lv_path = os.path.join(api_path, 'latest-content') + if not os.path.isdir(lv_path): + os.mkdir(lv_path) + + for release in self.lv: + if release == 'updated_at': + continue + + self._write_json( + os.path.join(lv_path, release), + self.lv[release] + ) + + release_path = os.path.join( + lv_path, + "%s-content" % release + ) + + if not os.path.isdir(release_path): + os.mkdir(release_path) + + for osys in self.lv[release]['downloads']: + self._write_json( + os.path.join(release_path, osys), + self.lv[release]['downloads'][osys] + ) - app.run(debug=True) - - -class AvailableResources(Resource): - def __init__(self, resources): - """ Set initial data. """ - self.resources = resources - - def get(self): - """ Return available resources on the API. """ - return self.resources - - -class Providers(Resource): - def __init__(self, providers, links): - """ Set initial data. """ - self.providers = providers - self.links = links - - def get(self, provider=None): - """ Return providers and links data. """ - - # we use arg to filter results by os and lc (in that order) - arg = parser.parse_args() - - if provider: - if arg['os']: - if arg['lc']: - # links by provider, os, and lc (in that order) - return self.links[provider][arg['os']][arg['lc']] - else: - # links by provider and os (in that order) - return self.links[provider][arg['os']] - else: - # links by provider - return self.links[provider] - else: - # list of providers - return self.providers - - -class LatestVersion(Resource): - def __init__(self, latest_version, releases): - """ Set initial data. """ - self.lv = latest_version - self.releases = releases - - def get(self, release=None): - """ Return latest version data. """ - - # we use arg to filter results by os and lc (in that order) - arg = parser.parse_args() - - if release: - if arg['os']: - if arg['lc']: - # tpo links by release, os and lc (in that order) - return self.lv[release]['downloads'][arg['os']][arg['lc']] - else: - # tpo links by release and os (in that order) - return self.lv[release]['downloads'][arg['os']] - else: - # version and tpo links by release - return self.lv[release] - else: - # list of releases - return self.releases - - -class Mirrors(Resource): - def __init__(self, mirrors): - """ Set initial data. """ - self.mirrors = mirrors + release_os_path = os.path.join( + release_path, + "%s-content" % osys + ) - def get(self): - """ Return mirrors data. """ - return self.mirrors + if not os.path.isdir(release_os_path): + os.mkdir(release_os_path) + + for lc in self.lv[release]['downloads'][osys]: + self._write_json( + os.path.join(release_os_path, lc), + self.lv[release]['downloads'][osys][lc] + ) + + # mirrors + self._write_json( + os.path.join(api_path, 'mirrors'), + self.mirrors + ) diff --git a/process_http.py b/process_http.py index 31475f1..ec20d5f 100644 --- a/process_http.py +++ b/process_http.py @@ -1,13 +1,14 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -import gettor.http +import gettor.http2 def main(): - api = gettor.http.HTTP('http.cfg') + api = gettor.http2.HTTP('http.cfg') api.load_data() - api.run() + # api.run() + api.build() if __name__ == '__main__': main() [View Less]

1 0

[gettor/develop] Merge branch 'develop' of ssh://git-rw.torproject.org/gettor into develop
by ilv＠torproject.org 31 Dec '15

31 Dec '15

commit b0261b785ec3bab2cc0324900036c5114590fab9 Merge: 3cc92b1 b2b07f3 Author: ilv <ilv(a)users.noreply.github.com> Date: Thu Dec 31 17:18:57 2015 -0300 Merge branch 'develop' of ssh://git-rw.torproject.org/gettor into develop lang/smtp/i18n/en/LC_MESSAGES/en.po | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)

1 0

[stem/master] Don't assign lambdas to variables
by atagar＠torproject.org 31 Dec '15

31 Dec '15

commit a03731a26e218ee6e02cf00b99ef3a38e105de8c Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Dec 30 10:28:32 2015 -0800 Don't assign lambdas to variables Stylistic complaint E731 of the new version of pep8... line 363 - E731 do not assign a lambda expression, use a def | _parse_client_versions_line = lambda descriptor, entries: setattr(descriptor, 'client_versions', _value('client-versions', entries).split(',')) Reason for … [View More]this is described at... http://stackoverflow.com/a/25010243/1067192 In the process of doing this realized we were mis-parsing a couple infrequently used server descriptor fields. --- docs/change_log.rst | 1 + stem/descriptor/__init__.py | 106 +++++++++++++-------------- stem/descriptor/microdescriptor.py | 10 ++- stem/descriptor/networkstatus.py | 10 +-- stem/descriptor/remote.py | 11 +-- stem/descriptor/server_descriptor.py | 11 +-- stem/version.py | 13 ++-- test/runner.py | 5 +- test/unit/descriptor/router_status_entry.py | 5 +- test/unit/descriptor/server_descriptor.py | 2 +- 10 files changed, 86 insertions(+), 88 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index fb0a71c..80938da 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -64,6 +64,7 @@ The following are only available within Stem's `git repository * Unable to read descriptors from data directories on Windows due to their CRLF newlines (:trac:`17051`) * TypeError under python3 when using 'use_mirrors = True' (:trac:`17083`) * Deprecated hidden service descriptor's *introduction_points_auth* field, which was never implemented in tor (:trac:`15190`, :spec:`9c218f9`) + * Fixed parsing of server descriptor's *allow-single-hop-exits* and *caches-extra-info* lines * **Utilities** diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 678325a..af96bde 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -188,53 +188,48 @@ def parse_file(descriptor_file, descriptor_type = None, validate = False, docume descriptor_path = getattr(descriptor_file, 'name', None) filename = '<undefined>' if descriptor_path is None else os.path.basename(descriptor_file.name) - file_parser = None - if descriptor_type is not None: - descriptor_type_match = re.match('^(\S+) (\d+).(\d+)$', descriptor_type) + def parse(descriptor_file): + if normalize_newlines: + descriptor_file = NewlineNormalizer(descriptor_file) - if descriptor_type_match: - desc_type, major_version, minor_version = descriptor_type_match.groups() - file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler, **kwargs) - else: - raise ValueError("The descriptor_type must be of the form '<type> <major_version>.<minor_version>'") - elif metrics_header_match: - # Metrics descriptor handling - - desc_type, major_version, minor_version = metrics_header_match.groups() - file_parser = lambda f: _parse_metrics_file(desc_type, int(major_version), int(minor_version), f, validate, document_handler, **kwargs) - else: - # Cached descriptor handling. These contain multiple descriptors per file. - - if normalize_newlines is None and stem.util.system.is_windows(): - normalize_newlines = True - - if filename == 'cached-descriptors' or filename == 'cached-descriptors.new': - file_parser = lambda f: stem.descriptor.server_descriptor._parse_file(f, validate = validate, **kwargs) - elif filename == 'cached-extrainfo' or filename == 'cached-extrainfo.new': - file_parser = lambda f: stem.descriptor.extrainfo_descriptor._parse_file(f, validate = validate, **kwargs) - elif filename == 'cached-microdescs' or filename == 'cached-microdescs.new': - file_parser = lambda f: stem.descriptor.microdescriptor._parse_file(f, validate = validate, **kwargs) - elif filename == 'cached-consensus': - file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, validate = validate, document_handler = document_handler, **kwargs) - elif filename == 'cached-microdesc-consensus': - file_parser = lambda f: stem.descriptor.networkstatus._parse_file(f, is_microdescriptor = True, validate = validate, document_handler = document_handler, **kwargs) - - if normalize_newlines: - descriptor_file = NewlineNormalizer(descriptor_file) - - if file_parser: - for desc in file_parser(descriptor_file): - if descriptor_path is not None: - desc._set_path(os.path.abspath(descriptor_path)) + if descriptor_type is not None: + descriptor_type_match = re.match('^(\S+) (\d+).(\d+)$', descriptor_type) - yield desc + if descriptor_type_match: + desc_type, major_version, minor_version = descriptor_type_match.groups() + return _parse_metrics_file(desc_type, int(major_version), int(minor_version), descriptor_file, validate, document_handler, **kwargs) + else: + raise ValueError("The descriptor_type must be of the form '<type> <major_version>.<minor_version>'") + elif metrics_header_match: + # Metrics descriptor handling - return + desc_type, major_version, minor_version = metrics_header_match.groups() + return _parse_metrics_file(desc_type, int(major_version), int(minor_version), descriptor_file, validate, document_handler, **kwargs) + else: + # Cached descriptor handling. These contain multiple descriptors per file. + + if normalize_newlines is None and stem.util.system.is_windows(): + descriptor_file = NewlineNormalizer(descriptor_file) + + if filename == 'cached-descriptors' or filename == 'cached-descriptors.new': + return stem.descriptor.server_descriptor._parse_file(descriptor_file, validate = validate, **kwargs) + elif filename == 'cached-extrainfo' or filename == 'cached-extrainfo.new': + return stem.descriptor.extrainfo_descriptor._parse_file(descriptor_file, validate = validate, **kwargs) + elif filename == 'cached-microdescs' or filename == 'cached-microdescs.new': + return stem.descriptor.microdescriptor._parse_file(descriptor_file, validate = validate, **kwargs) + elif filename == 'cached-consensus': + return stem.descriptor.networkstatus._parse_file(descriptor_file, validate = validate, document_handler = document_handler, **kwargs) + elif filename == 'cached-microdesc-consensus': + return stem.descriptor.networkstatus._parse_file(descriptor_file, is_microdescriptor = True, validate = validate, document_handler = document_handler, **kwargs) + else: + raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line)) - # Not recognized as a descriptor file. + for desc in parse(descriptor_file): + if descriptor_path is not None: + desc._set_path(os.path.abspath(descriptor_path)) - raise TypeError("Unable to determine the descriptor's type. filename: '%s', first line: '%s'" % (filename, first_line)) + yield desc def _parse_file_for_path(descriptor_file, *args, **kwargs): @@ -336,13 +331,18 @@ def _values(line, entries): return [entry[0] for entry in entries[line]] -def _parse_simple_line(keyword, attribute): +def _parse_simple_line(keyword, attribute, func = None): def _parse(descriptor, entries): - setattr(descriptor, attribute, _value(keyword, entries)) + value = _value(keyword, entries) + setattr(descriptor, attribute, func(value) if func else value) return _parse +def _parse_if_present(keyword, attribute): + return lambda descriptor, entries: setattr(descriptor, attribute, keyword in entries) + + def _parse_bytes_line(keyword, attribute): def _parse(descriptor, entries): line_match = re.search(stem.util.str_tools._to_bytes('^(opt )?%s(?:[%s]+(.*))?$' % (keyword, WHITESPACE)), descriptor.get_bytes(), re.MULTILINE) @@ -675,13 +675,7 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi **True** """ - if skip: - content = None - content_append = lambda x: None - else: - content = [] - content_append = content.append - + content = None if skip else [] ending_keyword = None if isinstance(keywords, (bytes, str_type)): @@ -690,8 +684,8 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi if ignore_first: first_line = descriptor_file.readline() - if first_line: - content_append(first_line) + if first_line and content is not None: + content.append(first_line) keyword_match = re.compile(SPECIFIC_KEYWORD_LINE % '|'.join(keywords)) @@ -713,12 +707,12 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi if not inclusive: descriptor_file.seek(last_position) - else: - content_append(line) + elif content is not None: + content.append(line) break - else: - content_append(line) + elif content is not None: + content.append(line) if include_ending_keyword: return (content, ending_keyword) diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py index b190635..3f74f70 100644 --- a/stem/descriptor/microdescriptor.py +++ b/stem/descriptor/microdescriptor.py @@ -72,7 +72,6 @@ from stem.descriptor import ( Descriptor, _get_descriptor_components, _read_until_keywords, - _value, _values, _parse_simple_line, _parse_key_block, @@ -180,11 +179,14 @@ def _parse_id_line(descriptor, entries): descriptor.identifiers = identities -_parse_digest = lambda descriptor, entries: setattr(descriptor, 'digest', hashlib.sha256(descriptor.get_bytes()).hexdigest().upper()) +def _parse_digest(descriptor, entries): + setattr(descriptor, 'digest', hashlib.sha256(descriptor.get_bytes()).hexdigest().upper()) + + _parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY') _parse_ntor_onion_key_line = _parse_simple_line('ntor-onion-key', 'ntor_onion_key') -_parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', _value('family', entries).split(' ')) -_parse_p6_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('p6', entries))) +_parse_family_line = _parse_simple_line('family', 'family', func = lambda v: v.split(' ')) +_parse_p6_line = _parse_simple_line('p6', 'exit_policy_v6', func = lambda v: stem.exit_policy.MicroExitPolicy(v)) class Microdescriptor(Descriptor): diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 103f72c..b471c1f 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -360,10 +360,10 @@ _parse_network_status_version_line = _parse_version_line('network-status-version _parse_fingerprint_line = _parse_forty_character_hex('fingerprint', 'fingerprint') _parse_contact_line = _parse_simple_line('contact', 'contact') _parse_dir_signing_key_line = _parse_key_block('dir-signing-key', 'signing_key', 'RSA PUBLIC KEY') -_parse_client_versions_line = lambda descriptor, entries: setattr(descriptor, 'client_versions', _value('client-versions', entries).split(',')) -_parse_server_versions_line = lambda descriptor, entries: setattr(descriptor, 'server_versions', _value('server-versions', entries).split(',')) +_parse_client_versions_line = _parse_simple_line('client-versions', 'client_versions', func = lambda v: v.split(',')) +_parse_server_versions_line = _parse_simple_line('server-versions', 'server_versions', func = lambda v: v.split(',')) _parse_published_line = _parse_timestamp_line('published', 'published') -_parse_dir_options_line = lambda descriptor, entries: setattr(descriptor, 'options', _value('dir-options', entries).split()) +_parse_dir_options_line = _parse_simple_line('dir-options', 'options', func = lambda v: v.split()) _parse_directory_signature_line = _parse_key_block('directory-signature', 'signature', 'SIGNATURE', value_attribute = 'signing_authority') @@ -685,8 +685,8 @@ _parse_header_fresh_until_line = _parse_timestamp_line('fresh-until', 'fresh_unt _parse_header_valid_until_line = _parse_timestamp_line('valid-until', 'valid_until') _parse_header_client_versions_line = _parse_versions_line('client-versions', 'client_versions') _parse_header_server_versions_line = _parse_versions_line('server-versions', 'server_versions') -_parse_header_known_flags_line = lambda descriptor, entries: setattr(descriptor, 'known_flags', [entry for entry in _value('known-flags', entries).split(' ') if entry]) -_parse_footer_bandwidth_weights_line = lambda descriptor, entries: setattr(descriptor, 'bandwidth_weights', _parse_int_mappings('bandwidth-weights', _value('bandwidth-weights', entries), True)) +_parse_header_known_flags_line = _parse_simple_line('known-flags', 'known_flags', func = lambda v: [entry for entry in v.split(' ') if entry]) +_parse_footer_bandwidth_weights_line = _parse_simple_line('bandwidth-weights', 'bandwidth_weights', func = lambda v: _parse_int_mappings('bandwidth-weights', v, True)) class NetworkStatusDocumentV3(NetworkStatusDocument): diff --git a/stem/descriptor/remote.py b/stem/descriptor/remote.py index b2b5b8e..4115550 100644 --- a/stem/descriptor/remote.py +++ b/stem/descriptor/remote.py @@ -96,11 +96,6 @@ from stem.util import log MAX_FINGERPRINTS = 96 MAX_MICRODESCRIPTOR_HASHES = 92 -# We commonly only want authorities that vote in the consensus, and hence have -# a v3ident. - -HAS_V3IDENT = lambda auth: auth.v3ident is not None - def _guess_descriptor_type(resource): # Attempts to determine the descriptor type based on the resource url. This @@ -345,7 +340,7 @@ class Query(object): """ if use_authority or not self.endpoints: - authority = random.choice(list(filter(HAS_V3IDENT, get_authorities().values()))) + authority = random.choice(list(filter(lambda auth: auth.v3ident is not None, get_authorities().values()))) address, dirport = authority.address, authority.dir_port else: address, dirport = random.choice(self.endpoints) @@ -395,7 +390,7 @@ class DescriptorDownloader(object): def __init__(self, use_mirrors = False, **default_args): self._default_args = default_args - authorities = filter(HAS_V3IDENT, get_authorities().values()) + authorities = filter(lambda auth: auth.v3ident is not None, get_authorities().values()) self._endpoints = [(auth.address, auth.dir_port) for auth in authorities] if use_mirrors: @@ -417,7 +412,7 @@ class DescriptorDownloader(object): :raises: **Exception** if unable to determine the directory mirrors """ - authorities = filter(HAS_V3IDENT, get_authorities().values()) + authorities = filter(lambda auth: auth.v3ident is not None, get_authorities().values()) new_endpoints = set([(auth.address, auth.dir_port) for auth in authorities]) consensus = list(self.get_consensus(document_handler = stem.descriptor.DocumentHandler.DOCUMENT).run())[0] diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index a555e12..c2a358f 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -54,6 +54,7 @@ from stem.descriptor import ( _value, _values, _parse_simple_line, + _parse_if_present, _parse_bytes_line, _parse_timestamp_line, _parse_forty_character_hex, @@ -394,11 +395,11 @@ _parse_contact_line = _parse_bytes_line('contact', 'contact') _parse_published_line = _parse_timestamp_line('published', 'published') _parse_read_history_line = functools.partial(_parse_history_line, 'read-history', 'read_history_end', 'read_history_interval', 'read_history_values') _parse_write_history_line = functools.partial(_parse_history_line, 'write-history', 'write_history_end', 'write_history_interval', 'write_history_values') -_parse_ipv6_policy_line = lambda descriptor, entries: setattr(descriptor, 'exit_policy_v6', stem.exit_policy.MicroExitPolicy(_value('ipv6-policy', entries))) -_parse_allow_single_hop_exits_line = lambda descriptor, entries: setattr(descriptor, 'allow_single_hop_exits', 'allow_single_hop_exits' in entries) -_parse_caches_extra_info_line = lambda descriptor, entries: setattr(descriptor, 'extra_info_cache', 'extra_info_cache' in entries) -_parse_family_line = lambda descriptor, entries: setattr(descriptor, 'family', set(_value('family', entries).split(' '))) -_parse_eventdns_line = lambda descriptor, entries: setattr(descriptor, 'eventdns', _value('eventdns', entries) == '1') +_parse_ipv6_policy_line = _parse_simple_line('ipv6-policy', 'exit_policy_v6', func = lambda v: stem.exit_policy.MicroExitPolicy(v)) +_parse_allow_single_hop_exits_line = _parse_if_present('allow-single-hop-exits', 'allow_single_hop_exits') +_parse_caches_extra_info_line = _parse_if_present('caches-extra-info', 'extra_info_cache') +_parse_family_line = _parse_simple_line('family', 'family', func = lambda v: set(v.split(' '))) +_parse_eventdns_line = _parse_simple_line('eventdns', 'eventdns', func = lambda v: v == '1') _parse_onion_key_line = _parse_key_block('onion-key', 'onion_key', 'RSA PUBLIC KEY') _parse_onion_key_crosscert_line = _parse_key_block('onion-key-crosscert', 'onion_key_crosscert', 'CROSSCERT') _parse_signing_key_line = _parse_key_block('signing-key', 'signing_key', 'RSA PUBLIC KEY') diff --git a/stem/version.py b/stem/version.py index 68ee40f..349d9bb 100644 --- a/stem/version.py +++ b/stem/version.py @@ -328,12 +328,13 @@ class _VersionRequirements(object): :param bool to_inclusive: if comparison is inclusive with the ending version """ - if from_inclusive and to_inclusive: - new_rule = lambda v: from_version <= v <= to_version - elif from_inclusive: - new_rule = lambda v: from_version <= v < to_version - else: - new_rule = lambda v: from_version < v < to_version + def new_rule(v): + if from_inclusive and to_inclusive: + return from_version <= v <= to_version + elif from_inclusive: + return from_version <= v < to_version + else: + return from_version < v < to_version self.rules.append(new_rule) diff --git a/test/runner.py b/test/runner.py index 0370abd..80f390e 100644 --- a/test/runner.py +++ b/test/runner.py @@ -713,10 +713,11 @@ class Runner(object): try: # wait to fully complete if we're running tests with network activity, # otherwise finish after local bootstraping + complete_percent = 100 if Target.ONLINE in self.attribute_targets else 5 - # prints output from tor's stdout while it starts up - print_init_line = lambda line: println(' %s' % line, SUBSTATUS) + def print_init_line(line): + println(' %s' % line, SUBSTATUS) torrc_dst = os.path.join(self._test_dir, 'torrc') self._tor_process = stem.process.launch_tor(tor_cmd, None, torrc_dst, complete_percent, print_init_line, take_ownership = True) diff --git a/test/unit/descriptor/router_status_entry.py b/test/unit/descriptor/router_status_entry.py index 73cda28..25ba99d 100644 --- a/test/unit/descriptor/router_status_entry.py +++ b/test/unit/descriptor/router_status_entry.py @@ -49,7 +49,10 @@ m 21 sha256=AVp41YVxKEJCaoEf0+77Cdvyw5YgpyDXdob0+LSv/pE def vote_document(): - mock_document = lambda x: x # just need anything with a __dict__ + class Stub(object): + pass + + mock_document = Stub() # just need anything with a __dict__ setattr(mock_document, 'is_vote', True) setattr(mock_document, 'is_consensus', False) return mock_document diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index d295884..cb587d0 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -376,7 +376,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertEqual(['1'], desc.circuit_protocols) self.assertEqual(False, desc.hibernating) self.assertEqual(False, desc.allow_single_hop_exits) - self.assertEqual(False, desc.extra_info_cache) + self.assertEqual(True, desc.extra_info_cache) self.assertEqual('BB1F13AA431421BEA29B840A2E33BB1C31C2990B', desc.extra_info_digest) self.assertEqual(None, desc.hidden_service_dir) self.assertEqual(set(), desc.family) [View Less]

1 0

[stem/master] Move imports to the top of files
by atagar＠torproject.org 31 Dec '15

31 Dec '15

commit e9060e62fa0317f229ec3bfdd3d821c7eb256a84 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Dec 31 08:45:15 2015 -0800 Move imports to the top of files Stylistic complaint E402 in the new version of pep8... line 870 - E402 module level import not at top of file | import stem.descriptor.extrainfo_descriptor Usually this is simply because I put __all__ blocks above them which is trivial to change. In a few cases though I reorder … [View More]imports for a good reason. As such, adding the ability to ignore more targeted pep8 issues. --- stem/__init__.py | 22 +++++++++++----------- stem/descriptor/__init__.py | 28 ++++++++++++++-------------- stem/interpreter/__init__.py | 14 +++++++------- stem/response/__init__.py | 10 +++++----- stem/util/__init__.py | 4 ++-- stem/util/test_tools.py | 28 ++++++++++++++++++++++++++-- test/settings.cfg | 8 ++++++++ test/unit/__init__.py | 7 +++---- test/unit/descriptor/__init__.py | 4 ++-- test/unit/util/__init__.py | 12 ++++++------ 10 files changed, 84 insertions(+), 53 deletions(-) diff --git a/stem/__init__.py b/stem/__init__.py index 273578a..e951ec2 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -464,6 +464,17 @@ Library for working with the tor process. ================= =========== """ +import stem.prereq + +if stem.prereq.is_python_3(): + str_type = str + int_type = int +else: + str_type = unicode + int_type = long + +import stem.util.enum + __version__ = '1.4.1-dev' __author__ = 'Damian Johnson' __contact__ = 'atagar(a)torproject.org' @@ -517,17 +528,6 @@ __all__ = [ 'TimeoutSetType', ] -import stem.prereq - -if stem.prereq.is_python_3(): - str_type = str - int_type = int -else: - str_type = unicode - int_type = long - -import stem.util.enum - # Constant to indicate an undefined argument default. Usually we'd use None for # this, but users will commonly provide None as the argument so need something # else fairly unique... diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index af96bde..e46f30d 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -36,20 +36,6 @@ Package for parsing and processing descriptor data. =================== =========== """ -__all__ = [ - 'export', - 'reader', - 'remote', - 'extrainfo_descriptor', - 'server_descriptor', - 'microdescriptor', - 'networkstatus', - 'router_status_entry', - 'tordnsel', - 'parse_file', - 'Descriptor', -] - import base64 import codecs import copy @@ -71,6 +57,20 @@ try: except ImportError: from stem.util.ordereddict import OrderedDict +__all__ = [ + 'export', + 'reader', + 'remote', + 'extrainfo_descriptor', + 'server_descriptor', + 'microdescriptor', + 'networkstatus', + 'router_status_entry', + 'tordnsel', + 'parse_file', + 'Descriptor', +] + KEYWORD_CHAR = 'a-zA-Z0-9-' WHITESPACE = ' \t' KEYWORD_LINE = re.compile('^([%s]+)(?:[%s]+(.*))?$' % (KEYWORD_CHAR, WHITESPACE)) diff --git a/stem/interpreter/__init__.py b/stem/interpreter/__init__.py index f8dce7e..195e32a 100644 --- a/stem/interpreter/__init__.py +++ b/stem/interpreter/__init__.py @@ -6,13 +6,6 @@ Interactive interpreter for interacting with Tor directly. This adds usability features such as tab completion, history, and IRC-style functions (like /help). """ -__all__ = [ - 'arguments', - 'autocomplete', - 'commands', - 'help', -] - import os import sys @@ -26,6 +19,13 @@ import stem.util.term from stem.util.term import Attr, Color, format +__all__ = [ + 'arguments', + 'autocomplete', + 'commands', + 'help', +] + PROMPT = format('>>> ', Color.GREEN, Attr.BOLD, Attr.READLINE_ESCAPE) STANDARD_OUTPUT = (Color.BLUE, ) diff --git a/stem/response/__init__.py b/stem/response/__init__.py index 0612007..1c391c0 100644 --- a/stem/response/__init__.py +++ b/stem/response/__init__.py @@ -30,6 +30,11 @@ Parses replies from the control socket. +- pop_mapping - removes and returns the next entry as a KEY=VALUE mapping """ +import re +import threading + +import stem.socket + __all__ = [ 'add_onion', 'events', @@ -43,16 +48,11 @@ __all__ = [ 'SingleLineResponse', ] -import re -import threading - try: from StringIO import StringIO except ImportError: from io import StringIO -import stem.socket - KEY_ARG = re.compile('^(\S+)=') # Escape sequences from the 'esc_for_log' function of tor's 'common/util.c'. diff --git a/stem/util/__init__.py b/stem/util/__init__.py index ecdf288..2ee9b9b 100644 --- a/stem/util/__init__.py +++ b/stem/util/__init__.py @@ -5,6 +5,8 @@ Utility functions used by the stem library. """ +import datetime + __all__ = [ 'conf', 'connection', @@ -20,8 +22,6 @@ __all__ = [ 'datetime_to_unix', ] -import datetime - def datetime_to_unix(timestamp): """ diff --git a/stem/util/test_tools.py b/stem/util/test_tools.py index bc54f94..533f8d3 100644 --- a/stem/util/test_tools.py +++ b/stem/util/test_tools.py @@ -139,6 +139,8 @@ def stylistic_issues(paths, check_newlines = False, check_exception_keyword = Fa pep8.ignore E121 pep8.ignore E501 + pep8.ignore run_tests.py => E402: import stem.util.enum + ... you can then run tests with... :: @@ -177,6 +179,26 @@ def stylistic_issues(paths, check_newlines = False, check_exception_keyword = Fa issues = {} + ignore_rules = [] + ignore_for_file = [] + + for rule in CONFIG['pep8.ignore']: + if '=>' in rule: + path, rule_entry = rule.split('=>', 1) + + if ':' in rule_entry: + rule, code = rule_entry.split(':', 1) + ignore_for_file.append((path.strip(), rule.strip(), code.strip())) + else: + ignore_rules.append(rule) + + def is_ignored(path, rule, code): + for ignored_path, ignored_rule, ignored_code in ignore_for_file: + if path.endswith(ignored_path) and ignored_rule == rule and ignored_code == code.strip(): + return True + + return False + if is_pep8_available(): import pep8 @@ -189,9 +211,11 @@ def stylistic_issues(paths, check_newlines = False, check_exception_keyword = Fa if code: line = linecache.getline(self.filename, line_number) - issues.setdefault(self.filename, []).append(Issue(line_number, text, line)) - style_checker = pep8.StyleGuide(ignore = CONFIG['pep8.ignore'], reporter = StyleReport) + if not is_ignored(self.filename, code, line): + issues.setdefault(self.filename, []).append(Issue(line_number, text, line)) + + style_checker = pep8.StyleGuide(ignore = ignore_rules, reporter = StyleReport) style_checker.check_files(list(_python_files(paths))) if check_newlines or check_exception_keyword: diff --git a/test/settings.cfg b/test/settings.cfg index 7443544..fd199e1 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -128,6 +128,14 @@ pep8.ignore E251 pep8.ignore E127 pep8.ignore E131 +pep8.ignore stem/__init__.py => E402: import stem.util.enum +pep8.ignore stem/descriptor/__init__.py => E402: import stem.descriptor.server_descriptor +pep8.ignore stem/descriptor/__init__.py => E402: import stem.descriptor.extrainfo_descriptor +pep8.ignore stem/descriptor/__init__.py => E402: import stem.descriptor.networkstatus +pep8.ignore stem/descriptor/__init__.py => E402: import stem.descriptor.microdescriptor +pep8.ignore stem/descriptor/__init__.py => E402: import stem.descriptor.tordnsel +pep8.ignore stem/descriptor/__init__.py => E402: import stem.descriptor.hidden_service_descriptor + # False positives from pyflakes. These are mappings between the path and the # issue. diff --git a/test/unit/__init__.py b/test/unit/__init__.py index d76ecfe..94cd059 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -2,6 +2,9 @@ Unit tests for the stem library. """ +import os +import test.util + __all__ = [ 'connection', 'control', @@ -13,10 +16,6 @@ __all__ = [ ] -import os -import test.util - - def exec_documentation_example(filename): path = os.path.join(test.util.STEM_BASE, 'docs', '_static', 'example', filename) diff --git a/test/unit/descriptor/__init__.py b/test/unit/descriptor/__init__.py index b182f42..b7696ac 100644 --- a/test/unit/descriptor/__init__.py +++ b/test/unit/descriptor/__init__.py @@ -2,6 +2,8 @@ Unit tests for stem.descriptor. """ +import os + __all__ = [ 'export', 'extrainfo_descriptor', @@ -12,8 +14,6 @@ __all__ = [ 'server_descriptor', ] -import os - DESCRIPTOR_TEST_DATA = os.path.join(os.path.dirname(__file__), 'data') diff --git a/test/unit/util/__init__.py b/test/unit/util/__init__.py index e755625..866ec4b 100644 --- a/test/unit/util/__init__.py +++ b/test/unit/util/__init__.py @@ -2,6 +2,12 @@ Unit tests for stem.util.* contents. """ +import datetime +import time +import unittest + +from stem.util import datetime_to_unix + __all__ = [ 'conf', 'connection', @@ -12,12 +18,6 @@ __all__ = [ 'tor_tools', ] -import datetime -import time -import unittest - -from stem.util import datetime_to_unix - class TestBaseUtil(unittest.TestCase): def test_datetime_to_unix(self): [View Less]

1 0

[stem/master] Re-raise the same type of exception we get in launch_tor()
by atagar＠torproject.org 30 Dec '15

30 Dec '15

commit ae9d81b279bb51b2f1c3d088b86c7aa310b9db11 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Dec 30 08:01:14 2015 -0800 Re-raise the same type of exception we get in launch_tor() On #17946 germn argues we should re-raise our original exception type and I can can see that. In normal scenarios we should *only* raise an OSError (as documented), but if the caller invokes an exception within us (like KeyboardInterrupt) then no reason not to give them … [View More]

1 0

[webwml/master] Actually, I hope that ES is mentioned just a couple more places than our blog.
by mikeperry＠torproject.org 30 Dec '15

30 Dec '15

commit 193905fab3ea55df61f29cdb350a5d808c93eab8 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Wed Dec 30 07:42:02 2015 -0800 Actually, I hope that ES is mentioned just a couple more places than our blog. --- .htaccess | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.htaccess b/.htaccess index 6081367..38ff2d5 100644 --- a/.htaccess +++ b/.htaccess @@ -67,7 +67,7 @@ RewriteRule ^donate/donate-hpbanner(.*) /donate/donate$1 [R=302,L] RewriteRule ^… [View More]

1 0

[webwml/master] Add donate link for ES.
by mikeperry＠torproject.org 30 Dec '15

30 Dec '15

commit 89eb452185839c491072d8d3010a5ec40cdad8c8 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Wed Dec 30 07:40:54 2015 -0800 Add donate link for ES. --- .htaccess | 1 + 1 file changed, 1 insertion(+) diff --git a/.htaccess b/.htaccess index fb59b90..6081367 100644 --- a/.htaccess +++ b/.htaccess @@ -67,6 +67,7 @@ RewriteRule ^donate/donate-hpbanner(.*) /donate/donate$1 [R=302,L] RewriteRule ^donate/donate-blog-benw(.*) /donate/donate$1 [R=302,L] RewriteRule ^… [View More]

1 0

[tor/master] change credit by request
by nickm＠torproject.org 30 Dec '15

30 Dec '15

commit cdbb04be102969bd2cece9daf42896e061cc8880 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Dec 30 09:16:11 2015 -0500 change credit by request --- changes/17944 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/17944 b/changes/17944 index b279506..0d8ed0a 100644 --- a/changes/17944 +++ b/changes/17944 @@ -1,3 +1,3 @@ o Minor features (portability): - Use timingsafe_memcmp() where available. Closes ticket 17944; - patch from "… [View More]

1 0

[stem/master] Unexpected exceptions could cause launch_tor() to leave a lingering process
by atagar＠torproject.org 29 Dec '15

29 Dec '15

commit 75f311db973412f3754eadacdd08104173f768a1 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Dec 29 10:06:16 2015 -0800 Unexpected exceptions could cause launch_tor() to leave a lingering process Good point from germn that KeyboardInterrupts and other unxpected exception types could cause launch_tor() to leave an orphaned tor process behind... https://trac.torproject.org/projects/tor/ticket/17946 Ensuring that when we raise an … [View More]exception it gets terminated. --- docs/change_log.rst | 1 + stem/control.py | 2 +- stem/process.py | 49 +++++++++++++++++++++++++++++++------------------ test/integ/process.py | 22 ++++++++++++++++++++++ 4 files changed, 55 insertions(+), 19 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index 87ea053..fb0a71c 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -50,6 +50,7 @@ The following are only available within Stem's `git repository * Added `support for NETWORK_LIVENESS events <api/response.html#stem.response.events.NetworkLivenessEvent>`_ (:spec:`44aac63`) * Added :func:`~stem.control.Controller.is_user_traffic_allowed` to the :class:`~stem.control.Controller` * Added the replica attribute to the :class:`~stem.response.events.HSDescEvent` (:spec:`4989e73`) + * :func:`~stem.process.launch_tor` could leave a lingering process during an unexpected exception (:trac:`17946`) * IPv6 addresses could trigger errors in :func:`~stem.control.Controller.get_listeners`, :class:`~stem.response.events.ORConnEvent`, and quite a few other things (:trac:`16174`) * Don't obscure stacktraces, most notably :class:`~stem.control.Controller` getter methods with default values * Classes with custom equality checks didn't provide a corresponding inequality method diff --git a/stem/control.py b/stem/control.py index 3b1edfa..e4509a9 100644 --- a/stem/control.py +++ b/stem/control.py @@ -1713,7 +1713,7 @@ class Controller(BaseController): raise stem.DescriptorUnavailable('Descriptor information is unavailable, tor might still be downloading it') return stem.descriptor.server_descriptor.RelayDescriptor(desc_content) - except Exception as exc: + except: if not self._is_server_descriptors_available(): raise ValueError(SERVER_DESCRIPTORS_UNSUPPORTED) diff --git a/stem/process.py b/stem/process.py index b8b1f92..f2d63f0 100644 --- a/stem/process.py +++ b/stem/process.py @@ -22,6 +22,7 @@ import os import re import signal import subprocess +import sys import tempfile import stem.prereq @@ -105,27 +106,26 @@ def launch_tor(tor_cmd = 'tor', args = None, torrc_path = None, completion_perce if take_ownership: runtime_args += ['__OwningControllerProcess', str(os.getpid())] - tor_process = subprocess.Popen(runtime_args, stdout = subprocess.PIPE, stdin = subprocess.PIPE, stderr = subprocess.PIPE) + tor_process = None - if stdin: - tor_process.stdin.write(stem.util.str_tools._to_bytes(stdin)) - tor_process.stdin.close() + try: + tor_process = subprocess.Popen(runtime_args, stdout = subprocess.PIPE, stdin = subprocess.PIPE, stderr = subprocess.PIPE) - if timeout: - def timeout_handler(signum, frame): - # terminates the uninitialized tor process and raise on timeout + if stdin: + tor_process.stdin.write(stem.util.str_tools._to_bytes(stdin)) + tor_process.stdin.close() - tor_process.kill() - raise OSError('reached a %i second timeout without success' % timeout) + if timeout: + def timeout_handler(signum, frame): + raise OSError('reached a %i second timeout without success' % timeout) - signal.signal(signal.SIGALRM, timeout_handler) - signal.alarm(timeout) + signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(timeout) - bootstrap_line = re.compile('Bootstrapped ([0-9]+)%: ') - problem_line = re.compile('\[(warn|err)\] (.*)$') - last_problem = 'Timed out' + bootstrap_line = re.compile('Bootstrapped ([0-9]+)%: ') + problem_line = re.compile('\[(warn|err)\] (.*)$') + last_problem = 'Timed out' - try: while True: # Tor's stdout will be read as ASCII bytes. This is fine for python 2, but # in python 3 that means it'll mismatch with other operations (for instance @@ -139,7 +139,6 @@ def launch_tor(tor_cmd = 'tor', args = None, torrc_path = None, completion_perce # this will provide empty results if the process is terminated if not init_line: - tor_process.kill() # ... but best make sure raise OSError('Process terminated: %s' % last_problem) # provide the caller with the initialization message if they want it @@ -162,12 +161,26 @@ def launch_tor(tor_cmd = 'tor', args = None, torrc_path = None, completion_perce msg = msg.split(': ')[-1].strip() last_problem = msg + except: + if tor_process: + tor_process.kill() # don't leave a lingering process + tor_process.wait() + + exc = sys.exc_info()[1] + + if type(exc) == OSError: + raise # something we're raising ourselves + else: + raise OSError('Unexpected exception while starting tor (%s): %s' % (type(exc).__name__, exc)) finally: if timeout: signal.alarm(0) # stop alarm - tor_process.stdout.close() - tor_process.stderr.close() + if tor_process and tor_process.stdout: + tor_process.stdout.close() + + if tor_process and tor_process.stderr: + tor_process.stderr.close() if temp_file: try: diff --git a/test/integ/process.py b/test/integ/process.py index eec1e1e..3b44ef5 100644 --- a/test/integ/process.py +++ b/test/integ/process.py @@ -196,6 +196,28 @@ class TestProcess(unittest.TestCase): self.assertTrue('UseBridges' in output) self.assertTrue('SocksPort' in output) + @patch('re.compile', Mock(side_effect = KeyboardInterrupt('nope'))) + def test_no_orphaned_process(self): + """ + Check that when an exception arises in the middle of spawning tor that we + don't leave a lingering process. + """ + + # We don't need to actually run tor for this test. Rather, any process will + # do the trick. Picking sleep so this'll clean itself up if our test fails. + + mock_tor_process = subprocess.Popen(['sleep', '60']) + + with patch('subprocess.Popen', Mock(return_value = mock_tor_process)): + try: + stem.process.launch_tor() + self.fail("tor shoudn't have started") + except OSError as exc: + if os.path.exists('/proc/%s' % mock_tor_process.pid): + self.fail("launch_tor() left a lingering tor process") + + self.assertEqual('Unexpected exception while starting tor (KeyboardInterrupt): nope', str(exc)) + def test_torrc_arguments(self): """ Pass configuration options on the commandline. [View Less]

1 0