November 2014 - tor-commits - lists.torproject.org

[tor-messenger-build/master] Remove update options during build
by sukhbir＠torproject.org 23 Nov '14

23 Nov '14

commit fae6d4ad33ca529266c89cb322ef7d5a276713cd Author: Sukhbir Singh <sukhbir(a)torproject.org> Date: Sun Nov 23 11:04:38 2014 +0530 Remove update options during build --- projects/instantbird/mozconfig-linux-i686 | 2 -- projects/instantbird/mozconfig-linux-x86_64 | 2 -- projects/instantbird/mozconfig-windows-i686 | 2 -- 3 files changed, 6 deletions(-) diff --git a/projects/instantbird/mozconfig-linux-i686 b/projects/instantbird/mozconfig-linux-i686 index 0e79df2..52fd2e9 100644 --- a/projects/instantbird/mozconfig-linux-i686 +++ b/projects/instantbird/mozconfig-linux-i686 @@ -15,8 +15,6 @@ mk_add_options MOZ_MAKE_FLAGS="-j$CONCURRENCY" ac_add_options --with-ccache ac_add_options --enable-application=im -ac_add_options --enable-update-channel=release -ac_add_options --enable-update-packaging ac_add_options --enable-optimize ac_add_options --disable-debug diff --git a/projects/instantbird/mozconfig-linux-x86_64 b/projects/instantbird/mozconfig-linux-x86_64 index cfd1bb7..d3cf703 100644 --- a/projects/instantbird/mozconfig-linux-x86_64 +++ b/projects/instantbird/mozconfig-linux-x86_64 @@ -12,8 +12,6 @@ mk_add_options MOZ_MAKE_FLAGS="-j$CONCURRENCY" ac_add_options --with-ccache ac_add_options --enable-application=im -ac_add_options --enable-update-channel=release -ac_add_options --enable-update-packaging ac_add_options --enable-optimize ac_add_options --disable-debug diff --git a/projects/instantbird/mozconfig-windows-i686 b/projects/instantbird/mozconfig-windows-i686 index f9d1d5e..11fea5c 100644 --- a/projects/instantbird/mozconfig-windows-i686 +++ b/projects/instantbird/mozconfig-windows-i686 @@ -9,8 +9,6 @@ ac_add_options --enable-strip mk_add_options MOZ_MAKE_FLAGS="-j4" ac_add_options --enable-application=im -ac_add_options --enable-update-channel=release -ac_add_options --enable-update-packaging ac_add_options --enable-optimize ac_add_options --disable-debug

1 0

[tor-messenger-build/master] Update mozconfigs to remove official branding
by sukhbir＠torproject.org 23 Nov '14

23 Nov '14

commit d6b4177805bdeb78a24eb841815801cf7e56ae91 Author: Sukhbir Singh <sukhbir(a)torproject.org> Date: Sun Nov 23 10:56:46 2014 +0530 Update mozconfigs to remove official branding --- projects/instantbird/mozconfig-linux-i686 | 4 ---- projects/instantbird/mozconfig-linux-x86_64 | 4 ---- projects/instantbird/mozconfig-windows-i686 | 4 ---- 3 files changed, 12 deletions(-) diff --git a/projects/instantbird/mozconfig-linux-i686 b/projects/instantbird/mozconfig-linux-i686 index 476ff74..0e79df2 100644 --- a/projects/instantbird/mozconfig-linux-i686 +++ b/projects/instantbird/mozconfig-linux-i686 @@ -1,8 +1,5 @@ no_tooltool=1 -export MOZILLA_OFFICIAL=1 -mk_add_options MOZILLA_OFFICIAL=1 - export CFLAGS="-m32 -gdwarf-2 -Wno-sign-compare" export CXXFLAGS="-m32 -gdwarf-2" export LDFLAGS=-m32 @@ -23,6 +20,5 @@ ac_add_options --enable-update-packaging ac_add_options --enable-optimize ac_add_options --disable-debug -ac_add_options --enable-official-branding ac_add_options --with-l10n-base=.. ac_add_options --host=i686-linux-gnu diff --git a/projects/instantbird/mozconfig-linux-x86_64 b/projects/instantbird/mozconfig-linux-x86_64 index f3a4c41..cfd1bb7 100644 --- a/projects/instantbird/mozconfig-linux-x86_64 +++ b/projects/instantbird/mozconfig-linux-x86_64 @@ -1,8 +1,5 @@ no_tooltool=1 -export MOZILLA_OFFICIAL=1 -mk_add_options MOZILLA_OFFICIAL=1 - export CFLAGS="-gdwarf-2 -Wno-sign-compare" export CXXFLAGS="-gdwarf-2" @@ -20,5 +17,4 @@ ac_add_options --enable-update-packaging ac_add_options --enable-optimize ac_add_options --disable-debug -ac_add_options --enable-official-branding ac_add_options --with-l10n-base=.. diff --git a/projects/instantbird/mozconfig-windows-i686 b/projects/instantbird/mozconfig-windows-i686 index 321c873..f9d1d5e 100644 --- a/projects/instantbird/mozconfig-windows-i686 +++ b/projects/instantbird/mozconfig-windows-i686 @@ -2,9 +2,6 @@ export CROSS_COMPILE=1 no_tooltool=1 -export MOZILLA_OFFICIAL=1 -mk_add_options MOZILLA_OFFICIAL=1 - ac_add_options --target=i686-w64-mingw32 ac_add_options --enable-default-toolkit=cairo-windows ac_add_options --enable-strip @@ -17,7 +14,6 @@ ac_add_options --enable-update-packaging ac_add_options --enable-optimize ac_add_options --disable-debug -ac_add_options --enable-official-branding ac_add_options --with-l10n-base=.. # sandbox currently fail to build with mingw

1 0

[tor-messenger-build/master] Remove the branding patch for now (use mozconfig later)
by sukhbir＠torproject.org 23 Nov '14

23 Nov '14

commit 8e6cceb35cd9997d017e4843e9848a69b9930f47 Author: Sukhbir Singh <sukhbir(a)torproject.org> Date: Sun Nov 23 09:23:46 2014 +0530 Remove the branding patch for now (use mozconfig later) --- projects/instantbird/branding.patch | 39 ----------------------------------- projects/instantbird/config | 1 - 2 files changed, 40 deletions(-) diff --git a/projects/instantbird/branding.patch b/projects/instantbird/branding.patch deleted file mode 100644 index 01ce618..0000000 --- a/projects/instantbird/branding.patch +++ /dev/null @@ -1,39 +0,0 @@ -# HG changeset patch -# User Sukhbir Singh <sukhbir(a)torproject.org> -# Date 1416650722 18000 -# Node ID 9d3cc50f1b5c0679d3397154f6c7c329aaf15cf5 -# Parent 84423e51b0535ccd21aff64f10176f3e8c05b7c3 -Update the branding - -diff --git a/im/branding/nightly/locales/en-US/brand.dtd b/im/branding/nightly/locales/en-US/brand.dtd ---- a/im/branding/nightly/locales/en-US/brand.dtd -+++ b/im/branding/nightly/locales/en-US/brand.dtd -@@ -1,10 +1,10 @@ -  - -  - --<!ENTITY brandShortName "Instantbird"> --<!ENTITY brandFullName "Instantbird - Nightly"> --<!ENTITY brandMotto "'Cause geeks can also do magic!"> --<!ENTITY vendorShortName "Instantbird"> -+<!ENTITY brandShortName "Tor Messenger"> -+<!ENTITY brandFullName "Tor Messenger - Alpha"> -+<!ENTITY brandMotto "Talk over the onion!"> -+<!ENTITY vendorShortName "Tor Messenger"> -diff --git a/im/branding/nightly/locales/en-US/brand.properties b/im/branding/nightly/locales/en-US/brand.properties ---- a/im/branding/nightly/locales/en-US/brand.properties -+++ b/im/branding/nightly/locales/en-US/brand.properties -@@ -1,7 +1,7 @@ - # This Source Code Form is subject to the terms of the Mozilla Public - # License, v. 2.0. If a copy of the MPL was not distributed with this - # file, You can obtain one at http://mozilla.org/MPL/2.0/. - --brandShortName=Instantbird --brandFullName=Instantbird - Nightly --vendorShortName=Instantbird -+brandShortName=Tor Messenger -+brandFullName=Tor Messenger - Alpha -+vendorShortName=Tor Messenger diff --git a/projects/instantbird/config b/projects/instantbird/config index d32a255..e7c8b61 100644 --- a/projects/instantbird/config +++ b/projects/instantbird/config @@ -61,7 +61,6 @@ input_files: - filename: accountcreation.patch - filename: links.patch - filename: picture.patch - - filename: branding.patch - filename: prepare-messages-for-displaying.patch - filename: fix-mingw-build.nsprpatch enable: '[% c("var/windows") %]'

1 0

[stem/master] stem/descriptor/extrainfo_descriptor.py: Compile regexps outside of loops.
by atagar＠torproject.org 23 Nov '14

23 Nov '14

commit 4669c89a4b3b36712dbf879287312dd4221b18da Author: Ossi Herrala <oherrala(a)gmail.com> Date: Wed Nov 19 20:58:16 2014 +0200 stem/descriptor/extrainfo_descriptor.py: Compile regexps outside of loops. With 24MB cached-extrainfo file this reduces function calls from 7457706 down to 6003241. before (lines sorted): 7457706 function calls (7450757 primitive calls) in 14.153 seconds 7457706 function calls (7450757 primitive calls) in 14.163 seconds 7457706 function calls (7450757 primitive calls) in 14.274 seconds 7457706 function calls (7450757 primitive calls) in 14.293 seconds 7457706 function calls (7450757 primitive calls) in 14.311 seconds 7457706 function calls (7450757 primitive calls) in 14.356 seconds 7457706 function calls (7450757 primitive calls) in 14.418 seconds 7457706 function calls (7450757 primitive calls) in 14.487 seconds 7457706 function calls (7450757 primitive calls) in 14.517 seconds 7457706 function calls (7450757 primitive calls) in 14.682 seconds after (lines sorted): 6003241 function calls (5996319 primitive calls) in 12.607 seconds 6003241 function calls (5996319 primitive calls) in 12.619 seconds 6003241 function calls (5996319 primitive calls) in 12.717 seconds 6003241 function calls (5996319 primitive calls) in 12.718 seconds 6003241 function calls (5996319 primitive calls) in 12.727 seconds 6003241 function calls (5996319 primitive calls) in 12.834 seconds 6003241 function calls (5996319 primitive calls) in 12.840 seconds 6003241 function calls (5996319 primitive calls) in 12.882 seconds 6003241 function calls (5996319 primitive calls) in 13.027 seconds 6003241 function calls (5996319 primitive calls) in 13.277 seconds --- stem/descriptor/extrainfo_descriptor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 0aa7724..31f0e20 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -151,6 +151,10 @@ SINGLE_FIELDS = ( ) +_timestamp_re = re.compile('^(.*) $([0-9]+) s$( .*)?$') +_locale_re = re.compile('^[a-zA-Z0-9\?]{2}$') + + def _parse_file(descriptor_file, is_bridge = False, validate = True, **kwargs): """ Iterates over the extra-info descriptors in a file. @@ -202,7 +206,7 @@ def _parse_timestamp_and_interval(keyword, content): """ line = '%s %s' % (keyword, content) - content_match = re.match('^(.*) $([0-9]+) s$( .*)?$', content) + content_match = _timestamp_re.match(content) if not content_match: raise ValueError('Malformed %s line: %s' % (keyword, line)) @@ -783,7 +787,7 @@ class ExtraInfoDescriptor(Descriptor): locale, count = entry.split('=', 1) - if re.match('^[a-zA-Z0-9\?]{2}$', locale) and count.isdigit(): + if _locale_re.match(locale) and count.isdigit(): locale_usage[locale] = int(count) elif validate: raise ValueError(error_msg)

1 0

[stem/master] stem/descriptor/extrainfo_descriptor.py: Optimize away calls to datetime.datetime.strptime()
by atagar＠torproject.org 23 Nov '14

23 Nov '14

commit c1dc0959feed16b9e0264c4b795fb7b5d0f16ef8 Author: Ossi Herrala <oherrala(a)gmail.com> Date: Thu Nov 20 22:08:43 2014 +0200 stem/descriptor/extrainfo_descriptor.py: Optimize away calls to datetime.datetime.strptime() datetime.datetime.strptime() seems to be slow. We can go faster with regexp and build the datetime.datetime object from the results. before (lines sorted): 6003241 function calls (5996319 primitive calls) in 12.508 seconds 6003241 function calls (5996319 primitive calls) in 12.564 seconds 6003241 function calls (5996319 primitive calls) in 12.677 seconds 6003241 function calls (5996319 primitive calls) in 12.720 seconds 6003241 function calls (5996319 primitive calls) in 12.871 seconds 6003241 function calls (5996319 primitive calls) in 12.871 seconds 6003241 function calls (5996319 primitive calls) in 12.935 seconds 6003241 function calls (5996319 primitive calls) in 12.982 seconds 6003241 function calls (5996319 primitive calls) in 13.230 seconds 6003241 function calls (5996319 primitive calls) in 13.663 seconds after (lines sorted): 5444306 function calls (5437532 primitive calls) in 11.417 seconds 5444306 function calls (5437532 primitive calls) in 11.419 seconds 5444306 function calls (5437532 primitive calls) in 11.494 seconds 5444306 function calls (5437532 primitive calls) in 11.568 seconds 5444306 function calls (5437532 primitive calls) in 11.593 seconds 5444306 function calls (5437532 primitive calls) in 11.629 seconds 5444306 function calls (5437532 primitive calls) in 11.630 seconds 5444306 function calls (5437532 primitive calls) in 11.649 seconds 5444306 function calls (5437532 primitive calls) in 11.751 seconds 5444306 function calls (5437532 primitive calls) in 12.247 seconds --- stem/descriptor/extrainfo_descriptor.py | 5 ++--- stem/util/str_tools.py | 36 ++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 31f0e20..b677888 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -69,7 +69,6 @@ Extra-info descriptors are available from a few sources... ===================== =========== """ -import datetime import hashlib import re @@ -220,7 +219,7 @@ def _parse_timestamp_and_interval(keyword, content): raise ValueError("%s line's interval wasn't a number: %s" % (keyword, line)) try: - timestamp = datetime.datetime.strptime(timestamp_str, '%Y-%m-%d %H:%M:%S') + timestamp = stem.util.str_tools._parse_timestamp(timestamp_str) return timestamp, int(interval), remainder except ValueError: raise ValueError("%s line's timestamp wasn't parsable: %s" % (keyword, line)) @@ -652,7 +651,7 @@ class ExtraInfoDescriptor(Descriptor): # "<keyword>" YYYY-MM-DD HH:MM:SS try: - timestamp = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') + timestamp = stem.util.str_tools._parse_timestamp(value) if keyword == 'published': self.published = timestamp diff --git a/stem/util/str_tools.py b/stem/util/str_tools.py index 16681d5..039447e 100644 --- a/stem/util/str_tools.py +++ b/stem/util/str_tools.py @@ -23,6 +23,7 @@ Toolkit for various string activity. import codecs import datetime +import re import sys import stem.prereq @@ -56,6 +57,8 @@ TIME_UNITS = ( (1.0, 's', ' second'), ) +_timestamp_re = re.compile(r'(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})') + if stem.prereq.is_python_3(): def _to_bytes_impl(msg): if isinstance(msg, str): @@ -436,6 +439,32 @@ def parse_short_time_label(label): raise ValueError('Non-numeric value in time entry: %s' % label) +def _parse_timestamp(entry): + """ + Parses the date and time that in format like like... + + :: + + 2012-11-08 16:48:41 + + :param str entry: timestamp to be parsed + + :returns: datetime for the time represented by the timestamp + + :raises: ValueError if the timestamp is malformed + """ + + if not isinstance(entry, (str, unicode)): + raise IOError('parse_iso_timestamp() input must be a str, got a %s' % type(entry)) + + try: + time = [int(x) for x in _timestamp_re.match(entry).groups()] + except AttributeError: + raise ValueError("Expected timestamp in format YYYY-MM-DD HH:MM:ss but got " + entry) + + return datetime.datetime(time[0], time[1], time[2], time[3], time[4], time[5]) + + def _parse_iso_timestamp(entry): """ Parses the ISO 8601 standard that provides for timestamps like... @@ -465,7 +494,12 @@ def _parse_iso_timestamp(entry): if len(microseconds) != 6 or not microseconds.isdigit(): raise ValueError("timestamp's microseconds should be six digits") - timestamp = datetime.datetime.strptime(timestamp_str, "%Y-%m-%dT%H:%M:%S") + if timestamp_str[10] == 'T': + timestamp_str = timestamp_str[:10] + ' ' + timestamp_str[11:] + else: + raise ValueError("timestamp didn't contain delimeter 'T' between date and time") + + timestamp = _parse_timestamp(timestamp_str) return timestamp + datetime.timedelta(microseconds = int(microseconds))

1 0

[stem/master] stem/util/tor_tools.py: Small optimizations to validators.
by atagar＠torproject.org 23 Nov '14

23 Nov '14

commit 7cd62935b7f6d5dcf41fb2a82da8a13853fe809c Author: Ossi Herrala <oherrala(a)gmail.com> Date: Fri Nov 21 02:00:42 2014 +0200 stem/util/tor_tools.py: Small optimizations to validators. * Remove calls to isinstance() and trust the called functions to throw exception instead. * Add more unit tests for cases found while poking around. --- stem/util/tor_tools.py | 35 +++++++++++++++++++++-------------- test/unit/util/tor_tools.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/stem/util/tor_tools.py b/stem/util/tor_tools.py index 9eb46a0..54579f6 100644 --- a/stem/util/tor_tools.py +++ b/stem/util/tor_tools.py @@ -51,15 +51,16 @@ def is_valid_fingerprint(entry, check_prefix = False): :returns: **True** if the string could be a relay fingerprint, **False** otherwise """ - if not isinstance(entry, (str, unicode)): - return False - elif check_prefix: - if not entry or entry[0] != '$': - return False + try: + if check_prefix: + if not entry or entry[0] != '$': + return False - entry = entry[1:] + entry = entry[1:] - return bool(FINGERPRINT_PATTERN.match(entry)) + return is_hex_digits(entry, 40) + except TypeError: + return False def is_valid_nickname(entry): @@ -71,11 +72,11 @@ def is_valid_nickname(entry): :returns: **True** if the string could be a nickname, **False** otherwise """ - if not isinstance(entry, (str, unicode)): + try: + return bool(NICKNAME_PATTERN.match(entry)) + except TypeError: return False - return bool(NICKNAME_PATTERN.match(entry)) - def is_valid_circuit_id(entry): """ @@ -84,11 +85,11 @@ def is_valid_circuit_id(entry): :returns: **True** if the string could be a circuit id, **False** otherwise """ - if not isinstance(entry, (str, unicode)): + try: + return bool(CIRC_ID_PATTERN.match(entry)) + except TypeError: return False - return bool(CIRC_ID_PATTERN.match(entry)) - def is_valid_stream_id(entry): """ @@ -123,4 +124,10 @@ def is_hex_digits(entry, count): :returns: **True** if the string matches this number """ - return bool(re.match('^%s{%i}$' % (HEX_DIGIT, count), entry)) + try: + if len(entry) != count: + return False + int(entry, 16) + except (ValueError, TypeError): + return False + return True diff --git a/test/unit/util/tor_tools.py b/test/unit/util/tor_tools.py index 6e3af86..80362f7 100644 --- a/test/unit/util/tor_tools.py +++ b/test/unit/util/tor_tools.py @@ -19,7 +19,10 @@ class TestTorTools(unittest.TestCase): ) invalid_fingerprints = ( + None, '', + 5, + ['A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB'], 'A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB', '$A7569A83B5706AB1B1A9CB52EFF7D2D32E4553E', '$A7569A83B5706AB1B1A9CB52EFF7D2D32E4553E33', @@ -46,6 +49,7 @@ class TestTorTools(unittest.TestCase): invalid_nicknames = ( None, '', + 5, 'toolongggggggggggggg', 'bad_character', ) @@ -81,3 +85,29 @@ class TestTorTools(unittest.TestCase): for circuit_id in invalid_circuit_ids: self.assertFalse(stem.util.tor_tools.is_valid_circuit_id(circuit_id)) + + def test_is_valid_hex_digits(self): + """ + Checks the is_valid_hex_digits function. + """ + + valid_hex_digits = ( + "12345", + "AbCdE", + ) + + invalid_hex_digits = ( + None, + '', + 5, + 'X', + "1234", + "ABCDEF", + [1, "2", (3, 4)] + ) + + for hex_digits in valid_hex_digits: + self.assertTrue(stem.util.tor_tools.is_hex_digits(hex_digits, 5)) + + for hex_digits in invalid_hex_digits: + self.assertFalse(stem.util.tor_tools.is_hex_digits(hex_digits, 5))

1 0

[stem/master] stem/util/connection.py: Optimize validators.
by atagar＠torproject.org 23 Nov '14

23 Nov '14

commit 25771febe94b0a3b3885b74222e2291618acb751 Author: Ossi Herrala <oherrala(a)gmail.com> Date: Fri Nov 21 02:07:18 2014 +0200 stem/util/connection.py: Optimize validators. * is_valid_ipv4_address() could do just fine without checking the address backwards (string -> packed -> string) but this breaks one unit test (IP '0.0.00.0'). * Add one missing test case for is_valid_port(). --- stem/util/connection.py | 69 ++++++++++++++---------------------------- test/unit/util/connection.py | 2 +- 2 files changed, 24 insertions(+), 47 deletions(-) diff --git a/stem/util/connection.py b/stem/util/connection.py index 09acdeb..d1fb5c2 100644 --- a/stem/util/connection.py +++ b/stem/util/connection.py @@ -46,6 +46,7 @@ import hmac import os import platform import re +import socket import stem.util.proc import stem.util.system @@ -333,23 +334,12 @@ def is_valid_ipv4_address(address): :returns: **True** if input is a valid IPv4 address, **False** otherwise """ - if not isinstance(address, (bytes, unicode)): - return False - - # checks if theres four period separated values - - if address.count('.') != 3: + try: + packed = socket.inet_pton(socket.AF_INET, address) + return socket.inet_ntop(socket.AF_INET, packed) == address + except socket.error: return False - # checks that each value in the octet are decimal values between 0-255 - for entry in address.split('.'): - if not entry.isdigit() or int(entry) < 0 or int(entry) > 255: - return False - elif entry[0] == '0' and len(entry) > 1: - return False # leading zeros, for instance in '1.2.3.001' - - return True - def is_valid_ipv6_address(address, allow_brackets = False): """ @@ -365,24 +355,11 @@ def is_valid_ipv6_address(address, allow_brackets = False): if address.startswith('[') and address.endswith(']'): address = address[1:-1] - # addresses are made up of eight colon separated groups of four hex digits - # with leading zeros being optional - # https://en.wikipedia.org/wiki/IPv6#Address_format - - colon_count = address.count(':') - - if colon_count > 7: - return False # too many groups - elif colon_count != 7 and '::' not in address: - return False # not enough groups and none are collapsed - elif address.count('::') > 1 or ':::' in address: - return False # multiple groupings of zeros can't be collapsed - - for entry in address.split(':'): - if not re.match('^[0-9a-fA-f]{0,4}$', entry): - return False - - return True + try: + socket.inet_pton(socket.AF_INET6, address) + return True + except socket.error: + return False def is_valid_port(entry, allow_zero = False): @@ -395,24 +372,24 @@ def is_valid_port(entry, allow_zero = False): :returns: **True** if input is an integer and within the valid port range, **False** otherwise """ - if isinstance(entry, list): + try: + value = int(entry) + if str(value) != str(entry): + return False # invalid leading char, e.g. space or zero + if allow_zero: + return value >= 0 and value < 65536 + else: + return value > 0 and value < 65536 + + except TypeError: + # Maybe entry is list to validate? for port in entry: if not is_valid_port(port, allow_zero): return False - return True - elif isinstance(entry, (bytes, unicode)): - if not entry.isdigit(): - return False - elif entry[0] == '0' and len(entry) > 1: - return False # leading zeros, ex '001' - entry = int(entry) - - if allow_zero and entry == 0: - return True - - return entry > 0 and entry < 65536 + except ValueError: + return False def is_private_address(address): diff --git a/test/unit/util/connection.py b/test/unit/util/connection.py index 0ae6f9b..0766c32 100644 --- a/test/unit/util/connection.py +++ b/test/unit/util/connection.py @@ -359,7 +359,7 @@ class TestConnection(unittest.TestCase): Checks the is_valid_port function. """ - valid_ports = (1, '1', 1234, '1234', 65535, '65535') + valid_ports = (1, '1', 1234, '1234', 65535, '65535', [1, '2']) invalid_ports = (0, '0', 65536, '65536', 'abc', '*', ' 15', '01') for port in valid_ports:

1 0

[stem/master] Replace calls to datetime.datetime.strptime() with stem.util.str_tools._parse_timestamp()
by atagar＠torproject.org 23 Nov '14

23 Nov '14

commit b1ffbb8827ea55e68a13ced877a9e44d83a8e22b Author: Ossi Herrala <oherrala(a)gmail.com> Date: Thu Nov 20 23:04:02 2014 +0200 Replace calls to datetime.datetime.strptime() with stem.util.str_tools._parse_timestamp() --- stem/control.py | 3 +-- stem/descriptor/networkstatus.py | 9 ++++----- stem/descriptor/router_status_entry.py | 3 +-- stem/descriptor/server_descriptor.py | 3 +-- stem/descriptor/tordnsel.py | 8 +++----- stem/response/events.py | 7 +++---- 6 files changed, 13 insertions(+), 20 deletions(-) diff --git a/stem/control.py b/stem/control.py index d815fcc..320fcab 100644 --- a/stem/control.py +++ b/stem/control.py @@ -221,7 +221,6 @@ If you're fine with allowing your script to raise exceptions then this can be mo import calendar import collections -import datetime import inspect import io import os @@ -1273,7 +1272,7 @@ class Controller(BaseController): used = self.get_info('accounting/bytes') left = self.get_info('accounting/bytes-left') - interval_end = datetime.datetime.strptime(interval_end, '%Y-%m-%d %H:%M:%S') + interval_end = stem.util.str_tools._parse_timestamp(interval_end) used_read, used_written = [int(val) for val in used.split(' ', 1)] left_read, left_written = [int(val) for val in left.split(' ', 1)] diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 1e516d3..6a15caf 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -49,7 +49,6 @@ For more information see :func:`~stem.descriptor.__init__.DocumentHandler`... DirectoryAuthority - Directory authority as defined in a v3 network status document """ -import datetime import io import stem.descriptor.router_status_entry @@ -417,7 +416,7 @@ class NetworkStatusDocumentV2(NetworkStatusDocument): self.server_versions.append(version_str) elif keyword == 'published': try: - self.published = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') + self.published = stem.util.str_tools._parse_timestamp(value) except ValueError: if validate: raise ValueError("Version 2 network status document's 'published' time wasn't parsable: %s" % value) @@ -699,7 +698,7 @@ class _DocumentHeader(object): raise ValueError("A network status document's consensus-method must be an integer, but was '%s'" % value) elif keyword in ('published', 'valid-after', 'fresh-until', 'valid-until'): try: - date_value = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') + date_value = stem.util.str_tools._parse_timestamp(value) if keyword == 'published': self.published = date_value @@ -1362,7 +1361,7 @@ class KeyCertificate(Descriptor): # "dir-key-expires" YYYY-MM-DD HH:MM:SS try: - date_value = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') + date_value = stem.util.str_tools._parse_timestamp(value) if keyword == 'dir-key-published': self.published = date_value @@ -1498,7 +1497,7 @@ class BridgeNetworkStatusDocument(NetworkStatusDocument): published_line = published_line.split(' ', 1)[1].strip() try: - self.published = datetime.datetime.strptime(published_line, '%Y-%m-%d %H:%M:%S') + self.published = stem.util.str_tools._parse_timestamp(published_line) except ValueError: if validate: raise ValueError("Bridge network status document's 'published' time wasn't parsable: %s" % published_line) diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py index 75ad646..3dd9026 100644 --- a/stem/descriptor/router_status_entry.py +++ b/stem/descriptor/router_status_entry.py @@ -21,7 +21,6 @@ sources... import base64 import binascii -import datetime import stem.exit_policy import stem.prereq @@ -523,7 +522,7 @@ def _parse_r_line(desc, value, validate, include_digest = True): try: published = '%s %s' % (r_comp[3], r_comp[4]) - desc.published = datetime.datetime.strptime(published, '%Y-%m-%d %H:%M:%S') + desc.published = stem.util.str_tools._parse_timestamp(published) except ValueError: if validate: raise ValueError("Publication time time wasn't parsable: r %s" % value) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index c1fed66..e92f0e0 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -34,7 +34,6 @@ etc). This information is provided from a few sources... import base64 import codecs -import datetime import hashlib import re @@ -460,7 +459,7 @@ class ServerDescriptor(Descriptor): # "published" YYYY-MM-DD HH:MM:SS try: - self.published = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') + self.published = stem.util.str_tools._parse_timestamp(value) except ValueError: if validate: raise ValueError("Published line's time wasn't parsable: %s" % line) diff --git a/stem/descriptor/tordnsel.py b/stem/descriptor/tordnsel.py index 72b3806..951b72f 100644 --- a/stem/descriptor/tordnsel.py +++ b/stem/descriptor/tordnsel.py @@ -10,8 +10,6 @@ exit list files. TorDNSEL - Exit list provided by TorDNSEL """ -import datetime - import stem.util.connection import stem.util.str_tools import stem.util.tor_tools @@ -89,13 +87,13 @@ class TorDNSEL(Descriptor): self.fingerprint = value elif keyword == 'Published': try: - self.published = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') + self.published = stem.util.str_tools._parse_timestamp(value) except ValueError: if validate: raise ValueError("Published time wasn't parsable: %s" % value) elif keyword == 'LastStatus': try: - self.last_status = datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') + self.last_status = stem.util.str_tools._parse_timestamp(value) except ValueError: if validate: raise ValueError("LastStatus time wasn't parsable: %s" % value) @@ -110,7 +108,7 @@ class TorDNSEL(Descriptor): raise ValueError('Unexpected block content: %s' % block_content) try: - date = datetime.datetime.strptime(date, '%Y-%m-%d %H:%M:%S') + date = stem.util.str_tools._parse_timestamp(date) self.exit_addresses.append((address, date)) except ValueError: if validate: diff --git a/stem/response/events.py b/stem/response/events.py index 0df830b..92dd281 100644 --- a/stem/response/events.py +++ b/stem/response/events.py @@ -1,7 +1,6 @@ # Copyright 2012-2014, Damian Johnson and The Tor Project # See LICENSE for licensing information -import datetime import io import re import time @@ -190,12 +189,12 @@ class AddrMapEvent(Event): self.expiry = None else: try: - self.expiry = datetime.datetime.strptime(self.expiry, '%Y-%m-%d %H:%M:%S') + self.expiry = stem.util.str_tools._parse_timestamp(self.expiry) except ValueError: raise stem.ProtocolError('Unable to parse date in ADDRMAP event: %s' % self) if self.utc_expiry is not None: - self.utc_expiry = datetime.datetime.strptime(self.utc_expiry, '%Y-%m-%d %H:%M:%S') + self.utc_expiry = stem.util.str_tools._parse_timestamp(self.utc_expiry) if self.cached is not None: if self.cached == 'YES': @@ -487,7 +486,7 @@ class ClientsSeenEvent(Event): def _parse(self): if self.start_time is not None: - self.start_time = datetime.datetime.strptime(self.start_time, '%Y-%m-%d %H:%M:%S') + self.start_time = stem.util.str_tools._parse_timestamp(self.start_time) if self.locales is not None: locale_to_count = {}

1 0

[stem/master] Performance improvements for parsing descriptor documents
by atagar＠torproject.org 23 Nov '14

23 Nov '14

commit c3205411cf5265473141bd81a941a49603bf63df Merge: 4bf82db 4b98091 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Nov 22 16:03:00 2014 -0800 Performance improvements for parsing descriptor documents Set of performance improvements from oherrala by regex match constants and avoiding isinstance() checks... https://trac.torproject.org/projects/tor/ticket/13821 docs/change_log.rst | 2 +- stem/control.py | 3 +- stem/descriptor/extrainfo_descriptor.py | 13 ++++-- stem/descriptor/networkstatus.py | 9 ++-- stem/descriptor/router_status_entry.py | 3 +- stem/descriptor/server_descriptor.py | 3 +- stem/descriptor/tordnsel.py | 8 ++-- stem/response/events.py | 7 ++- stem/util/connection.py | 77 +++++++++++-------------------- stem/util/str_tools.py | 42 +++++++++++++++-- stem/util/tor_tools.py | 40 +++++++++------- test/unit/util/connection.py | 4 +- test/unit/util/tor_tools.py | 30 ++++++++++++ 13 files changed, 143 insertions(+), 98 deletions(-)

1 0

[stem/master] Handling edge cases for earlier optimizations
by atagar＠torproject.org 23 Nov '14

23 Nov '14

commit 4b980915168e3c712e7352c81c9c00671da7a075 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Nov 22 15:54:49 2014 -0800 Handling edge cases for earlier optimizations Just a few minor edge cases... * The is_valid_address() functions could have an uncaught TypeError if given unexpected types. * Explicitly checking for lists in is_valid_port(). For instance, dictionaries could trip it up. Having a isinstance() check for just the list situation should still pretty much achieve the performance benefits this was after. --- docs/change_log.rst | 2 +- stem/util/connection.py | 22 ++++++++++++---------- stem/util/str_tools.py | 12 ++++++------ stem/util/tor_tools.py | 9 ++++----- test/unit/util/connection.py | 2 +- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index e0e765b..5a3ca34 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -53,7 +53,7 @@ The following are only available within Stem's `git repository * **Descriptors** - * Improved speed for parsing consensus documents by around 30% (:trac:`12859`) + * Improved speed for parsing consensus documents by around 40% (:trac:`12859` and :trac:`13821`) * Don't fail if consensus method 1 is not present, as it is no longer required (:spec:`fc8a6f0`) * Include '\*.new' files when reading from a Tor data directory (:trac:`13756`) * Updated the authorities we list, `replacing turtles with longclaw <https://lists.torproject.org/pipermail/tor-talk/2014-November/035650.html>`_ and `updating gabelmoo's address <https://lists.torproject.org/pipermail/tor-talk/2014-September/034898.html>`_ diff --git a/stem/util/connection.py b/stem/util/connection.py index d1fb5c2..df918d2 100644 --- a/stem/util/connection.py +++ b/stem/util/connection.py @@ -337,7 +337,7 @@ def is_valid_ipv4_address(address): try: packed = socket.inet_pton(socket.AF_INET, address) return socket.inet_ntop(socket.AF_INET, packed) == address - except socket.error: + except (TypeError, socket.error): return False @@ -358,7 +358,7 @@ def is_valid_ipv6_address(address, allow_brackets = False): try: socket.inet_pton(socket.AF_INET6, address) return True - except socket.error: + except (TypeError, socket.error): return False @@ -374,20 +374,22 @@ def is_valid_port(entry, allow_zero = False): try: value = int(entry) + if str(value) != str(entry): return False # invalid leading char, e.g. space or zero - if allow_zero: - return value >= 0 and value < 65536 + elif allow_zero and value == 0: + return True else: return value > 0 and value < 65536 - except TypeError: - # Maybe entry is list to validate? - for port in entry: - if not is_valid_port(port, allow_zero): - return False - return True + if isinstance(entry, (tuple, list)): + for port in entry: + if not is_valid_port(port, allow_zero): + return False + return True + else: + return False except ValueError: return False diff --git a/stem/util/str_tools.py b/stem/util/str_tools.py index 039447e..43d12d6 100644 --- a/stem/util/str_tools.py +++ b/stem/util/str_tools.py @@ -449,13 +449,13 @@ def _parse_timestamp(entry): :param str entry: timestamp to be parsed - :returns: datetime for the time represented by the timestamp + :returns: **datetime** for the time represented by the timestamp - :raises: ValueError if the timestamp is malformed + :raises: **ValueError** if the timestamp is malformed """ if not isinstance(entry, (str, unicode)): - raise IOError('parse_iso_timestamp() input must be a str, got a %s' % type(entry)) + raise ValueError('parse_timestamp() input must be a str, got a %s' % type(entry)) try: time = [int(x) for x in _timestamp_re.match(entry).groups()] @@ -475,12 +475,12 @@ def _parse_iso_timestamp(entry): :param str entry: timestamp to be parsed - :returns: datetime for the time represented by the timestamp + :returns: **datetime** for the time represented by the timestamp - :raises: ValueError if the timestamp is malformed + :raises: **ValueError** if the timestamp is malformed """ - if not isinstance(entry, str): + if not isinstance(entry, (str, unicode)): raise ValueError('parse_iso_timestamp() input must be a str, got a %s' % type(entry)) # based after suggestions from... diff --git a/stem/util/tor_tools.py b/stem/util/tor_tools.py index 54579f6..7b02e6c 100644 --- a/stem/util/tor_tools.py +++ b/stem/util/tor_tools.py @@ -33,8 +33,6 @@ import re # case insensitive. Tor doesn't define this in the spec so flipping a coin # and going with case insensitive. -HEX_DIGIT = '[0-9a-fA-F]' -FINGERPRINT_PATTERN = re.compile('^%s{40}$' % HEX_DIGIT) NICKNAME_PATTERN = re.compile('^[a-zA-Z0-9]{1,19}$') CIRC_ID_PATTERN = re.compile('^[a-zA-Z0-9]{1,16}$') @@ -121,13 +119,14 @@ def is_hex_digits(entry, count): :param str entry: string to be checked :param int count: number of hex digits to be checked for - :returns: **True** if the string matches this number + :returns: **True** if the given number of hex digits, **False** otherwise """ try: if len(entry) != count: return False - int(entry, 16) + + int(entry, 16) # attempt to convert it as hex + return True except (ValueError, TypeError): return False - return True diff --git a/test/unit/util/connection.py b/test/unit/util/connection.py index 0766c32..f0359eb 100644 --- a/test/unit/util/connection.py +++ b/test/unit/util/connection.py @@ -360,7 +360,7 @@ class TestConnection(unittest.TestCase): """ valid_ports = (1, '1', 1234, '1234', 65535, '65535', [1, '2']) - invalid_ports = (0, '0', 65536, '65536', 'abc', '*', ' 15', '01') + invalid_ports = (0, '0', 65536, '65536', 'abc', '*', ' 15', '01', True, {}) for port in valid_ports: self.assertTrue(stem.util.connection.is_valid_port(port))

1 0