commit 99c4ad9db437dc528f1e7134001842d30a073a8a Author: Damian Johnson atagar@torproject.org Date: Wed Jul 18 04:00:34 2018 -0700

Cache immutable class hash values

Immutable classes have immutable cache values, and as frequntly used equality/dictionary values it's useful to cache these. --- stem/__init__.py | 12 ++++++++++-- stem/client/cell.py | 46 +++++++++++++++------------------------------- stem/client/datatype.py | 14 ++++++++++---- stem/directory.py | 17 +++++++++++++---- stem/manual.py | 15 +++++++++++---- stem/util/__init__.py | 7 +++---- stem/version.py | 5 ++--- 7 files changed, 64 insertions(+), 52 deletions(-)

diff --git a/stem/__init__.py b/stem/__init__.py index 7d84423a..6da73320 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -572,9 +572,13 @@ class Endpoint(object):

self.address = address self.port = int(port) + self._hash = None

def __hash__(self): - return stem.util._hash_attr(self, 'address', 'port') + if self._hash is None: + self._hash = stem.util._hash_attr(self, 'address', 'port') + + return self._hash

def __eq__(self, other): return hash(self) == hash(other) if isinstance(other, Endpoint) else False @@ -593,9 +597,13 @@ class ORPort(Endpoint): def __init__(self, address, port, link_protocols = None): super(ORPort, self).__init__(address, port) self.link_protocols = link_protocols + self._hash = None

def __hash__(self): - return stem.util._hash_attr(self, 'link_protocols', parent = Endpoint) + if self._hash is None: + self._hash = stem.util._hash_attr(self, 'link_protocols', parent = True) + + return self._hash

class DirPort(Endpoint): diff --git a/stem/client/cell.py b/stem/client/cell.py index 948201fd..51e2fbd3 100644 --- a/stem/client/cell.py +++ b/stem/client/cell.py @@ -46,7 +46,7 @@ import stem.util

from stem import UNDEFINED from stem.client.datatype import HASH_LEN, ZERO, LinkProtocol, Address, Certificate, CloseReason, RelayCommand, Size, split -from stem.util import _hash_attr, datetime_to_unix, str_tools +from stem.util import datetime_to_unix, str_tools

FIXED_PAYLOAD_LEN = 509 # PAYLOAD_LEN, per tor-spec section 0.2 AUTH_CHALLENGE_SIZE = 32 @@ -95,6 +95,7 @@ class Cell(object): def __init__(self, unused = b''): super(Cell, self).__init__() self.unused = unused + self._hash = None

@staticmethod def by_name(name): @@ -252,6 +253,9 @@ class Cell(object):

raise NotImplementedError('Unpacking not yet implemented for %s cells' % cls.NAME)

+ def __hash__(self): + return self._hash if self._hash else super(type(self), self).__hash__() + def __eq__(self, other): return hash(self) == hash(other) if isinstance(other, Cell) else False

@@ -290,6 +294,7 @@ class PaddingCell(Cell):

super(PaddingCell, self).__init__() self.payload = payload + self._hash = stem.util._hash_attr(self, 'payload')

def pack(self, link_protocol): return PaddingCell._pack(link_protocol, self.payload) @@ -298,9 +303,6 @@ class PaddingCell(Cell): def _unpack(cls, content, circ_id, link_protocol): return PaddingCell(content)

- def __hash__(self): - return _hash_attr(self, 'payload') -

class CreateCell(CircuitCell): NAME = 'CREATE' @@ -358,6 +360,7 @@ class RelayCell(CircuitCell): self.stream_id = stream_id self.digest = digest self.data = str_tools._to_bytes(data) + self._hash = stem.util._hash_attr(self, 'command_int', 'stream_id', 'digest', 'data')

if digest == 0: if not stream_id and self.command in STREAM_ID_REQUIRED: @@ -390,9 +393,6 @@ class RelayCell(CircuitCell):

return RelayCell(circ_id, command, data, digest, stream_id, recognized, unused)

- def __hash__(self): - return _hash_attr(self, 'command_int', 'stream_id', 'digest', 'data') -

class DestroyCell(CircuitCell): """ @@ -409,6 +409,7 @@ class DestroyCell(CircuitCell): def __init__(self, circ_id, reason = CloseReason.NONE, unused = b''): super(DestroyCell, self).__init__(circ_id, unused) self.reason, self.reason_int = CloseReason.get(reason) + self._hash = stem.util._hash_attr(self, 'circ_id', 'reason_int')

def pack(self, link_protocol): return DestroyCell._pack(link_protocol, Size.CHAR.pack(self.reason_int), self.unused, self.circ_id) @@ -418,9 +419,6 @@ class DestroyCell(CircuitCell): reason, unused = Size.CHAR.pop(content) return DestroyCell(circ_id, reason, unused)

- def __hash__(self): - return _hash_attr(self, 'circ_id', 'reason_int') -

class CreateFastCell(CircuitCell): """ @@ -442,6 +440,7 @@ class CreateFastCell(CircuitCell):

super(CreateFastCell, self).__init__(circ_id, unused) self.key_material = key_material + self._hash = stem.util._hash_attr(self, 'circ_id', 'key_material')

def pack(self, link_protocol): return CreateFastCell._pack(link_protocol, self.key_material, self.unused, self.circ_id) @@ -455,9 +454,6 @@ class CreateFastCell(CircuitCell):

return CreateFastCell(circ_id, key_material, unused)

- def __hash__(self): - return _hash_attr(self, 'circ_id', 'key_material') -

class CreatedFastCell(CircuitCell): """ @@ -483,6 +479,7 @@ class CreatedFastCell(CircuitCell): super(CreatedFastCell, self).__init__(circ_id, unused) self.key_material = key_material self.derivative_key = derivative_key + self._hash = stem.util._hash_attr(self, 'circ_id', 'derivative_key', 'key_material')

def pack(self, link_protocol): return CreatedFastCell._pack(link_protocol, self.key_material + self.derivative_key, self.unused, self.circ_id) @@ -497,9 +494,6 @@ class CreatedFastCell(CircuitCell):

return CreatedFastCell(circ_id, derivative_key, key_material, content)

- def __hash__(self): - return _hash_attr(self, 'circ_id', 'derivative_key', 'key_material') -

class VersionsCell(Cell): """ @@ -515,6 +509,7 @@ class VersionsCell(Cell): def __init__(self, versions): super(VersionsCell, self).__init__() self.versions = versions + self._hash = stem.util._hash_attr(self, 'versions')

def pack(self, link_protocol): payload = b''.join([Size.SHORT.pack(v) for v in self.versions]) @@ -530,9 +525,6 @@ class VersionsCell(Cell):

return VersionsCell(link_protocols)

- def __hash__(self): - return _hash_attr(self, 'versions') -

class NetinfoCell(Cell): """ @@ -552,6 +544,7 @@ class NetinfoCell(Cell): self.timestamp = timestamp if timestamp else datetime.datetime.now() self.receiver_address = receiver_address self.sender_addresses = sender_addresses + self._hash = stem.util._hash_attr(self, 'timestamp', 'receiver_address', 'sender_addresses')

def pack(self, link_protocol): payload = bytearray() @@ -578,9 +571,6 @@ class NetinfoCell(Cell):

return NetinfoCell(receiver_address, sender_addresses, datetime.datetime.utcfromtimestamp(timestamp), unused = content)

- def __hash__(self): - return _hash_attr(self, 'timestamp', 'receiver_address', 'sender_addresses') -

class RelayEarlyCell(CircuitCell): NAME = 'RELAY_EARLY' @@ -639,6 +629,7 @@ class VPaddingCell(Cell):

super(VPaddingCell, self).__init__() self.payload = payload if payload is not None else os.urandom(size) + self._hash = stem.util._hash_attr(self, 'payload')

def pack(self, link_protocol): return VPaddingCell._pack(link_protocol, self.payload) @@ -647,9 +638,6 @@ class VPaddingCell(Cell): def _unpack(cls, content, circ_id, link_protocol): return VPaddingCell(payload = content)

- def __hash__(self): - return _hash_attr(self, 'payload') -

class CertsCell(Cell): """ @@ -665,6 +653,7 @@ class CertsCell(Cell): def __init__(self, certs, unused = b''): super(CertsCell, self).__init__(unused) self.certificates = certs + self._hash = stem.util._hash_attr(self, 'certificates')

def pack(self, link_protocol): return CertsCell._pack(link_protocol, Size.CHAR.pack(len(self.certificates)) + b''.join([cert.pack() for cert in self.certificates]), self.unused) @@ -683,9 +672,6 @@ class CertsCell(Cell):

return CertsCell(certs, unused = content)

- def __hash__(self): - return _hash_attr(self, 'certificates') -

class AuthChallengeCell(Cell): """ @@ -709,6 +695,7 @@ class AuthChallengeCell(Cell): super(AuthChallengeCell, self).__init__(unused) self.challenge = challenge self.methods = methods + self._hash = stem.util._hash_attr(self, 'challenge', 'methods')

def pack(self, link_protocol): payload = bytearray() @@ -740,9 +727,6 @@ class AuthChallengeCell(Cell):

return AuthChallengeCell(methods, challenge, unused = content)

- def __hash__(self): - return _hash_attr(self, 'challenge', 'methods') -

class AuthenticateCell(Cell): NAME = 'AUTHENTICATE' diff --git a/stem/client/datatype.py b/stem/client/datatype.py index 2ee624cb..22827245 100644 --- a/stem/client/datatype.py +++ b/stem/client/datatype.py @@ -121,8 +121,6 @@ import stem.util import stem.util.connection import stem.util.enum

-from stem.util import _hash_attr - ZERO = b'\x00' HASH_LEN = 20 KEY_LEN = 16 @@ -382,6 +380,7 @@ class Address(Field): raise ValueError("'%s' isn't an IPv4 or IPv6 address" % value)

self.type, self.type_int = AddrType.get(addr_type) + self._hash = None

if self.type == AddrType.IPv4: if stem.util.connection.is_valid_ipv4_address(value): @@ -431,7 +430,10 @@ class Address(Field): return Address(addr_value, addr_type), content

def __hash__(self): - return _hash_attr(self, 'type_int', 'value_bin') + if self._hash is None: + self._hash = stem.util._hash_attr(self, 'type_int', 'value_bin') + + return self._hash

class Certificate(Field): @@ -446,6 +448,7 @@ class Certificate(Field): def __init__(self, cert_type, value): self.type, self.type_int = CertType.get(cert_type) self.value = value + self._hash = None

def pack(self): cell = bytearray() @@ -466,7 +469,10 @@ class Certificate(Field): return Certificate(cert_type, cert_bytes), content

def __hash__(self): - return _hash_attr(self, 'type_int', 'value') + if self._hash is None: + self._hash = stem.util._hash_attr(self, 'type_int', 'value') + + return self._hash

class KDF(collections.namedtuple('KDF', ['key_hash', 'forward_digest', 'backward_digest', 'forward_key', 'backward_key'])): diff --git a/stem/directory.py b/stem/directory.py index efd37641..020ac696 100644 --- a/stem/directory.py +++ b/stem/directory.py @@ -41,9 +41,10 @@ as follows... import os import re

+import stem.util import stem.util.conf

-from stem.util import _hash_attr, connection, str_tools, tor_tools +from stem.util import connection, str_tools, tor_tools

try: # added in python 2.7 @@ -217,7 +218,7 @@ class Directory(object): raise NotImplementedError('Unsupported Operation: this should be implemented by the Directory subclass')

def __hash__(self): - return _hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint', 'nickname', 'orport_v6') + return stem.util._hash_attr(self, 'address', 'or_port', 'dir_port', 'fingerprint', 'nickname', 'orport_v6')

def __eq__(self, other): return hash(self) == hash(other) if isinstance(other, Directory) else False @@ -254,6 +255,7 @@ class Authority(Directory):

self.v3ident = v3ident self.is_bandwidth_authority = is_bandwidth_authority + self._hash = None

@staticmethod def from_cache(): @@ -313,7 +315,10 @@ class Authority(Directory): return section_lines

def __hash__(self): - return _hash_attr(self, 'v3ident', 'is_bandwidth_authority', parent = Directory) + if self._hash is None: + self._hash = stem.util._hash_attr(self, 'v3ident', 'is_bandwidth_authority', parent = True) + + return self._hash

def __eq__(self, other): return hash(self) == hash(other) if isinstance(other, Authority) else False @@ -365,6 +370,7 @@ class Fallback(Directory): super(Fallback, self).__init__(address, or_port, dir_port, fingerprint, nickname, orport_v6) self.has_extrainfo = has_extrainfo self.header = OrderedDict(header) if header else OrderedDict() + self._hash = None

@staticmethod def from_cache(path = FALLBACK_CACHE_PATH): @@ -514,7 +520,10 @@ class Fallback(Directory): conf.save(path)

def __hash__(self): - return _hash_attr(self, 'has_extrainfo', 'header', parent = Directory) + if self._hash is None: + self._hash = stem.util._hash_attr(self, 'has_extrainfo', 'header', parent = True) + + return self._hash

def __eq__(self, other): return hash(self) == hash(other) if isinstance(other, Fallback) else False diff --git a/stem/manual.py b/stem/manual.py index 573542da..458014b0 100644 --- a/stem/manual.py +++ b/stem/manual.py @@ -54,13 +54,12 @@ import sys import tempfile

import stem.prereq +import stem.util import stem.util.conf import stem.util.enum import stem.util.log import stem.util.system

-from stem.util import _hash_attr - try: # added in python 2.7 from collections import OrderedDict @@ -189,9 +188,13 @@ class ConfigOption(object): self.usage = usage self.summary = summary self.description = description + self._hash = None

def __hash__(self): - return _hash_attr(self, 'name', 'category', 'usage', 'summary', 'description') + if self._hash is None: + self._hash = stem.util._hash_attr(self, 'name', 'category', 'usage', 'summary', 'description') + + return self._hash

def __eq__(self, other): return hash(self) == hash(other) if isinstance(other, ConfigOption) else False @@ -381,6 +384,7 @@ class Manual(object): self.man_commit = None self.stem_commit = None self.schema = None + self._hash = None

@staticmethod def from_cache(path = None): @@ -654,7 +658,10 @@ class Manual(object): conf.save(path)

def __hash__(self): - return _hash_attr(self, 'name', 'synopsis', 'description', 'commandline_options', 'signals', 'files', 'config_options') + if self._hash is None: + self._hash = stem.util._hash_attr(self, 'name', 'synopsis', 'description', 'commandline_options', 'signals', 'files', 'config_options') + + return self._hash

def __eq__(self, other): return hash(self) == hash(other) if isinstance(other, Manual) else False diff --git a/stem/util/__init__.py b/stem/util/__init__.py index b812fc13..cc40bd0a 100644 --- a/stem/util/__init__.py +++ b/stem/util/__init__.py @@ -130,12 +130,11 @@ def _hash_attr(obj, *attributes, **kwargs):

:param Object obj: object to be hashed :param list attributes: attribute names to take into account - :param class parent: parent object to include in the hash value + :param bool parent: include parent's hash value """

- # TODO: deal with this parent thing - - my_hash = hash(str(type(obj))) if kwargs.get('parent') is None else kwargs.get('parent').__hash__(obj) + my_hash = super(type(obj), obj).__hash__() if kwargs.get('parent') else 0 + my_hash = my_hash * 1024 + hash(str(type(obj)))

for attr in attributes: val = getattr(obj, attr) diff --git a/stem/version.py b/stem/version.py index 9036effb..ed1a3a38 100644 --- a/stem/version.py +++ b/stem/version.py @@ -83,11 +83,10 @@ easily parsed and compared, for instance... import os import re

+import stem.util import stem.util.enum import stem.util.system

-from stem.util import _hash_attr - try: # added in python 3.2 from functools import lru_cache @@ -252,7 +251,7 @@ class Version(object):

def __hash__(self): if self._hash is None: - self._hash = _hash_attr(self, 'major', 'minor', 'micro', 'patch', 'status') + self._hash = stem.util._hash_attr(self, 'major', 'minor', 'micro', 'patch', 'status')

return self._hash