January 2018 - tor-commits - lists.torproject.org

[stem/master] Drop incorrect Cell docs
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 269084dc6af7fd590b0192bca4f2ad739781d520 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 14 13:08:52 2018 -0800 Drop incorrect Cell docs Removing outdated docs about these attributes. --- stem/client/cell.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/stem/client/cell.py b/stem/client/cell.py index 995b482a..872f0998 100644 --- a/stem/client/cell.py +++ b/stem/client/cell.py @@ -51,13 +51,6 @@ FIXED_PAYLOAD_LEN = 509 class Cell(object): """ Metadata for ORPort cells. - - :var str name: command of the cell - :var int value: integer value of the command on the wire - :var bool fixed_size: **True** if cells have a fixed length, - **False** if variable - :var bool for_circuit: **True** if command is for a circuit, - **False** otherwise """ NAME = 'UNKNOWN'

1 0

[stem/master] Unpack only a single cell
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit e548a715bc50c4c1b8a773d49b1c6d00442a1ea7 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 16 09:46:44 2018 -0800 Unpack only a single cell When first establishing a link the first cell we receive (VERSIONS) determines the link protocol version used to unpack all future cells. As such we definitely want an 'unpack just one cell' function like endosome. --- stem/client/cell.py | 33 ++++++++++++++------------------- test/unit/client/cell.py | 16 ++++++++++------ 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/stem/client/cell.py b/stem/client/cell.py index 98ab198c..8a6480c7 100644 --- a/stem/client/cell.py +++ b/stem/client/cell.py @@ -95,39 +95,34 @@ class Cell(object): @staticmethod def unpack(content, link_version): """ - Unpacks encoded bytes into a series of cells. + Unpacks the first cell. :param bytes content: payload to decode :param int link_version: link protocol version - :returns: **list** of :class:`~stem.client.cell.Cell` subclasses + :returns: (:class:`~stem.client.cell.Cell`, remainder) tuple :raises: * ValueError if content is malformed * NotImplementedError if unable to unpack this cell type """ - cells = [] + circ_id, content = Size.SHORT.pop(content) if link_version < 4 else Size.LONG.pop(content) + command, content = Size.CHAR.pop(content) + cls = Cell.by_value(command) - while content: - circ_id, content = Size.SHORT.pop(content) if link_version < 4 else Size.LONG.pop(content) - command, content = Size.CHAR.pop(content) - cls = Cell.by_value(command) - - if cls.IS_FIXED_SIZE: - payload_len = FIXED_PAYLOAD_LEN - else: - payload_len, content = Size.SHORT.pop(content) - - if len(content) < payload_len: - raise ValueError('%s cell should have a payload of %i bytes, but only had %i' % (cls.NAME, payload_len, len(content))) + if cls.IS_FIXED_SIZE: + payload_len = FIXED_PAYLOAD_LEN + else: + payload_len, content = Size.SHORT.pop(content) - payload = content[:payload_len] - content = content[payload_len:] + if len(content) < payload_len: + raise ValueError('%s cell should have a payload of %i bytes, but only had %i' % (cls.NAME, payload_len, len(content))) - cells.append(cls._unpack(payload, link_version, circ_id)) + payload = content[:payload_len] + content = content[payload_len:] - return cells + return cls._unpack(payload, link_version, circ_id), content @classmethod def _pack(cls, link_version, payload, circ_id = 0): diff --git a/test/unit/client/cell.py b/test/unit/client/cell.py index cf7eb684..a0eab2e5 100644 --- a/test/unit/client/cell.py +++ b/test/unit/client/cell.py @@ -86,11 +86,12 @@ class TestCell(unittest.TestCase): (7, '\x1a\xa5\xb3\xbd\x88\xb1C'), ) - link_cells = Cell.unpack(test_data('new_link_cells'), 2) - self.assertEqual(4, len(link_cells)) - self.assertEqual(VersionsCell([3, 4, 5]), link_cells[0]) + content = test_data('new_link_cells') - certs_cell = link_cells[1] + version_cell, content = Cell.unpack(content, 2) + self.assertEqual(VersionsCell([3, 4, 5]), version_cell) + + certs_cell, content = Cell.unpack(content, 2) self.assertEqual(CertsCell, type(certs_cell)) self.assertEqual(len(expected_certs), len(certs_cell.certificates)) @@ -98,14 +99,17 @@ class TestCell(unittest.TestCase): self.assertEqual(cert_type, certs_cell.certificates[i].type) self.assertTrue(certs_cell.certificates[i].value.startswith(cert_prefix)) - self.assertEqual(AuthChallengeCell('\x89Y\t\x99\xb2\x1e\xd9*V\xb6\x1bn\n\x05\xd8/\xe3QH\x85\x13Z\x17\xfc\x1c\x00{\xa9\xae\x83^K', [1, 3]), link_cells[2]) + auth_challenge_cell, content = Cell.unpack(content, 2) + self.assertEqual(AuthChallengeCell('\x89Y\t\x99\xb2\x1e\xd9*V\xb6\x1bn\n\x05\xd8/\xe3QH\x85\x13Z\x17\xfc\x1c\x00{\xa9\xae\x83^K', [1, 3]), auth_challenge_cell) - netinfo_cell = link_cells[3] + netinfo_cell, content = Cell.unpack(content, 2) self.assertEqual(NetinfoCell, type(netinfo_cell)) self.assertEqual(datetime.datetime(2018, 1, 14, 1, 46, 56), netinfo_cell.timestamp) self.assertEqual(Address(type='IPv4', type_int=4, value='127.0.0.1', value_bin='\x7f\x00\x00\x01'), netinfo_cell.receiver_address) self.assertEqual([Address(type='IPv4', type_int=4, value='97.113.15.2', value_bin='aq\x0f\x02')], netinfo_cell.sender_addresses) + self.assertEqual('', content) # check that we've consumed all of the bytes + def test_padding_packing(self): for cell_bytes, payload in PADDING_CELLS.items(): self.assertEqual(cell_bytes, PaddingCell.pack(2, payload))

1 0

[stem/master] Rename an internal connection helper
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 598ca5cb181938e816dd26b86edd8108d158c242 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 15 09:01:09 2018 -0800 Rename an internal connection helper --- stem/exit_policy.py | 2 +- stem/util/connection.py | 8 ++++---- test/unit/util/connection.py | 10 +++++----- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/stem/exit_policy.py b/stem/exit_policy.py index fd8c15c2..4ef66666 100644 --- a/stem/exit_policy.py +++ b/stem/exit_policy.py @@ -910,7 +910,7 @@ class ExitPolicyRule(object): def _get_mask_bin(self): # provides an integer representation of our mask - return int(stem.util.connection._get_address_binary(self.get_mask(False)), 2) + return int(stem.util.connection._address_to_binary(self.get_mask(False)), 2) @lru_cache() def _get_address_bin(self): diff --git a/stem/util/connection.py b/stem/util/connection.py index 464a9ca1..d68c6124 100644 --- a/stem/util/connection.py +++ b/stem/util/connection.py @@ -558,7 +558,7 @@ def address_to_int(address): # TODO: Could be neat to also use this for serialization if we also had an # int_to_address() function. - return int(_get_address_binary(address), 2) + return int(_address_to_binary(address), 2) def expand_ipv6_address(address): @@ -597,7 +597,7 @@ def expand_ipv6_address(address): # # '5.9.158.75' => '0509:9e4b' - ipv4_bin = _get_address_binary(address[ipv4_start:ipv4_end]) + ipv4_bin = _address_to_binary(address[ipv4_start:ipv4_end]) groupings = [ipv4_bin[16 * i:16 * (i + 1)] for i in range(2)] ipv6_snippet = ':'.join(['%04x' % int(group, 2) for group in groupings]) @@ -691,7 +691,7 @@ def _get_masked_bits(mask): raise ValueError("'%s' is an invalid subnet mask" % mask) # converts octets to binary representation - mask_bin = _get_address_binary(mask) + mask_bin = _address_to_binary(mask) mask_match = re.match('^(1*)(0*)$', mask_bin) if mask_match: @@ -713,7 +713,7 @@ def _get_binary(value, bits): return ''.join([str((value >> y) & 1) for y in range(bits - 1, -1, -1)]) -def _get_address_binary(address): +def _address_to_binary(address): """ Provides the binary value for an IPv4 or IPv6 address. diff --git a/test/unit/util/connection.py b/test/unit/util/connection.py index 98d5c3f3..58a5cb82 100644 --- a/test/unit/util/connection.py +++ b/test/unit/util/connection.py @@ -602,9 +602,9 @@ class TestConnection(unittest.TestCase): self.assertRaises(ValueError, stem.util.connection._get_masked_bits, 'blarg') self.assertRaises(ValueError, stem.util.connection._get_masked_bits, '255.255.0.255') - def test_get_address_binary(self): + def test_address_to_binary(self): """ - Checks the _get_address_binary function. + Checks the _address_to_binary function. """ test_values = { @@ -619,7 +619,7 @@ class TestConnection(unittest.TestCase): } for test_arg, expected in test_values.items(): - self.assertEqual(expected, stem.util.connection._get_address_binary(test_arg)) + self.assertEqual(expected, stem.util.connection._address_to_binary(test_arg)) - self.assertRaises(ValueError, stem.util.connection._get_address_binary, '') - self.assertRaises(ValueError, stem.util.connection._get_address_binary, 'blarg') + self.assertRaises(ValueError, stem.util.connection._address_to_binary, '') + self.assertRaises(ValueError, stem.util.connection._address_to_binary, 'blarg')

1 0

[stem/master] Implement AUTH_CHALLENGE cells
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 06767ca7c64a3d399e75cb92610cf80197fe7039 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 14 17:31:37 2018 -0800 Implement AUTH_CHALLENGE cells --- stem/client/cell.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++-- test/unit/client/cell.py | 19 ++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/stem/client/cell.py b/stem/client/cell.py index 872f0998..df34508d 100644 --- a/stem/client/cell.py +++ b/stem/client/cell.py @@ -46,6 +46,7 @@ from stem import UNDEFINED from stem.client import ZERO, Certificate, Size FIXED_PAYLOAD_LEN = 509 +AUTH_CHALLENGE_SIZE = 32 class Cell(object): @@ -444,18 +445,74 @@ class CertsCell(Cell): if cert_size > len(content): raise ValueError('CERTS cell should have a certificate with %i bytes, but only had %i remaining' % (cert_size, len(content))) - cert_bytes = content[:cert_size] - content = content[cert_size:] + cert_bytes, content = content[:cert_size], content[cert_size:] certs.append(Certificate(cert_type, cert_bytes)) return CertsCell(certs) class AuthChallengeCell(Cell): + """ + First step of the authentication handshake. + + :var bytes challenge: random bytes for us to sign to authenticate + :var list methods: authentication methods supported by the relay we're + communicating with + """ + NAME = 'AUTH_CHALLENGE' VALUE = 130 IS_FIXED_SIZE = False + def __init__(self, challenge, methods): + self.challenge = challenge + self.methods = methods + + @classmethod + def pack(cls, link_version, methods, challenge = None): + """ + Provides an authentication challenge. + + :param int link_version: link protocol version + :param list methods: authentication methods we support + :param bytes challenge: randomized string for the receiver to sign + + :returns: **bytes** with a payload for this challenge + """ + + if challenge is None: + challenge = os.urandom(AUTH_CHALLENGE_SIZE) + elif len(challenge) != AUTH_CHALLENGE_SIZE: + raise ValueError('AUTH_CHALLENGE must be %i bytes, but was %i' % (AUTH_CHALLENGE_SIZE, len(challenge))) + + payload = io.BytesIO() + payload.write(challenge) + payload.write(Size.SHORT.pack(len(methods))) + + for method in methods: + payload.write(Size.SHORT.pack(method)) + + return cls._pack(link_version, payload.getvalue()) + + @classmethod + def _unpack(cls, content, circ_id, link_version): + if len(content) < AUTH_CHALLENGE_SIZE + 2: + raise ValueError('AUTH_CHALLENGE payload should be at least 34 bytes, but was %i' % len(content)) + + challenge, content = content[:AUTH_CHALLENGE_SIZE], content[AUTH_CHALLENGE_SIZE:] + method_count, content = Size.SHORT.pop(content) + + if len(content) < method_count * 2: + raise ValueError('AUTH_CHALLENGE should have %i methods, but only had %i bytes for it' % (method_count, len(content))) + + methods = [] + + for i in range(method_count): + method, content = Size.SHORT.pop(content) + methods.append(method) + + return AuthChallengeCell(challenge, methods) + class AuthenticateCell(Cell): NAME = 'AUTHENTICATE' diff --git a/test/unit/client/cell.py b/test/unit/client/cell.py index 0d2b677d..f95a664e 100644 --- a/test/unit/client/cell.py +++ b/test/unit/client/cell.py @@ -15,9 +15,11 @@ from stem.client.cell import ( VersionsCell, VPaddingCell, CertsCell, + AuthChallengeCell, ) RANDOM_PAYLOAD = os.urandom(FIXED_PAYLOAD_LEN) +CHALLENGE = '\x89Y\t\x99\xb2\x1e\xd9*V\xb6\x1bn\n\x05\xd8/\xe3QH\x85\x13Z\x17\xfc\x1c\x00{\xa9\xae\x83^K' PADDING_CELLS = { '\x00\x00\x00' + RANDOM_PAYLOAD: RANDOM_PAYLOAD, @@ -42,6 +44,10 @@ CERTS_CELLS = { '\x00\x00\x81\x00\x05\x01\x01\x00\x01\x08': [Certificate(type = 1, value = '\x08')], } +AUTH_CHALLENGE_CELLS = { + '\x00\x00\x82\x00&%s\x00\x02\x00\x01\x00\x03' % CHALLENGE: (CHALLENGE, [1, 3]), +} + class TestCell(unittest.TestCase): def test_by_name(self): @@ -69,7 +75,7 @@ class TestCell(unittest.TestCase): def test_unpack_for_new_link(self): # TODO: we need to support more cell types before we can test this - self.assertRaisesRegexp(NotImplementedError, 'Unpacking not yet implemented for AUTH_CHALLENGE cells', Cell.unpack, test_data('new_link_cells'), 2) + self.assertRaisesRegexp(NotImplementedError, 'Unpacking not yet implemented for NETINFO cells', Cell.unpack, test_data('new_link_cells'), 2) def test_padding_packing(self): for cell_bytes, payload in PADDING_CELLS.items(): @@ -101,3 +107,14 @@ class TestCell(unittest.TestCase): self.assertRaisesRegexp(ValueError, 'CERTS cell should have a certificate with 3 bytes, but only had 1 remaining', Cell.unpack, '\x00\x00\x81\x00\x05\x01\x01\x00\x03\x08', 2) self.assertRaisesRegexp(ValueError, 'CERTS cell indicates it should have 2 certificates, but only contained 1', Cell.unpack, '\x00\x00\x81\x00\x05\x02\x01\x00\x01\x08', 2) + + def test_auth_challenge_packing(self): + for cell_bytes, (challenge, methods) in AUTH_CHALLENGE_CELLS.items(): + self.assertEqual(cell_bytes, AuthChallengeCell.pack(2, methods, challenge)) + + cell = Cell.unpack(cell_bytes, 2)[0] + self.assertEqual(challenge, cell.challenge) + self.assertEqual(methods, cell.methods) + + self.assertRaisesRegexp(ValueError, 'AUTH_CHALLENGE cell should have a payload of 38 bytes, but only had 16', Cell.unpack, '\x00\x00\x82\x00&%s\x00\x02\x00\x01\x00\x03' % CHALLENGE[:10], 2) + self.assertRaisesRegexp(ValueError, 'AUTH_CHALLENGE should have 3 methods, but only had 4 bytes for it', Cell.unpack, '\x00\x00\x82\x00&%s\x00\x03\x00\x01\x00\x03' % CHALLENGE, 2)

1 0

[stem/master] Subclass Size under a Field superclass
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 8f0ab3bca64d6d64efa808c5c1d51af063cd91c2 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jan 18 10:04:46 2018 -0800 Subclass Size under a Field superclass No functional changes yet. Just laying groundwork for the others. --- stem/client/__init__.py | 104 +++++++++++++++++++++++++++++++----------------- test/settings.cfg | 1 + 2 files changed, 69 insertions(+), 36 deletions(-) diff --git a/stem/client/__init__.py b/stem/client/__init__.py index d458e068..223956f5 100644 --- a/stem/client/__init__.py +++ b/stem/client/__init__.py @@ -71,6 +71,13 @@ AddrType = stem.util.enum.UppercaseEnum( 'UNKNOWN', ) +CertType = stem.util.enum.UppercaseEnum( + 'LINK', + 'IDENTITY', + 'AUTHENTICATE', + 'UNKNOWN', +) + ADDR_INT = { 0: AddrType.HOSTNAME, 4: AddrType.IPv4, @@ -93,7 +100,57 @@ def split(content, size): return content[:size], content[size:] -class Size(object): +class Field(object): + """ + Packable and unpackable datatype. + """ + + def pack(self): + """ + Encodes field into bytes. + + :returns: **bytes** that can be communicated over Tor's ORPort + + :raises: **ValueError** if incorrect type or size + """ + + raise NotImplementedError('Not yet available') + + @classmethod + def unpack(cls, packed): + """ + Decodes bytes into a field of this type. + + :param bytes packed: content to decode + + :returns: instance of this class + + :raises: **ValueError** if packed data is malformed + """ + + unpacked, remainder = cls.pop(packed) + + if remainder: + raise ValueError('%s is the wrong size for a %s field' % (repr(packed), cls.__name__)) + + return unpacked + + @staticmethod + def pop(packed): + """ + Decodes bytes as this field type, providing it and the remainder. + + :param bytes packed: content to decode + + :returns: tuple of the form (unpacked, remainder) + + :raises: **ValueError** if packed data is malformed + """ + + raise NotImplementedError('Not yet available') + + +class Size(Field): """ Unsigned `struct.pack format <https://docs.python.org/2/library/struct.html#format-characters>` for @@ -114,15 +171,11 @@ class Size(object): self.size = size self.format = pack_format - def pack(self, content): - """ - Encodes bytes into a packed field. - - :param int content: content to encode - - :raises: **ValueError** if incorrect type or size - """ + @staticmethod + def pop(packed): + raise NotImplementedError("Use our constant's unpack() and pop() instead") + def pack(self, content): if not isinstance(content, int): raise ValueError('Size.pack encodes an integer, but was a %s' % type(content).__name__) @@ -133,35 +186,14 @@ class Size(object): return packed - def unpack(self, content): - """ - Decodes packed data into bytes. - - :param bytes content: content to encode - - :returns: **int** with the unpacked value - - :raises: **ValueError** if packed data isn't of the right size - """ - - if self.size != len(content): - raise ValueError('%s is the wrong size for a %s field' % (repr(content), self.name)) - - return struct.unpack(self.format, content)[0] - - def pop(self, content): - """ - Decodes the first characters as this data type, providing it and the - remainder. - - :param bytes content: content to encode - - :returns: tuple of the form (unpacked, remainder) + def unpack(self, packed): + if self.size != len(packed): + raise ValueError('%s is the wrong size for a %s field' % (repr(packed), self.name)) - :raises: **ValueError** if packed data isn't of the right size - """ + return struct.unpack(self.format, packed)[0] - return self.unpack(content[:self.size]), content[self.size:] + def pop(self, packed): + return self.unpack(packed[:self.size]), packed[self.size:] class Certificate(collections.namedtuple('Certificate', ['type', 'value'])): diff --git a/test/settings.cfg b/test/settings.cfg index 78e77dd1..28694b60 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -146,6 +146,7 @@ pycodestyle.ignore test/unit/util/connection.py => W291: _tor tor 158 # issue. pyflakes.ignore run_tests.py => 'unittest' imported but unused +pyflakes.ignore stem/client/__init__.py => redefinition of unused 'pop' from * pyflakes.ignore stem/util/__init__.py => undefined name 'long' pyflakes.ignore stem/util/__init__.py => undefined name 'unicode' pyflakes.ignore stem/control.py => undefined name 'controller'

1 0

[stem/master] Revert "Add an IntString class"
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 29b3bd2fcbdfae2e08abeae0786766222a2c3e9e Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jan 18 09:55:28 2018 -0800 Revert "Add an IntString class" Nah. This was an interesting idea and would indeed make usage a tad nicer but it violates the principal of least surprise. This reverts commit 3f8493851bd538bc4aa3d868b880271697ecf516. --- docs/change_log.rst | 4 ---- stem/util/str_tools.py | 26 -------------------------- test/unit/doctest.py | 1 - test/unit/util/str_tools.py | 6 ------ 4 files changed, 37 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index ebdf8e3e..fcfc0115 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -56,10 +56,6 @@ The following are only available within Stem's `git repository * `Fallback directory v2 support <https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html>`_, which adds *nickname* and *extrainfo* * Reduced maximum descriptors fetched by the remote module to match tor's new limit (:trac:`24743`) - * **Utilities** - - * Added :class:`~stem.util.str_tools.IntString` - * **Website** * Added `terminal styling <tutorials/east_of_the_sun.html#terminal-styling>`_ to our utilities tutorial diff --git a/stem/util/str_tools.py b/stem/util/str_tools.py index 108d21ea..bc0446fa 100644 --- a/stem/util/str_tools.py +++ b/stem/util/str_tools.py @@ -12,8 +12,6 @@ Toolkit for various string activity. :: - IntString - string that can also be used as an integer - crop - shortens string to a given length size_label - human readable label for a number of bytes @@ -187,30 +185,6 @@ def _split_by_length(msg, size): Ending = stem.util.enum.Enum('ELLIPSE', 'HYPHEN') -class IntString(str): - """ - String that can be used as an integer as well... - - :: - - >>> s = IntString('me', 5) - >>> print('hi %s, the number is %i' % (s, s)) - hi me, the number is 5 - - .. versionadded:: 1.7.0 - """ - - def __new__(self, str_value, int_value): - return str.__new__(self, str_value) - - def __init__(self, str_value, int_value): - self._str_value = str_value - self._int_value = int_value - - def __int__(self): - return self._int_value - - def crop(msg, size, min_word_length = 4, min_crop = 0, ending = Ending.ELLIPSE, get_remainder = False): """ Shortens a string to a given length. diff --git a/test/unit/doctest.py b/test/unit/doctest.py index 9dda1422..7e40e873 100644 --- a/test/unit/doctest.py +++ b/test/unit/doctest.py @@ -67,7 +67,6 @@ class TestDocumentation(unittest.TestCase): args['globs'] = { '_to_camel_case': stem.util.str_tools._to_camel_case, '_split_by_length': stem.util.str_tools._split_by_length, - 'IntString': stem.util.str_tools.IntString, 'crop': stem.util.str_tools.crop, 'size_label': stem.util.str_tools.size_label, 'time_label': stem.util.str_tools.time_label, diff --git a/test/unit/util/str_tools.py b/test/unit/util/str_tools.py index 7349bd56..ee270717 100644 --- a/test/unit/util/str_tools.py +++ b/test/unit/util/str_tools.py @@ -50,12 +50,6 @@ class TestStrTools(unittest.TestCase): self.assertEqual(['hello'], str_tools._split_by_length('hello', 5)) self.assertEqual(['hello'], str_tools._split_by_length('hello', 6)) - def test_int_string(self): - s = str_tools.IntString('me', 5) - self.assertEqual('me', s) - self.assertEqual(5, int(s)) - self.assertEqual('hi me, the number is 5', 'hi %s, the number is %i' % (s, s)) - def test_crop(self): # test the pydoc examples self.assertEqual('This is a looo...', str_tools.crop('This is a looooong message', 17))

1 0

[stem/master] Drop TTL from addresses
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 5619151f3a26b689449a007346bca7eeb8d23e07 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 15 10:23:22 2018 -0800 Drop TTL from addresses Damn, that cost me a few hours. Netinfo addresses are of the form specified in section 6.4 of the tor-spec but they actually lack the TTL field. I'll clarify the spec in a bit. --- stem/client/__init__.py | 11 +++-------- test/unit/client/types.py | 3 +-- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/stem/client/__init__.py b/stem/client/__init__.py index 931dbab1..271237d8 100644 --- a/stem/client/__init__.py +++ b/stem/client/__init__.py @@ -79,7 +79,7 @@ class Certificate(collections.namedtuple('Certificate', ['type', 'value'])): """ -class Address(collections.namedtuple('Address', ['type', 'type_int', 'value', 'value_bin', 'ttl'])): +class Address(collections.namedtuple('Address', ['type', 'type_int', 'value', 'value_bin'])): """ Relay address. @@ -87,7 +87,6 @@ class Address(collections.namedtuple('Address', ['type', 'type_int', 'value', 'v :var int type_int: integer value of the address type :var unicode value: address value :var bytes value_bin: encoded address value - :var int ttl: seconds the record can be validly cached for """ @staticmethod @@ -103,20 +102,16 @@ class Address(collections.namedtuple('Address', ['type', 'type_int', 'value', 'v if len(content) < addr_length: raise ValueError('Address specified a payload of %i bytes, but only had %i' % (addr_length, len(content))) - elif len(content) < addr_length + 4: - raise ValueError('Address missing a TTL at its end') - - address_bin, content = content[:addr_length], content[addr_length:] - ttl, content = Size.LONG.pop(content) # TODO: add support for other address types + address_bin, content = content[:addr_length], content[addr_length:] address = None if addr_type == AddrType.IPv4 and len(address_bin) == 4: address = '.'.join([str(Size.CHAR.unpack(address_bin[i])) for i in range(4)]) - return Address(addr_type, addr_type_int, address, address_bin, ttl), content + return Address(addr_type, addr_type_int, address, address_bin), content class Size(object): diff --git a/test/unit/client/types.py b/test/unit/client/types.py index fdf4e085..a85a0b8d 100644 --- a/test/unit/client/types.py +++ b/test/unit/client/types.py @@ -10,10 +10,9 @@ from stem.client import Address class TestClientTypes(unittest.TestCase): def test_address_ipv4(self): addr, content = Address.pop('\x04\x04\x7f\x00\x00\x01\x01\x04\x04aq\x0f\x02\x00\x00\x00\x00') - self.assertEqual('q\x0f\x02\x00\x00\x00\x00', content) + self.assertEqual('\x01\x04\x04aq\x0f\x02\x00\x00\x00\x00', content) self.assertEqual('IPv4', addr.type) self.assertEqual(4, addr.type_int) self.assertEqual('127.0.0.1', addr.value) self.assertEqual('\x7f\x00\x00\x01', addr.value_bin) - self.assertEqual(17040481, addr.ttl)

1 0

[stem/master] Stub Netinfo packing function
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 9952f9341174c12df63cf5b34a8d7e3e1a4ae2ed Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 16 10:36:38 2018 -0800 Stub Netinfo packing function Not yet tested or even fully implemented yet since this requires the ability to pack an Address. --- stem/client/__init__.py | 8 ++++++++ stem/client/cell.py | 26 ++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/stem/client/__init__.py b/stem/client/__init__.py index 271237d8..d15f0aa4 100644 --- a/stem/client/__init__.py +++ b/stem/client/__init__.py @@ -90,6 +90,14 @@ class Address(collections.namedtuple('Address', ['type', 'type_int', 'value', 'v """ @staticmethod + def pack(addr): + """ + Bytes payload for an address. + """ + + raise NotImplementedError('Not yet available') + + @staticmethod def pop(content): if not content: raise ValueError('Payload empty where an address was expected') diff --git a/stem/client/cell.py b/stem/client/cell.py index 8a6480c7..bc44298f 100644 --- a/stem/client/cell.py +++ b/stem/client/cell.py @@ -45,7 +45,7 @@ import sys from stem import UNDEFINED from stem.client import ZERO, Address, Certificate, Size -from stem.util import _hash_attr +from stem.util import _hash_attr, datetime_to_unix FIXED_PAYLOAD_LEN = 509 AUTH_CHALLENGE_SIZE = 32 @@ -352,7 +352,29 @@ class NetinfoCell(Cell): @classmethod def pack(cls, link_version, receiver_address, sender_addresses, timestamp = None): - raise NotImplementedError('Netinfo packing not yet implemented') + """ + Payload about our timestamp and versions. + + :param int link_version: link protocol version + :param stem.client.Address receiver_address: address of the receiver + :param list sender_addresses: our addresses + :param datetime timestamp: current time according to our clock + + :returns: **bytes** with a payload for these versions + """ + + if timestamp is None: + timestamp = datetime.datetime.now() + + payload = io.BytesIO() + payload.write(Size.LONG.pack(int(datetime_to_unix(timestamp)))) + payload.write(Address.pack(receiver_address)) + payload.write(Size.CHAR.pack(len(sender_addresses))) + + for addr in sender_addresses: + payload.write(Address.pack(addr)) + + return cls._pack(link_version, payload.getvalue()) @classmethod def _unpack(cls, content, circ_id, link_version):

1 0

[stem/master] Implement PADDING and VPADDING cells
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 9fb787cd89b10254df58deb0d190ac5fdbe58c16 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Jan 14 13:06:00 2018 -0800 Implement PADDING and VPADDING cells Couple particularly easy cell types. --- stem/client/cell.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++-- test/unit/client/cell.py | 38 +++++++++++++++++++++++++-- 2 files changed, 102 insertions(+), 4 deletions(-) diff --git a/stem/client/cell.py b/stem/client/cell.py index 4fe7952e..995b482a 100644 --- a/stem/client/cell.py +++ b/stem/client/cell.py @@ -38,6 +38,8 @@ Messages communicated over a Tor relay's ORPort. import inspect import io +import os +import random import sys from stem import UNDEFINED @@ -219,10 +221,36 @@ class CircuitCell(Cell): class PaddingCell(Cell): + """ + Randomized content to either keep activity going on a circuit. + + :var bytes payload: randomized payload + """ + NAME = 'PADDING' VALUE = 0 IS_FIXED_SIZE = True + def __init__(self, payload): + self.payload = payload + + @classmethod + def pack(cls, link_version, payload = None): + """ + Provides a randomized padding payload. + + :param int link_version: link protocol version + :param bytes payload: padding payload + + :returns: **bytes** with randomized content + """ + + return cls._pack(link_version, payload if payload else os.urandom(FIXED_PAYLOAD_LEN)) + + @classmethod + def _unpack(cls, content, circ_id, link_version): + return PaddingCell(content) + class CreateCell(CircuitCell): NAME = 'CREATE' @@ -332,10 +360,46 @@ class PaddingNegotiateCell(Cell): class VPaddingCell(Cell): + """ + Variable length randomized content to either keep activity going on a circuit. + + :var bytes payload: randomized payload + """ + NAME = 'VPADDING' VALUE = 128 IS_FIXED_SIZE = False + def __init__(self, payload): + self.payload = payload + + @classmethod + def pack(cls, link_version, size = None, payload = None): + """ + Provides a randomized padding payload. If no size or payload is provided + then this provides padding of an arbitrarily chosen size between 128-1024. + + :param int link_version: link protocol version + :param int size: number of bytes to pad + :param bytes payload: padding payload + + :returns: **bytes** with randomized content + + :raises: **ValueError** if both a size and payload are provided, and they + mismatch + """ + + if payload is None: + payload = os.urandom(size) if size else os.urandom(random.randint(128, 1024)) + elif size is not None and size != len(payload): + raise ValueError('VPaddingCell.pack caller specified both a size of %i bytes and payload of %i bytes' % (size, len(payload))) + + return cls._pack(link_version, payload) + + @classmethod + def _unpack(cls, content, circ_id, link_version): + return VPaddingCell(content) + class CertsCell(Cell): """ @@ -352,12 +416,12 @@ class CertsCell(Cell): self.certificates = certs @classmethod - def pack(cls, certs, link_version): + def pack(cls, link_version, certs): """ Provides the payload for a series of certificates. - :param list certs: series of :class:`~stem.client.Certificate` for the cell :param int link_version: link protocol version + :param list certs: series of :class:`~stem.client.Certificate` for the cell :returns: **bytes** with a payload for these versions """ diff --git a/test/unit/client/cell.py b/test/unit/client/cell.py index b26549e2..0d2b677d 100644 --- a/test/unit/client/cell.py +++ b/test/unit/client/cell.py @@ -2,18 +2,40 @@ Unit tests for the stem.client.cell. """ +import os import unittest from stem.client import Certificate -from stem.client.cell import Cell, VersionsCell, CertsCell from test.unit.client import test_data +from stem.client.cell import ( + FIXED_PAYLOAD_LEN, + Cell, + PaddingCell, + VersionsCell, + VPaddingCell, + CertsCell, +) + +RANDOM_PAYLOAD = os.urandom(FIXED_PAYLOAD_LEN) + +PADDING_CELLS = { + '\x00\x00\x00' + RANDOM_PAYLOAD: RANDOM_PAYLOAD, +} + VERSIONS_CELLS = { '\x00\x00\x07\x00\x00': [], '\x00\x00\x07\x00\x02\x00\x01': [1], '\x00\x00\x07\x00\x06\x00\x01\x00\x02\x00\x03': [1, 2, 3], } +VPADDING_CELLS = { + '\x00\x00\x80\x00\x00': '', + '\x00\x00\x80\x00\x01\x08': '\x08', + '\x00\x00\x80\x00\x02\x08\x11': '\x08\x11', + '\x00\x00\x80\x01\xfd' + RANDOM_PAYLOAD: RANDOM_PAYLOAD, +} + CERTS_CELLS = { '\x00\x00\x81\x00\x01\x00': [], '\x00\x00\x81\x00\x04\x01\x01\x00\x00': [Certificate(type = 1, value = '')], @@ -49,14 +71,26 @@ class TestCell(unittest.TestCase): # TODO: we need to support more cell types before we can test this self.assertRaisesRegexp(NotImplementedError, 'Unpacking not yet implemented for AUTH_CHALLENGE cells', Cell.unpack, test_data('new_link_cells'), 2) + def test_padding_packing(self): + for cell_bytes, payload in PADDING_CELLS.items(): + self.assertEqual(cell_bytes, PaddingCell.pack(2, payload)) + self.assertEqual(payload, Cell.unpack(cell_bytes, 2)[0].payload) + def test_versions_packing(self): for cell_bytes, versions in VERSIONS_CELLS.items(): self.assertEqual(cell_bytes, VersionsCell.pack(versions)) self.assertEqual(versions, Cell.unpack(cell_bytes, 2)[0].versions) + def test_vpadding_packing(self): + for cell_bytes, payload in VPADDING_CELLS.items(): + self.assertEqual(cell_bytes, VPaddingCell.pack(2, payload = payload)) + self.assertEqual(payload, Cell.unpack(cell_bytes, 2)[0].payload) + + self.assertRaisesRegexp(ValueError, 'VPaddingCell.pack caller specified both a size of 5 bytes and payload of 1 bytes', VPaddingCell.pack, 2, 5, '\x02') + def test_certs_packing(self): for cell_bytes, certs in CERTS_CELLS.items(): - self.assertEqual(cell_bytes, CertsCell.pack(certs, 2)) + self.assertEqual(cell_bytes, CertsCell.pack(2, certs)) self.assertEqual(certs, Cell.unpack(cell_bytes, 2)[0].certificates) # extra bytes after the last certificate should be ignored

1 0

[stem/master] Unpack NETINFO cells
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 25fb815e8ab7ee0218ea7b9c44cd9700c3b50e51 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 15 12:07:21 2018 -0800 Unpack NETINFO cells Adding support for unpacking NETINFO cells, but not packing. Not terribly hard to add, but also not needed right now. --- stem/client/cell.py | 62 +++++++++++++++++++++++++++++++++++++++++++++++- stem/util/__init__.py | 3 +++ test/unit/client/cell.py | 42 +++++++++++++++++++++++++++++--- 3 files changed, 103 insertions(+), 4 deletions(-) diff --git a/stem/client/cell.py b/stem/client/cell.py index df34508d..98ab198c 100644 --- a/stem/client/cell.py +++ b/stem/client/cell.py @@ -36,6 +36,7 @@ Messages communicated over a Tor relay's ORPort. +- unpack - Decodes bytes for this cell class. """ +import datetime import inspect import io import os @@ -43,7 +44,8 @@ import random import sys from stem import UNDEFINED -from stem.client import ZERO, Certificate, Size +from stem.client import ZERO, Address, Certificate, Size +from stem.util import _hash_attr FIXED_PAYLOAD_LEN = 509 AUTH_CHALLENGE_SIZE = 32 @@ -184,6 +186,12 @@ class Cell(object): raise NotImplementedError('Unpacking not yet implemented for %s cells' % cls.NAME) + def __eq__(self, other): + return hash(self) == hash(other) if isinstance(other, Cell) else False + + def __ne__(self, other): + return not self == other + class CircuitCell(Cell): """ @@ -245,6 +253,9 @@ class PaddingCell(Cell): def _unpack(cls, content, circ_id, link_version): return PaddingCell(content) + def __hash__(self): + return _hash_attr(self, 'payload') + class CreateCell(CircuitCell): NAME = 'CREATE' @@ -322,12 +333,52 @@ class VersionsCell(Cell): return VersionsCell(link_versions) + def __hash__(self): + return _hash_attr(self, 'versions') + class NetinfoCell(Cell): + """ + Information relays exchange about each other. + + :var datetime timestamp: current time + :var stem.client.Address receiver_address: receiver's OR address + :var list sender_addresses: sender's OR addresses + """ + NAME = 'NETINFO' VALUE = 8 IS_FIXED_SIZE = True + def __init__(self, timestamp, receiver_address, sender_addresses): + self.timestamp = timestamp + self.receiver_address = receiver_address + self.sender_addresses = sender_addresses + + @classmethod + def pack(cls, link_version, receiver_address, sender_addresses, timestamp = None): + raise NotImplementedError('Netinfo packing not yet implemented') + + @classmethod + def _unpack(cls, content, circ_id, link_version): + if len(content) < 4: + raise ValueError('NETINFO cell expected to start with a timestamp') + + timestamp, content = Size.LONG.pop(content) + receiver_address, content = Address.pop(content) + + sender_addresses = [] + sender_addr_count, content = Size.CHAR.pop(content) + + for i in range(sender_addr_count): + addr, content = Address.pop(content) + sender_addresses.append(addr) + + return NetinfoCell(datetime.datetime.utcfromtimestamp(timestamp), receiver_address, sender_addresses) + + def __hash__(self): + return _hash_attr(self, 'timestamp', 'receiver_address', 'sender_addresses') + class RelayEarlyCell(CircuitCell): NAME = 'RELAY_EARLY' @@ -394,6 +445,9 @@ class VPaddingCell(Cell): def _unpack(cls, content, circ_id, link_version): return VPaddingCell(content) + def __hash__(self): + return _hash_attr(self, 'payload') + class CertsCell(Cell): """ @@ -450,6 +504,9 @@ class CertsCell(Cell): return CertsCell(certs) + def __hash__(self): + return _hash_attr(self, 'certificates') + class AuthChallengeCell(Cell): """ @@ -513,6 +570,9 @@ class AuthChallengeCell(Cell): return AuthChallengeCell(challenge, methods) + def __hash__(self): + return _hash_attr(self, 'challenge', 'methods') + class AuthenticateCell(Cell): NAME = 'AUTHENTICATE' diff --git a/stem/util/__init__.py b/stem/util/__init__.py index 242a3110..b206dc73 100644 --- a/stem/util/__init__.py +++ b/stem/util/__init__.py @@ -69,6 +69,9 @@ def _hash_attr(obj, *attributes, **kwargs): if isinstance(attr_value, dict): for k in sorted(attr_value.keys()): my_hash = (my_hash + hash(k)) * 1024 + hash(attr_value[k]) + elif isinstance(attr_value, (list, tuple)): + for entry in attr_value: + my_hash = (my_hash + hash(entry)) * 1024 else: my_hash += hash(attr_value) diff --git a/test/unit/client/cell.py b/test/unit/client/cell.py index f95a664e..cf7eb684 100644 --- a/test/unit/client/cell.py +++ b/test/unit/client/cell.py @@ -2,10 +2,11 @@ Unit tests for the stem.client.cell. """ +import datetime import os import unittest -from stem.client import Certificate +from stem.client import ZERO, Address, Certificate from test.unit.client import test_data from stem.client.cell import ( @@ -13,6 +14,7 @@ from stem.client.cell import ( Cell, PaddingCell, VersionsCell, + NetinfoCell, VPaddingCell, CertsCell, AuthChallengeCell, @@ -31,6 +33,8 @@ VERSIONS_CELLS = { '\x00\x00\x07\x00\x06\x00\x01\x00\x02\x00\x03': [1, 2, 3], } +NETINFO_CELL = '\x00\x00\x08ZZ\xb6\x90\x04\x04\x7f\x00\x00\x01\x01\x04\x04aq\x0f\x02' + ZERO * (FIXED_PAYLOAD_LEN - 17) + VPADDING_CELLS = { '\x00\x00\x80\x00\x00': '', '\x00\x00\x80\x00\x01\x08': '\x08', @@ -74,8 +78,33 @@ class TestCell(unittest.TestCase): self.assertRaisesRegexp(NotImplementedError, 'Unpacking not yet implemented for AUTHORIZE cells', Cell.unpack, '\x00\x00\x84\x00\x06\x00\x01\x00\x02\x00\x03', 2) def test_unpack_for_new_link(self): - # TODO: we need to support more cell types before we can test this - self.assertRaisesRegexp(NotImplementedError, 'Unpacking not yet implemented for NETINFO cells', Cell.unpack, test_data('new_link_cells'), 2) + expected_certs = ( + (1, '0\x82\x02F0\x82\x01\xaf'), + (2, '0\x82\x01\xc90\x82\x012'), + (4, '\x01\x04\x00\x06m\x1f'), + (5, '\x01\x05\x00\x06m\n\x01'), + (7, '\x1a\xa5\xb3\xbd\x88\xb1C'), + ) + + link_cells = Cell.unpack(test_data('new_link_cells'), 2) + self.assertEqual(4, len(link_cells)) + self.assertEqual(VersionsCell([3, 4, 5]), link_cells[0]) + + certs_cell = link_cells[1] + self.assertEqual(CertsCell, type(certs_cell)) + self.assertEqual(len(expected_certs), len(certs_cell.certificates)) + + for i, (cert_type, cert_prefix) in enumerate(expected_certs): + self.assertEqual(cert_type, certs_cell.certificates[i].type) + self.assertTrue(certs_cell.certificates[i].value.startswith(cert_prefix)) + + self.assertEqual(AuthChallengeCell('\x89Y\t\x99\xb2\x1e\xd9*V\xb6\x1bn\n\x05\xd8/\xe3QH\x85\x13Z\x17\xfc\x1c\x00{\xa9\xae\x83^K', [1, 3]), link_cells[2]) + + netinfo_cell = link_cells[3] + self.assertEqual(NetinfoCell, type(netinfo_cell)) + self.assertEqual(datetime.datetime(2018, 1, 14, 1, 46, 56), netinfo_cell.timestamp) + self.assertEqual(Address(type='IPv4', type_int=4, value='127.0.0.1', value_bin='\x7f\x00\x00\x01'), netinfo_cell.receiver_address) + self.assertEqual([Address(type='IPv4', type_int=4, value='97.113.15.2', value_bin='aq\x0f\x02')], netinfo_cell.sender_addresses) def test_padding_packing(self): for cell_bytes, payload in PADDING_CELLS.items(): @@ -87,6 +116,13 @@ class TestCell(unittest.TestCase): self.assertEqual(cell_bytes, VersionsCell.pack(versions)) self.assertEqual(versions, Cell.unpack(cell_bytes, 2)[0].versions) + def test_netinfo_packing(self): + cell = Cell.unpack(NETINFO_CELL, 2)[0] + + self.assertEqual(datetime.datetime(2018, 1, 14, 1, 46, 56), cell.timestamp) + self.assertEqual(Address(type='IPv4', type_int=4, value='127.0.0.1', value_bin='\x7f\x00\x00\x01'), cell.receiver_address) + self.assertEqual([Address(type='IPv4', type_int=4, value='97.113.15.2', value_bin='aq\x0f\x02')], cell.sender_addresses) + def test_vpadding_packing(self): for cell_bytes, payload in VPADDING_CELLS.items(): self.assertEqual(cell_bytes, VPaddingCell.pack(2, payload = payload))

1 0