January 2018 - tor-commits - lists.torproject.org

[stem/master] Add an IntString class
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 3f8493851bd538bc4aa3d868b880271697ecf516 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 17 11:37:40 2018 -0800 Add an IntString class Most stem.client enumerations are backed by integer values. This class will let us avoid storing two variables for all of those fields. --- docs/change_log.rst | 4 ++++ stem/util/str_tools.py | 26 ++++++++++++++++++++++++++ test/unit/doctest.py | 1 + test/unit/util/str_tools.py | 6 ++++++ 4 files changed, 37 insertions(+) diff --git a/docs/change_log.rst b/docs/change_log.rst index fcfc0115..ebdf8e3e 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -56,6 +56,10 @@ The following are only available within Stem's `git repository * `Fallback directory v2 support <https://lists.torproject.org/pipermail/tor-dev/2017-December/012721.html>`_, which adds *nickname* and *extrainfo* * Reduced maximum descriptors fetched by the remote module to match tor's new limit (:trac:`24743`) + * **Utilities** + + * Added :class:`~stem.util.str_tools.IntString` + * **Website** * Added `terminal styling <tutorials/east_of_the_sun.html#terminal-styling>`_ to our utilities tutorial diff --git a/stem/util/str_tools.py b/stem/util/str_tools.py index bc0446fa..108d21ea 100644 --- a/stem/util/str_tools.py +++ b/stem/util/str_tools.py @@ -12,6 +12,8 @@ Toolkit for various string activity. :: + IntString - string that can also be used as an integer + crop - shortens string to a given length size_label - human readable label for a number of bytes @@ -185,6 +187,30 @@ def _split_by_length(msg, size): Ending = stem.util.enum.Enum('ELLIPSE', 'HYPHEN') +class IntString(str): + """ + String that can be used as an integer as well... + + :: + + >>> s = IntString('me', 5) + >>> print('hi %s, the number is %i' % (s, s)) + hi me, the number is 5 + + .. versionadded:: 1.7.0 + """ + + def __new__(self, str_value, int_value): + return str.__new__(self, str_value) + + def __init__(self, str_value, int_value): + self._str_value = str_value + self._int_value = int_value + + def __int__(self): + return self._int_value + + def crop(msg, size, min_word_length = 4, min_crop = 0, ending = Ending.ELLIPSE, get_remainder = False): """ Shortens a string to a given length. diff --git a/test/unit/doctest.py b/test/unit/doctest.py index 7e40e873..9dda1422 100644 --- a/test/unit/doctest.py +++ b/test/unit/doctest.py @@ -67,6 +67,7 @@ class TestDocumentation(unittest.TestCase): args['globs'] = { '_to_camel_case': stem.util.str_tools._to_camel_case, '_split_by_length': stem.util.str_tools._split_by_length, + 'IntString': stem.util.str_tools.IntString, 'crop': stem.util.str_tools.crop, 'size_label': stem.util.str_tools.size_label, 'time_label': stem.util.str_tools.time_label, diff --git a/test/unit/util/str_tools.py b/test/unit/util/str_tools.py index ee270717..7349bd56 100644 --- a/test/unit/util/str_tools.py +++ b/test/unit/util/str_tools.py @@ -50,6 +50,12 @@ class TestStrTools(unittest.TestCase): self.assertEqual(['hello'], str_tools._split_by_length('hello', 5)) self.assertEqual(['hello'], str_tools._split_by_length('hello', 6)) + def test_int_string(self): + s = str_tools.IntString('me', 5) + self.assertEqual('me', s) + self.assertEqual(5, int(s)) + self.assertEqual('hi me, the number is 5', 'hi %s, the number is %i' % (s, s)) + def test_crop(self): # test the pydoc examples self.assertEqual('This is a looo...', str_tools.crop('This is a looooong message', 17))

1 0

[stem/master] Rename test module
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 3e2de2a6b6607773606faa549c2dd3622bc183fd Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 16 18:33:29 2018 -0800 Rename test module --- test/settings.cfg | 2 +- test/unit/client/{types.py => address.py} | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/settings.cfg b/test/settings.cfg index 96e2d302..78e77dd1 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -229,8 +229,8 @@ test.unit_tests |test.unit.response.authchallenge.TestAuthChallengeResponse |test.unit.response.protocolinfo.TestProtocolInfoResponse |test.unit.response.mapaddress.TestMapAddressResponse -|test.unit.client.types.TestClientTypes |test.unit.client.size.TestSize +|test.unit.client.address.TestAddress |test.unit.client.cell.TestCell |test.unit.connection.authentication.TestAuthenticate |test.unit.connection.connect.TestConnect diff --git a/test/unit/client/types.py b/test/unit/client/address.py similarity index 78% rename from test/unit/client/types.py rename to test/unit/client/address.py index a85a0b8d..bb844264 100644 --- a/test/unit/client/types.py +++ b/test/unit/client/address.py @@ -1,5 +1,5 @@ """ -Unit tests for the types in stem.client. +Unit tests for stem.client.Address. """ import unittest @@ -7,8 +7,8 @@ import unittest from stem.client import Address -class TestClientTypes(unittest.TestCase): - def test_address_ipv4(self): +class TestAddress(unittest.TestCase): + def test_ipv4(self): addr, content = Address.pop('\x04\x04\x7f\x00\x00\x01\x01\x04\x04aq\x0f\x02\x00\x00\x00\x00') self.assertEqual('\x01\x04\x04aq\x0f\x02\x00\x00\x00\x00', content)

1 0

[stem/master] Just provide bytes for address errors
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 407ed1cef0444290a7ab4cb80db460727cddd4b0 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jan 20 12:12:59 2018 -0800 Just provide bytes for address errors Spec doesn't specify what form errors take and I don't have any examples. Just providing the bytes the socket gives us. We might change the value in the future to be something more useful. --- stem/client/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/stem/client/__init__.py b/stem/client/__init__.py index b0da7aae..7705e59d 100644 --- a/stem/client/__init__.py +++ b/stem/client/__init__.py @@ -266,7 +266,12 @@ class Address(Field): self.value = ':'.join(['%04x' % Size.SHORT.unpack(value[i * 2:(i + 1) * 2]) for i in range(8)]) self.value_bin = value else: - self.value, self.value_bin = None, None # TODO: implement + # The spec doesn't really tell us what form to expect errors to be. For + # now just leaving the value unset so we can fill it in later when we + # know what would be most useful. + + self.value = None + self.value_bin = value @staticmethod def pop(content):

1 0

[stem/master] Support Address packing
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit e8490b15d6585a8fb33d6d8d7002c69cf68ba120 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jan 20 12:39:45 2018 -0800 Support Address packing --- stem/client/__init__.py | 8 ++++++++ test/unit/client/address.py | 18 +++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/stem/client/__init__.py b/stem/client/__init__.py index 7705e59d..13952a00 100644 --- a/stem/client/__init__.py +++ b/stem/client/__init__.py @@ -52,6 +52,7 @@ a wrapper for :class:`~stem.socket.RelaySocket`, much the same way as """ import collections +import io import struct import stem.util.connection @@ -273,6 +274,13 @@ class Address(Field): self.value = None self.value_bin = value + def pack(self): + cell = io.BytesIO() + cell.write(Size.CHAR.pack(self.type_int)) + cell.write(Size.CHAR.pack(len(self.value_bin))) + cell.write(self.value_bin) + return cell.getvalue() + @staticmethod def pop(content): if not content: diff --git a/test/unit/client/address.py b/test/unit/client/address.py index 689b77a0..acda251b 100644 --- a/test/unit/client/address.py +++ b/test/unit/client/address.py @@ -33,7 +33,23 @@ class TestAddress(unittest.TestCase): self.assertRaisesRegexp(ValueError, re.escape("Packed IPv4 addresses should be four bytes, but was: '\\x7f\\x00'"), Address, 4, '\x7f\x00') self.assertRaisesRegexp(ValueError, re.escape("Packed IPv6 addresses should be sixteen bytes, but was: '\\x7f\\x00'"), Address, 6, '\x7f\x00') - def test_pop_ipv4(self): + def test_unknown_type(self): + addr = Address(12, 'hello') + self.assertEqual(AddrType.UNKNOWN, addr.type) + self.assertEqual(12, addr.type_int) + self.assertEqual(None, addr.value) + self.assertEqual('hello', addr.value_bin) + + def test_packing(self): + test_data = { + '\x04\x04\x7f\x00\x00\x01': Address(AddrType.IPv4, '127.0.0.1'), + '\x06\x10 \x01\r\xb8\x00\x00\x00\x00\x00\x00\xff\x00\x00B\x83)': Address(AddrType.IPv6, '2001:0db8:0000:0000:0000:ff00:0042:8329'), + } + + for cell_bytes, address in test_data.items(): + self.assertEqual(cell_bytes, address.pack()) + self.assertEqual(address, Address.unpack(cell_bytes)) + addr, content = Address.pop('\x04\x04\x7f\x00\x00\x01\x01\x04\x04aq\x0f\x02\x00\x00\x00\x00') self.assertEqual('\x01\x04\x04aq\x0f\x02\x00\x00\x00\x00', content)

1 0

[stem/master] Change Certificate to a Field subclass
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 7981a765e15c0e4b7b4154c5c236974b8a3cdfea Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jan 20 15:26:32 2018 -0800 Change Certificate to a Field subclass Moving packing and unpacking of certificates into the type so we fit the pattern we're forming. --- stem/client/__init__.py | 71 +++++++++++++++++++++++++++++------------ stem/client/cell.py | 20 ++---------- test/settings.cfg | 1 + test/unit/client/__init__.py | 1 + test/unit/client/cell.py | 21 ++++++------ test/unit/client/certificate.py | 39 ++++++++++++++++++++++ 6 files changed, 106 insertions(+), 47 deletions(-) diff --git a/stem/client/__init__.py b/stem/client/__init__.py index 13952a00..1fd05f9d 100644 --- a/stem/client/__init__.py +++ b/stem/client/__init__.py @@ -51,7 +51,6 @@ a wrapper for :class:`~stem.socket.RelaySocket`, much the same way as ===================== =========== """ -import collections import io import struct @@ -198,24 +197,6 @@ class Size(Field): return self.unpack(packed[:self.size]), packed[self.size:] -class Certificate(collections.namedtuple('Certificate', ['type', 'value'])): - """ - Relay certificate as defined in tor-spec section 4.2. Certificate types - are... - - ==================== =========== - Type Value Description - ==================== =========== - 1 Link key certificate certified by RSA1024 identity - 2 RSA1024 Identity certificate - 3 RSA1024 AUTHENTICATE cell link certificate - ==================== =========== - - :var int type: certificate type - :var bytes value: certificate value - """ - - class Address(Field): """ Relay address. @@ -299,10 +280,60 @@ class Address(Field): return Address(addr_type, addr_value), content def __hash__(self): - # no need to include value or type since they're derived from these return _hash_attr(self, 'type_int', 'value_bin') +class Certificate(Field): + """ + Relay certificate as defined in tor-spec section 4.2. + + :var stem.client.CertType type: certificate type + :var int type_int: integer value of the certificate type + :var bytes value: certificate value + """ + + TYPE_FOR_INT = { + 1: CertType.LINK, + 2: CertType.IDENTITY, + 3: CertType.AUTHENTICATE, + } + + INT_FOR_TYPE = dict((v, k) for k, v in TYPE_FOR_INT.items()) + + def __init__(self, cert_type, value): + if isinstance(cert_type, int): + self.type = Certificate.TYPE_FOR_INT.get(cert_type, CertType.UNKNOWN) + self.type_int = cert_type + elif cert_type in CertType: + self.type = cert_type + self.type_int = Certificate.INT_FOR_TYPE.get(cert_type, -1) + else: + raise ValueError('Invalid certificate type: %s' % cert_type) + + self.value = value + + def pack(self): + cell = io.BytesIO() + cell.write(Size.CHAR.pack(self.type_int)) + cell.write(Size.SHORT.pack(len(self.value))) + cell.write(self.value) + return cell.getvalue() + + @staticmethod + def pop(content): + cert_type, content = Size.CHAR.pop(content) + cert_size, content = Size.SHORT.pop(content) + + if cert_size > len(content): + raise ValueError('CERTS cell should have a certificate with %i bytes, but only had %i remaining' % (cert_size, len(content))) + + cert_bytes, content = split(content, cert_size) + return Certificate(cert_type, cert_bytes), content + + def __hash__(self): + return _hash_attr(self, 'type_int', 'value') + + setattr(Size, 'CHAR', Size('CHAR', 1, '!B')) setattr(Size, 'SHORT', Size('SHORT', 2, '!H')) setattr(Size, 'LONG', Size('LONG', 4, '!L')) diff --git a/stem/client/cell.py b/stem/client/cell.py index 332e3f16..074d2079 100644 --- a/stem/client/cell.py +++ b/stem/client/cell.py @@ -489,15 +489,7 @@ class CertsCell(Cell): :returns: **bytes** with a payload for these versions """ - payload = io.BytesIO() - payload.write(Size.CHAR.pack(len(certs))) - - for cert in certs: - payload.write(Size.CHAR.pack(cert.type)) - payload.write(Size.SHORT.pack(len(cert.value))) - payload.write(cert.value) - - return cls._pack(link_version, payload.getvalue()) + return cls._pack(link_version, Size.CHAR.pack(len(certs)) + ''.join([cert.pack() for cert in certs])) @classmethod def _unpack(cls, content, circ_id, link_version): @@ -508,14 +500,8 @@ class CertsCell(Cell): if not content: raise ValueError('CERTS cell indicates it should have %i certificates, but only contained %i' % (cert_count, len(certs))) - cert_type, content = Size.CHAR.pop(content) - cert_size, content = Size.SHORT.pop(content) - - if cert_size > len(content): - raise ValueError('CERTS cell should have a certificate with %i bytes, but only had %i remaining' % (cert_size, len(content))) - - cert_bytes, content = split(content, cert_size) - certs.append(Certificate(cert_type, cert_bytes)) + cert, content = Certificate.pop(content) + certs.append(cert) return CertsCell(certs) diff --git a/test/settings.cfg b/test/settings.cfg index 28694b60..518ae2b2 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -232,6 +232,7 @@ test.unit_tests |test.unit.response.mapaddress.TestMapAddressResponse |test.unit.client.size.TestSize |test.unit.client.address.TestAddress +|test.unit.client.certificate.TestCertificate |test.unit.client.cell.TestCell |test.unit.connection.authentication.TestAuthenticate |test.unit.connection.connect.TestConnect diff --git a/test/unit/client/__init__.py b/test/unit/client/__init__.py index b36b72af..7e745352 100644 --- a/test/unit/client/__init__.py +++ b/test/unit/client/__init__.py @@ -5,6 +5,7 @@ Unit tests for stem.client.* contents. __all__ = [ 'address', 'cell', + 'certificate', 'size', ] diff --git a/test/unit/client/cell.py b/test/unit/client/cell.py index c9aea5bc..7f3318ee 100644 --- a/test/unit/client/cell.py +++ b/test/unit/client/cell.py @@ -6,7 +6,7 @@ import datetime import os import unittest -from stem.client import ZERO, AddrType, Address, Certificate +from stem.client import ZERO, AddrType, CertType, Address, Certificate from test.unit.client import test_data from stem.client.cell import ( @@ -44,8 +44,8 @@ VPADDING_CELLS = { CERTS_CELLS = { '\x00\x00\x81\x00\x01\x00': [], - '\x00\x00\x81\x00\x04\x01\x01\x00\x00': [Certificate(type = 1, value = '')], - '\x00\x00\x81\x00\x05\x01\x01\x00\x01\x08': [Certificate(type = 1, value = '\x08')], + '\x00\x00\x81\x00\x04\x01\x01\x00\x00': [Certificate(1, '')], + '\x00\x00\x81\x00\x05\x01\x01\x00\x01\x08': [Certificate(1, '\x08')], } AUTH_CHALLENGE_CELLS = { @@ -79,11 +79,11 @@ class TestCell(unittest.TestCase): def test_unpack_for_new_link(self): expected_certs = ( - (1, '0\x82\x02F0\x82\x01\xaf'), - (2, '0\x82\x01\xc90\x82\x012'), - (4, '\x01\x04\x00\x06m\x1f'), - (5, '\x01\x05\x00\x06m\n\x01'), - (7, '\x1a\xa5\xb3\xbd\x88\xb1C'), + (CertType.LINK, 1, '0\x82\x02F0\x82\x01\xaf'), + (CertType.IDENTITY, 2, '0\x82\x01\xc90\x82\x012'), + (CertType.UNKNOWN, 4, '\x01\x04\x00\x06m\x1f'), + (CertType.UNKNOWN, 5, '\x01\x05\x00\x06m\n\x01'), + (CertType.UNKNOWN, 7, '\x1a\xa5\xb3\xbd\x88\xb1C'), ) content = test_data('new_link_cells') @@ -95,8 +95,9 @@ class TestCell(unittest.TestCase): self.assertEqual(CertsCell, type(certs_cell)) self.assertEqual(len(expected_certs), len(certs_cell.certificates)) - for i, (cert_type, cert_prefix) in enumerate(expected_certs): + for i, (cert_type, cert_type_int, cert_prefix) in enumerate(expected_certs): self.assertEqual(cert_type, certs_cell.certificates[i].type) + self.assertEqual(cert_type_int, certs_cell.certificates[i].type_int) self.assertTrue(certs_cell.certificates[i].value.startswith(cert_prefix)) auth_challenge_cell, content = Cell.unpack(content, 2) @@ -141,7 +142,7 @@ class TestCell(unittest.TestCase): # extra bytes after the last certificate should be ignored - self.assertEqual([Certificate(type = 1, value = '\x08')], Cell.unpack('\x00\x00\x81\x00\x07\x01\x01\x00\x01\x08\x06\x04', 2)[0].certificates) + self.assertEqual([Certificate(1, '\x08')], Cell.unpack('\x00\x00\x81\x00\x07\x01\x01\x00\x01\x08\x06\x04', 2)[0].certificates) # ... but truncated or missing certificates should error diff --git a/test/unit/client/certificate.py b/test/unit/client/certificate.py new file mode 100644 index 00000000..873de51d --- /dev/null +++ b/test/unit/client/certificate.py @@ -0,0 +1,39 @@ +""" +Unit tests for stem.client.Certificate. +""" + +import unittest + +from stem.client import CertType, Certificate + + +class TestCertificate(unittest.TestCase): + def test_constructor(self): + test_data = ( + ((1, '\x7f\x00\x00\x01'), (CertType.LINK, 1, '\x7f\x00\x00\x01')), + ((2, '\x7f\x00\x00\x01'), (CertType.IDENTITY, 2, '\x7f\x00\x00\x01')), + ((3, '\x7f\x00\x00\x01'), (CertType.AUTHENTICATE, 3, '\x7f\x00\x00\x01')), + ((4, '\x7f\x00\x00\x01'), (CertType.UNKNOWN, 4, '\x7f\x00\x00\x01')), + ((CertType.IDENTITY, '\x7f\x00\x00\x01'), (CertType.IDENTITY, 2, '\x7f\x00\x00\x01')), + ) + + for (cert_type, cert_value), (expected_type, expected_type_int, expected_value) in test_data: + cert = Certificate(cert_type, cert_value) + self.assertEqual(expected_type, cert.type) + self.assertEqual(expected_type_int, cert.type_int) + self.assertEqual(expected_value, cert.value) + + def test_unknown_type(self): + cert = Certificate(12, 'hello') + self.assertEqual(CertType.UNKNOWN, cert.type) + self.assertEqual(12, cert.type_int) + self.assertEqual('hello', cert.value) + + def test_packing(self): + cert, content = Certificate.pop('\x02\x00\x04\x00\x00\x01\x01\x04\x04aq\x0f\x02\x00\x00\x00\x00') + self.assertEqual('\x04\x04aq\x0f\x02\x00\x00\x00\x00', content) + + self.assertEqual(CertType.IDENTITY, cert.type) + self.assertEqual(2, cert.type_int) + self.assertEqual('\x00\x00\x01\x01', cert.value) + self.assertEqual('\x02\x00\x04\x00\x00\x01\x01', cert.pack())

1 0

[stem/master] Support packing Netinfo cells
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit c24e22402e66b62fdb206a46ec541449b3e7cd6b Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jan 20 15:37:37 2018 -0800 Support packing Netinfo cells Now that addresses can be packed supporting Netinfo cells is simple. --- stem/client/cell.py | 4 ++-- test/unit/client/cell.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/stem/client/cell.py b/stem/client/cell.py index 074d2079..3a596b9c 100644 --- a/stem/client/cell.py +++ b/stem/client/cell.py @@ -366,11 +366,11 @@ class NetinfoCell(Cell): payload = io.BytesIO() payload.write(Size.LONG.pack(int(datetime_to_unix(timestamp)))) - payload.write(Address.pack(receiver_address)) + payload.write(receiver_address.pack()) payload.write(Size.CHAR.pack(len(sender_addresses))) for addr in sender_addresses: - payload.write(Address.pack(addr)) + payload.write(addr.pack()) return cls._pack(link_version, payload.getvalue()) diff --git a/test/unit/client/cell.py b/test/unit/client/cell.py index 7f3318ee..22de54d9 100644 --- a/test/unit/client/cell.py +++ b/test/unit/client/cell.py @@ -33,7 +33,9 @@ VERSIONS_CELLS = { '\x00\x00\x07\x00\x06\x00\x01\x00\x02\x00\x03': [1, 2, 3], } -NETINFO_CELL = '\x00\x00\x08ZZ\xb6\x90\x04\x04\x7f\x00\x00\x01\x01\x04\x04aq\x0f\x02' + ZERO * (FIXED_PAYLOAD_LEN - 17) +NETINFO_CELLS = { + '\x00\x00\x08ZZ\xb6\x90\x04\x04\x7f\x00\x00\x01\x01\x04\x04aq\x0f\x02' + ZERO * (FIXED_PAYLOAD_LEN - 17): (datetime.datetime(2018, 1, 14, 1, 46, 56), Address(AddrType.IPv4, '127.0.0.1'), [Address(AddrType.IPv4, '97.113.15.2')]), +} VPADDING_CELLS = { '\x00\x00\x80\x00\x00': '', @@ -122,11 +124,13 @@ class TestCell(unittest.TestCase): self.assertEqual(versions, Cell.unpack(cell_bytes, 2)[0].versions) def test_netinfo_packing(self): - cell = Cell.unpack(NETINFO_CELL, 2)[0] + for cell_bytes, (timestamp, receiver_address, sender_addresses) in NETINFO_CELLS.items(): + self.assertEqual(cell_bytes, NetinfoCell.pack(2, receiver_address, sender_addresses, timestamp)) - self.assertEqual(datetime.datetime(2018, 1, 14, 1, 46, 56), cell.timestamp) - self.assertEqual(Address(AddrType.IPv4, '127.0.0.1'), cell.receiver_address) - self.assertEqual([Address(AddrType.IPv4, '97.113.15.2')], cell.sender_addresses) + cell = Cell.unpack(cell_bytes, 2)[0] + self.assertEqual(timestamp, cell.timestamp) + self.assertEqual(receiver_address, cell.receiver_address) + self.assertEqual(sender_addresses, cell.sender_addresses) def test_vpadding_packing(self): for cell_bytes, payload in VPADDING_CELLS.items():

1 0

[stem/master] Initial stem.client module
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit b2c8810dc94b0e2e62c0e482158de3d33504a709 Merge: ea2752c5 c24e2240 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Jan 20 17:53:41 2018 -0800 Initial stem.client module Far from complete. Thus far this only provides low level support for establishing links, including packing/unpacking for the following cell types... * PADDING * VPADDING * VERSIONS * NETINFO * CERTS * AUTH_CHALLENGE This is based on Tim's fantastic python proof of concept for ORPort communication, Endosome... https://github.com/teor2345/endosome Once this is pushed I'll be shooting him a pull request to take advantage of our new functionality. Next up is support for additional cell types and a high level class for clients to use (everything here are building blocks, *not* what end users will want to leverage). docs/change_log.rst | 2 + setup.py | 2 +- stem/client/__init__.py | 340 ++++++++++++++++++++ stem/client/cell.py | 587 +++++++++++++++++++++++++++++++++++ stem/connection.py | 2 +- stem/control.py | 4 +- stem/exit_policy.py | 2 +- stem/socket.py | 364 +++++++++++++++------- stem/util/__init__.py | 3 + stem/util/connection.py | 8 +- test/integ/control/controller.py | 4 +- test/settings.cfg | 5 + test/unit/__init__.py | 1 + test/unit/client/__init__.py | 27 ++ test/unit/client/address.py | 59 ++++ test/unit/client/cell.py | 165 ++++++++++ test/unit/client/certificate.py | 39 +++ test/unit/client/data/new_link_cells | Bin 0 -> 2043 bytes test/unit/client/size.py | 50 +++ test/unit/util/connection.py | 10 +- 20 files changed, 1542 insertions(+), 132 deletions(-)

1 0

[stem/master] Drop max_response_size argument from RelaySocket's recv
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 58c788780bb54a114745a0ddf8fbe4666f20dd97 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 1 12:08:46 2018 -0800 Drop max_response_size argument from RelaySocket's recv Turns out endosome never actually used it. I'll need to ask Tim where the constant came from. --- stem/socket.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/stem/socket.py b/stem/socket.py index 20fbf6d0..fe902f0a 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -387,12 +387,10 @@ class RelaySocket(BaseSocket): self._send(message, lambda s, sf, msg: _write_to_socket(sf, msg)) - def recv(self, max_response_size = MAX_READ_BUFFER_LEN): + def recv(self): """ Receives a message from the relay. - :param int max_response_size: maximum bytes to return - :returns: bytes for the message received :raises: @@ -400,10 +398,10 @@ class RelaySocket(BaseSocket): * :class:`stem.SocketClosed` if the socket closes before we receive a complete message """ - # TODO: Not really sure what we'll want here. To start with just copying - # endosome's behavior. + # TODO: Is MAX_READ_BUFFER_LEN defined in the spec? Not sure where it came + # from. - return self._recv(lambda s, sf: s.recv(max_response_size)) + return self._recv(lambda s, sf: s.recv(MAX_READ_BUFFER_LEN)) def is_localhost(self): return self.address == '127.0.0.1'

1 0

[stem/master] PackType enum
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit c9ea8a70622e80cf1ba9a960976244aa1a52ef73 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 1 16:40:30 2018 -0800 PackType enum Enum for a more readable counterpart for endosome's PACK_FMT. --- stem/client.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/stem/client.py b/stem/client.py index e1011f96..4da12bba 100644 --- a/stem/client.py +++ b/stem/client.py @@ -9,6 +9,21 @@ providing higher level functions in much the same way as our .. versionadded:: 1.7.0 +.. data:: PackType (enum) + + Unsigned `struct.pack format + <https://docs.python.org/2/library/struct.html#format-characters>` for + network-order fields. + + ==================== =========== + PackType Description + ==================== =========== + CHAR Unsigned char (1 byte) + SHORT Unsigned short (2 bytes) + LONG Unsigned long (4 bytes) + LONG_LONG Unsigned long long (8 bytes) + ==================== =========== + **Module Overview:** :: @@ -16,6 +31,15 @@ providing higher level functions in much the same way as our Relay - Connection with a relay's ORPort. """ +from stem.util import enum + +PackType = enum.Enum( + ('CHAR', '!B'), # 1 byte + ('SHORT', '!H'), # 2 bytes + ('LONG', '!L'), # 4 bytes + ('LONG_LONG', '!Q'), # 8 bytes +) + class Relay(object): """

1 0

[stem/master] Move thread safe socket communication to BaseSocket
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 21e2db3f038cb8b2f9f8c2318d4a7be561886bc1 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 31 12:28:52 2017 -0800 Move thread safe socket communication to BaseSocket Not certain yet this is what we want, but relay ORPort communication looks similar in some basic ways to controller communication. We'll want thread safe sockets for both so making a base class that provides just that. --- stem/socket.py | 207 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 109 insertions(+), 98 deletions(-) diff --git a/stem/socket.py b/stem/socket.py index b05524b5..38354ec8 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -46,16 +46,18 @@ Tor... :: - ControlSocket - Socket wrapper that speaks the tor control protocol. - |- ControlPort - Control connection via a port. - | |- get_address - provides the ip address of our socket - | +- get_port - provides the port of our socket + BaseSocket - Thread safe socket. + |- ControlSocket - Socket wrapper that speaks the tor control protocol. + | |- ControlPort - Control connection via a port. + | | |- get_address - provides the ip address of our socket + | | +- get_port - provides the port of our socket + | | + | |- ControlSocketFile - Control connection via a local file socket. + | | +- get_socket_path - provides the path of the socket we connect to + | | + | |- send - sends a message to the socket + | +- recv - receives a ControlMessage from the socket | - |- ControlSocketFile - Control connection via a local file socket. - | +- get_socket_path - provides the path of the socket we connect to - | - |- send - sends a message to the socket - |- recv - receives a ControlMessage from the socket |- is_alive - reports if the socket is known to be closed |- is_localhost - returns if the socket is for the local system or not |- connect - connects a new socket @@ -89,14 +91,9 @@ ERROR_MSG = 'Error while receiving a control message (%s): %s' TRUNCATE_LOGS = 10 -class ControlSocket(object): +class BaseSocket(object): """ - Wrapper for a socket connection that speaks the Tor control protocol. To the - better part this transparently handles the formatting for sending and - receiving complete messages. All methods are thread safe. - - Callers should not instantiate this class directly, but rather use subclasses - which are expected to implement the **_make_socket()** method. + Thread safe socket, providing common socket functionality. """ def __init__(self): @@ -111,95 +108,21 @@ class ControlSocket(object): self._send_lock = threading.RLock() self._recv_lock = threading.RLock() - def send(self, message, raw = False): - """ - Formats and sends a message to the control socket. For more information see - the :func:`~stem.socket.send_message` function. - - :param str message: message to be formatted and sent to the socket - :param bool raw: leaves the message formatting untouched, passing it to the socket as-is - - :raises: - * :class:`stem.SocketError` if a problem arises in using the socket - * :class:`stem.SocketClosed` if the socket is known to be shut down - """ - - with self._send_lock: - try: - if not self.is_alive(): - raise stem.SocketClosed() - - send_message(self._socket_file, message, raw) - except stem.SocketClosed: - # if send_message raises a SocketClosed then we should properly shut - # everything down - - if self.is_alive(): - self.close() - - raise - - def recv(self): - """ - Receives a message from the control socket, blocking until we've received - one. For more information see the :func:`~stem.socket.recv_message` function. - - :returns: :class:`~stem.response.ControlMessage` for the message received - - :raises: - * :class:`stem.ProtocolError` the content from the socket is malformed - * :class:`stem.SocketClosed` if the socket closes before we receive a complete message - """ - - with self._recv_lock: - try: - # makes a temporary reference to the _socket_file because connect() - # and close() may set or unset it - - socket_file = self._socket_file - - if not socket_file: - raise stem.SocketClosed() - - return recv_message(socket_file) - except stem.SocketClosed: - # If recv_message raises a SocketClosed then we should properly shut - # everything down. However, there's a couple cases where this will - # cause deadlock... - # - # * This SocketClosed was *caused by* a close() call, which is joining - # on our thread. - # - # * A send() call that's currently in flight is about to call close(), - # also attempting to join on us. - # - # To resolve this we make a non-blocking call to acquire the send lock. - # If we get it then great, we can close safely. If not then one of the - # above are in progress and we leave the close to them. - - if self.is_alive(): - if self._send_lock.acquire(False): - self.close() - self._send_lock.release() - - raise - def is_alive(self): """ Checks if the socket is known to be closed. We won't be aware if it is until we either use it or have explicitily shut it down. In practice a socket derived from a port knows about its disconnection - after a failed :func:`~stem.socket.ControlSocket.recv` call. Socket file - derived connections know after either a - :func:`~stem.socket.ControlSocket.send` or - :func:`~stem.socket.ControlSocket.recv`. + after failing to receive data, whereas socket file derived connections + know after either sending or receiving data. This means that to have reliable detection for when we're disconnected you need to continually pull from the socket (which is part of what the :class:`~stem.control.BaseController` does). - :returns: **bool** that's **True** if our socket is connected and **False** otherwise + :returns: **bool** that's **True** if our socket is connected and **False** + otherwise """ return self._is_alive @@ -208,7 +131,8 @@ class ControlSocket(object): """ Returns if the connection is for the local system or not. - :returns: **bool** that's **True** if the connection is for the local host and **False** otherwise + :returns: **bool** that's **True** if the connection is for the local host + and **False** otherwise """ return False @@ -307,8 +231,8 @@ class ControlSocket(object): """ The send lock is useful to classes that interact with us at a deep level because it's used to lock :func:`stem.socket.ControlSocket.connect` / - :func:`stem.socket.ControlSocket.close`, and by extension our - :func:`stem.socket.ControlSocket.is_alive` state changes. + :func:`stem.socket.BaseSocket.close`, and by extension our + :func:`stem.socket.BaseSocket.is_alive` state changes. :returns: **threading.RLock** that governs sending messages to our socket and state changes @@ -347,7 +271,94 @@ class ControlSocket(object): * **NotImplementedError** if not implemented by a subclass """ - raise NotImplementedError('Unsupported Operation: this should be implemented by the ControlSocket subclass') + raise NotImplementedError('Unsupported Operation: this should be implemented by the BaseSocket subclass') + + +class ControlSocket(BaseSocket): + """ + Wrapper for a socket connection that speaks the Tor control protocol. To the + better part this transparently handles the formatting for sending and + receiving complete messages. + + Callers should not instantiate this class directly, but rather use subclasses + which are expected to implement the **_make_socket()** method. + """ + + def __init__(self): + super(ControlSocket, self).__init__() + + def send(self, message, raw = False): + """ + Formats and sends a message to the control socket. For more information see + the :func:`~stem.socket.send_message` function. + + :param str message: message to be formatted and sent to the socket + :param bool raw: leaves the message formatting untouched, passing it to the socket as-is + + :raises: + * :class:`stem.SocketError` if a problem arises in using the socket + * :class:`stem.SocketClosed` if the socket is known to be shut down + """ + + with self._send_lock: + try: + if not self.is_alive(): + raise stem.SocketClosed() + + send_message(self._socket_file, message, raw) + except stem.SocketClosed: + # if send_message raises a SocketClosed then we should properly shut + # everything down + + if self.is_alive(): + self.close() + + raise + + def recv(self): + """ + Receives a message from the control socket, blocking until we've received + one. For more information see the :func:`~stem.socket.recv_message` function. + + :returns: :class:`~stem.response.ControlMessage` for the message received + + :raises: + * :class:`stem.ProtocolError` the content from the socket is malformed + * :class:`stem.SocketClosed` if the socket closes before we receive a complete message + """ + + with self._recv_lock: + try: + # makes a temporary reference to the _socket_file because connect() + # and close() may set or unset it + + socket_file = self._socket_file + + if not socket_file: + raise stem.SocketClosed() + + return recv_message(socket_file) + except stem.SocketClosed: + # If recv_message raises a SocketClosed then we should properly shut + # everything down. However, there's a couple cases where this will + # cause deadlock... + # + # * This SocketClosed was *caused by* a close() call, which is joining + # on our thread. + # + # * A send() call that's currently in flight is about to call close(), + # also attempting to join on us. + # + # To resolve this we make a non-blocking call to acquire the send lock. + # If we get it then great, we can close safely. If not then one of the + # above are in progress and we leave the close to them. + + if self.is_alive(): + if self._send_lock.acquire(False): + self.close() + self._send_lock.release() + + raise class ControlPort(ControlSocket):

1 0