January 2018 - tor-commits - lists.torproject.org

[stem/master] Move thread safe socket communication to BaseSocket
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 21e2db3f038cb8b2f9f8c2318d4a7be561886bc1 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 31 12:28:52 2017 -0800 Move thread safe socket communication to BaseSocket Not certain yet this is what we want, but relay ORPort communication looks similar in some basic ways to controller communication. We'll want thread safe sockets for both so making a base class that provides just that. --- stem/socket.py | 207 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 109 insertions(+), 98 deletions(-) diff --git a/stem/socket.py b/stem/socket.py index b05524b5..38354ec8 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -46,16 +46,18 @@ Tor... :: - ControlSocket - Socket wrapper that speaks the tor control protocol. - |- ControlPort - Control connection via a port. - | |- get_address - provides the ip address of our socket - | +- get_port - provides the port of our socket + BaseSocket - Thread safe socket. + |- ControlSocket - Socket wrapper that speaks the tor control protocol. + | |- ControlPort - Control connection via a port. + | | |- get_address - provides the ip address of our socket + | | +- get_port - provides the port of our socket + | | + | |- ControlSocketFile - Control connection via a local file socket. + | | +- get_socket_path - provides the path of the socket we connect to + | | + | |- send - sends a message to the socket + | +- recv - receives a ControlMessage from the socket | - |- ControlSocketFile - Control connection via a local file socket. - | +- get_socket_path - provides the path of the socket we connect to - | - |- send - sends a message to the socket - |- recv - receives a ControlMessage from the socket |- is_alive - reports if the socket is known to be closed |- is_localhost - returns if the socket is for the local system or not |- connect - connects a new socket @@ -89,14 +91,9 @@ ERROR_MSG = 'Error while receiving a control message (%s): %s' TRUNCATE_LOGS = 10 -class ControlSocket(object): +class BaseSocket(object): """ - Wrapper for a socket connection that speaks the Tor control protocol. To the - better part this transparently handles the formatting for sending and - receiving complete messages. All methods are thread safe. - - Callers should not instantiate this class directly, but rather use subclasses - which are expected to implement the **_make_socket()** method. + Thread safe socket, providing common socket functionality. """ def __init__(self): @@ -111,95 +108,21 @@ class ControlSocket(object): self._send_lock = threading.RLock() self._recv_lock = threading.RLock() - def send(self, message, raw = False): - """ - Formats and sends a message to the control socket. For more information see - the :func:`~stem.socket.send_message` function. - - :param str message: message to be formatted and sent to the socket - :param bool raw: leaves the message formatting untouched, passing it to the socket as-is - - :raises: - * :class:`stem.SocketError` if a problem arises in using the socket - * :class:`stem.SocketClosed` if the socket is known to be shut down - """ - - with self._send_lock: - try: - if not self.is_alive(): - raise stem.SocketClosed() - - send_message(self._socket_file, message, raw) - except stem.SocketClosed: - # if send_message raises a SocketClosed then we should properly shut - # everything down - - if self.is_alive(): - self.close() - - raise - - def recv(self): - """ - Receives a message from the control socket, blocking until we've received - one. For more information see the :func:`~stem.socket.recv_message` function. - - :returns: :class:`~stem.response.ControlMessage` for the message received - - :raises: - * :class:`stem.ProtocolError` the content from the socket is malformed - * :class:`stem.SocketClosed` if the socket closes before we receive a complete message - """ - - with self._recv_lock: - try: - # makes a temporary reference to the _socket_file because connect() - # and close() may set or unset it - - socket_file = self._socket_file - - if not socket_file: - raise stem.SocketClosed() - - return recv_message(socket_file) - except stem.SocketClosed: - # If recv_message raises a SocketClosed then we should properly shut - # everything down. However, there's a couple cases where this will - # cause deadlock... - # - # * This SocketClosed was *caused by* a close() call, which is joining - # on our thread. - # - # * A send() call that's currently in flight is about to call close(), - # also attempting to join on us. - # - # To resolve this we make a non-blocking call to acquire the send lock. - # If we get it then great, we can close safely. If not then one of the - # above are in progress and we leave the close to them. - - if self.is_alive(): - if self._send_lock.acquire(False): - self.close() - self._send_lock.release() - - raise - def is_alive(self): """ Checks if the socket is known to be closed. We won't be aware if it is until we either use it or have explicitily shut it down. In practice a socket derived from a port knows about its disconnection - after a failed :func:`~stem.socket.ControlSocket.recv` call. Socket file - derived connections know after either a - :func:`~stem.socket.ControlSocket.send` or - :func:`~stem.socket.ControlSocket.recv`. + after failing to receive data, whereas socket file derived connections + know after either sending or receiving data. This means that to have reliable detection for when we're disconnected you need to continually pull from the socket (which is part of what the :class:`~stem.control.BaseController` does). - :returns: **bool** that's **True** if our socket is connected and **False** otherwise + :returns: **bool** that's **True** if our socket is connected and **False** + otherwise """ return self._is_alive @@ -208,7 +131,8 @@ class ControlSocket(object): """ Returns if the connection is for the local system or not. - :returns: **bool** that's **True** if the connection is for the local host and **False** otherwise + :returns: **bool** that's **True** if the connection is for the local host + and **False** otherwise """ return False @@ -307,8 +231,8 @@ class ControlSocket(object): """ The send lock is useful to classes that interact with us at a deep level because it's used to lock :func:`stem.socket.ControlSocket.connect` / - :func:`stem.socket.ControlSocket.close`, and by extension our - :func:`stem.socket.ControlSocket.is_alive` state changes. + :func:`stem.socket.BaseSocket.close`, and by extension our + :func:`stem.socket.BaseSocket.is_alive` state changes. :returns: **threading.RLock** that governs sending messages to our socket and state changes @@ -347,7 +271,94 @@ class ControlSocket(object): * **NotImplementedError** if not implemented by a subclass """ - raise NotImplementedError('Unsupported Operation: this should be implemented by the ControlSocket subclass') + raise NotImplementedError('Unsupported Operation: this should be implemented by the BaseSocket subclass') + + +class ControlSocket(BaseSocket): + """ + Wrapper for a socket connection that speaks the Tor control protocol. To the + better part this transparently handles the formatting for sending and + receiving complete messages. + + Callers should not instantiate this class directly, but rather use subclasses + which are expected to implement the **_make_socket()** method. + """ + + def __init__(self): + super(ControlSocket, self).__init__() + + def send(self, message, raw = False): + """ + Formats and sends a message to the control socket. For more information see + the :func:`~stem.socket.send_message` function. + + :param str message: message to be formatted and sent to the socket + :param bool raw: leaves the message formatting untouched, passing it to the socket as-is + + :raises: + * :class:`stem.SocketError` if a problem arises in using the socket + * :class:`stem.SocketClosed` if the socket is known to be shut down + """ + + with self._send_lock: + try: + if not self.is_alive(): + raise stem.SocketClosed() + + send_message(self._socket_file, message, raw) + except stem.SocketClosed: + # if send_message raises a SocketClosed then we should properly shut + # everything down + + if self.is_alive(): + self.close() + + raise + + def recv(self): + """ + Receives a message from the control socket, blocking until we've received + one. For more information see the :func:`~stem.socket.recv_message` function. + + :returns: :class:`~stem.response.ControlMessage` for the message received + + :raises: + * :class:`stem.ProtocolError` the content from the socket is malformed + * :class:`stem.SocketClosed` if the socket closes before we receive a complete message + """ + + with self._recv_lock: + try: + # makes a temporary reference to the _socket_file because connect() + # and close() may set or unset it + + socket_file = self._socket_file + + if not socket_file: + raise stem.SocketClosed() + + return recv_message(socket_file) + except stem.SocketClosed: + # If recv_message raises a SocketClosed then we should properly shut + # everything down. However, there's a couple cases where this will + # cause deadlock... + # + # * This SocketClosed was *caused by* a close() call, which is joining + # on our thread. + # + # * A send() call that's currently in flight is about to call close(), + # also attempting to join on us. + # + # To resolve this we make a non-blocking call to acquire the send lock. + # If we get it then great, we can close safely. If not then one of the + # above are in progress and we leave the close to them. + + if self.is_alive(): + if self._send_lock.acquire(False): + self.close() + self._send_lock.release() + + raise class ControlPort(ControlSocket):

1 0

[stem/master] Replace get_address, get_port, and get_socket_path with attributes
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit cfcf763c69b04fa6b24c8737a92dad91a743e395 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 31 13:32:32 2017 -0800 Replace get_address, get_port, and get_socket_path with attributes These are methods because I wanted to emphasize that they're read-only, but getter methods like this are pretty javay. Just providing attributes instead. --- docs/change_log.rst | 1 + stem/connection.py | 2 +- stem/control.py | 4 ++-- stem/socket.py | 36 +++++++++++++++++++++++------------- test/integ/control/controller.py | 4 ++-- 5 files changed, 29 insertions(+), 18 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index 2d2490b5..68dd35fc 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -48,6 +48,7 @@ The following are only available within Stem's `git repository * Added support for limiting the maximum number of streams to :func:`~stem.control.Controller.create_ephemeral_hidden_service` (:spec:`2fcb1c2`) * Stacktrace if :func:`stem.connection.connect` had a string port argument + * Replaced socket's :func:`~stem.socket.ControlPort.get_address`, :func:`~stem.socket.ControlPort.get_port`, and :func:`~stem.socket.ControlSocketFile.get_socket_path` with attributes * **Descriptors** diff --git a/stem/connection.py b/stem/connection.py index 3aad575a..aba11c5e 100644 --- a/stem/connection.py +++ b/stem/connection.py @@ -375,7 +375,7 @@ def _connect_auth(control_socket, password, password_prompt, chroot_path, contro return controller(control_socket, is_authenticated = True) except IncorrectSocketType: if isinstance(control_socket, stem.socket.ControlPort): - print(CONNECT_MESSAGES['wrong_port_type'].format(port = control_socket.get_port())) + print(CONNECT_MESSAGES['wrong_port_type'].format(port = control_socket.port)) else: print(CONNECT_MESSAGES['wrong_socket_type']) diff --git a/stem/control.py b/stem/control.py index a4297677..94ad5eaf 100644 --- a/stem/control.py +++ b/stem/control.py @@ -1606,9 +1606,9 @@ class Controller(BaseController): control_socket = self.get_socket() if isinstance(control_socket, stem.socket.ControlPort): - pid = stem.util.system.pid_by_port(control_socket.get_port()) + pid = stem.util.system.pid_by_port(control_socket.port) elif isinstance(control_socket, stem.socket.ControlSocketFile): - pid = stem.util.system.pid_by_open_file(control_socket.get_socket_path()) + pid = stem.util.system.pid_by_open_file(control_socket.path) if pid: self._set_cache({'pid': pid}) diff --git a/stem/socket.py b/stem/socket.py index 38354ec8..68ca36bb 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -49,11 +49,7 @@ Tor... BaseSocket - Thread safe socket. |- ControlSocket - Socket wrapper that speaks the tor control protocol. | |- ControlPort - Control connection via a port. - | | |- get_address - provides the ip address of our socket - | | +- get_port - provides the port of our socket - | | | |- ControlSocketFile - Control connection via a local file socket. - | | +- get_socket_path - provides the path of the socket we connect to | | | |- send - sends a message to the socket | +- recv - receives a ControlMessage from the socket @@ -365,6 +361,9 @@ class ControlPort(ControlSocket): """ Control connection to tor. For more information see tor's ControlPort torrc option. + + :var str address: address our socket connects to + :var int port: ControlPort our socket connects to """ def __init__(self, address = '127.0.0.1', port = 9051, connect = True): @@ -380,8 +379,8 @@ class ControlPort(ControlSocket): """ super(ControlPort, self).__init__() - self._control_addr = address - self._control_port = port + self.address = address + self.port = port if connect: self.connect() @@ -390,27 +389,33 @@ class ControlPort(ControlSocket): """ Provides the ip address our socket connects to. + .. deprecated:: 1.7.0 + Use the **address** attribute instead. + :returns: str with the ip address of our socket """ - return self._control_addr + return self.address def get_port(self): """ Provides the port our socket connects to. + .. deprecated:: 1.7.0 + Use the **port** attribute instead. + :returns: int with the port of our socket """ - return self._control_port + return self.port def is_localhost(self): - return self._control_addr == '127.0.0.1' + return self.address == '127.0.0.1' def _make_socket(self): try: control_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - control_socket.connect((self._control_addr, self._control_port)) + control_socket.connect((self.address, self.port)) return control_socket except socket.error as exc: raise stem.SocketError(exc) @@ -420,6 +425,8 @@ class ControlSocketFile(ControlSocket): """ Control connection to tor. For more information see tor's ControlSocket torrc option. + + :var str path: filesystem path of the socket we connect to """ def __init__(self, path = '/var/run/tor/control', connect = True): @@ -434,7 +441,7 @@ class ControlSocketFile(ControlSocket): """ super(ControlSocketFile, self).__init__() - self._socket_path = path + self.path = path if connect: self.connect() @@ -443,10 +450,13 @@ class ControlSocketFile(ControlSocket): """ Provides the path our socket connects to. + .. deprecated:: 1.7.0 + Use the **path** attribute instead. + :returns: str with the path for our control socket """ - return self._socket_path + return self.path def is_localhost(self): return True @@ -454,7 +464,7 @@ class ControlSocketFile(ControlSocket): def _make_socket(self): try: control_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - control_socket.connect(self._socket_path) + control_socket.connect(self.path) return control_socket except socket.error as exc: raise stem.SocketError(exc) diff --git a/test/integ/control/controller.py b/test/integ/control/controller.py index af7b384f..b2ba77a5 100644 --- a/test/integ/control/controller.py +++ b/test/integ/control/controller.py @@ -364,10 +364,10 @@ class TestController(unittest.TestCase): control_socket = controller.get_socket() if isinstance(control_socket, stem.socket.ControlPort): - connection_value = str(control_socket.get_port()) + connection_value = str(control_socket.port) config_key = 'ControlPort' elif isinstance(control_socket, stem.socket.ControlSocketFile): - connection_value = str(control_socket.get_socket_path()) + connection_value = control_socket.path config_key = 'ControlSocket' # successful single query

1 0

[stem/master] Stub initial Relay class
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit c01f50f51adfb12969d3d76b265c8b81f7ef2bc2 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 1 13:37:09 2018 -0800 Stub initial Relay class Still feeling out the direction we'll go, but I suspect we're gonna have a Relay class providing high level methods the same way as our Controller does. --- stem/client.py | 24 ++++++++++++++++++++++++ stem/socket.py | 4 +++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/stem/client.py b/stem/client.py new file mode 100644 index 00000000..e1011f96 --- /dev/null +++ b/stem/client.py @@ -0,0 +1,24 @@ +# Copyright 2018, Damian Johnson and The Tor Project +# See LICENSE for licensing information + +""" +Module for interacting with the ORPort provided by Tor relays. The +:class:`~stem.client.Relay` is a wrapper for :class:`~stem.socket.RelaySocket`, +providing higher level functions in much the same way as our +:class:`~stem.control.Controller` wraps :class:`~stem.socket.ControlSocket`. + +.. versionadded:: 1.7.0 + +**Module Overview:** + +:: + + Relay - Connection with a relay's ORPort. +""" + + +class Relay(object): + """ + Connection with a `Tor relay's ORPort + <https://gitweb.torproject.org/torspec.git/tree/tor-spec.txt>`_. + """ diff --git a/stem/socket.py b/stem/socket.py index fe902f0a..7a2ec9de 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -2,7 +2,7 @@ # See LICENSE for licensing information """ -Supports communication with sockets speaking the Tor control protocol. This +Supports communication with sockets speaking Tor protocols. This allows us to send messages as basic strings, and receive responses as :class:`~stem.response.ControlMessage` instances. @@ -351,6 +351,8 @@ class RelaySocket(BaseSocket): <https://gitweb.torproject.org/torspec.git/tree/tor-spec.txt>`_ to a Tor relay. + .. versionadded:: 1.7.0 + :var str address: address our socket connects to :var int port: ORPort our socket connects to """

1 0

[stem/master] Read directly from socket with RelaySocket
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 4d3f18df7243ac149ab402386f8b7f7695a4e51d Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 1 11:20:57 2018 -0800 Read directly from socket with RelaySocket Ick, struggled with this for a while. The file's read() retrieves data until a given amount of data has been reached or the socket closed, whereas recv simply reads whatever is available. Struggled for a while to get read() to behave like recv() but no luck. Just having the RelaySocket read directly from the socket instead... https://stackoverflow.com/questions/6946629/can-i-get-a-socket-makefile-to-… --- stem/socket.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/stem/socket.py b/stem/socket.py index 01626068..20fbf6d0 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -238,7 +238,7 @@ class BaseSocket(object): :: - my_handler(socket_file, message) + my_handler(socket, socket_file, message) """ with self._send_lock: @@ -246,7 +246,7 @@ class BaseSocket(object): if not self.is_alive(): raise stem.SocketClosed() - handler(self._socket_file, message) + handler(self._socket, self._socket_file, message) except stem.SocketClosed: # if send_message raises a SocketClosed then we should properly shut # everything down @@ -262,7 +262,7 @@ class BaseSocket(object): :: - my_handler(socket_file) + my_handler(socket, socket_file) """ with self._recv_lock: @@ -270,12 +270,12 @@ class BaseSocket(object): # makes a temporary reference to the _socket_file because connect() # and close() may set or unset it - socket_file = self._socket_file + my_socket, my_socket_file = self._socket, self._socket_file - if not socket_file: + if not my_socket or not my_socket_file: raise stem.SocketClosed() - return handler(socket_file) + return handler(my_socket, my_socket_file) except stem.SocketClosed: # If recv_message raises a SocketClosed then we should properly shut # everything down. However, there's a couple cases where this will @@ -385,7 +385,7 @@ class RelaySocket(BaseSocket): * :class:`stem.SocketClosed` if the socket is known to be shut down """ - self._send(message, _write_to_socket) + self._send(message, lambda s, sf, msg: _write_to_socket(sf, msg)) def recv(self, max_response_size = MAX_READ_BUFFER_LEN): """ @@ -403,10 +403,7 @@ class RelaySocket(BaseSocket): # TODO: Not really sure what we'll want here. To start with just copying # endosome's behavior. - def _read(control_file): - return control_file.read(max_response_size) - - return self._recv(_read) + return self._recv(lambda s, sf: s.recv(max_response_size)) def is_localhost(self): return self.address == '127.0.0.1' @@ -450,7 +447,7 @@ class ControlSocket(BaseSocket): * :class:`stem.SocketClosed` if the socket is known to be shut down """ - self._send(message, send_message) + self._send(message, lambda s, sf, msg: send_message(sf, msg)) def recv(self): """ @@ -464,7 +461,7 @@ class ControlSocket(BaseSocket): * :class:`stem.SocketClosed` if the socket closes before we receive a complete message """ - return self._recv(recv_message) + return self._recv(lambda s, sf: recv_message(sf)) class ControlPort(ControlSocket):

1 0

[stem/master] Stub initial RelaySocket
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit a11428262d283dcfb02d70fd2e99080cdc1033fc Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 31 14:34:05 2017 -0800 Stub initial RelaySocket Ok, time to start adding ORPort support. Adding a socket class similar to our ControlSocket. Unfotunately we need to fill this out a bit more before it'll be testable. --- stem/socket.py | 105 ++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 96 insertions(+), 9 deletions(-) diff --git a/stem/socket.py b/stem/socket.py index 6bf13a39..1b9cea5f 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -47,6 +47,10 @@ Tor... :: BaseSocket - Thread safe socket. + |- RelaySocket - Socket for a relay's ORPort. + | |- send - sends a message to the socket + | +- recv - receives a response from the socket + | |- ControlSocket - Socket wrapper that speaks the tor control protocol. | |- ControlPort - Control connection via a port. | |- ControlSocketFile - Control connection via a local file socket. @@ -86,6 +90,10 @@ ERROR_MSG = 'Error while receiving a control message (%s): %s' TRUNCATE_LOGS = 10 +# maximum number of bytes to read at a time from a relay socket + +MAX_READ_BUFFER_LEN = 10 * 1024 * 1024 + class BaseSocket(object): """ @@ -336,6 +344,81 @@ class BaseSocket(object): raise NotImplementedError('Unsupported Operation: this should be implemented by the BaseSocket subclass') +class RelaySocket(BaseSocket): + """ + `Link-level connection + <https://gitweb.torproject.org/torspec.git/tree/tor-spec.txt>`_ to a Tor + relay. + + :var str address: address our socket connects to + :var int port: ORPort our socket connects to + """ + + def __init__(self, address = '127.0.0.1', port = 9050, connect = True): + """ + RelaySocket constructor. + + :param str address: ip address of the relay + :param int port: orport of the relay + :param bool connect: connects to the socket if True, leaves it unconnected otherwise + + :raises: :class:`stem.SocketError` if connect is **True** and we're + unable to establish a connection + """ + + super(RelaySocket, self).__init__() + self.address = address + self.port = port + + if connect: + self.connect() + + def send(self, message): + """ + Sends a message to the relay's ORPort. + + :param str message: message to be formatted and sent to the socket + + :raises: + * :class:`stem.SocketError` if a problem arises in using the socket + * :class:`stem.SocketClosed` if the socket is known to be shut down + """ + + self._send(message, _write_to_socket) + + def recv(self, max_response_size = MAX_READ_BUFFER_LEN): + """ + Receives a message from the relay. + + :param int max_response_size: maximum bytes to return + + :returns: bytes for the message received + + :raises: + * :class:`stem.ProtocolError` the content from the socket is malformed + * :class:`stem.SocketClosed` if the socket closes before we receive a complete message + """ + + # TODO: Not really sure what we'll want here. To start with just copying + # endosome's behavior. + + def _read(control_file): + return control_file.read(max_response_size) + + return self._recv(_read) + + def is_localhost(self): + return self.address == '127.0.0.1' + + def _make_socket(self): + try: + relay_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + relay_socket.connect((self.address, self.port)) + return relay_socket + except socket.error as exc: + raise stem.SocketError(exc) + + class ControlSocket(BaseSocket): """ Wrapper for a socket connection that speaks the Tor control protocol. To the @@ -531,16 +614,20 @@ def send_message(control_file, message, raw = False): if not raw: message = send_formatting(message) - try: - control_file.write(stem.util.str_tools._to_bytes(message)) - control_file.flush() + _write_to_socket(control_file, message) - if log.is_tracing(): - log_message = message.replace('\r\n', '\n').rstrip() - msg_div = '\n' if '\n' in log_message else ' ' - log.trace('Sent to tor:%s%s' % (msg_div, log_message)) + if log.is_tracing(): + log_message = message.replace('\r\n', '\n').rstrip() + msg_div = '\n' if '\n' in log_message else ' ' + log.trace('Sent to tor:%s%s' % (msg_div, log_message)) + + +def _write_to_socket(socket_file, message): + try: + socket_file.write(stem.util.str_tools._to_bytes(message)) + socket_file.flush() except socket.error as exc: - log.info('Failed to send message: %s' % exc) + log.info('Failed to send: %s' % exc) # When sending there doesn't seem to be a reliable method for # distinguishing between failures from a disconnect verses other things. @@ -554,7 +641,7 @@ def send_message(control_file, message, raw = False): # if the control_file has been closed then flush will receive: # AttributeError: 'NoneType' object has no attribute 'sendall' - log.info('Failed to send message: file has been closed') + log.info('Failed to send: file has been closed') raise stem.SocketClosed('file has been closed')

1 0

[stem/master] Add cell metadata
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 309cb3f1200bce50134b653c18b182d107f5ef08 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 3 13:50:49 2018 -0800 Add cell metadata Adding the CELL_COMMAND table from endosome. --- stem/client.py | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++ test/settings.cfg | 1 + test/unit/__init__.py | 1 + test/unit/client.py | 24 +++++++++++++++++++++++ 4 files changed, 80 insertions(+) diff --git a/stem/client.py b/stem/client.py index 8c5ad506..73c91ef5 100644 --- a/stem/client.py +++ b/stem/client.py @@ -31,6 +31,7 @@ providing higher level functions in much the same way as our Relay - Connection with a relay's ORPort. """ +import collections import struct from stem.util import enum @@ -43,6 +44,41 @@ PackType = enum.Enum( ) +class CellAttributes(collections.namedtuple('CellAttributes', ['name', 'value', 'fixed_length', 'for_circuit'])): + """ + Metadata for cells tor will accept on its ORPort. + + :var str name: name of the cell type + :var int value: integer value of the command on the wire + :var bool fixed_length: **True** if cells have a fixed length, + **False** if variable + :var bool for_circuit: **True** if command is for a circuit, **False** + otherwise + """ + + +CELL_ATTR = ( + CellAttributes('PADDING', 0, True, False), # Padding (section 7.2) + CellAttributes('CREATE', 1, True, True), # Create a circuit (section 5.1) + CellAttributes('CREATED', 2, True, True), # Acknowledge create (section 5.1) + CellAttributes('RELAY', 3, True, True), # End-to-end data (section 5.5 and 6) + CellAttributes('DESTROY', 4, True, True), # Stop using a circuit (section 5.4) + CellAttributes('CREATE_FAST', 5, True, True), # Create a circuit, no PK (section 5.1) + CellAttributes('CREATED_FAST', 6, True, True), # Circuit created, no PK (section 5.1) + CellAttributes('VERSIONS', 7, False, False), # Negotiate proto version (section 4) + CellAttributes('NETINFO', 8, True, False), # Time and address info (section 4.5) + CellAttributes('RELAY_EARLY', 9, True, True), # End-to-end data; limited (section 5.6) + CellAttributes('CREATE2', 10, True, True), # Extended CREATE cell (section 5.1) + CellAttributes('CREATED2', 11, True, True), # Extended CREATED cell (section 5.1) + CellAttributes('PADDING_NEGOTIATE', 12, True, False), # Padding negotiation (section 7.2) + CellAttributes('VPADDING', 128, False, False), # Variable-length padding (section 7.2) + CellAttributes('CERTS', 129, False, False), # Certificates (section 4.2) + CellAttributes('AUTH_CHALLENGE', 130, False, False), # Challenge value (section 4.3) + CellAttributes('AUTHENTICATE', 131, False, False), # Client authentication (section 4.5) + CellAttributes('AUTHORIZE', 132, False, False), # Client authorization (not yet used) +) + + class Relay(object): """ Connection with a `Tor relay's ORPort @@ -50,6 +86,24 @@ class Relay(object): """ +def cell_attributes(cell_type): + """ + Provides attributes of the given cell type. + + :parm str,int cell_type: cell type as either a string or integer + + :raise: **ValueError** if cell type is invalid + """ + + param = 'value' if isinstance(cell_type, int) else 'name' + + for attr in CELL_ATTR: + if getattr(attr, param) == cell_type: + return attr + + raise ValueError("'%s' isn't a valid cell type" % cell_type) + + def serialize_versions(versions): """ Provides the payload for a series of link versions. diff --git a/test/settings.cfg b/test/settings.cfg index 4e9b2ab9..501dff9c 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -229,6 +229,7 @@ test.unit_tests |test.unit.response.authchallenge.TestAuthChallengeResponse |test.unit.response.protocolinfo.TestProtocolInfoResponse |test.unit.response.mapaddress.TestMapAddressResponse +|test.unit.client.TestClient |test.unit.connection.authentication.TestAuthenticate |test.unit.connection.connect.TestConnect |test.unit.control.controller.TestControl diff --git a/test/unit/__init__.py b/test/unit/__init__.py index 6cdbe98b..34fcbb19 100644 --- a/test/unit/__init__.py +++ b/test/unit/__init__.py @@ -6,6 +6,7 @@ import os import test __all__ = [ + 'client', 'connection', 'control', 'descriptor', diff --git a/test/unit/client.py b/test/unit/client.py new file mode 100644 index 00000000..4761cf8e --- /dev/null +++ b/test/unit/client.py @@ -0,0 +1,24 @@ +""" +Unit tests for the stem.client. +""" + +import unittest + +import stem.client + + +class TestClient(unittest.TestCase): + def test_cell_attributes(self): + attr = stem.client.cell_attributes('NETINFO') + + self.assertEqual('NETINFO', attr.name) + self.assertEqual(8, attr.value) + self.assertEqual(True, attr.fixed_length) + self.assertEqual(False, attr.for_circuit) + + self.assertEqual(10, stem.client.cell_attributes('CREATE2').value) + self.assertEqual('CREATE2', stem.client.cell_attributes(10).name) + + self.assertRaises(ValueError, stem.client.cell_attributes, 'NOPE') + self.assertRaises(ValueError, stem.client.cell_attributes, 85) + self.assertRaises(ValueError, stem.client.cell_attributes, None)

1 0

[stem/master] Move send and recv implementation to BaseSocket
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit f0d78cf8e509531aad79da2f73ea3a6af3c9548e Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 31 14:12:58 2017 -0800 Move send and recv implementation to BaseSocket Boilerplate for sending and receiving is common for all sockets. Only thing that differs is message formatting and parsing. --- stem/socket.py | 114 ++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 68 insertions(+), 46 deletions(-) diff --git a/stem/socket.py b/stem/socket.py index 8ff316a7..6bf13a39 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -223,6 +223,72 @@ class BaseSocket(object): if is_change: self._close() + def _send(self, message, handler): + """ + Send message in a thread safe manner. Handler is expected to be of the form... + + :: + + my_handler(socket_file, message) + """ + + with self._send_lock: + try: + if not self.is_alive(): + raise stem.SocketClosed() + + handler(self._socket_file, message) + except stem.SocketClosed: + # if send_message raises a SocketClosed then we should properly shut + # everything down + + if self.is_alive(): + self.close() + + raise + + def _recv(self, handler): + """ + Receives a message in a thread safe manner. Handler is expected to be of the form... + + :: + + my_handler(socket_file) + """ + + with self._recv_lock: + try: + # makes a temporary reference to the _socket_file because connect() + # and close() may set or unset it + + socket_file = self._socket_file + + if not socket_file: + raise stem.SocketClosed() + + return handler(socket_file) + except stem.SocketClosed: + # If recv_message raises a SocketClosed then we should properly shut + # everything down. However, there's a couple cases where this will + # cause deadlock... + # + # * This SocketClosed was *caused by* a close() call, which is joining + # on our thread. + # + # * A send() call that's currently in flight is about to call close(), + # also attempting to join on us. + # + # To resolve this we make a non-blocking call to acquire the send lock. + # If we get it then great, we can close safely. If not then one of the + # above are in progress and we leave the close to them. + + if self.is_alive(): + if self._send_lock.acquire(False): + self.close() + self._send_lock.release() + + raise + def _get_send_lock(self): """ The send lock is useful to classes that interact with us at a deep level @@ -300,20 +366,7 @@ class ControlSocket(BaseSocket): * :class:`stem.SocketClosed` if the socket is known to be shut down """ - with self._send_lock: - try: - if not self.is_alive(): - raise stem.SocketClosed() - - send_message(self._socket_file, message) - except stem.SocketClosed: - # if send_message raises a SocketClosed then we should properly shut - # everything down - - if self.is_alive(): - self.close() - - raise + self._send(message, send_message) def recv(self): """ @@ -327,38 +380,7 @@ class ControlSocket(BaseSocket): * :class:`stem.SocketClosed` if the socket closes before we receive a complete message """ - with self._recv_lock: - try: - # makes a temporary reference to the _socket_file because connect() - # and close() may set or unset it - - socket_file = self._socket_file - - if not socket_file: - raise stem.SocketClosed() - - return recv_message(socket_file) - except stem.SocketClosed: - # If recv_message raises a SocketClosed then we should properly shut - # everything down. However, there's a couple cases where this will - # cause deadlock... - # - # * This SocketClosed was *caused by* a close() call, which is joining - # on our thread. - # - # * A send() call that's currently in flight is about to call close(), - # also attempting to join on us. - # - # To resolve this we make a non-blocking call to acquire the send lock. - # If we get it then great, we can close safely. If not then one of the - # above are in progress and we leave the close to them. - - if self.is_alive(): - if self._send_lock.acquire(False): - self.close() - self._send_lock.release() - - raise + return self._recv(recv_message) class ControlPort(ControlSocket):

1 0

[stem/master] Rename fixed_length to fixed_size
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit b0cf9dbaacab439cf10e3977c0bf9e63c268e6a5 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 3 23:29:25 2018 -0800 Rename fixed_length to fixed_size It's shorter. :P --- stem/client.py | 8 ++++---- test/unit/client.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stem/client.py b/stem/client.py index 73c91ef5..136fc240 100644 --- a/stem/client.py +++ b/stem/client.py @@ -44,16 +44,16 @@ PackType = enum.Enum( ) -class CellAttributes(collections.namedtuple('CellAttributes', ['name', 'value', 'fixed_length', 'for_circuit'])): +class CellAttributes(collections.namedtuple('CellAttributes', ['name', 'value', 'fixed_size', 'for_circuit'])): """ Metadata for cells tor will accept on its ORPort. :var str name: name of the cell type :var int value: integer value of the command on the wire - :var bool fixed_length: **True** if cells have a fixed length, + :var bool fixed_size: **True** if cells have a fixed length, **False** if variable - :var bool for_circuit: **True** if command is for a circuit, **False** - otherwise + :var bool for_circuit: **True** if command is for a circuit, + **False** otherwise """ diff --git a/test/unit/client.py b/test/unit/client.py index 4761cf8e..204c55fd 100644 --- a/test/unit/client.py +++ b/test/unit/client.py @@ -13,7 +13,7 @@ class TestClient(unittest.TestCase): self.assertEqual('NETINFO', attr.name) self.assertEqual(8, attr.value) - self.assertEqual(True, attr.fixed_length) + self.assertEqual(True, attr.fixed_size) self.assertEqual(False, attr.for_circuit) self.assertEqual(10, stem.client.cell_attributes('CREATE2').value)

1 0

[stem/master] Fully remove 'raw' argument
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 333eb6c03bf8ff90bcedf08945d5c10d3113784d Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 31 14:00:31 2017 -0800 Fully remove 'raw' argument To provide backward compatability we deprecate rather than outright remove things, but in this case it's low enough level that it's probably safe to simply drop it. Unfortunatley keeping raw would complicate other refactoring I have in mind. --- stem/socket.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/stem/socket.py b/stem/socket.py index 7020cdf6..8ff316a7 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -283,13 +283,13 @@ class ControlSocket(BaseSocket): def __init__(self): super(ControlSocket, self).__init__() - def send(self, message, raw = None): + def send(self, message): """ Formats and sends a message to the control socket. For more information see the :func:`~stem.socket.send_message` function. .. deprecated:: 1.7.0 - The **raw** argument is unhelpful and will be removed. Use + The **raw** argument was unhelpful and be removed. Use :func:`stem.socket.send_message` if you need this level of control instead. @@ -305,10 +305,7 @@ class ControlSocket(BaseSocket): if not self.is_alive(): raise stem.SocketClosed() - if raw is None: - send_message(self._socket_file, message) - else: - send_message(self._socket_file, message, raw) + send_message(self._socket_file, message) except stem.SocketClosed: # if send_message raises a SocketClosed then we should properly shut # everything down

1 0

[stem/master] Add serialize_versions helper
by atagar＠torproject.org 21 Jan '18

21 Jan '18

commit 222bff37e5fb896341a3a4157c5608d44b8315e0 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jan 3 13:11:19 2018 -0800 Add serialize_versions helper Probably not what we're gonna have long term, but lets start moving some endosome functionality over. --- stem/client.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/stem/client.py b/stem/client.py index 4da12bba..8c5ad506 100644 --- a/stem/client.py +++ b/stem/client.py @@ -31,6 +31,8 @@ providing higher level functions in much the same way as our Relay - Connection with a relay's ORPort. """ +import struct + from stem.util import enum PackType = enum.Enum( @@ -46,3 +48,15 @@ class Relay(object): Connection with a `Tor relay's ORPort <https://gitweb.torproject.org/torspec.git/tree/tor-spec.txt>`_. """ + + +def serialize_versions(versions): + """ + Provides the payload for a series of link versions. + + :param list versions: link versions to serialize + + :returns: **bytes** with a payload for these versions + """ + + return b''.join([struct.pack(PackType.SHORT, v) for v in versions])

1 0