July 2020 - tor-commits - lists.torproject.org

[stem/master] Migrate to asyncio
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit 6f8f8d5c26079ed0e1e08bbb5815f2f39891a46e Merge: 8634aa04 46dec7b9 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jul 15 18:10:33 2020 -0700 Migrate to asyncio Python 3.5 added asyncio, an asynchronous IO framework similar to Twisted... https://www.python.org/dev/peps/pep-0492/ This added new python keywords (async and await) which give callers more control over how they await asynchronous operations (for instance, asyncio.wait_for() to apply a timeout). This branch migrates our stem.control, stem.client, and stem.descriptor.remote modules from a synchronous to asynchronous implementation. Usually this would preclude us from being used by non-asyncio users, but this also adds a Synchronous mixin that allows us to be used from either context. In other words, internally Stem is now an asynchronous library that is usable by asyncio users, while retaining its ability to be used by synchronous users in the exact same way we always have. Win-win for everyone. Many thanks to Illia for all his hard work on this branch! .gitignore | 3 + run_tests.py | 19 +- stem/client/__init__.py | 93 ++-- stem/connection.py | 227 ++++++--- stem/control.py | 648 ++++++++++++++----------- stem/descriptor/remote.py | 266 +++++----- stem/interpreter/__init__.py | 5 +- stem/interpreter/autocomplete.py | 8 +- stem/interpreter/commands.py | 12 +- stem/interpreter/help.py | 9 +- stem/response/__init__.py | 2 +- stem/socket.py | 441 +++++++++-------- stem/util/__init__.py | 226 ++++++++- stem/util/test_tools.py | 61 ++- test/integ/client/connection.py | 38 +- test/integ/connection/authentication.py | 143 +++--- test/integ/connection/connect.py | 17 +- test/integ/control/base_controller.py | 133 ++--- test/integ/control/controller.py | 828 ++++++++++++++++++-------------- test/integ/process.py | 28 +- test/integ/response/protocolinfo.py | 43 +- test/integ/socket/control_message.py | 84 ++-- test/integ/socket/control_socket.py | 84 ++-- test/integ/util/connection.py | 6 +- test/integ/util/proc.py | 6 +- test/integ/version.py | 12 +- test/runner.py | 48 +- test/settings.cfg | 11 + test/unit/connection/authentication.py | 36 +- test/unit/connection/connect.py | 54 ++- test/unit/control/controller.py | 206 ++++---- test/unit/descriptor/remote.py | 230 ++++----- test/unit/response/control_message.py | 10 +- test/unit/util/synchronous.py | 237 +++++++++ 34 files changed, 2602 insertions(+), 1672 deletions(-)

1 0

[stem/master] Permit all controller types by connect()
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit 46dec7b974a7a835f9b1ff58e5d905f75110afde Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jul 15 17:04:04 2020 -0700 Permit all controller types by connect() Our 'controller' argument's limitations were an artifact of having a separate sync/async Controller. We can also drop our _connect_async() helper, which became just an alias. --- stem/connection.py | 24 ++++++++---------------- stem/socket.py | 3 +++ 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/stem/connection.py b/stem/connection.py index 8495da2a..1d386594 100644 --- a/stem/connection.py +++ b/stem/connection.py @@ -257,8 +257,8 @@ def connect(control_port: Tuple[str, Union[str, int]] = ('127.0.0.1', 'default') :param password_prompt: prompt for the controller password if it wasn't supplied :param chroot_path: path prefix if in a chroot environment - :param controller: :class:`~stem.control.Controller` subclass to be - returned + :param controller: :class:`~stem.control.BaseController` subclass to be + returned, this provides a :class:`~stem.socket.ControlSocket` if **None** :returns: authenticated control connection, the type based on the controller argument @@ -267,9 +267,8 @@ def connect(control_port: Tuple[str, Union[str, int]] = ('127.0.0.1', 'default') """ # TODO: change this function's API so we can provide a concrete type - - if controller is None or not issubclass(controller, stem.control.Controller): - raise ValueError('Controller should be a stem.control.Controller subclass.') + # TODO: 'loop_thread' gets orphaned, causing the thread to linger until we + # change this function's API loop = asyncio.new_event_loop() loop_thread = threading.Thread(target = loop.run_forever, name = 'asyncio') @@ -277,7 +276,7 @@ def connect(control_port: Tuple[str, Union[str, int]] = ('127.0.0.1', 'default') loop_thread.start() try: - connection = asyncio.run_coroutine_threadsafe(_connect_async(control_port, control_socket, password, password_prompt, chroot_path, controller), loop).result() + connection = asyncio.run_coroutine_threadsafe(connect_async(control_port, control_socket, password, password_prompt, chroot_path, controller), loop).result() if connection is None and loop_thread.is_alive(): loop.call_soon_threadsafe(loop.stop) @@ -319,7 +318,7 @@ async def connect_async(control_port: Tuple[str, Union[str, int]] = ('127.0.0.1' supplied :param chroot_path: path prefix if in a chroot environment :param controller: :class:`~stem.control.BaseController` subclass to be - returned + returned, this provides a :class:`~stem.socket.ControlSocket` if **None** :returns: authenticated control connection, the type based on the controller argument @@ -327,12 +326,6 @@ async def connect_async(control_port: Tuple[str, Union[str, int]] = ('127.0.0.1' **control_port** and **control_socket** are **None** """ - if controller and not issubclass(controller, stem.control.BaseController): - raise ValueError('The provided controller should be a stem.control.BaseController subclass.') - return await _connect_async(control_port, control_socket, password, password_prompt, chroot_path, controller) - - -async def _connect_async(control_port: Tuple[str, Union[str, int]], control_socket: str, password: Optional[str], password_prompt: bool, chroot_path: Optional[str], controller: Type[Union[stem.control.BaseController, stem.control.Controller]]) -> Any: if control_port is None and control_socket is None: raise ValueError('Neither a control port nor control socket were provided. Nothing to connect to.') elif control_port: @@ -394,9 +387,8 @@ async def _connect_auth(control_socket: stem.socket.ControlSocket, password: str :param password_prompt: prompt for the controller password if it wasn't supplied :param chroot_path: path prefix if in a chroot environment - :param controller: :class:`~stem.control.BaseController` or - :class:`~stem.control.Controller` subclass to be returned, this provides a - :class:`~stem.socket.ControlSocket` if **None** + :param controller: :class:`~stem.control.BaseController` subclass to be + returned, this provides a :class:`~stem.socket.ControlSocket` if **None** :returns: authenticated control connection, the type based on the controller argument """ diff --git a/stem/socket.py b/stem/socket.py index 8de13ba0..a19a8f25 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -687,6 +687,9 @@ def recv_message_from_bytes_io(reader: BinaryIO, arrived_at: Optional[float] = N a complete message """ + # TODO: We should deduplicate this with recv_message(), but separating IO + # from the low level aspects of this parsing will be difficult. + parsed_content = [] # type: List[Tuple[str, str, bytes]] raw_content = bytearray() first_line = True

1 0

[stem/master] Make `Controller.close_stream` asynchronous
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit 6e0e2f4559bf0ed1ca4ef649652f6e2749ca4221 Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Tue Apr 21 22:23:35 2020 +0300 Make `Controller.close_stream` asynchronous --- stem/control.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stem/control.py b/stem/control.py index 8eaa8bd8..b36497a5 100644 --- a/stem/control.py +++ b/stem/control.py @@ -3618,7 +3618,7 @@ class Controller(BaseController): else: raise stem.ProtocolError('ATTACHSTREAM returned unexpected response code: %s' % response.code) - def close_stream(self, stream_id: str, reason: stem.RelayEndReason = stem.RelayEndReason.MISC, flag: str = '') -> None: + async def close_stream(self, stream_id: str, reason: stem.RelayEndReason = stem.RelayEndReason.MISC, flag: str = '') -> None: """ Closes the specified stream. @@ -3634,7 +3634,7 @@ class Controller(BaseController): # there's a single value offset between RelayEndReason.index_of() and the # value that tor expects since tor's value starts with the index of one - response = stem.response._convert_to_single_line(self.msg('CLOSESTREAM %s %s %s' % (stream_id, stem.RelayEndReason.index_of(reason) + 1, flag))) + response = stem.response._convert_to_single_line(await self.msg('CLOSESTREAM %s %s %s' % (stream_id, stem.RelayEndReason.index_of(reason) + 1, flag))) if not response.is_ok(): if response.code in ('512', '552'):

1 0

[stem/master] Make `Controller.enable_feature` asynchronous
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit 0355c06962d9c807b6330d36eb363d6cfb5ba162 Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Tue Apr 21 22:27:26 2020 +0300 Make `Controller.enable_feature` asynchronous --- stem/control.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stem/control.py b/stem/control.py index b36497a5..f8fae01e 100644 --- a/stem/control.py +++ b/stem/control.py @@ -3320,7 +3320,7 @@ class Controller(BaseController): return feature in self._enabled_features - def enable_feature(self, features: Union[str, Sequence[str]]) -> None: + async def enable_feature(self, features: Union[str, Sequence[str]]) -> None: """ Enables features that are disabled by default to maintain backward compatibility. Once enabled, a feature cannot be disabled and a new @@ -3337,7 +3337,7 @@ class Controller(BaseController): if isinstance(features, (bytes, str)): features = [features] - response = stem.response._convert_to_single_line(self.msg('USEFEATURE %s' % ' '.join(features))) + response = stem.response._convert_to_single_line(await self.msg('USEFEATURE %s' % ' '.join(features))) if not response.is_ok(): if response.code == '552':

1 0

[stem/master] Make `Controller.create_ephemeral_hidden_service` asynchronous
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit 83cfe4cf517e72d5ff28731da849462ac13d7b6a Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Tue Apr 21 22:59:20 2020 +0300 Make `Controller.create_ephemeral_hidden_service` asynchronous --- stem/control.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/stem/control.py b/stem/control.py index 618408c3..dce1584b 100644 --- a/stem/control.py +++ b/stem/control.py @@ -2832,7 +2832,7 @@ class Controller(BaseController): return [r for r in result if r] # drop any empty responses (GETINFO is blank if unset) - def create_ephemeral_hidden_service(self, ports: Union[int, Sequence[int], Mapping[int, str]], key_type: str = 'NEW', key_content: str = 'BEST', discard_key: bool = False, detached: bool = False, await_publication: bool = False, timeout: Optional[float] = None, basic_auth: Optional[Mapping[str, str]] = None, max_streams: Optional[int] = None) -> stem.response.add_onion.AddOnionResponse: + async def create_ephemeral_hidden_service(self, ports: Union[int, Sequence[int], Mapping[int, str]], key_type: str = 'NEW', key_content: str = 'BEST', discard_key: bool = False, detached: bool = False, await_publication: bool = False, timeout: Optional[float] = None, basic_auth: Optional[Mapping[str, str]] = None, max_streams: Optional[int] = None) -> stem.response.add_onion.AddOnionResponse: """ Creates a new hidden service. Unlike :func:`~stem.control.Controller.create_hidden_service` this style of @@ -2934,15 +2934,15 @@ class Controller(BaseController): * :class:`stem.Timeout` if **timeout** was reached """ - hs_desc_queue = queue.Queue() # type: queue.Queue[stem.response.events.Event] + hs_desc_queue = asyncio.Queue() # type: asyncio.Queue[stem.response.events.Event] hs_desc_listener = None start_time = time.time() if await_publication: - def hs_desc_listener(event: stem.response.events.Event) -> None: - hs_desc_queue.put(event) + async def hs_desc_listener(event: stem.response.events.Event) -> None: + await hs_desc_queue.put(event) - self.add_event_listener(hs_desc_listener, EventType.HS_DESC) + await self.add_event_listener(hs_desc_listener, EventType.HS_DESC) request = 'ADD_ONION %s:%s' % (key_type, key_content) @@ -2960,7 +2960,7 @@ class Controller(BaseController): if max_streams is not None: flags.append('MaxStreamsCloseCircuit') - if self.get_conf('HiddenServiceSingleHopMode', None) == '1' and self.get_conf('HiddenServiceNonAnonymousMode', None) == '1': + if (await self.get_conf('HiddenServiceSingleHopMode', None)) == '1' and (await self.get_conf('HiddenServiceNonAnonymousMode', None)) == '1': flags.append('NonAnonymous') if flags: @@ -2987,7 +2987,7 @@ class Controller(BaseController): else: request += ' ClientAuth=%s' % client_name - response = stem.response._convert_to_add_onion(stem.response._convert_to_add_onion(self.msg(request))) + response = stem.response._convert_to_add_onion(await self.msg(request)) if await_publication: # We should receive five UPLOAD events, followed by up to another five @@ -3000,7 +3000,7 @@ class Controller(BaseController): try: while True: - event = _get_with_timeout(hs_desc_queue, timeout, start_time) + event = await _get_with_timeout(hs_desc_queue, timeout, start_time) if event.action == stem.HSDescAction.UPLOAD and event.address == response.service_id: directories_uploaded_to.append(event.directory_fingerprint) @@ -3012,7 +3012,7 @@ class Controller(BaseController): if len(directories_uploaded_to) == len(failures): raise stem.OperationFailed(message = 'Failed to upload our hidden service descriptor to %s' % ', '.join(failures)) finally: - self.remove_event_listener(hs_desc_listener) + await self.remove_event_listener(hs_desc_listener) return response

1 0

[stem/master] Make `Controller.remove_ephemeral_hidden_service` asynchronous
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit 211c726637f744b201a2871a61090dfaad6decc1 Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Tue Apr 21 22:59:48 2020 +0300 Make `Controller.remove_ephemeral_hidden_service` asynchronous --- stem/control.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stem/control.py b/stem/control.py index dce1584b..bcf34f6c 100644 --- a/stem/control.py +++ b/stem/control.py @@ -3016,7 +3016,7 @@ class Controller(BaseController): return response - def remove_ephemeral_hidden_service(self, service_id: str) -> bool: + async def remove_ephemeral_hidden_service(self, service_id: str) -> bool: """ Discontinues a given hidden service that was created with :func:`~stem.control.Controller.create_ephemeral_hidden_service`. @@ -3031,7 +3031,7 @@ class Controller(BaseController): :raises: :class:`stem.ControllerError` if the call fails """ - response = stem.response._convert_to_single_line(self.msg('DEL_ONION %s' % service_id)) + response = stem.response._convert_to_single_line(await self.msg('DEL_ONION %s' % service_id)) if response.is_ok(): return True

1 0

[stem/master] Make `load_conf` and `save_conf` of `Controller` asynchronous
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit 8e8814552a68cd9926363e4953f969144667448a Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Tue Apr 21 22:40:45 2020 +0300 Make `load_conf` and `save_conf` of `Controller` asynchronous --- stem/control.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stem/control.py b/stem/control.py index f8fae01e..93b7b924 100644 --- a/stem/control.py +++ b/stem/control.py @@ -3259,7 +3259,7 @@ class Controller(BaseController): self._request_cache = {} self._last_newnym = 0.0 - def load_conf(self, configtext: str) -> None: + async def load_conf(self, configtext: str) -> None: """ Sends the configuration text to Tor and loads it as if it has been read from the torrc. @@ -3269,7 +3269,7 @@ class Controller(BaseController): :raises: :class:`stem.ControllerError` if the call fails """ - response = stem.response._convert_to_single_line(self.msg('LOADCONF\n%s' % configtext)) + response = stem.response._convert_to_single_line(async self.msg('LOADCONF\n%s' % configtext)) if response.code in ('552', '553'): if response.code == '552' and response.message.startswith('Invalid config file: Failed to parse/validate config: Unknown option'): @@ -3278,7 +3278,7 @@ class Controller(BaseController): elif not response.is_ok(): raise stem.ProtocolError('+LOADCONF Received unexpected response\n%s' % str(response)) - def save_conf(self, force: bool = False) -> None: + async def save_conf(self, force: bool = False) -> None: """ Saves the current configuration options into the active torrc file. @@ -3294,7 +3294,7 @@ class Controller(BaseController): the configuration file """ - response = stem.response._convert_to_single_line(self.msg('SAVECONF FORCE' if force else 'SAVECONF')) + response = stem.response._convert_to_single_line(await self.msg('SAVECONF FORCE' if force else 'SAVECONF')) if response.is_ok(): pass

1 0

[stem/master] Make `Controller.remove_hidden_service` asynchronous
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit fab5e14bfaad34ec6eb4f675d2f0b2a40b8e6246 Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Tue Apr 21 23:14:20 2020 +0300 Make `Controller.remove_hidden_service` asynchronous --- stem/control.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stem/control.py b/stem/control.py index 78c70991..6648727f 100644 --- a/stem/control.py +++ b/stem/control.py @@ -2750,7 +2750,7 @@ class Controller(BaseController): config = conf, ) - def remove_hidden_service(self, path: str, port: Optional[int] = None) -> bool: + async def remove_hidden_service(self, path: str, port: Optional[int] = None) -> bool: """ Discontinues a given hidden service. @@ -2769,7 +2769,7 @@ class Controller(BaseController): raise ValueError("%s isn't a valid port number" % port) port = int(port) if port else None - conf = self.get_hidden_service_conf() + conf = await self.get_hidden_service_conf() if path not in conf: return False @@ -2788,7 +2788,7 @@ class Controller(BaseController): if not conf[path]['HiddenServicePort']: del conf[path] # no ports left - self.set_hidden_service_conf(conf) + await self.set_hidden_service_conf(conf) return True @with_default()

1 0

[stem/master] Make `Controller.list_ephemeral_hidden_services` asynchronous
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit c3411da131f40e1ab79c3f7b6ee5a45d06b444f6 Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Tue Apr 21 22:58:07 2020 +0300 Make `Controller.list_ephemeral_hidden_services` asynchronous --- stem/control.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stem/control.py b/stem/control.py index 93b7b924..618408c3 100644 --- a/stem/control.py +++ b/stem/control.py @@ -2792,7 +2792,7 @@ class Controller(BaseController): return True @with_default() - def list_ephemeral_hidden_services(self, default: Any = UNDEFINED, our_services: bool = True, detached: bool = False) -> Sequence[str]: + async def list_ephemeral_hidden_services(self, default: Any = UNDEFINED, our_services: bool = True, detached: bool = False) -> Sequence[str]: """ list_ephemeral_hidden_services(default = UNDEFINED, our_services = True, detached = False) @@ -2821,11 +2821,11 @@ class Controller(BaseController): result = [] if our_services: - result += self.get_info('onions/current').split('\n') + result += (await self.get_info('onions/current')).split('\n') if detached: try: - result += self.get_info('onions/detached').split('\n') + result += (await self.get_info('onions/detached')).split('\n') except (stem.ProtocolError, stem.OperationFailed) as exc: if 'No onion services of the specified type.' not in str(exc): raise

1 0

[stem/master] Rename `Controller` to `AsyncController`
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit 8ef1d67abae624c6219064af75c7203bc67231c2 Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Thu Apr 23 20:18:28 2020 +0300 Rename `Controller` to `AsyncController` --- stem/control.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stem/control.py b/stem/control.py index 6648727f..b7050626 100644 --- a/stem/control.py +++ b/stem/control.py @@ -995,7 +995,7 @@ class BaseController(object): self._event_notice.clear() -class Controller(BaseController): +class AsyncController(BaseController): """ Connection with Tor's control socket. This is built on top of the BaseController and provides a more user friendly API for library users. @@ -1033,7 +1033,7 @@ class Controller(BaseController): else: control_port = stem.socket.ControlPort(address, int(port)) - return Controller(control_port) + return AsyncController(control_port) @staticmethod def from_socket_file(path: str = '/var/run/tor/control') -> 'stem.control.Controller': @@ -1048,7 +1048,7 @@ class Controller(BaseController): """ control_socket = stem.socket.ControlSocketFile(path) - return Controller(control_socket) + return AsyncController(control_socket) def __init__(self, control_socket: stem.socket.ControlSocket, is_authenticated: bool = False) -> None: self._is_caching_enabled = True @@ -1066,7 +1066,7 @@ class Controller(BaseController): self._last_address_exc = None # type: Optional[BaseException] self._last_fingerprint_exc = None # type: Optional[BaseException] - super(Controller, self).__init__(control_socket, is_authenticated) + super(AsyncController, self).__init__(control_socket, is_authenticated) def _sighup_listener(event: stem.response.events.SignalEvent) -> None: if event.signal == Signal.RELOAD: @@ -1100,7 +1100,7 @@ class Controller(BaseController): async def close(self) -> None: self.clear_cache() - await super(Controller, self).close() + await super(AsyncController, self).close() async def authenticate(self, *args: Any, **kwargs: Any) -> None: """ @@ -3772,7 +3772,7 @@ class Controller(BaseController): await self.msg('DROPGUARDS') async def _post_authentication(self) -> None: - await super(Controller, self)._post_authentication() + await super(AsyncController, self)._post_authentication() # try to re-attach event listeners to the new instance

1 0