July 2020 - tor-commits - lists.torproject.org

[stem/master] Move `get_socket` to `_BaseControllerSocketMixin`
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit f238b5b4978fc968fcd31b205ab80272a12629b7 Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Thu Apr 23 20:56:33 2020 +0300 Move `get_socket` to `_BaseControllerSocketMixin` --- stem/control.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/stem/control.py b/stem/control.py index e264e4b5..50dff89f 100644 --- a/stem/control.py +++ b/stem/control.py @@ -589,6 +589,16 @@ class _BaseControllerSocketMixin: return self._socket.connection_time() + def get_socket(self): + """ + Provides the socket used to speak with the tor process. Communicating with + the socket directly isn't advised since it may confuse this controller. + + :returns: :class:`~stem.socket.ControlSocket` we're communicating with + """ + + return self._socket + class BaseController(_BaseControllerSocketMixin): """ @@ -768,16 +778,6 @@ class BaseController(_BaseControllerSocketMixin): if t.is_alive() and threading.current_thread() != t: t.join() - def get_socket(self) -> stem.socket.ControlSocket: - """ - Provides the socket used to speak with the tor process. Communicating with - the socket directly isn't advised since it may confuse this controller. - - :returns: :class:`~stem.socket.ControlSocket` we're communicating with - """ - - return self._socket - def get_latest_heartbeat(self) -> float: """ Provides the unix timestamp for when we last heard from tor. This is zero

1 0

[stem/master] Remove a redundant blank line
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit f4be2a7807babaaaec8c320edc18c3073447d95b Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Sun Apr 26 21:11:39 2020 +0300 Remove a redundant blank line --- stem/control.py | 1 - 1 file changed, 1 deletion(-) diff --git a/stem/control.py b/stem/control.py index d055058e..db539539 100644 --- a/stem/control.py +++ b/stem/control.py @@ -934,7 +934,6 @@ class BaseController(_BaseControllerSocketMixin): for coroutine in (self._reader_loop(), self._event_loop()): self._asyncio_loop_tasks.append(self._asyncio_loop.create_task(coroutine)) - async def _reader_loop(self) -> None: """ Continually pulls from the control socket, directing the messages into

1 0

[stem/master] Fix `_connect_auth`
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit 95d22a73d6f3b7c182ba50ec72697af536072e3b Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Sun Apr 26 21:11:10 2020 +0300 Fix `_connect_auth` --- stem/connection.py | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/stem/connection.py b/stem/connection.py index eac38e2f..3f7983de 100644 --- a/stem/connection.py +++ b/stem/connection.py @@ -127,11 +127,13 @@ fine-grained control over the authentication process. For instance... ============== =========== """ +import asyncio import binascii import getpass import hashlib import hmac import os +import threading import stem.control import stem.response @@ -316,8 +318,23 @@ def _connect_auth(control_socket: stem.socket.ControlSocket, password: str, pass :returns: authenticated control connection, the type based on the controller argument """ + if controller: + loop = controller._asyncio_loop + asyncio_thread = None + else: + loop = asyncio.new_event_loop() + asyncio_thread = threading.Thread(target=loop.run_forever, name='async_auth') + asyncio_thread.setDaemon(True) + asyncio_thread.start() + + def run_coroutine(coroutine): + asyncio.run_coroutine_threadsafe(coroutine, loop).result() + + def close_control_socket(): + run_coroutine(control_socket.close()) + try: - authenticate(control_socket, password, chroot_path) + run_coroutine(authenticate(control_socket, password, chroot_path)) if controller is None: return control_socket @@ -329,41 +346,47 @@ def _connect_auth(control_socket: stem.socket.ControlSocket, password: str, pass else: print(CONNECT_MESSAGES['wrong_socket_type']) - control_socket.close() + close_control_socket() return None except UnrecognizedAuthMethods as exc: print(CONNECT_MESSAGES['uncrcognized_auth_type'].format(auth_methods = ', '.join(exc.unknown_auth_methods))) - control_socket.close() + close_control_socket() return None except IncorrectPassword: print(CONNECT_MESSAGES['incorrect_password']) - control_socket.close() + close_control_socket() return None except MissingPassword: if password is not None: - control_socket.close() + close_control_socket() raise ValueError(CONNECT_MESSAGES['missing_password_bug']) if password_prompt: try: password = getpass.getpass(CONNECT_MESSAGES['password_prompt'] + ' ') except KeyboardInterrupt: - control_socket.close() + close_control_socket() return None return _connect_auth(control_socket, password, password_prompt, chroot_path, controller) else: print(CONNECT_MESSAGES['needs_password']) - control_socket.close() + close_control_socket() return None except UnreadableCookieFile as exc: print(CONNECT_MESSAGES['unreadable_cookie_file'].format(path = exc.cookie_path, issue = str(exc))) - control_socket.close() + close_control_socket() return None except AuthenticationFailure as exc: print(CONNECT_MESSAGES['general_auth_failure'].format(error = exc)) - control_socket.close() + close_control_socket() return None + finally: + if asyncio_thread: + loop.call_soon_threadsafe(loop.stop) + if asyncio_thread.is_alive(): + asyncio_thread.join() + loop.close() async def authenticate(controller: Union[stem.control.BaseController, stem.socket.ControlSocket], password: Optional[str] = None, chroot_path: Optional[str] = None, protocolinfo_response: Optional[stem.response.protocolinfo.ProtocolInfoResponse] = None) -> None:

1 0

[stem/master] Fix a problem with a mock object not recognized as a coroutine function
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit f90cb98741405f6a9d882238a89d9360665992c1 Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Sun Apr 26 21:48:35 2020 +0300 Fix a problem with a mock object not recognized as a coroutine function --- stem/control.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/stem/control.py b/stem/control.py index 3a687f1c..3eb706e0 100644 --- a/stem/control.py +++ b/stem/control.py @@ -3834,10 +3834,9 @@ class AsyncController(_ControllerClassMethodMixin, BaseController): if listener_type == event_type: for listener in event_listeners: try: - if asyncio.iscoroutinefunction(listener): - await listener(event_message) - else: - listener(event_message) + potential_coroutine = listener(event_message) + if asyncio.iscoroutine(potential_coroutine): + await potential_coroutine except Exception as exc: log.warn('Event listener raised an uncaught exception (%s): %s' % (exc, event))

1 0

[stem/master] Use a new event loop for wrapping the asynchronous controller
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit 9e08ea9068fd3176cb0c5c6c0aa4f845037e5478 Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Sun Apr 26 21:45:03 2020 +0300 Use a new event loop for wrapping the asynchronous controller --- stem/control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stem/control.py b/stem/control.py index db539539..95fba5bf 100644 --- a/stem/control.py +++ b/stem/control.py @@ -3899,7 +3899,7 @@ class Controller(_ControllerClassMethodMixin, _BaseControllerSocketMixin): return instance def __init__(self, control_socket: 'stem.socket.ControlSocket', is_authenticated: bool = False) -> None: - self._asyncio_loop = asyncio.get_event_loop() + self._asyncio_loop = asyncio.new_event_loop() self._asyncio_thread = threading.Thread(target=self._asyncio_loop.run_forever, name='asyncio') self._asyncio_thread.setDaemon(True)

1 0

[stem/master] Fix message synchronization
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit 8d18e6bb83c02ecb1a98e4cafbde7113f4e55730 Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Mon Apr 27 18:33:21 2020 +0300 Fix message synchronization --- stem/control.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/stem/control.py b/stem/control.py index 3eb706e0..d3e074b7 100644 --- a/stem/control.py +++ b/stem/control.py @@ -553,6 +553,27 @@ def event_description(event: str) -> str: return EVENT_DESCRIPTIONS.get(event.lower()) +class _MsgLock: + def __init__(self): + self._r_lock = threading.RLock() + self._async_lock = asyncio.Lock() + + async def acquire(self): + await self._async_lock.acquire() + self._r_lock.acquire() + + def release(self): + self._r_lock.release() + self._async_lock.release() + + async def __aenter__(self): + await self.acquire() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + self.release() + + class _BaseControllerSocketMixin: def is_alive(self) -> bool: """ @@ -622,7 +643,7 @@ class BaseController(_BaseControllerSocketMixin): self._asyncio_loop = asyncio.get_event_loop() self._asyncio_loop_tasks = [] - self._msg_lock = threading.RLock() + self._msg_lock = _MsgLock() self._status_listeners = [] # type: List[Tuple[Callable[[stem.control.BaseController, stem.control.State, float], None], bool]] # tuples of the form (callback, spawn_thread) self._status_listeners_lock = threading.RLock() @@ -669,7 +690,7 @@ class BaseController(_BaseControllerSocketMixin): * :class:`stem.SocketClosed` if the socket is shut down """ - with self._msg_lock: + async with self._msg_lock: # If our _reply_queue isn't empty then one of a few things happened... # # - Our connection was closed and probably re-restablished. This was @@ -1128,7 +1149,7 @@ class AsyncController(_ControllerClassMethodMixin, BaseController): * :class:`stem.connection.AuthenticationFailure` if unable to authenticate """ - with self._msg_lock: + async with self._msg_lock: await self.connect() self.clear_cache() await self.authenticate(*args, **kwargs)

1 0

[stem/master] Fix `stem.connection.connect`
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit a8e92bbb1532f25da843a29b1b771eeb80c477bd Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Mon Apr 27 18:57:02 2020 +0300 Fix `stem.connection.connect` --- stem/connection.py | 39 +++++++++++++++++++-------------------- stem/control.py | 10 +++++++--- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/stem/connection.py b/stem/connection.py index 3f7983de..6b985725 100644 --- a/stem/connection.py +++ b/stem/connection.py @@ -267,10 +267,17 @@ def connect(control_port: Tuple[str, Union[str, int]] = ('127.0.0.1', 'default') control_connection = None # type: Optional[stem.socket.ControlSocket] error_msg = '' + async_controller_thread = stem.control._AsyncControllerThread() + async_controller_thread.start() + + def connect_socket(socket): + asyncio.run_coroutine_threadsafe(socket.connect(), async_controller_thread.loop).result() + if control_socket: if os.path.exists(control_socket): try: control_connection = stem.socket.ControlSocketFile(control_socket) + connect_socket(control_connection) except stem.SocketError as exc: error_msg = CONNECT_MESSAGES['unable_to_use_socket'].format(path = control_socket, error = exc) else: @@ -284,6 +291,7 @@ def connect(control_port: Tuple[str, Union[str, int]] = ('127.0.0.1', 'default') control_connection = _connection_for_default_port(address) else: control_connection = stem.socket.ControlPort(address, int(port)) + connect_socket(control_connection) except stem.SocketError as exc: error_msg = CONNECT_MESSAGES['unable_to_use_port'].format(address = address, port = port, error = exc) @@ -297,12 +305,14 @@ def connect(control_port: Tuple[str, Union[str, int]] = ('127.0.0.1', 'default') error_msg = CONNECT_MESSAGES['no_control_port'] if is_tor_running else CONNECT_MESSAGES['tor_isnt_running'] print(error_msg) + if async_controller_thread.is_alive(): + async_controller_thread.join() return None - return _connect_auth(control_connection, password, password_prompt, chroot_path, controller) + return _connect_auth(control_connection, password, password_prompt, chroot_path, controller, async_controller_thread) -def _connect_auth(control_socket: stem.socket.ControlSocket, password: str, password_prompt: bool, chroot_path: str, controller: Optional[Type[stem.control.BaseController]]) -> Any: +def _connect_auth(control_socket: stem.socket.ControlSocket, password: str, password_prompt: bool, chroot_path: str, controller: Optional[Type[stem.control.BaseController]], async_controller_thread: 'threading.Thread') -> Any: """ Helper for the connect_* functions that authenticates the socket and constructs the controller. @@ -318,20 +328,13 @@ def _connect_auth(control_socket: stem.socket.ControlSocket, password: str, pass :returns: authenticated control connection, the type based on the controller argument """ - if controller: - loop = controller._asyncio_loop - asyncio_thread = None - else: - loop = asyncio.new_event_loop() - asyncio_thread = threading.Thread(target=loop.run_forever, name='async_auth') - asyncio_thread.setDaemon(True) - asyncio_thread.start() - def run_coroutine(coroutine): - asyncio.run_coroutine_threadsafe(coroutine, loop).result() + asyncio.run_coroutine_threadsafe(coroutine, async_controller_thread.loop).result() def close_control_socket(): run_coroutine(control_socket.close()) + if async_controller_thread.is_alive(): + async_controller_thread.join() try: run_coroutine(authenticate(control_socket, password, chroot_path)) @@ -339,7 +342,7 @@ def _connect_auth(control_socket: stem.socket.ControlSocket, password: str, pass if controller is None: return control_socket else: - return controller(control_socket, is_authenticated = True) + return controller(control_socket, is_authenticated = True, started_async_controller_thread = async_controller_thread) except IncorrectSocketType: if isinstance(control_socket, stem.socket.ControlPort): print(CONNECT_MESSAGES['wrong_port_type'].format(port = control_socket.port)) @@ -381,12 +384,6 @@ def _connect_auth(control_socket: stem.socket.ControlSocket, password: str, pass print(CONNECT_MESSAGES['general_auth_failure'].format(error = exc)) close_control_socket() return None - finally: - if asyncio_thread: - loop.call_soon_threadsafe(loop.stop) - if asyncio_thread.is_alive(): - asyncio_thread.join() - loop.close() async def authenticate(controller: Union[stem.control.BaseController, stem.socket.ControlSocket], password: Optional[str] = None, chroot_path: Optional[str] = None, protocolinfo_response: Optional[stem.response.protocolinfo.ProtocolInfoResponse] = None) -> None: @@ -994,7 +991,9 @@ async def get_protocolinfo(controller: Union[stem.control.BaseController, stem.s # next followed by authentication. Transparently reconnect if that happens. if not protocolinfo_response or str(protocolinfo_response) == 'Authentication required.': - controller.connect() + potential_coroutine = controller.connect() + if asyncio.iscoroutine(potential_coroutine): + await potential_coroutine try: protocolinfo_response = await _msg(controller, 'PROTOCOLINFO 1') diff --git a/stem/control.py b/stem/control.py index 20173a7a..21a89a5a 100644 --- a/stem/control.py +++ b/stem/control.py @@ -3933,9 +3933,13 @@ class Controller(_ControllerClassMethodMixin, _BaseControllerSocketMixin): instance.connect() return instance - def __init__(self, control_socket: 'stem.socket.ControlSocket', is_authenticated: bool = False) -> None: - self._async_controller_thread = _AsyncControllerThread() - self._async_controller_thread.start() + def __init__(self, control_socket: 'stem.socket.ControlSocket', is_authenticated: bool = False, started_async_controller_thread: Optional['threading.Thread'] = None) -> None: + def __init__(self, control_socket, is_authenticated = False, started_async_controller_thread = None): + if started_async_controller_thread: + self._async_controller_thread = started_async_controller_thread + else: + self._async_controller_thread = _AsyncControllerThread() + self._async_controller_thread.start() self._asyncio_loop = self._async_controller_thread.loop self._async_controller = self._init_async_controller(control_socket, is_authenticated)

1 0

[stem/master] Commit a missing file
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit dc6a98c30566184eb60cfccfca7fca5ca494869d Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Mon Apr 27 19:18:24 2020 +0300 Commit a missing file --- test/unit/async_util.py | 26 ++++++++++++++++++++++++++ test/unit/connection/authentication.py | 2 +- test/unit/connection/connect.py | 2 +- test/unit/control/controller.py | 2 +- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/test/unit/async_util.py b/test/unit/async_util.py new file mode 100644 index 00000000..aa8a6dd0 --- /dev/null +++ b/test/unit/async_util.py @@ -0,0 +1,26 @@ +import asyncio +import functools + + +def async_test(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + loop = asyncio.new_event_loop() + try: + result = loop.run_until_complete(func(*args, **kwargs)) + finally: + loop.close() + return result + return wrapper + + +def coro_func_returning_value(return_value): + async def coroutine_func(*args, **kwargs): + return return_value + return coroutine_func + + +def coro_func_raising_exc(exc): + async def coroutine_func(*args, **kwargs): + raise exc + return coroutine_func diff --git a/test/unit/connection/authentication.py b/test/unit/connection/authentication.py index 596fa50c..8df38c8f 100644 --- a/test/unit/connection/authentication.py +++ b/test/unit/connection/authentication.py @@ -18,7 +18,7 @@ from unittest.mock import patch from stem.response import ControlMessage from stem.util import log -from test.unit.util.asynchronous import ( +from test.unit.async_util import ( async_test, coro_func_raising_exc, coro_func_returning_value, diff --git a/test/unit/connection/connect.py b/test/unit/connection/connect.py index d2a22f18..aa91736b 100644 --- a/test/unit/connection/connect.py +++ b/test/unit/connection/connect.py @@ -11,7 +11,7 @@ import stem.socket from unittest.mock import Mock, patch -from test.unit.util.asynchronous import coro_func_raising_exc, coro_func_returning_value +from test.unit.async_util import coro_func_raising_exc, coro_func_returning_value class TestConnect(unittest.TestCase): diff --git a/test/unit/control/controller.py b/test/unit/control/controller.py index d09b5ca8..99bd5f19 100644 --- a/test/unit/control/controller.py +++ b/test/unit/control/controller.py @@ -21,7 +21,7 @@ from stem import ControllerError, DescriptorUnavailable, InvalidArguments, Inval from stem.control import MALFORMED_EVENTS, _parse_circ_path, Listener, Controller, EventType from stem.response import ControlMessage from stem.exit_policy import ExitPolicy -from test.unit.util.asynchronous import ( +from test.unit.async_util import ( async_test, coro_func_raising_exc, coro_func_returning_value,

1 0

[stem/master] Remove an old comment
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit 8df1419a4e4d2057fb896989df39c9e6db1226e9 Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Thu Apr 30 19:26:16 2020 +0300 Remove an old comment --- stem/control.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/stem/control.py b/stem/control.py index a87e88ca..98631428 100644 --- a/stem/control.py +++ b/stem/control.py @@ -891,8 +891,6 @@ class BaseController(_BaseControllerSocketMixin): self._event_notice.set() self._is_authenticated = False - # joins on our threads if it's safe to do so - self._notify_status_listeners(State.CLOSED) await self._socket_close()

1 0

[stem/master] Start initializing the asynchronous controller in the dedicated thread
by atagar＠torproject.org 16 Jul '20

16 Jul '20

commit bc2c978a47e37b2f4a9b2ee581acec2e454fe0cd Author: Illia Volochii <illia.volochii(a)gmail.com> Date: Sun Apr 26 21:46:41 2020 +0300 Start initializing the asynchronous controller in the dedicated thread --- stem/control.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/stem/control.py b/stem/control.py index 0105f9d4..3a687f1c 100644 --- a/stem/control.py +++ b/stem/control.py @@ -3908,9 +3908,15 @@ class Controller(_ControllerClassMethodMixin, _BaseControllerSocketMixin): self._asyncio_thread.setDaemon(True) self._asyncio_thread.start() - self._async_controller = AsyncController(control_socket, is_authenticated) + self._async_controller = self._init_async_controller(control_socket, is_authenticated) self._socket = self._async_controller._socket + def _init_async_controller(self, control_socket: 'stem.socket.ControlSocket', is_authenticated: bool) -> 'stem.control.AsyncController': + async def init_async_controller(): + return AsyncController(control_socket, is_authenticated) + + return asyncio.run_coroutine_threadsafe(init_async_controller(), self._asyncio_loop).result() + def _execute_async_method(self, method_name: str, *args: Any, **kwargs: Any) -> Any: return asyncio.run_coroutine_threadsafe( getattr(self._async_controller, method_name)(*args, **kwargs),

1 0