December 2012 - tor-commits - lists.torproject.org

[stem/master] Allowing zero Terget port in STREAM events
by atagar＠torproject.org 03 Dec '12

03 Dec '12

commit 4959378f9aa83d96bc1431d72bea6fa7a1b6e2c7 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 2 11:51:34 2012 -0800 Allowing zero Terget port in STREAM events The STREAM event's target port can be zero in DNS_REQUEST requests. Caught thanks to robinson on... https://trac.torproject.org/7598 --- stem/response/events.py | 2 +- test/unit/response/events.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletions(-) diff --git a/stem/response/events.py b/stem/response/events.py index 96764a0..598f8d1 100644 --- a/stem/response/events.py +++ b/stem/response/events.py @@ -497,7 +497,7 @@ class StreamEvent(Event): address, port = self.target.split(':', 1) - if not connection.is_valid_port(port): + if not connection.is_valid_port(port, allow_zero = True): raise stem.ProtocolError("Target location's port is invalid: %s" % self) self.target_address = address diff --git a/test/unit/response/events.py b/test/unit/response/events.py index 54d4777..57a3f58 100644 --- a/test/unit/response/events.py +++ b/test/unit/response/events.py @@ -113,6 +113,10 @@ STREAM_DIR_FETCH = "650 STREAM 14 NEW 0 \ 176.28.51.238.$649F2D0ACF418F7CFC6539AB2257EB2D5297BAFA.exit:443 \ SOURCE_ADDR=(Tor_internal):0 PURPOSE=DIR_FETCH" +STREAM_DNS_REQUEST = "650 STREAM 1113 NEW 0 www.google.com:0 \ +SOURCE_ADDR=127.0.0.1:15297 \ +PURPOSE=DNS_REQUEST" + # ORCONN events from starting tor 0.2.2.39 via TBB ORCONN_CONNECTED = "650 ORCONN $7ED90E2833EE38A75795BA9237B0A4560E51E1A0=GreenDragon CONNECTED" @@ -670,4 +674,22 @@ class TestEvents(unittest.TestCase): self.assertEqual("(Tor_internal)", event.source_address) self.assertEqual(0, event.source_port) self.assertEqual(StreamPurpose.DIR_FETCH, event.purpose) + + event = _get_event(STREAM_DNS_REQUEST) + + self.assertTrue(isinstance(event, stem.response.events.StreamEvent)) + self.assertEqual(STREAM_DNS_REQUEST.lstrip("650 "), str(event)) + self.assertEqual("1113", event.id) + self.assertEqual(StreamStatus.NEW, event.status) + self.assertEqual(None, event.circ_id) + self.assertEqual("www.google.com:0", event.target) + self.assertEqual("www.google.com", event.target_address) + self.assertEqual(0, event.target_port) + self.assertEqual(None, event.reason) + self.assertEqual(None, event.remote_reason) + self.assertEqual(None, event.source) + self.assertEqual("127.0.0.1:15297", event.source_addr) + self.assertEqual("127.0.0.1", event.source_address) + self.assertEqual(15297, event.source_port) + self.assertEqual(StreamPurpose.DNS_REQUEST, event.purpose)

1 0

[stem/master] Handling typo in STATUS_SERVER event name
by atagar＠torproject.org 03 Dec '12

03 Dec '12

commit aeebbe762579951b4a78ddc8c68347d2d181fc49 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 2 02:05:09 2012 -0800 Handling typo in STATUS_SERVER event name Prior to tor 0.2.0.22 STATUS_SERVER events claimed to be 'STATUS_SEVER'. Accepting both names as per the spec's suggestion. --- stem/response/events.py | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/stem/response/events.py b/stem/response/events.py index bf9db4c..d580829 100644 --- a/stem/response/events.py +++ b/stem/response/events.py @@ -539,5 +539,8 @@ EVENT_TYPE_TO_CLASS = { "STATUS_SERVER": StatusEvent, "STREAM": StreamEvent, "WARN": LogEvent, + + # accounting for a bug in tor 0.2.0.22 + "STATUS_SEVER": StatusEvent, }

1 0

[stem/master] Support for NS events
by atagar＠torproject.org 03 Dec '12

03 Dec '12

commit 92af0a5c0f9c6924cfddfb6388f581f3b6014732 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 2 16:16:20 2012 -0800 Support for NS events Implementation and testing for NS events. To get test data I wiped my cached_consensus, then listened for NS events while bootstraping. The first time I did this it provided a flood of entries (probably the whole consensus), but then when I restarted tor and tried again it just proved one. --- stem/control.py | 1 + stem/response/events.py | 22 ++++++++++++++++++++++ test/unit/response/events.py | 20 ++++++++++++++++++++ 3 files changed, 43 insertions(+), 0 deletions(-) diff --git a/stem/control.py b/stem/control.py index c2c014d..5fd69c7 100644 --- a/stem/control.py +++ b/stem/control.py @@ -89,6 +89,7 @@ providing its own for interacting at a higher level. **INFO** :class:`stem.response.events.LogEvent` **NEWDESC** :class:`stem.response.events.NewDescEvent` **NOTICE** :class:`stem.response.events.LogEvent` + **NS** :class:`stem.response.events.NetworkStatusEvent` **ORCONN** :class:`stem.response.events.ORConnEvent` **STATUS_CLIENT** :class:`stem.response.events.StatusEvent` **STATUS_GENERAL** :class:`stem.response.events.StatusEvent` diff --git a/stem/response/events.py b/stem/response/events.py index bca9557..1bdfd02 100644 --- a/stem/response/events.py +++ b/stem/response/events.py @@ -1,9 +1,11 @@ import re import datetime +import StringIO import stem import stem.control import stem.response +import stem.descriptor.router_status_entry from stem.util import connection, log, str_tools, tor_tools @@ -360,6 +362,25 @@ class LogEvent(Event): log_id = "event.logging.unknown_runlevel.%s" % self.runlevel log.log_once(log_id, log.INFO, unrecognized_msg % ('runlevel', self.runlevel)) +class NetworkStatusEvent(Event): + """ + Event for when our copy of the consensus has changed. This was introduced in + tor version 0.1.2.3. + + :param list desc: :class:`~stem.descriptor.router_status_entry.RouterStatusEntryV3` for the changed descriptors + """ + + _SKIP_PARSING = True + + def _parse(self): + content = str(self).lstrip("NS\n") + + self.desc = list(stem.descriptor.router_status_entry.parse_file( + StringIO.StringIO(content), + True, + entry_class = stem.descriptor.router_status_entry.RouterStatusEntryV3, + )) + class NewDescEvent(Event): """ Event that indicates that a new descriptor is available. @@ -567,6 +588,7 @@ EVENT_TYPE_TO_CLASS = { "INFO": LogEvent, "NEWDESC": NewDescEvent, "NOTICE": LogEvent, + "NS": NetworkStatusEvent, "ORCONN": ORConnEvent, "STATUS_CLIENT": StatusEvent, "STATUS_GENERAL": StatusEvent, diff --git a/test/unit/response/events.py b/test/unit/response/events.py index 2a64a27..84a1e7b 100644 --- a/test/unit/response/events.py +++ b/test/unit/response/events.py @@ -47,6 +47,15 @@ GUARD_NEW = "650 GUARD ENTRY $36B5DBA788246E8369DBAF58577C6BC044A9A374 NEW" GUARD_GOOD = "650 GUARD ENTRY $5D0034A368E0ABAF663D21847E1C9B6CFA09752A GOOD" GUARD_BAD = "650 GUARD ENTRY $5D0034A368E0ABAF663D21847E1C9B6CFA09752A BAD" +# NS event from tor v0.2.1.30. + +NS_EVENT = """650+NS +r whnetz dbBxYcJriTTrcxsuy4PUZcMRwCA VStM7KAIH/mXXoGDUpoGB1OXufg 2012-12-02 21:03:56 141.70.120.13 9001 9030 +s Fast HSDir Named Stable V2Dir Valid +. +650 OK +""" + # STATUS_* events that I was able to easily trigger. Most came from starting # TBB, then listening while it bootstrapped. @@ -380,6 +389,17 @@ class TestEvents(unittest.TestCase): self.assertEqual(NEWDESC_MULTIPLE.lstrip("650 "), str(event)) self.assertEqual(expected_relays, event.relays) + def test_ns_event(self): + expected_desc = mocking.get_router_status_entry_v3({ + "r": "whnetz dbBxYcJriTTrcxsuy4PUZcMRwCA VStM7KAIH/mXXoGDUpoGB1OXufg 2012-12-02 21:03:56 141.70.120.13 9001 9030", + "s": "Fast HSDir Named Stable V2Dir Valid", + }) + + event = _get_event(NS_EVENT) + + self.assertTrue(isinstance(event, stem.response.events.NetworkStatusEvent)) + self.assertEqual([expected_desc], event.desc) + def test_orconn_event(self): event = _get_event(ORCONN_CONNECTED)

1 0

[stem/master] Testing for STATUS_* events
by atagar＠torproject.org 03 Dec '12

03 Dec '12

commit e59de80010d1894c5eecb7d4b9d953ee195a5259 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 2 03:00:51 2012 -0800 Testing for STATUS_* events I had grand plans to have a StatusEvent subclass for each of the causes. However, on reflection that would encompass a bundle of work that's almost as large as all of the other events put together. It might still be a neat thing to do someday, but there's plenty of higher priority things to do first. Adding tests for all of the event examples that I collected earlier. This includes a bit of a hack to make quoted key/value mappings work. Previously we've only parsed quoted mappings if they belonged to the _QUOTED listing, but STATUS_* events sprinkle them all around so for forward compatability we can't simply enumerate them. --- stem/response/events.py | 31 ++++++-- test/unit/response/events.py | 159 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 182 insertions(+), 8 deletions(-) diff --git a/stem/response/events.py b/stem/response/events.py index d580829..96764a0 100644 --- a/stem/response/events.py +++ b/stem/response/events.py @@ -34,6 +34,11 @@ class Event(stem.response.ControlMessage): _QUOTED = () _SKIP_PARSING = False + # If set then we'll parse anything that looks like a quoted key/value + # mapping, reguardless of if it shows up in _QUOTED. + + _PERMISSIVE_QUOTED_MAPPINGS = False + def _parse_message(self, arrived_at): if not str(self).strip(): raise stem.ProtocolError("Received a blank tor event. Events must at the very least have a type.") @@ -73,13 +78,24 @@ class Event(stem.response.ControlMessage): content = str(self) - for keyword in set(self._QUOTED).intersection(set(self._KEYWORD_ARGS.keys())): - match = re.match("^(.*) %s=\"(.*)\"(.*)$" % keyword, content) - - if match: - prefix, value, suffix = match.groups() - content = prefix + suffix - self.keyword_args[keyword] = value + if self._PERMISSIVE_QUOTED_MAPPINGS: + while True: + match = re.match("^(.*) (\S*)=\"(.*)\"(.*)$", content) + + if match: + prefix, keyword, value, suffix = match.groups() + content = prefix + suffix + self.keyword_args[keyword] = value + else: + break + else: + for keyword in set(self._QUOTED).intersection(set(self._KEYWORD_ARGS.keys())): + match = re.match("^(.*) %s=\"(.*)\"(.*)$" % keyword, content) + + if match: + prefix, value, suffix = match.groups() + content = prefix + suffix + self.keyword_args[keyword] = value fields = content.split()[1:] @@ -423,6 +439,7 @@ class StatusEvent(Event): """ _POSITIONAL_ARGS = ("runlevel", "action") + _PERMISSIVE_QUOTED_MAPPINGS = True def _parse(self): if self.type == 'STATUS_GENERAL': diff --git a/test/unit/response/events.py b/test/unit/response/events.py index fc796a2..54d4777 100644 --- a/test/unit/response/events.py +++ b/test/unit/response/events.py @@ -378,7 +378,7 @@ class TestEvents(unittest.TestCase): self.assertEqual(ORClosureReason.DONE, event.reason) self.assertEqual(None, event.circ_count) - def test_status_event(self): + def test_status_event_consensus_arrived(self): event = _get_event(STATUS_CLIENT_CONSENSUS_ARRIVED) self.assertTrue(isinstance(event, stem.response.events.StatusEvent)) @@ -387,6 +387,163 @@ class TestEvents(unittest.TestCase): self.assertEqual(Runlevel.NOTICE, event.runlevel) self.assertEqual("CONSENSUS_ARRIVED", event.action) + def test_status_event_enough_dir_info(self): + event = _get_event(STATUS_CLIENT_ENOUGH_DIR_INFO) + + self.assertEqual(StatusType.CLIENT, event.status_type) + self.assertEqual(Runlevel.NOTICE, event.runlevel) + self.assertEqual("ENOUGH_DIR_INFO", event.action) + + def test_status_event_circuit_established(self): + event = _get_event(STATUS_CLIENT_CIRC_ESTABLISHED) + + self.assertEqual(StatusType.CLIENT, event.status_type) + self.assertEqual(Runlevel.NOTICE, event.runlevel) + self.assertEqual("CIRCUIT_ESTABLISHED", event.action) + + def test_status_event_bootstrap_descriptors(self): + event = _get_event(STATUS_CLIENT_BOOTSTRAP_DESCRIPTORS) + + self.assertEqual(StatusType.CLIENT, event.status_type) + self.assertEqual(Runlevel.NOTICE, event.runlevel) + self.assertEqual("BOOTSTRAP", event.action) + + expected_attr = { + 'PROGRESS': '53', + 'TAG': 'loading_descriptors', + 'SUMMARY': 'Loading relay descriptors', + } + + self.assertEqual(expected_attr, event.keyword_args) + + def test_status_event_bootstrap_stuck(self): + event = _get_event(STATUS_CLIENT_BOOTSTRAP_STUCK) + + self.assertEqual(StatusType.CLIENT, event.status_type) + self.assertEqual(Runlevel.WARN, event.runlevel) + self.assertEqual("BOOTSTRAP", event.action) + + expected_attr = { + 'PROGRESS': '80', + 'TAG': 'conn_or', + 'SUMMARY': 'Connecting to the Tor network', + 'WARNING': 'Network is unreachable', + 'REASON': 'NOROUTE', + 'COUNT': '5', + 'RECOMMENDATION': 'warn', + } + + self.assertEqual(expected_attr, event.keyword_args) + + def test_status_event_bootstrap_connecting(self): + event = _get_event(STATUS_CLIENT_BOOTSTRAP_CONNECTING) + + self.assertEqual(StatusType.CLIENT, event.status_type) + self.assertEqual(Runlevel.NOTICE, event.runlevel) + self.assertEqual("BOOTSTRAP", event.action) + + expected_attr = { + 'PROGRESS': '80', + 'TAG': 'conn_or', + 'SUMMARY': 'Connecting to the Tor network', + } + + self.assertEqual(expected_attr, event.keyword_args) + + def test_status_event_bootstrap_first_handshake(self): + event = _get_event(STATUS_CLIENT_BOOTSTRAP_FIRST_HANDSHAKE) + + self.assertEqual(StatusType.CLIENT, event.status_type) + self.assertEqual(Runlevel.NOTICE, event.runlevel) + self.assertEqual("BOOTSTRAP", event.action) + + expected_attr = { + 'PROGRESS': '85', + 'TAG': 'handshake_or', + 'SUMMARY': 'Finishing handshake with first hop', + } + + self.assertEqual(expected_attr, event.keyword_args) + + def test_status_event_bootstrap_established(self): + event = _get_event(STATUS_CLIENT_BOOTSTRAP_ESTABLISHED) + + self.assertEqual(StatusType.CLIENT, event.status_type) + self.assertEqual(Runlevel.NOTICE, event.runlevel) + self.assertEqual("BOOTSTRAP", event.action) + + expected_attr = { + 'PROGRESS': '90', + 'TAG': 'circuit_create', + 'SUMMARY': 'Establishing a Tor circuit', + } + + self.assertEqual(expected_attr, event.keyword_args) + + def test_status_event_bootstrap_done(self): + event = _get_event(STATUS_CLIENT_BOOTSTRAP_DONE) + + self.assertEqual(StatusType.CLIENT, event.status_type) + self.assertEqual(Runlevel.NOTICE, event.runlevel) + self.assertEqual("BOOTSTRAP", event.action) + + expected_attr = { + 'PROGRESS': '100', + 'TAG': 'done', + 'SUMMARY': 'Done', + } + + self.assertEqual(expected_attr, event.keyword_args) + + def test_status_event_bootstrap_check_reachability(self): + event = _get_event(STATUS_SERVER_CHECK_REACHABILITY) + + self.assertEqual(StatusType.SERVER, event.status_type) + self.assertEqual(Runlevel.NOTICE, event.runlevel) + self.assertEqual("CHECKING_REACHABILITY", event.action) + + expected_attr = { + 'ORADDRESS': '71.35.143.230:9050', + } + + self.assertEqual(expected_attr, event.keyword_args) + + def test_status_event_dns_timeout(self): + event = _get_event(STATUS_SERVER_DNS_TIMEOUT) + + self.assertEqual(StatusType.SERVER, event.status_type) + self.assertEqual(Runlevel.NOTICE, event.runlevel) + self.assertEqual("NAMESERVER_STATUS", event.action) + + expected_attr = { + 'NS': '205.171.3.25', + 'STATUS': 'DOWN', + 'ERR': 'request timed out.', + } + + self.assertEqual(expected_attr, event.keyword_args) + + def test_status_event_dns_down(self): + event = _get_event(STATUS_SERVER_DNS_DOWN) + + self.assertEqual(StatusType.SERVER, event.status_type) + self.assertEqual(Runlevel.WARN, event.runlevel) + self.assertEqual("NAMESERVER_ALL_DOWN", event.action) + + def test_status_event_dns_up(self): + event = _get_event(STATUS_SERVER_DNS_UP) + + self.assertEqual(StatusType.SERVER, event.status_type) + self.assertEqual(Runlevel.NOTICE, event.runlevel) + self.assertEqual("NAMESERVER_STATUS", event.action) + + expected_attr = { + 'NS': '205.171.3.25', + 'STATUS': 'UP', + } + + self.assertEqual(expected_attr, event.keyword_args) + def test_stream_event(self): event = _get_event(STREAM_NEW)

1 0

[stem/master] Support for STREAM events with a DIR_FETCH purpose
by atagar＠torproject.org 03 Dec '12

03 Dec '12

commit 1af965f1d12f5a58e921559901a2b7f0ca1e4cba Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Nov 30 23:04:07 2012 -0800 Support for STREAM events with a DIR_FETCH purpose The SOURCE_ADDR field of STREAM events can have zero as their port. Fixing this and adding a test. This was caught thanks to robinson on... https://trac.torproject.org/7598 --- stem/response/events.py | 2 +- test/unit/response/events.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletions(-) diff --git a/stem/response/events.py b/stem/response/events.py index ab0e5ef..bf9db4c 100644 --- a/stem/response/events.py +++ b/stem/response/events.py @@ -495,7 +495,7 @@ class StreamEvent(Event): address, port = self.source_addr.split(':', 1) - if not connection.is_valid_port(port): + if not connection.is_valid_port(port, allow_zero = True): raise stem.ProtocolError("Source location's port is invalid: %s" % self) self.source_address = address diff --git a/test/unit/response/events.py b/test/unit/response/events.py index 41a9d0e..fc796a2 100644 --- a/test/unit/response/events.py +++ b/test/unit/response/events.py @@ -109,6 +109,10 @@ STREAM_SUCCEEDED = "650 STREAM 18 SUCCEEDED 26 74.125.227.129:443" STREAM_CLOSED_RESET = "650 STREAM 21 CLOSED 26 74.125.227.129:443 REASON=CONNRESET" STREAM_CLOSED_DONE = "650 STREAM 25 CLOSED 26 199.7.52.72:80 REASON=DONE" +STREAM_DIR_FETCH = "650 STREAM 14 NEW 0 \ +176.28.51.238.$649F2D0ACF418F7CFC6539AB2257EB2D5297BAFA.exit:443 \ +SOURCE_ADDR=(Tor_internal):0 PURPOSE=DIR_FETCH" + # ORCONN events from starting tor 0.2.2.39 via TBB ORCONN_CONNECTED = "650 ORCONN $7ED90E2833EE38A75795BA9237B0A4560E51E1A0=GreenDragon CONNECTED" @@ -491,4 +495,22 @@ class TestEvents(unittest.TestCase): self.assertEqual(None, event.source_address) self.assertEqual(None, event.source_port) self.assertEqual(None, event.purpose) + + event = _get_event(STREAM_DIR_FETCH) + + self.assertTrue(isinstance(event, stem.response.events.StreamEvent)) + self.assertEqual(STREAM_DIR_FETCH.lstrip("650 "), str(event)) + self.assertEqual("14", event.id) + self.assertEqual(StreamStatus.NEW, event.status) + self.assertEqual(None, event.circ_id) + self.assertEqual("176.28.51.238.$649F2D0ACF418F7CFC6539AB2257EB2D5297BAFA.exit:443", event.target) + self.assertEqual("176.28.51.238.$649F2D0ACF418F7CFC6539AB2257EB2D5297BAFA.exit", event.target_address) + self.assertEqual(443, event.target_port) + self.assertEqual(None, event.reason) + self.assertEqual(None, event.remote_reason) + self.assertEqual(None, event.source) + self.assertEqual("(Tor_internal):0", event.source_addr) + self.assertEqual("(Tor_internal)", event.source_address) + self.assertEqual(0, event.source_port) + self.assertEqual(StreamPurpose.DIR_FETCH, event.purpose)

1 0

[stem/master] Support for GUARD events
by atagar＠torproject.org 03 Dec '12

03 Dec '12

commit 2585969d6efbd8b1a6660d5de1ede437792dc22b Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 2 14:02:21 2012 -0800 Support for GUARD events Implementation and testing for GUARD events. The spec is a little vauge, so we're missing enum descriptions and a good understanding of the 'Name' attribute... https://trac.torproject.org/7619 Got the test data by listening for GUARD events while bootstrapping... 650 GUARD ENTRY $5D0034A368E0ABAF663D21847E1C9B6CFA09752A BAD 650 GUARD ENTRY $071C2D9D0E03D75FF30DCCA850BB29CA4A4BB383 BAD 650 GUARD ENTRY $36B5DBA788246E8369DBAF58577C6BC044A9A374 NEW 650 GUARD ENTRY $071C2D9D0E03D75FF30DCCA850BB29CA4A4BB383 GOOD 650 GUARD ENTRY $5D0034A368E0ABAF663D21847E1C9B6CFA09752A GOOD --- stem/__init__.py | 46 ++++++++++++++++++++++++++++++++++++++++++ stem/control.py | 1 + stem/response/events.py | 17 +++++++++++++++ test/unit/response/events.py | 25 ++++++++++++++++++++++ 4 files changed, 89 insertions(+), 0 deletions(-) diff --git a/stem/__init__.py b/stem/__init__.py index c0b47fd..815e82c 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -260,6 +260,37 @@ Library for working with the tor process. **CLIENT** related to our activity as a tor client **SERVER** related to our activity as a tor relay ============= =========== + +.. data:: GuardType (enum) + + Use a guard relay can be for. Tor may provide types not in this enum. + + Enum descriptions are pending... + https://trac.torproject.org/7619 + + =========== =========== + GuardType Description + =========== =========== + **ENTRY** unknown + =========== =========== + +.. data:: GuardStatus (enum) + + Status a guard relay can have. Tor may provide types not in this enum. + + Enum descriptions are pending... + https://trac.torproject.org/7619 + + ============= =========== + GuardStatus Description + ============= =========== + **NEW** unknown + **UP** unknown + **DOWN** unknown + **BAD** unknown + **GOOD** unknown + **DROPPED** unknown + ============= =========== """ __version__ = '0.0.1' @@ -301,6 +332,8 @@ __all__ = [ "ORClosureReason", "AuthDescriptorAction", "StatusType", + "GuardType", + "GuardStatus", ] import stem.util.enum @@ -496,3 +529,16 @@ StatusType = stem.util.enum.UppercaseEnum( "SERVER", ) +GuardType = stem.util.enum.UppercaseEnum( + "ENTRY", +) + +GuardStatus = stem.util.enum.UppercaseEnum( + "NEW", + "UP", + "DOWN", + "BAD", + "GOOD", + "DROPPED", +) + diff --git a/stem/control.py b/stem/control.py index 45b7df8..c2c014d 100644 --- a/stem/control.py +++ b/stem/control.py @@ -85,6 +85,7 @@ providing its own for interacting at a higher level. **DEBUG** :class:`stem.response.events.LogEvent` **DESCCHANGED** :class:`stem.response.events.DescChangedEvent` **ERR** :class:`stem.response.events.LogEvent` + **GUARD** :class:`stem.response.events.GuardEvent` **INFO** :class:`stem.response.events.LogEvent` **NEWDESC** :class:`stem.response.events.NewDescEvent` **NOTICE** :class:`stem.response.events.LogEvent` diff --git a/stem/response/events.py b/stem/response/events.py index 598f8d1..bca9557 100644 --- a/stem/response/events.py +++ b/stem/response/events.py @@ -317,6 +317,22 @@ class DescChangedEvent(Event): pass +class GuardEvent(Event): + """ + Event that indicates that our guard relays have changed. + + :var stem.GuardType guard_type: purpose the guard relay is for + :var str name: nickname or fingerprint of the guard relay + :var stem.GuardStatus status: status of the guard relay + """ + + # TODO: We should replace the 'name' field with a fingerprint or nickname + # attribute once we know what it can be... + # + # https://trac.torproject.org/7619 + + _POSITIONAL_ARGS = ("guard_type", "name", "status") + class LogEvent(Event): """ Tor logging event. These are the most visible kind of event since, by @@ -547,6 +563,7 @@ EVENT_TYPE_TO_CLASS = { "DEBUG": LogEvent, "DESCCHANGED": DescChangedEvent, "ERR": LogEvent, + "GUARD": GuardEvent, "INFO": LogEvent, "NEWDESC": NewDescEvent, "NOTICE": LogEvent, diff --git a/test/unit/response/events.py b/test/unit/response/events.py index 57a3f58..2a64a27 100644 --- a/test/unit/response/events.py +++ b/test/unit/response/events.py @@ -41,6 +41,12 @@ $E57A476CD4DFBD99B4EE52A100A58610AD6E80B9,hamburgerphone" CIRC_BUILT_OLD = "650 CIRC 1 BUILT \ $E57A476CD4DFBD99B4EE52A100A58610AD6E80B9,hamburgerphone,PrivacyRepublic14" +# GUARD events from tor v0.2.1.30. + +GUARD_NEW = "650 GUARD ENTRY $36B5DBA788246E8369DBAF58577C6BC044A9A374 NEW" +GUARD_GOOD = "650 GUARD ENTRY $5D0034A368E0ABAF663D21847E1C9B6CFA09752A GOOD" +GUARD_BAD = "650 GUARD ENTRY $5D0034A368E0ABAF663D21847E1C9B6CFA09752A BAD" + # STATUS_* events that I was able to easily trigger. Most came from starting # TBB, then listening while it bootstrapped. @@ -339,6 +345,25 @@ class TestEvents(unittest.TestCase): self.assertEqual([], event.positional_args) self.assertEqual({}, event.keyword_args) + def test_guard_event(self): + event = _get_event(GUARD_NEW) + + self.assertTrue(isinstance(event, stem.response.events.GuardEvent)) + self.assertEqual(GUARD_NEW.lstrip("650 "), str(event)) + self.assertEqual(GuardType.ENTRY, event.guard_type) + self.assertEqual("$36B5DBA788246E8369DBAF58577C6BC044A9A374", event.name) + self.assertEqual(GuardStatus.NEW, event.status) + + event = _get_event(GUARD_GOOD) + self.assertEqual(GuardType.ENTRY, event.guard_type) + self.assertEqual("$5D0034A368E0ABAF663D21847E1C9B6CFA09752A", event.name) + self.assertEqual(GuardStatus.GOOD, event.status) + + event = _get_event(GUARD_BAD) + self.assertEqual(GuardType.ENTRY, event.guard_type) + self.assertEqual("$5D0034A368E0ABAF663D21847E1C9B6CFA09752A", event.name) + self.assertEqual(GuardStatus.BAD, event.status) + def test_newdesc_event(self): event = _get_event(NEWDESC_SINGLE) expected_relays = (("B3FA3110CC6F42443F039220C134CBD2FC4F0493", "Sakura"),)

1 0

[stem/master] Support for NEWCONSENSUS events
by atagar＠torproject.org 03 Dec '12

03 Dec '12

commit c1e88e4257c606593ecd2abf4212cc5266b34bf4 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 2 18:24:54 2012 -0800 Support for NEWCONSENSUS events Implementation and testing for NEWCONSENSUS events. This is pretty much the exact same thing from our perspective as NS events. --- stem/response/events.py | 24 ++++++++++++++++++++++++ test/unit/response/events.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 0 deletions(-) diff --git a/stem/response/events.py b/stem/response/events.py index 49a8169..522f9cc 100644 --- a/stem/response/events.py +++ b/stem/response/events.py @@ -438,6 +438,29 @@ class NetworkStatusEvent(Event): entry_class = stem.descriptor.router_status_entry.RouterStatusEntryV3, )) +class NewConsensusEvent(Event): + """ + Event for when we have a new consensus. This is similar to + :class:`~stem.response.events.NetworkStatusEvent`, except that it contains + the whole consensus so anything not listed is implicitly no longer + recommended. + + This was introduced in tor version 0.2.1.13. + + :param list desc: :class:`~stem.descriptor.router_status_entry.RouterStatusEntryV3` for the changed descriptors + """ + + _SKIP_PARSING = True + + def _parse(self): + content = str(self).lstrip("NEWCONSENSUS\n") + + self.desc = list(stem.descriptor.router_status_entry.parse_file( + StringIO.StringIO(content), + True, + entry_class = stem.descriptor.router_status_entry.RouterStatusEntryV3, + )) + class NewDescEvent(Event): """ Event that indicates that a new descriptor is available. @@ -644,6 +667,7 @@ EVENT_TYPE_TO_CLASS = { "ERR": LogEvent, "GUARD": GuardEvent, "INFO": LogEvent, + "NEWCONSENSUS": NewConsensusEvent, "NEWDESC": NewDescEvent, "NOTICE": LogEvent, "NS": NetworkStatusEvent, diff --git a/test/unit/response/events.py b/test/unit/response/events.py index 0b225d9..6f18676 100644 --- a/test/unit/response/events.py +++ b/test/unit/response/events.py @@ -54,6 +54,17 @@ GUARD_NEW = "650 GUARD ENTRY $36B5DBA788246E8369DBAF58577C6BC044A9A374 NEW" GUARD_GOOD = "650 GUARD ENTRY $5D0034A368E0ABAF663D21847E1C9B6CFA09752A GOOD" GUARD_BAD = "650 GUARD ENTRY $5D0034A368E0ABAF663D21847E1C9B6CFA09752A BAD" +# NEWCONSENSUS event from v0.2.1.30. + +NEWCONSENSUS_EVENT = """650+NEWCONSENSUS +r Beaver /96bKo4soysolMgKn5Hex2nyFSY pAJH9dSBp/CG6sPhhVY/5bLaVPM 2012-12-02 22:02:45 77.223.43.54 9001 0 +s Fast Named Running Stable Valid +r Unnamed /+fJRWjmIGNAL2C5rRZHq3R91tA 7AnpZjfdBpYzXnMNm+w1bTsFF6Y 2012-12-02 17:51:10 91.121.184.87 9001 0 +s Fast Guard Running Stable Valid +. +650 OK +""" + # NS event from tor v0.2.1.30. NS_EVENT = """650+NS @@ -405,6 +416,24 @@ class TestEvents(unittest.TestCase): self.assertEqual(NEWDESC_MULTIPLE.lstrip("650 "), str(event)) self.assertEqual(expected_relays, event.relays) + def test_new_consensus_event(self): + expected_desc = [] + + expected_desc.append(mocking.get_router_status_entry_v3({ + "r": "Beaver /96bKo4soysolMgKn5Hex2nyFSY pAJH9dSBp/CG6sPhhVY/5bLaVPM 2012-12-02 22:02:45 77.223.43.54 9001 0", + "s": "Fast Named Running Stable Valid", + })) + + expected_desc.append(mocking.get_router_status_entry_v3({ + "r": "Unnamed /+fJRWjmIGNAL2C5rRZHq3R91tA 7AnpZjfdBpYzXnMNm+w1bTsFF6Y 2012-12-02 17:51:10 91.121.184.87 9001 0", + "s": "Fast Guard Running Stable Valid", + })) + + event = _get_event(NEWCONSENSUS_EVENT) + + self.assertTrue(isinstance(event, stem.response.events.NewConsensusEvent)) + self.assertEqual(expected_desc, event.desc) + def test_ns_event(self): expected_desc = mocking.get_router_status_entry_v3({ "r": "whnetz dbBxYcJriTTrcxsuy4PUZcMRwCA VStM7KAIH/mXXoGDUpoGB1OXufg 2012-12-02 21:03:56 141.70.120.13 9001 9030",

1 0

[stem/master] Support for CLIENTS_SEEN events
by atagar＠torproject.org 03 Dec '12

03 Dec '12

commit 92419a54b83d80d7b124482df9bdf42fb931c7a0 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 2 17:23:02 2012 -0800 Support for CLIENTS_SEEN events Implementation and testing for CLINETS_SEEN events. This would be a difficult event to get examples for, but luckily the spec has one. --- stem/control.py | 1 + stem/response/events.py | 58 ++++++++++++++++++++++++++++++++++++++++++ test/unit/response/events.py | 16 +++++++++++ 3 files changed, 75 insertions(+), 0 deletions(-) diff --git a/stem/control.py b/stem/control.py index 5fd69c7..ec727ce 100644 --- a/stem/control.py +++ b/stem/control.py @@ -82,6 +82,7 @@ providing its own for interacting at a higher level. **AUTHDIR_NEWDESCS** :class:`stem.response.events.AuthDirNewDescEvent` **BW** :class:`stem.response.events.BandwidthEvent` **CIRC** :class:`stem.response.events.CircuitEvent` + **CLIENTS_SEEN** :class:`stem.response.events.ClientsSeenEvent` **DEBUG** :class:`stem.response.events.LogEvent` **DESCCHANGED** :class:`stem.response.events.DescChangedEvent` **ERR** :class:`stem.response.events.LogEvent` diff --git a/stem/response/events.py b/stem/response/events.py index 1bdfd02..49a8169 100644 --- a/stem/response/events.py +++ b/stem/response/events.py @@ -311,6 +311,63 @@ class CircuitEvent(Event): log_id = "event.circ.unknown_remote_reason.%s" % self.remote_reason log.log_once(log_id, log.INFO, unrecognized_msg % ('remote reason', self.remote_reason)) +class ClientsSeenEvent(Event): + """ + Periodic event on bridge relays that provides a summary of our users. + + :var datetime start_time: time in UTC that we started collecting these stats + :var dict locales: mapping of country codes to a rounded count for the number of users + :var dict ip_versions: mapping of ip protocols to a rounded count for the number of users + """ + + _KEYWORD_ARGS = { + "TimeStarted": "start_time", + "CountrySummary": "locales", + "IPVersions": "ip_versions", + } + + _QUOTED = ("TimeStarted",) + + def _parse(self): + if self.start_time != None: + self.start_time = datetime.datetime.strptime(self.start_time, "%Y-%m-%d %H:%M:%S") + + if self.locales != None: + locale_to_count = {} + + for entry in self.locales.split(','): + if not '=' in entry: + raise stem.ProtocolError("The CLIENTS_SEEN's CountrySummary should be a comma separated listing of '<locale>=<count>' mappings: %s" % self) + + locale, count = entry.split('=', 1) + + if len(locale) != 2: + raise stem.ProtocolError("Locales should be a two character code, got '%s': %s" % (locale, self)) + elif not count.isdigit(): + raise stem.ProtocolError("Locale count was non-numeric (%s): %s" % (count, self)) + elif locale in locale_to_count: + raise stem.ProtocolError("CountrySummary had multiple mappings for '%s': %s" % (locale, self)) + + locale_to_count[locale] = int(count) + + self.locales = locale_to_count + + if self.ip_versions != None: + protocol_to_count = {} + + for entry in self.ip_versions.split(','): + if not '=' in entry: + raise stem.ProtocolError("The CLIENTS_SEEN's IPVersions should be a comma separated listing of '<protocol>=<count>' mappings: %s" % self) + + protocol, count = entry.split('=', 1) + + if not count.isdigit(): + raise stem.ProtocolError("IP protocol count was non-numeric (%s): %s" % (count, self)) + + protocol_to_count[protocol] = int(count) + + self.ip_versions = protocol_to_count + class DescChangedEvent(Event): """ Event that indicates that our descriptor has changed. This was first added in @@ -581,6 +638,7 @@ EVENT_TYPE_TO_CLASS = { "AUTHDIR_NEWDESCS": AuthDirNewDescEvent, "BW": BandwidthEvent, "CIRC": CircuitEvent, + "CLIENTS_SEEN": ClientsSeenEvent, "DEBUG": LogEvent, "DESCCHANGED": DescChangedEvent, "ERR": LogEvent, diff --git a/test/unit/response/events.py b/test/unit/response/events.py index 84a1e7b..0b225d9 100644 --- a/test/unit/response/events.py +++ b/test/unit/response/events.py @@ -41,6 +41,13 @@ $E57A476CD4DFBD99B4EE52A100A58610AD6E80B9,hamburgerphone" CIRC_BUILT_OLD = "650 CIRC 1 BUILT \ $E57A476CD4DFBD99B4EE52A100A58610AD6E80B9,hamburgerphone,PrivacyRepublic14" +# CLIENTS_SEEN example from the spec + +CLIENTS_SEEN_EVENT = '650 CLIENTS_SEEN \ +TimeStarted="2008-12-25 23:50:43" \ +CountrySummary=us=16,de=8,uk=8 \ +IPVersions=v4=16,v6=40' + # GUARD events from tor v0.2.1.30. GUARD_NEW = "650 GUARD ENTRY $36B5DBA788246E8369DBAF58577C6BC044A9A374 NEW" @@ -343,6 +350,15 @@ class TestEvents(unittest.TestCase): self.assertEqual(None, event.reason) self.assertEqual(None, event.remote_reason) + def test_clients_seen_event(self): + event = _get_event(CLIENTS_SEEN_EVENT) + + self.assertTrue(isinstance(event, stem.response.events.ClientsSeenEvent)) + self.assertEqual(CLIENTS_SEEN_EVENT.lstrip("650 "), str(event)) + self.assertEqual(datetime.datetime(2008, 12, 25, 23, 50, 43), event.start_time) + self.assertEqual({'us': 16, 'de': 8, 'uk': 8}, event.locales) + self.assertEqual({'v4': 16, 'v6': 40}, event.ip_versions) + def test_descchanged_event(self): # all we can check for is that the event is properly parsed as a # DescChangedEvent instance

1 0

[stem/master] Tor event handling
by atagar＠torproject.org 03 Dec '12

03 Dec '12

commit 42872dd08e81d6b341654ab85969df2ed77a7397 Merge: e1772bb c1e88e4 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Dec 2 18:30:14 2012 -0800 Tor event handling Merging support for tor's asynchronous events. This isn't entirely complete, missing... * STREAM_BW (being added in https://trac.torproject.org/7616) * BUILDTIMEOUT_SET * SIGNAL * CONF_CHANGED * CIRC_MINOR ... but better not to let the perfect be the enemy of good. This branch was partly code reviewed by robinson on... https://trac.torproject.org/7328 docs/api/control.rst | 4 +- docs/api/response.rst | 5 + run_tests.py | 4 + stem/__init__.py | 450 ++++++++++++++++++ stem/control.py | 257 ++++++++++- stem/descriptor/extrainfo_descriptor.py | 2 +- stem/descriptor/server_descriptor.py | 2 +- stem/response/__init__.py | 3 + stem/response/events.py | 684 +++++++++++++++++++++++++++ stem/util/str_tools.py | 36 ++ stem/util/tor_tools.py | 18 + test/integ/control/controller.py | 51 ++ test/unit/__init__.py | 1 + test/unit/control/__init__.py | 6 + test/unit/control/controller.py | 50 ++ test/unit/response/__init__.py | 1 + test/unit/response/events.py | 785 +++++++++++++++++++++++++++++++ test/unit/util/str_tools.py | 31 ++ 18 files changed, 2378 insertions(+), 12 deletions(-)

1 0

[stem/master] Adding event handling support to the Controller
by atagar＠torproject.org 03 Dec '12

03 Dec '12

commit ba280f94da9962e1a2b1bfaf22d5fbd17ffa3103 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Nov 4 15:53:44 2012 -0800 Adding event handling support to the Controller My initial plan was to add event listener support similar to TorCtl but with automated handling of the SETEVENT calls. While working on this meejah suggested accepting functors instead (like txtorcon). On reflection this is both much nicer for our callers and easier for us to support. Adding functions for adding and removing listeners, with a simple integ test. Next up will be the Event classes... --- stem/control.py | 97 ++++++++++++++++++++++++++++++++++++++ test/integ/control/controller.py | 45 +++++++++++++++++ 2 files changed, 142 insertions(+), 0 deletions(-) diff --git a/stem/control.py b/stem/control.py index 1476598..ca3730b 100644 --- a/stem/control.py +++ b/stem/control.py @@ -13,6 +13,8 @@ providing its own for interacting at a higher level. | |- from_port - Provides a Controller based on a port connection. | +- from_socket_file - Provides a Controller based on a socket file connection. | + |- add_event_listener - attaches an event listener to be notified of tor events + |- remove_event_listener - removes a listener so it isn't notified of further events |- is_caching_enabled - true if the controller has enabled caching |- is_geoip_unavailable - true if we've discovered our geoip db to be unavailable |- clear_cache - clears any cached results @@ -73,6 +75,34 @@ import stem.util.log as log State = stem.util.enum.Enum("INIT", "RESET", "CLOSED") +EventType = stem.util.enum.UppercaseEnum( + "CIRC", + "STREAM", + "ORCONN", + "BW", + "DEBUG", + "INFO", + "NOTICE", + "WARN", + "ERR", + "NEWDESC", + "ADDRMAP", + "AUTHDIR_NEWDESCS", + "DESCCHANGED", + "STATUS_GENERAL", + "STATUS_CLIENT", + "STATUS_SERVER", + "GUARD", + "NS", + "STREAM_BW", + "CLIENTS_SEEN", + "NEWCONSENSUS", + "BUILDTIMEOUT_SET", + "SIGNAL", + "CONF_CHANGED", + "CIRC_MINOR", +) + # Constant to indicate an undefined argument default. Usually we'd use None for # this, but users will commonly provide None as the argument so need something # else fairly unique... @@ -508,6 +538,11 @@ class Controller(BaseController): self._is_caching_enabled = enable_caching self._request_cache = {} + # mapping of event types to their listeners + + self._event_listeners = {} + self._event_listeners_lock = threading.RLock() + # number of sequential 'GETINFO ip-to-country/*' lookups that have failed self._geoip_failure_count = 0 self.enabled_features = [] @@ -524,6 +559,57 @@ class Controller(BaseController): super(Controller, self).close() + def add_event_listener(self, listener, *events): + """ + Directs further tor controller events to a given function. The function is + expected to take a single argument, which is a + :class:`~stem.response.events.Event` subclass. + + If a new control connection is initialized then this listener will be + reattached. + + :param functor listener: function to be called when an event is received + :param stem.control.EventType events: event types to be listened for + + :raises: :class:`stem.socket.ControllerError` if unable to set the events + """ + + with self._event_listeners_lock: + for event_type in events: + self._event_listeners.setdefault(event_type, []).append(listener) + + if self.is_alive(): + response = self.msg("SETEVENTS %s" % " ".join(self._event_listeners.keys())) + + if not response.is_ok(): + raise stem.socket.ProtocolError("SETEVENTS received unexpected response\n%s" % response) + + def remove_event_listener(self, listener): + """ + Stops a listener from being notified of further tor events. + + :param stem.control.EventListener listener: listener to be removed + + :raises: :class:`stem.socket.ControllerError` if unable to set the events + """ + + with self._event_listeners_lock: + event_types_changed = False + + for event_type, event_listeners in self._event_listeners.items(): + if listener in event_listeners: + event_listeners.remove(listener) + + if len(event_listeners) == 0: + event_types_changed = True + del self._event_listeners[event_type] + + if event_types_changed: + response = self.msg("SETEVENTS %s" % " ".join(self._event_listeners.keys())) + + if not response.is_ok(): + raise stem.socket.ProtocolError("SETEVENTS received unexpected response\n%s" % response) + def is_caching_enabled(self): """ **True** if caching has been enabled, **False** otherwise. @@ -1320,6 +1406,17 @@ class Controller(BaseController): stem.response.convert("MAPADDRESS", response) return response.entries + + def _handle_event(self, event_message): + # TODO: parse the event_message into a stem.response.events.Event class + + event_message_type = str(event_message).split()[0] + + with self._event_listeners_lock: + for event_type, event_listeners in self._event_listeners.items(): + if event_type == event_message_type: + for listener in event_listeners: + listener(event_message) def _case_insensitive_lookup(entries, key, default = UNDEFINED): """ diff --git a/test/integ/control/controller.py b/test/integ/control/controller.py index 1ba06bb..e5d7782 100644 --- a/test/integ/control/controller.py +++ b/test/integ/control/controller.py @@ -6,6 +6,7 @@ from __future__ import with_statement import os import re +import time import shutil import socket import unittest @@ -20,6 +21,8 @@ import test.util import stem.descriptor.router_status_entry import stem.descriptor.reader +from stem.control import EventType + class TestController(unittest.TestCase): def test_from_port(self): """ @@ -47,6 +50,48 @@ class TestController(unittest.TestCase): else: self.assertRaises(stem.SocketError, stem.control.Controller.from_socket_file, test.runner.CONTROL_SOCKET_PATH) + def test_event_handling(self): + """ + Add a couple listeners for various events and make sure that they receive + them. Then remove the listeners. + """ + + if test.runner.require_control(self): return + + event_buffer1, event_buffer2 = [], [] + + def listener1(event): + event_buffer1.append(event) + + def listener2(event): + event_buffer2.append(event) + + runner = test.runner.get_runner() + with runner.get_tor_controller() as controller: + controller.add_event_listener(listener1, EventType.BW) + controller.add_event_listener(listener2, EventType.BW, EventType.DEBUG) + + # BW events occure at the rate of one per second, so wait a bit to let + # some accumulate. + + # TODO: check that the type of events in event_buffer1 are BandwidthEvent + # instances when we have proper event types + + time.sleep(3) + + self.assertTrue(len(event_buffer1) >= 2) + self.assertTrue(len(event_buffer2) >= 2) + + # Checking that a listener's no longer called after being removed. + + controller.remove_event_listener(listener2) + + buffer2_size = len(event_buffer2) + time.sleep(2) + + self.assertTrue(len(event_buffer1) >= 4) + self.assertEqual(buffer2_size, len(event_buffer2)) + def test_getinfo(self): """ Exercises GETINFO with valid and invalid queries.

1 0