May 2012 - tor-commits - lists.torproject.org

[stem/master] Testing for the controller's asynchronous event handling
by atagar＠torproject.org 06 May '12

06 May '12

commit 80809adcf7023d91c08f97ee9e7affcb3a3bc748 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 5 19:17:55 2012 -0700 Testing for the controller's asynchronous event handling Adding a test that hammers the control socket with queries while also listening for BW events. This also tests for and fixes a bug where listeners wouldn't get all of the enqueued events if the controller was closed. --- stem/control.py | 4 ++- test/integ/control/base_controller.py | 56 +++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletions(-) diff --git a/stem/control.py b/stem/control.py index b3bd29c..9b912e5 100644 --- a/stem/control.py +++ b/stem/control.py @@ -414,11 +414,13 @@ class BaseController: socket. """ - while self.is_alive(): + while True: try: event_message = self._event_queue.get_nowait() self._handle_event(event_message) except Queue.Empty: + if not self.is_alive(): break + self._event_notice.wait() self._event_notice.clear() diff --git a/test/integ/control/base_controller.py b/test/integ/control/base_controller.py index 940ef99..cb5eaf2 100644 --- a/test/integ/control/base_controller.py +++ b/test/integ/control/base_controller.py @@ -2,6 +2,7 @@ Integration tests for the stem.control.BaseController class. """ +import re import time import unittest import threading @@ -135,6 +136,61 @@ class TestBaseController(unittest.TestCase): for msg_thread in message_threads: msg_thread.join() + def test_asynchronous_event_handling(self): + """ + Check that we can both receive asynchronous events while hammering our + socket with queries, and checks that when a controller is closed the + listeners will still receive all of the enqueued events. + """ + + class ControlledListener(stem.control.BaseController): + """ + Controller that blocks event handling until told to do so. + """ + + def __init__(self, control_socket): + stem.control.BaseController.__init__(self, control_socket) + self.received_events = [] + self.receive_notice = threading.Event() + + def _handle_event(self, event_message): + self.receive_notice.wait() + self.received_events.append(event_message) + + with test.runner.get_runner().get_tor_socket() as control_socket: + controller = ControlledListener(control_socket) + controller.msg("SETEVENTS BW") + + # Wait for a couple events for events to be enqueued. Doing a bunch of + # GETINFO queries while waiting to better exercise the asynchronous event + # handling. + + start_time = time.time() + + while (time.time() - start_time) < 2: + test.runner.exercise_controller(self, controller) + + # Concurrently shut down the controller. We need to do this in another + # thread because it'll block on the event handling, which in turn is + # currently blocking on the reveive_notice. + + close_thread = threading.Thread(target = controller.close, name = "Closing controller") + close_thread.setDaemon(True) + close_thread.start() + + # Finally start handling the BW events that we've received. We should + # have at least a couple of them. + + controller.receive_notice.set() + close_thread.join() + + self.assertTrue(len(controller.received_events) >= 2) + + for bw_event in controller.received_events: + self.assertTrue(re.match("BW [0-9]+ [0-9]+", str(bw_event))) + self.assertTrue(re.match("650 BW [0-9]+ [0-9]+\r\n", bw_event.raw_content())) + self.assertEquals(("650", " "), bw_event.content()[0][:2]) + def test_status_notifications(self): """ Checks basic functionality of the add_status_listener() and

1 0

[stem/master] Mislabeled server descriptors as being version 3
by atagar＠torproject.org 06 May '12

06 May '12

commit cffc3c1be1c4c2a00ce41325a37038d83c8a80a1 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 5 15:47:06 2012 -0700 Mislabeled server descriptors as being version 3 We're using version 3 of the dir-spec, but server descriptors themselves are still using their initial version. Correcting the class names and documentation that labeled them as being version 3. Thanks to Karsten for the correction! --- stem/descriptor/__init__.py | 2 +- stem/descriptor/server_descriptor.py | 36 ++++++++++++------------ test/integ/descriptor/server_descriptor.py | 16 +++++----- test/unit/descriptor/server_descriptor.py | 42 ++++++++++++++-------------- 4 files changed, 48 insertions(+), 48 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 21f40c2..2af1fb0 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -41,7 +41,7 @@ def parse_file(path, descriptor_file): descriptor_file.seek(0) if filename == "cached-descriptors" or first_line.startswith("router "): - for desc in stem.descriptor.server_descriptor.parse_file_v3(descriptor_file): + for desc in stem.descriptor.server_descriptor.parse_file(descriptor_file): desc._set_path(path) yield desc else: diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index d56b453..5a187df 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -7,13 +7,13 @@ etc). This information is provided from a few sources... - the 'cached-descriptors' file in tor's data directory - tor metrics, at https://metrics.torproject.org/data.html -parse_file_v3 - Iterates over the server descriptors in a file. -ServerDescriptorV3 - Tor server descriptor, version 3. - | |- RelayDescriptorV3 - Server descriptor for a relay. +parse_file - Iterates over the server descriptors in a file. +ServerDescriptor - Tor server descriptor. + | |- RelayDescriptor - Server descriptor for a relay. | | |- is_valid - checks the signature against the descriptor content | | +- digest - calculates the digest value for our content | | - | +- BridgeDescriptorV3 - Scrubbed server descriptor for a bridge. + | +- BridgeDescriptor - Scrubbed server descriptor for a bridge. | |- is_scrubbed - checks if our content has been properly scrubbed | +- get_scrubbing_issues - description of issues with our scrubbing | @@ -66,10 +66,10 @@ SINGLE_FIELDS = ( "allow-single-hop-exits", ) -def parse_file_v3(descriptor_file, validate = True): +def parse_file(descriptor_file, validate = True): """ - Iterates over the version 3 server descriptors in a file. This can read - either relay or bridge v3 server descriptors. + Iterates over the server descriptors in a file. This can read either relay or + bridge server descriptors. Arguments: descriptor_file (file) - file with descriptor content @@ -77,7 +77,7 @@ def parse_file_v3(descriptor_file, validate = True): True, skips these checks otherwise Returns: - iterator for ServerDescriptorV3 instances in the file + iterator for ServerDescriptor instances in the file Raises: ValueError if the contents is malformed and validate is True @@ -95,7 +95,7 @@ def parse_file_v3(descriptor_file, validate = True): descriptor_file.seek(0) if first_line.startswith("router Unnamed 10."): - yield BridgeDescriptorV3(descriptor_file.read()) + yield BridgeDescriptor(descriptor_file.read()) return # Handler for relay descriptors @@ -137,7 +137,7 @@ def parse_file_v3(descriptor_file, validate = True): annotations = map(str.strip, annotations) descriptor_text = "".join(descriptor_content) - descriptor = RelayDescriptorV3(descriptor_text, validate, annotations) + descriptor = RelayDescriptor(descriptor_text, validate, annotations) yield descriptor else: break # done parsing descriptors @@ -175,9 +175,9 @@ def _read_until_keyword(keyword, descriptor_file, inclusive = False): return content -class ServerDescriptorV3(stem.descriptor.Descriptor): +class ServerDescriptor(stem.descriptor.Descriptor): """ - Common parent for version 3 server descriptors. + Common parent for server descriptors. Attributes: nickname (str) - relay's nickname (*) @@ -599,7 +599,7 @@ class ServerDescriptorV3(stem.descriptor.Descriptor): def _first_keyword(self): return None def _last_keyword(self): return None -class RelayDescriptorV3(ServerDescriptorV3): +class RelayDescriptor(ServerDescriptor): """ Version 3 server descriptor, as specified in... https://gitweb.torproject.org/torspec.git/blob/HEAD:/dir-spec.txt @@ -618,7 +618,7 @@ class RelayDescriptorV3(ServerDescriptorV3): self.signature = None self._digest = None - ServerDescriptorV3.__init__(self, raw_contents, validate, annotations) + ServerDescriptor.__init__(self, raw_contents, validate, annotations) def is_valid(self): """ @@ -680,7 +680,7 @@ class RelayDescriptorV3(ServerDescriptorV3): self.signature = block_contents del entries["router-signature"] - ServerDescriptorV3._parse(self, entries, validate) + ServerDescriptor._parse(self, entries, validate) def _required_fields(self): return REQUIRED_FIELDS @@ -694,7 +694,7 @@ class RelayDescriptorV3(ServerDescriptorV3): def _last_keyword(self): return "router-signature" -class BridgeDescriptorV3(ServerDescriptorV3): +class BridgeDescriptor(ServerDescriptor): """ Version 3 bridge descriptor, as specified in... https://metrics.torproject.org/formats.html#bridgedesc @@ -708,7 +708,7 @@ class BridgeDescriptorV3(ServerDescriptorV3): def __init__(self, raw_contents, validate = True, annotations = None): self.address_alt = [] self._scrubbing_issues = None - ServerDescriptorV3.__init__(self, raw_contents, validate, annotations) + ServerDescriptor.__init__(self, raw_contents, validate, annotations) def _parse(self, entries, validate): entries = dict(entries) @@ -744,7 +744,7 @@ class BridgeDescriptorV3(ServerDescriptorV3): del entries["or-address"] - ServerDescriptorV3._parse(self, entries, validate) + ServerDescriptor._parse(self, entries, validate) def is_scrubbed(self): """ diff --git a/test/integ/descriptor/server_descriptor.py b/test/integ/descriptor/server_descriptor.py index 84732b8..5ad700f 100644 --- a/test/integ/descriptor/server_descriptor.py +++ b/test/integ/descriptor/server_descriptor.py @@ -74,7 +74,7 @@ dskLSPz8beUW7bzwDjR6EVNGpyoZde83Ejvau+5F2c6cGnlu91fiZN3suE88iE6e Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= -----END SIGNATURE-----""" - desc = stem.descriptor.server_descriptor.RelayDescriptorV3(descriptor_contents) + desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) self.assertEquals("caerSidi", desc.nickname) self.assertEquals("A7569A83B5706AB1B1A9CB52EFF7D2D32E4553EB", desc.fingerprint) self.assertEquals("71.35.133.197", desc.address) @@ -116,7 +116,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_contents = descriptor_file.read() descriptor_file.close() - desc = stem.descriptor.server_descriptor.RelayDescriptorV3(descriptor_contents) + desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) self.assertEquals("krypton", desc.nickname) self.assertEquals("3E2F63E2356F52318B536A12B6445373808A5D6C", desc.fingerprint) self.assertEquals("212.37.39.59", desc.address) @@ -181,7 +181,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= RAN_CACHED_DESCRIPTOR_TEST = True with open(descriptor_path) as descriptor_file: - for desc in stem.descriptor.server_descriptor.parse_file_v3(descriptor_file): + for desc in stem.descriptor.server_descriptor.parse_file(descriptor_file): # the following attributes should be deprecated, and not appear in the wild self.assertEquals(None, desc.read_history) self.assertEquals(None, desc.write_history) @@ -214,7 +214,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= expected_published = datetime.datetime(2012, 3, 21, 16, 28, 14) expected_contact = "2048R/F171EC1F Johan Bl\xc3\xa5b\xc3\xa4ck \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf" - desc = stem.descriptor.server_descriptor.RelayDescriptorV3(descriptor_contents) + desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) self.assertEquals("torrelay389752132", desc.nickname) self.assertEquals("FEBC7F992AC418BBE42BC13FE94EFCFE6549197E", desc.fingerprint) self.assertEquals("130.243.230.116", desc.address) @@ -253,7 +253,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_contents = descriptor_file.read() descriptor_file.close() - desc = stem.descriptor.server_descriptor.RelayDescriptorV3(descriptor_contents) + desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) self.assertEquals("pogonip", desc.nickname) self.assertEquals("6DABD62BC65D4E6FE620293157FC76968DAB9C9B", desc.fingerprint) @@ -280,7 +280,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= descriptor_contents = descriptor_file.read() descriptor_file.close() - desc = stem.descriptor.server_descriptor.RelayDescriptorV3(descriptor_contents) + desc = stem.descriptor.server_descriptor.RelayDescriptor(descriptor_contents) self.assertEquals("TipTor", desc.nickname) self.assertEquals("137962D4931DBF08A24E843288B8A155D6D2AEDD", desc.fingerprint) @@ -290,7 +290,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= # should appear descriptor_contents = descriptor_contents.replace("Tor 0.1.1.25", "Tor 0.1.2.7") - self.assertRaises(ValueError, stem.descriptor.server_descriptor.RelayDescriptorV3, descriptor_contents) + self.assertRaises(ValueError, stem.descriptor.server_descriptor.RelayDescriptor, descriptor_contents) def test_bridge_descriptor(self): """ @@ -311,7 +311,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= "$8C8A470D7C23151665A7B84E75E89FCC205A3304", ] - desc = stem.descriptor.server_descriptor.BridgeDescriptorV3(descriptor_contents) + desc = stem.descriptor.server_descriptor.BridgeDescriptor(descriptor_contents) self.assertEquals("Unnamed", desc.nickname) self.assertEquals("AE54E28ED069CDF45F3009F963EE3B3D6FA26A2E", desc.fingerprint) self.assertEquals("10.45.227.253", desc.address) diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 101cdcd..e3b2ec6 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -7,7 +7,7 @@ import StringIO import unittest import stem.descriptor.server_descriptor -from stem.descriptor.server_descriptor import RelayDescriptorV3, BridgeDescriptorV3 +from stem.descriptor.server_descriptor import RelayDescriptor, BridgeDescriptor CRYPTO_BLOB = """ MIGJAoGBAJv5IIWQ+WDWYUdyA/0L8qbIkEVH/cwryZWoIaPAzINfrw1WfNZGtBmg @@ -80,7 +80,7 @@ class TestServerDescriptor(unittest.TestCase): """ desc_text = _make_descriptor() - desc = RelayDescriptorV3(desc_text) + desc = RelayDescriptor(desc_text) self.assertEquals("caerSidi", desc.nickname) self.assertEquals("71.35.133.197", desc.address) @@ -95,7 +95,7 @@ class TestServerDescriptor(unittest.TestCase): """ desc_text = _make_descriptor({"opt": "contact www.atagar.com/contact/"}) - desc = RelayDescriptorV3(desc_text) + desc = RelayDescriptor(desc_text) self.assertEquals("www.atagar.com/contact/", desc.contact) def test_unrecognized_line(self): @@ -104,7 +104,7 @@ class TestServerDescriptor(unittest.TestCase): """ desc_text = _make_descriptor({"pepperjack": "is oh so tasty!"}) - desc = RelayDescriptorV3(desc_text) + desc = RelayDescriptor(desc_text) self.assertEquals(["pepperjack is oh so tasty!"], desc.get_unrecognized_lines()) def test_proceeding_line(self): @@ -185,12 +185,12 @@ class TestServerDescriptor(unittest.TestCase): """ desc_text = _make_descriptor({"platform": ""}) - desc = RelayDescriptorV3(desc_text, validate = False) + desc = RelayDescriptor(desc_text, validate = False) self.assertEquals("", desc.platform) # does the same but with 'platform ' replaced with 'platform' desc_text = desc_text.replace("platform ", "platform") - desc = RelayDescriptorV3(desc_text, validate = False) + desc = RelayDescriptor(desc_text, validate = False) self.assertEquals("", desc.platform) def test_protocols_no_circuit_versions(self): @@ -212,7 +212,7 @@ class TestServerDescriptor(unittest.TestCase): desc_text = _make_descriptor({"published": "2012-02-29 04:03:19"}) expected_published = datetime.datetime(2012, 2, 29, 4, 3, 19) - self.assertEquals(expected_published, RelayDescriptorV3(desc_text).published) + self.assertEquals(expected_published, RelayDescriptor(desc_text).published) def test_published_no_time(self): """ @@ -232,7 +232,7 @@ class TestServerDescriptor(unittest.TestCase): for field in ("read-history", "write-history"): value = "2005-12-16 18:00:48 (900 s) 81,8848,8927,8927,83,8848" desc_text = _make_descriptor({"opt %s" % field: value}) - desc = RelayDescriptorV3(desc_text) + desc = RelayDescriptor(desc_text) if field == "read-history": attr = (desc.read_history, desc.read_history_end, @@ -256,7 +256,7 @@ class TestServerDescriptor(unittest.TestCase): value = "2005-12-17 01:23:11 (900 s) " desc_text = _make_descriptor({"opt read-history": value}) - desc = RelayDescriptorV3(desc_text) + desc = RelayDescriptor(desc_text) self.assertEquals(value, desc.read_history) self.assertEquals(datetime.datetime(2005, 12, 17, 1, 23, 11), desc.read_history_end) self.assertEquals(900, desc.read_history_interval) @@ -271,8 +271,8 @@ class TestServerDescriptor(unittest.TestCase): desc_text += _make_descriptor() desc_text += "\ntrailing text that should be ignored, ho hum" - # running parse_file_v3 should provide an iterator with a single descriptor - desc_iter = stem.descriptor.server_descriptor.parse_file_v3(StringIO.StringIO(desc_text)) + # running parse_file should provide an iterator with a single descriptor + desc_iter = stem.descriptor.server_descriptor.parse_file(StringIO.StringIO(desc_text)) desc_entries = list(desc_iter) self.assertEquals(1, len(desc_entries)) desc = desc_entries[0] @@ -299,10 +299,10 @@ class TestServerDescriptor(unittest.TestCase): for attr in stem.descriptor.server_descriptor.REQUIRED_FIELDS: desc_text = _make_descriptor(exclude = [attr]) - self.assertRaises(ValueError, RelayDescriptorV3, desc_text) + self.assertRaises(ValueError, RelayDescriptor, desc_text) # check that we can still construct it without validation - desc = RelayDescriptorV3(desc_text, validate = False) + desc = RelayDescriptor(desc_text, validate = False) # for one of them checks that the corresponding values are None if attr == "router": @@ -318,7 +318,7 @@ class TestServerDescriptor(unittest.TestCase): """ desc_text = _make_descriptor(is_bridge = True) - desc = BridgeDescriptorV3(desc_text) + desc = BridgeDescriptor(desc_text) self.assertEquals("Unnamed", desc.nickname) self.assertEquals("10.45.227.253", desc.address) @@ -347,7 +347,7 @@ class TestServerDescriptor(unittest.TestCase): for attr in unsanitized_attr: desc_text = _make_descriptor(attr, is_bridge = True) - desc = BridgeDescriptorV3(desc_text) + desc = BridgeDescriptor(desc_text) self.assertFalse(desc.is_scrubbed()) def test_bridge_unsanitized_relay(self): @@ -357,7 +357,7 @@ class TestServerDescriptor(unittest.TestCase): """ desc_text = _make_descriptor() - desc = BridgeDescriptorV3(desc_text) + desc = BridgeDescriptor(desc_text) self.assertFalse(desc.is_scrubbed()) def test_or_address_v4(self): @@ -366,7 +366,7 @@ class TestServerDescriptor(unittest.TestCase): """ desc_text = _make_descriptor({"or-address": "10.45.227.253:9001"}, is_bridge = True) - desc = BridgeDescriptorV3(desc_text) + desc = BridgeDescriptor(desc_text) self.assertEquals([("10.45.227.253", 9001, False)], desc.address_alt) def test_or_address_v6(self): @@ -375,7 +375,7 @@ class TestServerDescriptor(unittest.TestCase): """ desc_text = _make_descriptor({"or-address": "[fd9f:2e19:3bcf::02:9970]:9001"}, is_bridge = True) - desc = BridgeDescriptorV3(desc_text) + desc = BridgeDescriptor(desc_text) self.assertEquals([("fd9f:2e19:3bcf::02:9970", 9001, True)], desc.address_alt) def test_or_address_multiple(self): @@ -394,7 +394,7 @@ class TestServerDescriptor(unittest.TestCase): ("fd9f:2e19:3bcf::02:9970", 443, True), ] - desc = BridgeDescriptorV3(desc_text) + desc = BridgeDescriptor(desc_text) self.assertEquals(expected_address_alt, desc.address_alt) def _expect_invalid_attr(self, desc_text, attr = None, expected_value = None): @@ -404,8 +404,8 @@ class TestServerDescriptor(unittest.TestCase): value when we're constructed without validation. """ - self.assertRaises(ValueError, RelayDescriptorV3, desc_text) - desc = RelayDescriptorV3(desc_text, validate = False) + self.assertRaises(ValueError, RelayDescriptor, desc_text) + desc = RelayDescriptor(desc_text, validate = False) if attr: # check that the invalid attribute matches the expected value when

1 0

[stem/master] Parsing descriptors with a 127.0.0.1 address as bridges
by atagar＠torproject.org 06 May '12

06 May '12

commit 477d448f5f19d9d5336e3297943b14567bb55c52 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 5 15:51:47 2012 -0700 Parsing descriptors with a 127.0.0.1 address as bridges A prior scrubbing scheme for bridge descriptors scrubbed their addresses to be 127.0.0.1 rather than 10.x.x.x. Parsing both as bridges so we're compatible with both schemes. Thanks to Karsten for pointing this out. --- stem/descriptor/server_descriptor.py | 9 +++++---- 1 files changed, 5 insertions(+), 4 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 5a187df..6b53866 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -87,14 +87,15 @@ def parse_file(descriptor_file, validate = True): # Handler for bridge descriptors # # Bridge descriptors are scrubbed so their nickname is 'Unnamed' and their - # ip address is in the 10.x.x.x space, which is normally reserved for - # private networks. Bride descriptors only come from metrics so a file only - # contains a single descriptor. + # ip address is in the 10.x.x.x space (priorly '127.0.0.1'), which is + # normally reserved for private networks. Bridge descriptors only come from + # metrics so a file only contains a single descriptor. first_line = descriptor_file.readline() descriptor_file.seek(0) - if first_line.startswith("router Unnamed 10."): + if first_line.startswith("router Unnamed 10.") or \ + first_line.startswith("router Unnamed 127.0.0.1"): yield BridgeDescriptor(descriptor_file.read()) return

1 0

[stem/master] Separating bridge scrubbing validation into method
by atagar＠torproject.org 06 May '12

06 May '12

commit 3a5e9979e40a400b8f4861406bc75af32578c30c Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 5 15:27:59 2012 -0700 Separating bridge scrubbing validation into method Bridge descriptor scrubbing is a moving target and it's quite possible that we'll need to parse descriptors that conform to a newer or older scheme. Hence making scrubbing validation part of the constructor is inappropriate. Moving it to a separate method instead. This includes two methods, one to simply check if we think that the scrubbing is right, and another to get descriptions of the issues. This is of limited usefulness to callers, so we might need to switch to an exception hierchy later. However, we don't have any use cases that care to check the scrubbing yet so leaving that alone for now. --- stem/descriptor/server_descriptor.py | 75 +++++++++++++++++++---------- test/unit/descriptor/server_descriptor.py | 11 ++-- 2 files changed, 54 insertions(+), 32 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index f3023a2..d56b453 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -14,6 +14,8 @@ ServerDescriptorV3 - Tor server descriptor, version 3. | | +- digest - calculates the digest value for our content | | | +- BridgeDescriptorV3 - Scrubbed server descriptor for a bridge. + | |- is_scrubbed - checks if our content has been properly scrubbed + | +- get_scrubbing_issues - description of issues with our scrubbing | |- get_unrecognized_lines - lines with unrecognized content |- get_annotations - dictionary of content prior to the descriptor entry @@ -705,6 +707,7 @@ class BridgeDescriptorV3(ServerDescriptorV3): def __init__(self, raw_contents, validate = True, annotations = None): self.address_alt = [] + self._scrubbing_issues = None ServerDescriptorV3.__init__(self, raw_contents, validate, annotations) def _parse(self, entries, validate): @@ -742,36 +745,56 @@ class BridgeDescriptorV3(ServerDescriptorV3): del entries["or-address"] ServerDescriptorV3._parse(self, entries, validate) - if validate: self._check_scrubbing() - def _check_scrubbing(self): + def is_scrubbed(self): """ - Checks that our attributes have been scrubbed in accordance with the bridge - descriptor specification. + Checks if we've been properly scrubbed in accordance with the bridge + descriptor specification. Validation is a moving target so this may not + be fully up to date. + + Returns: + True if we're scrubbed, False otherwise + """ + + return self.get_scrubbing_issues() == [] + + def get_scrubbing_issues(self): + """ + Provides issues with our scrubbing. + + Returns: + list of strings which describe issues we have with our scrubbing, this + list is empty if we're properly scrubbed """ - if self.nickname != "Unnamed": - raise ValueError("Router line's nickname should be scrubbed to be 'Unnamed': %s" % self.nickname) - elif not self.address.startswith("10."): - raise ValueError("Router line's address should be scrubbed to be '10.x.x.x': %s" % self.address) - elif self.contact and self.contact != "somebody": - raise ValueError("Contact line should be scrubbed to be 'somebody', but instead had '%s'" % self.contact) - - for address, _, is_ipv6 in self.address_alt: - if not is_ipv6 and not address.startswith("10."): - raise ValueError("or-address line's address should be scrubbed to be '10.x.x.x': %s" % address) - elif is_ipv6 and not address.startswith("fd9f:2e19:3bcf::"): - # TODO: this check isn't quite right because we aren't checking that - # the next grouping of hex digits contains 1-2 digits - raise ValueError("or-address line's address should be scrubbed to be 'fd9f:2e19:3bcf::xx:xxxx': %s" % address) - - for line in self.get_unrecognized_lines(): - if line.startswith("onion-key "): - raise ValueError("Bridge descriptors should have their onion-key scrubbed: %s" % line) - elif line.startswith("signing-key "): - raise ValueError("Bridge descriptors should have their signing-key scrubbed: %s" % line) - elif line.startswith("router-signature "): - raise ValueError("Bridge descriptors should have their signature scrubbed: %s" % line) + if self._scrubbing_issues == None: + issues = [] + if self.nickname != "Unnamed": + issues.append("Router line's nickname should be scrubbed to be 'Unnamed': %s" % self.nickname) + elif not self.address.startswith("10."): + issues.append("Router line's address should be scrubbed to be '10.x.x.x': %s" % self.address) + elif self.contact and self.contact != "somebody": + issues.append("Contact line should be scrubbed to be 'somebody', but instead had '%s'" % self.contact) + + for address, _, is_ipv6 in self.address_alt: + if not is_ipv6 and not address.startswith("10."): + issues.append("or-address line's address should be scrubbed to be '10.x.x.x': %s" % address) + elif is_ipv6 and not address.startswith("fd9f:2e19:3bcf::"): + # TODO: this check isn't quite right because we aren't checking that + # the next grouping of hex digits contains 1-2 digits + issues.append("or-address line's address should be scrubbed to be 'fd9f:2e19:3bcf::xx:xxxx': %s" % address) + + for line in self.get_unrecognized_lines(): + if line.startswith("onion-key "): + issues.append("Bridge descriptors should have their onion-key scrubbed: %s" % line) + elif line.startswith("signing-key "): + issues.append("Bridge descriptors should have their signing-key scrubbed: %s" % line) + elif line.startswith("router-signature "): + issues.append("Bridge descriptors should have their signature scrubbed: %s" % line) + + self._scrubbing_issues = issues + + return self._scrubbing_issues def _required_fields(self): # bridge required fields are the same as a relay descriptor, minus items diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index fceb773..101cdcd 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -346,11 +346,9 @@ class TestServerDescriptor(unittest.TestCase): ] for attr in unsanitized_attr: - try: - desc_text = _make_descriptor(attr, is_bridge = True) - BridgeDescriptorV3(desc_text) - self.fail("Unsanitized attribute wasn't detected: %s %s" % attr.items()[0]) - except ValueError: pass + desc_text = _make_descriptor(attr, is_bridge = True) + desc = BridgeDescriptorV3(desc_text) + self.assertFalse(desc.is_scrubbed()) def test_bridge_unsanitized_relay(self): """ @@ -359,7 +357,8 @@ class TestServerDescriptor(unittest.TestCase): """ desc_text = _make_descriptor() - self.assertRaises(ValueError, BridgeDescriptorV3, desc_text) + desc = BridgeDescriptorV3(desc_text) + self.assertFalse(desc.is_scrubbed()) def test_or_address_v4(self): """

1 0

[stem/master] Supporting controllers in the connection module
by atagar＠torproject.org 06 May '12

06 May '12

commit af691f5b54d1f571ee51d8c67010ea19e4c672fd Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 5 17:48:29 2012 -0700 Supporting controllers in the connection module Users will be using a BaseController subclass unless they really need to work at a low level, in which case they'll be using a ControlSocket. Making the connection module (which does authentication) support both. --- stem/connection.py | 86 +++++++++++++++++++------------ test/integ/connection/authentication.py | 26 +++++++--- test/integ/connection/connect.py | 4 +- test/runner.py | 38 +++++++++++--- 4 files changed, 103 insertions(+), 51 deletions(-) diff --git a/stem/connection.py b/stem/connection.py index d950049..825edf5 100644 --- a/stem/connection.py +++ b/stem/connection.py @@ -277,7 +277,7 @@ def _connect(control_socket, password, chroot_path, controller): print "Unable to authenticate: %s" % exc return None -def authenticate(control_socket, password = None, chroot_path = None, protocolinfo_response = None): +def authenticate(controller, password = None, chroot_path = None, protocolinfo_response = None): """ Authenticates to a control socket using the information provided by a PROTOCOLINFO response. In practice this will often be all we need to @@ -288,7 +288,8 @@ def authenticate(control_socket, password = None, chroot_path = None, protocolin about, then have a AuthenticationFailure catch-all at the end. Arguments: - control_socket (stem.socket.ControlSocket) - socket to be authenticated + controller (stem.socket.ControlSocket or stem.control.BaseController) - + tor controller connection to be authenticated password (str) - passphrase to present to the socket if it uses password authentication (skips password auth if None) chroot_path (str) - path prefix if in a chroot environment @@ -302,8 +303,8 @@ def authenticate(control_socket, password = None, chroot_path = None, protocolin follows... stem.connection.IncorrectSocketType - The control_socket does not speak the tor control protocol. Most often - this happened because the user confused the SocksPort or ORPort with the + The controller does not speak the tor control protocol. Most often this + happened because the user confused the SocksPort or ORPort with the ControlPort. stem.connection.UnrecognizedAuthMethods @@ -355,7 +356,7 @@ def authenticate(control_socket, password = None, chroot_path = None, protocolin if not protocolinfo_response: try: - protocolinfo_response = get_protocolinfo(control_socket) + protocolinfo_response = get_protocolinfo(controller) except stem.socket.ProtocolError: raise IncorrectSocketType("unable to use the control socket") except stem.socket.SocketError, exc: @@ -397,16 +398,16 @@ def authenticate(control_socket, password = None, chroot_path = None, protocolin try: if auth_type == AuthMethod.NONE: - authenticate_none(control_socket, False) + authenticate_none(controller, False) elif auth_type == AuthMethod.PASSWORD: - authenticate_password(control_socket, password, False) + authenticate_password(controller, password, False) elif auth_type == AuthMethod.COOKIE: cookie_path = protocolinfo_response.cookie_path if chroot_path: cookie_path = os.path.join(chroot_path, cookie_path.lstrip(os.path.sep)) - authenticate_cookie(control_socket, cookie_path, False) + authenticate_cookie(controller, cookie_path, False) return # success! except OpenAuthRejected, exc: @@ -439,7 +440,7 @@ def authenticate(control_socket, password = None, chroot_path = None, protocolin raise AssertionError("BUG: Authentication failed without providing a recognized exception: %s" % str(auth_exceptions)) -def authenticate_none(control_socket, suppress_ctl_errors = True): +def authenticate_none(controller, suppress_ctl_errors = True): """ Authenticates to an open control socket. All control connections need to authenticate before they can be used, even if tor hasn't been configured to @@ -452,7 +453,8 @@ def authenticate_none(control_socket, suppress_ctl_errors = True): For general usage use the authenticate() function instead. Arguments: - control_socket (stem.socket.ControlSocket) - socket to be authenticated + controller (stem.socket.ControlSocket or stem.control.BaseController) - + tor controller connection suppress_ctl_errors (bool) - reports raised stem.socket.ControllerError as authentication rejection if True, otherwise they're re-raised @@ -462,23 +464,22 @@ def authenticate_none(control_socket, suppress_ctl_errors = True): """ try: - control_socket.send("AUTHENTICATE") - auth_response = control_socket.recv() + auth_response = _msg(controller, "AUTHENTICATE") # if we got anything but an OK response then error if str(auth_response) != "OK": - try: control_socket.connect() + try: controller.connect() except: pass raise OpenAuthRejected(str(auth_response), auth_response) except stem.socket.ControllerError, exc: - try: control_socket.connect() + try: controller.connect() except: pass if not suppress_ctl_errors: raise exc else: raise OpenAuthRejected("Socket failed (%s)" % exc) -def authenticate_password(control_socket, password, suppress_ctl_errors = True): +def authenticate_password(controller, password, suppress_ctl_errors = True): """ Authenticates to a control socket that uses a password (via the HashedControlPassword torrc option). Quotes in the password are escaped. @@ -495,7 +496,8 @@ def authenticate_password(control_socket, password, suppress_ctl_errors = True): future versions. Arguments: - control_socket (stem.socket.ControlSocket) - socket to be authenticated + controller (stem.socket.ControlSocket or stem.control.BaseController) - + tor controller connection password (str) - passphrase to present to the socket suppress_ctl_errors (bool) - reports raised stem.socket.ControllerError as authentication rejection if True, otherwise they're re-raised @@ -514,12 +516,11 @@ def authenticate_password(control_socket, password, suppress_ctl_errors = True): password = password.replace('"', '\\"') try: - control_socket.send("AUTHENTICATE \"%s\"" % password) - auth_response = control_socket.recv() + auth_response = _msg(controller, "AUTHENTICATE \"%s\"" % password) # if we got anything but an OK response then error if str(auth_response) != "OK": - try: control_socket.connect() + try: controller.connect() except: pass # all we have to go on is the error message from tor... @@ -531,13 +532,13 @@ def authenticate_password(control_socket, password, suppress_ctl_errors = True): else: raise PasswordAuthRejected(str(auth_response), auth_response) except stem.socket.ControllerError, exc: - try: control_socket.connect() + try: controller.connect() except: pass if not suppress_ctl_errors: raise exc else: raise PasswordAuthRejected("Socket failed (%s)" % exc) -def authenticate_cookie(control_socket, cookie_path, suppress_ctl_errors = True): +def authenticate_cookie(controller, cookie_path, suppress_ctl_errors = True): """ Authenticates to a control socket that uses the contents of an authentication cookie (generated via the CookieAuthentication torrc option). This does basic @@ -559,7 +560,8 @@ def authenticate_cookie(control_socket, cookie_path, suppress_ctl_errors = True) future versions. Arguments: - control_socket (stem.socket.ControlSocket) - socket to be authenticated + controller (stem.socket.ControlSocket or stem.control.BaseController) - + tor controller connection cookie_path (str) - path of the authentication cookie to send to tor suppress_ctl_errors (bool) - reports raised stem.socket.ControllerError as authentication rejection if True, otherwise they're re-raised @@ -599,12 +601,12 @@ def authenticate_cookie(control_socket, cookie_path, suppress_ctl_errors = True) raise UnreadableCookieFile("Authentication failed: unable to read '%s' (%s)" % (cookie_path, exc), cookie_path) try: - control_socket.send("AUTHENTICATE %s" % binascii.b2a_hex(auth_cookie_contents)) - auth_response = control_socket.recv() + msg = "AUTHENTICATE %s" % binascii.b2a_hex(auth_cookie_contents) + auth_response = _msg(controller, msg) # if we got anything but an OK response then error if str(auth_response) != "OK": - try: control_socket.connect() + try: controller.connect() except: pass # all we have to go on is the error message from tor... @@ -617,20 +619,21 @@ def authenticate_cookie(control_socket, cookie_path, suppress_ctl_errors = True) else: raise CookieAuthRejected(str(auth_response), cookie_path, auth_response) except stem.socket.ControllerError, exc: - try: control_socket.connect() + try: controller.connect() except: pass if not suppress_ctl_errors: raise exc else: raise CookieAuthRejected("Socket failed (%s)" % exc, cookie_path) -def get_protocolinfo(control_socket): +def get_protocolinfo(controller): """ Issues a PROTOCOLINFO query to a control socket, getting information about the tor process running on it. If the socket is already closed then it is first reconnected. Arguments: - control_socket (stem.socket.ControlSocket) - connected tor control socket + controller (stem.socket.ControlSocket or stem.control.BaseController) - + tor controller connection Returns: stem.connection.ProtocolInfoResponse provided by tor @@ -642,8 +645,7 @@ def get_protocolinfo(control_socket): """ try: - control_socket.send("PROTOCOLINFO 1") - protocolinfo_response = control_socket.recv() + protocolinfo_response = _msg(controller, "PROTOCOLINFO 1") except: protocolinfo_response = None @@ -651,17 +653,22 @@ def get_protocolinfo(control_socket): # next followed by authentication. Transparently reconnect if that happens. if not protocolinfo_response or str(protocolinfo_response) == "Authentication required.": - control_socket.connect() + controller.connect() try: - control_socket.send("PROTOCOLINFO 1") - protocolinfo_response = control_socket.recv() + protocolinfo_response = _msg(controller, "PROTOCOLINFO 1") except stem.socket.SocketClosed, exc: raise stem.socket.SocketError(exc) ProtocolInfoResponse.convert(protocolinfo_response) - # attempt ot expand relative cookie paths via the control port or socket file + # attempt to expand relative cookie paths via the control port or socket file + + if isinstance(controller, stem.socket.ControlSocket): + control_socket = controller + else: + control_socket = controller.get_socket() + if isinstance(control_socket, stem.socket.ControlPort): if control_socket.get_address() == "127.0.0.1": pid_method = stem.util.system.get_pid_by_port @@ -672,6 +679,17 @@ def get_protocolinfo(control_socket): return protocolinfo_response +def _msg(controller, message): + """ + Sends and receives a message with either a ControlSocket or BaseController. + """ + + if isinstance(controller, stem.socket.ControlSocket): + controller.send(message) + return controller.recv() + else: + return controller.msg(message) + def _expand_cookie_path(protocolinfo_response, pid_resolver, pid_resolution_arg): """ Attempts to expand a relative cookie path with the given pid resolver. This diff --git a/test/integ/connection/authentication.py b/test/integ/connection/authentication.py index f80b848..c209d2b 100644 --- a/test/integ/connection/authentication.py +++ b/test/integ/connection/authentication.py @@ -80,7 +80,7 @@ class TestAuthenticate(unittest.TestCase): def setUp(self): test.runner.require_control(self) - def test_authenticate_general(self): + def test_authenticate_general_socket(self): """ Tests that the authenticate function can authenticate to our socket. """ @@ -88,7 +88,17 @@ class TestAuthenticate(unittest.TestCase): runner = test.runner.get_runner() with runner.get_tor_socket(False) as control_socket: stem.connection.authenticate(control_socket, test.runner.CONTROL_PASSWORD, runner.get_chroot()) - test.runner.exercise_socket(self, control_socket) + test.runner.exercise_controller(self, control_socket) + + def test_authenticate_general_controller(self): + """ + Tests that the authenticate function can authenticate via a Controller. + """ + + runner = test.runner.get_runner() + with runner.get_tor_controller(False) as controller: + stem.connection.authenticate(controller, test.runner.CONTROL_PASSWORD, runner.get_chroot()) + test.runner.exercise_controller(self, controller) def test_authenticate_general_example(self): """ @@ -108,7 +118,7 @@ class TestAuthenticate(unittest.TestCase): try: # this authenticate call should work for everything but password-only auth stem.connection.authenticate(control_socket, chroot_path = runner.get_chroot()) - test.runner.exercise_socket(self, control_socket) + test.runner.exercise_controller(self, control_socket) except stem.connection.IncorrectSocketType: self.fail() except stem.connection.MissingPassword: @@ -117,7 +127,7 @@ class TestAuthenticate(unittest.TestCase): try: stem.connection.authenticate_password(control_socket, controller_password) - test.runner.exercise_socket(self, control_socket) + test.runner.exercise_controller(self, control_socket) except stem.connection.PasswordAuthFailed: self.fail() except stem.connection.AuthenticationFailure: @@ -144,7 +154,7 @@ class TestAuthenticate(unittest.TestCase): self.assertRaises(stem.connection.MissingPassword, stem.connection.authenticate, control_socket) else: stem.connection.authenticate(control_socket, chroot_path = runner.get_chroot()) - test.runner.exercise_socket(self, control_socket) + test.runner.exercise_controller(self, control_socket) # tests with the incorrect password with runner.get_tor_socket(False) as control_socket: @@ -152,12 +162,12 @@ class TestAuthenticate(unittest.TestCase): self.assertRaises(stem.connection.IncorrectPassword, stem.connection.authenticate, control_socket, "blarg") else: stem.connection.authenticate(control_socket, "blarg", runner.get_chroot()) - test.runner.exercise_socket(self, control_socket) + test.runner.exercise_controller(self, control_socket) # tests with the right password with runner.get_tor_socket(False) as control_socket: stem.connection.authenticate(control_socket, test.runner.CONTROL_PASSWORD, runner.get_chroot()) - test.runner.exercise_socket(self, control_socket) + test.runner.exercise_controller(self, control_socket) def test_authenticate_none(self): """ @@ -299,7 +309,7 @@ class TestAuthenticate(unittest.TestCase): elif auth_type == stem.connection.AuthMethod.COOKIE: stem.connection.authenticate_cookie(control_socket, auth_arg) - test.runner.exercise_socket(self, control_socket) + test.runner.exercise_controller(self, control_socket) except stem.connection.AuthenticationFailure, exc: # authentication functions should re-attach on failure self.assertTrue(control_socket.is_alive()) diff --git a/test/integ/connection/connect.py b/test/integ/connection/connect.py index 29cf2dc..ac66ceb 100644 --- a/test/integ/connection/connect.py +++ b/test/integ/connection/connect.py @@ -34,7 +34,7 @@ class TestConnect(unittest.TestCase): controller = stem.connection.Controller.NONE) if test.runner.Torrc.PORT in runner.get_options(): - test.runner.exercise_socket(self, control_socket) + test.runner.exercise_controller(self, control_socket) control_socket.close() else: self.assertEquals(control_socket, None) @@ -53,7 +53,7 @@ class TestConnect(unittest.TestCase): controller = stem.connection.Controller.NONE) if test.runner.Torrc.SOCKET in runner.get_options(): - test.runner.exercise_socket(self, control_socket) + test.runner.exercise_controller(self, control_socket) control_socket.close() else: self.assertEquals(control_socket, None) diff --git a/test/runner.py b/test/runner.py index 6702b8b..66636f2 100644 --- a/test/runner.py +++ b/test/runner.py @@ -8,7 +8,7 @@ TorInaccessable - Tor can't be queried for the information require_control - skips the test unless tor provides a controller endpoint require_version - skips the test unless we meet a tor version requirement -exercise_socket - Does a basic sanity check that a control socket can be used +exercise_controller - basic sanity check that a controller connection can be used get_runner - Singleton for fetching our runtime context. Runner - Runtime context for our integration tests. @@ -25,7 +25,8 @@ Runner - Runtime context for our integration tests. |- get_tor_cwd - current working directory of our tor process |- get_chroot - provides the path of our emulated chroot if we have one |- get_pid - process id of our tor process - |- get_tor_socket - provides a socket to the tor instance + |- get_tor_socket - provides a socket to our test instance + |- get_tor_controller - provides a controller for our test instance |- get_tor_version - provides the version of tor we're running against +- get_tor_command - provides the command used to start tor """ @@ -115,14 +116,15 @@ def require_version(test_case, req_version): if get_runner().get_tor_version() < req_version: test_case.skipTest("(requires %s)" % req_version) -def exercise_socket(test_case, control_socket): +def exercise_controller(test_case, controller): """ Checks that we can now use the socket by issuing a 'GETINFO config-file' query. Arguments: test_case (unittest.TestCase) - test being ran - control_socket (stem.socket.ControlSocket) - socket to be tested + controller (stem.socket.ControlSocket or stem.control.BaseController) - + tor controller connection to be authenticated """ runner = get_runner() @@ -131,8 +133,12 @@ def exercise_socket(test_case, control_socket): if chroot_path and torrc_path.startswith(chroot_path): torrc_path = torrc_path[len(chroot_path):] - control_socket.send("GETINFO config-file") - config_file_response = control_socket.recv() + if isinstance(controller, stem.socket.ControlSocket): + controller.send("GETINFO config-file") + config_file_response = controller.recv() + else: + config_file_response = controller.msg("GETINFO config-file") + test_case.assertEquals("config-file=%s\nOK" % torrc_path, str(config_file_response)) def get_runner(): @@ -429,7 +435,7 @@ class Runner: def get_tor_socket(self, authenticate = True): """ - Provides a socket connected to the tor test instance's control socket. + Provides a socket connected to our tor test instance. Arguments: authenticate (bool) - if True then the socket is authenticated @@ -452,6 +458,24 @@ class Runner: return control_socket + def get_tor_controller(self, authenticate = True): + """ + Provides a controller connected to our tor test instance. + + Arguments: + authenticate (bool) - if True then the socket is authenticated + + Returns: + stem.socket.BaseController connected with our testing instance + + Raises: + TorInaccessable if tor can't be connected to + """ + + # TODO: replace with our general controller when we have one + control_socket = self.get_tor_socket(authenticate) + return stem.control.BaseController(control_socket) + def get_tor_version(self): """ Queries our test instance for tor's version.

1 0

[stem/master] Making descriptor reader persistence more convenient
by atagar＠torproject.org 06 May '12

06 May '12

commit 10cb30b546016f975559bdf1d8dd785ee516ae84 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 5 16:54:24 2012 -0700 Making descriptor reader persistence more convenient Our current functions for loading/saving processed file listings are fine if you want error handling and a great deal of control. However, I suspect that most callers would prefer for this to be an attribute of the reader itself. Adding an argument that performs best-effort persistance of our processed files listing. --- stem/descriptor/reader.py | 22 +++++++++++++++++++++- test/integ/descriptor/reader.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletions(-) diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index 0bddd53..8f00f5f 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -191,17 +191,25 @@ class DescriptorReader: waiting for our caller to fetch some of them. This is included to avoid unbounded memory usage. + Our persistence_path argument is a convenient method to persist the listing + of files we have processed between runs, however it doesn't allow for error + handling. If you want that then use the load/save_processed_files functions + instead. + Arguments: targets (list) - paths for files or directories to be read from follow_links (bool) - determines if we'll follow symlinks when traversing directories buffer_size (int) - descriptors we'll buffer before waiting for some to be read, this is unbounded if zero + persistence_path (str) - if set we will load and save processed file + listings from this path, errors are ignored """ - def __init__(self, targets, follow_links = False, buffer_size = 100): + def __init__(self, targets, follow_links = False, buffer_size = 100, persistence_path = None): self._targets = targets self._follow_links = follow_links + self._persistence_path = persistence_path self._skip_listeners = [] self._processed_files = {} @@ -218,6 +226,12 @@ class DescriptorReader: # FINISHED entry is used by the reading thread to indicate the end. self._unreturned_descriptors = Queue.Queue(buffer_size) + + if self._persistence_path: + try: + processed_files = load_processed_files(self._persistence_path) + self.set_processed_files(processed_files) + except: pass def get_processed_files(self): """ @@ -311,6 +325,12 @@ class DescriptorReader: self._reader_thread.join() self._reader_thread = None + + if self._persistence_path: + try: + processed_files = self.get_processed_files() + save_processed_files(self._persistence_path, processed_files) + except: pass def _read_descriptor_files(self): new_processed_files = {} diff --git a/test/integ/descriptor/reader.py b/test/integ/descriptor/reader.py index 5c84704..0e3467c 100644 --- a/test/integ/descriptor/reader.py +++ b/test/integ/descriptor/reader.py @@ -207,6 +207,35 @@ class TestDescriptorReader(unittest.TestCase): time.sleep(0.01) self.assertTrue(reader.get_buffered_descriptor_count() <= 2) + def test_persistence_path(self): + """ + Check that the persistence_path argument loads and saves a a processed + files listing. + """ + + persistence_path = _get_processed_files_path() + descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "example_descriptor") + + # First run where the persistence_path doesn't yet exist. This just tests + # the saving functionality. + + reader = stem.descriptor.reader.DescriptorReader([descriptor_path], persistence_path = persistence_path) + with reader: self.assertEqual(1, len(list(reader))) + + # check that we've saved reading example_descriptor + self.assertTrue(os.path.exists(persistence_path)) + + with open(persistence_path) as persistence_file: + persistance_file_contents = persistence_file.read() + self.assertTrue(persistance_file_contents.startswith(descriptor_path)) + + # Try running again with a new reader but the same persistance path, if it + # reads and takes the persistence_path into account then it won't read the + # descriptor file. This in essence just tests its loading functionality. + + reader = stem.descriptor.reader.DescriptorReader([descriptor_path], persistence_path = persistence_path) + with reader: self.assertEqual(0, len(list(reader))) + def test_archived_uncompressed(self): """ Checks that we can read descriptors from an uncompressed archive.

1 0

[stem/master] Making decriptor reader accept path or list of paths
by atagar＠torproject.org 06 May '12

06 May '12

commit 81f272b9e227b36fe93037cbd1d8ef2265403269 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat May 5 17:06:12 2012 -0700 Making decriptor reader accept path or list of paths More often than not people will want to read descriptors from a single path rather than a list of paths. There's no need to make them wrap that path in a list before calling us. --- stem/descriptor/reader.py | 9 ++++++--- test/integ/descriptor/reader.py | 30 +++++++++++++++--------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/stem/descriptor/reader.py b/stem/descriptor/reader.py index 8f00f5f..1110854 100644 --- a/stem/descriptor/reader.py +++ b/stem/descriptor/reader.py @@ -197,7 +197,8 @@ class DescriptorReader: instead. Arguments: - targets (list) - paths for files or directories to be read from + target (str, list) - path or list of paths for files or directories to be + read from follow_links (bool) - determines if we'll follow symlinks when traversing directories buffer_size (int) - descriptors we'll buffer before waiting for some to @@ -206,8 +207,10 @@ class DescriptorReader: listings from this path, errors are ignored """ - def __init__(self, targets, follow_links = False, buffer_size = 100, persistence_path = None): - self._targets = targets + def __init__(self, target, follow_links = False, buffer_size = 100, persistence_path = None): + if isinstance(target, str): self._targets = [target] + else: self._targets = target + self._follow_links = follow_links self._persistence_path = persistence_path self._skip_listeners = [] diff --git a/test/integ/descriptor/reader.py b/test/integ/descriptor/reader.py index 0e3467c..0d4a638 100644 --- a/test/integ/descriptor/reader.py +++ b/test/integ/descriptor/reader.py @@ -155,7 +155,7 @@ class TestDescriptorReader(unittest.TestCase): for i in xrange(15): remaining_entries = list(descriptor_entries) - with stem.descriptor.reader.DescriptorReader([descriptor_path]) as reader: + with stem.descriptor.reader.DescriptorReader(descriptor_path) as reader: for descriptor in reader: descriptor_str = str(descriptor) @@ -175,7 +175,7 @@ class TestDescriptorReader(unittest.TestCase): """ descriptor_path = os.path.join(DESCRIPTOR_TEST_DATA, "example_descriptor") - reader = stem.descriptor.reader.DescriptorReader([descriptor_path]) + reader = stem.descriptor.reader.DescriptorReader(descriptor_path) with reader: self.assertEquals(1, len(list(reader))) @@ -200,7 +200,7 @@ class TestDescriptorReader(unittest.TestCase): them. """ - reader = stem.descriptor.reader.DescriptorReader([DESCRIPTOR_TEST_DATA], buffer_size = 2) + reader = stem.descriptor.reader.DescriptorReader(DESCRIPTOR_TEST_DATA, buffer_size = 2) with reader: self.assertTrue(reader.get_buffered_descriptor_count() <= 2) @@ -219,7 +219,7 @@ class TestDescriptorReader(unittest.TestCase): # First run where the persistence_path doesn't yet exist. This just tests # the saving functionality. - reader = stem.descriptor.reader.DescriptorReader([descriptor_path], persistence_path = persistence_path) + reader = stem.descriptor.reader.DescriptorReader(descriptor_path, persistence_path = persistence_path) with reader: self.assertEqual(1, len(list(reader))) # check that we've saved reading example_descriptor @@ -233,7 +233,7 @@ class TestDescriptorReader(unittest.TestCase): # reads and takes the persistence_path into account then it won't read the # descriptor file. This in essence just tests its loading functionality. - reader = stem.descriptor.reader.DescriptorReader([descriptor_path], persistence_path = persistence_path) + reader = stem.descriptor.reader.DescriptorReader(descriptor_path, persistence_path = persistence_path) with reader: self.assertEqual(0, len(list(reader))) def test_archived_uncompressed(self): @@ -244,7 +244,7 @@ class TestDescriptorReader(unittest.TestCase): expected_results = _get_raw_tar_descriptors() test_path = os.path.join(DESCRIPTOR_TEST_DATA, "descriptor_archive.tar") - with stem.descriptor.reader.DescriptorReader([test_path]) as reader: + with stem.descriptor.reader.DescriptorReader(test_path) as reader: read_descriptors = [str(desc) for desc in list(reader)] self.assertEquals(expected_results, read_descriptors) @@ -256,7 +256,7 @@ class TestDescriptorReader(unittest.TestCase): expected_results = _get_raw_tar_descriptors() test_path = os.path.join(DESCRIPTOR_TEST_DATA, "descriptor_archive.tar.gz") - with stem.descriptor.reader.DescriptorReader([test_path]) as reader: + with stem.descriptor.reader.DescriptorReader(test_path) as reader: read_descriptors = [str(desc) for desc in list(reader)] self.assertEquals(expected_results, read_descriptors) @@ -268,7 +268,7 @@ class TestDescriptorReader(unittest.TestCase): expected_results = _get_raw_tar_descriptors() test_path = os.path.join(DESCRIPTOR_TEST_DATA, "descriptor_archive.tar.bz2") - with stem.descriptor.reader.DescriptorReader([test_path]) as reader: + with stem.descriptor.reader.DescriptorReader(test_path) as reader: read_descriptors = [str(desc) for desc in list(reader)] self.assertEquals(expected_results, read_descriptors) @@ -279,7 +279,7 @@ class TestDescriptorReader(unittest.TestCase): """ is_test_running = True - reader = stem.descriptor.reader.DescriptorReader(["/"]) + reader = stem.descriptor.reader.DescriptorReader("/") # Fails the test after a couple seconds if we don't finish successfully. # Depending on what we're blocked on this might not work when the test @@ -311,7 +311,7 @@ class TestDescriptorReader(unittest.TestCase): last_modified = int(os.stat(path).st_mtime) expected_results[path] = last_modified - reader = stem.descriptor.reader.DescriptorReader([DESCRIPTOR_TEST_DATA]) + reader = stem.descriptor.reader.DescriptorReader(DESCRIPTOR_TEST_DATA) with reader: list(reader) # iterates over all of the descriptors self.assertEquals(expected_results, reader.get_processed_files()) @@ -323,7 +323,7 @@ class TestDescriptorReader(unittest.TestCase): """ skip_listener = SkipListener() - reader = stem.descriptor.reader.DescriptorReader([DESCRIPTOR_TEST_DATA]) + reader = stem.descriptor.reader.DescriptorReader(DESCRIPTOR_TEST_DATA) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors @@ -350,7 +350,7 @@ class TestDescriptorReader(unittest.TestCase): initial_processed_files = {test_path: sys.maxint} skip_listener = SkipListener() - reader = stem.descriptor.reader.DescriptorReader([test_path]) + reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) reader.set_processed_files(initial_processed_files) @@ -380,7 +380,7 @@ class TestDescriptorReader(unittest.TestCase): test_file.close() skip_listener = SkipListener() - reader = stem.descriptor.reader.DescriptorReader([test_path]) + reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors @@ -409,7 +409,7 @@ class TestDescriptorReader(unittest.TestCase): os.chmod(test_path, 0077) # remove read permissions skip_listener = SkipListener() - reader = stem.descriptor.reader.DescriptorReader([test_path]) + reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors @@ -431,7 +431,7 @@ class TestDescriptorReader(unittest.TestCase): test_path = "/non-existant/path" skip_listener = SkipListener() - reader = stem.descriptor.reader.DescriptorReader([test_path]) + reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors

1 0

[metrics-db/master] Actually parse assignments*.gz, too.
by karsten＠torproject.org 05 May '12

05 May '12

commit 7406f4df4f836b96e329bee69171a6ca196ad51b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat May 5 19:49:04 2012 +0200 Actually parse assignments*.gz, too. --- .../ernie/db/BridgePoolAssignmentsProcessor.java | 19 +++++++++++++++---- 1 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java index 1eef418..f43723f 100644 --- a/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java +++ b/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java @@ -9,6 +9,7 @@ import java.util.logging.*; import org.apache.commons.codec.*; import org.apache.commons.codec.binary.*; import org.apache.commons.codec.digest.*; +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; public class BridgePoolAssignmentsProcessor { @@ -32,7 +33,7 @@ public class BridgePoolAssignmentsProcessor { File file = files.pop(); if (file.isDirectory()) { files.addAll(Arrays.asList(file.listFiles())); - } else if (!file.getName().endsWith(".gz")) { + } else { assignmentFiles.add(file); } } @@ -45,8 +46,14 @@ public class BridgePoolAssignmentsProcessor { filenameFormat.setTimeZone(TimeZone.getTimeZone("UTC")); for (File assignmentFile : assignmentFiles) { try { - BufferedReader br = new BufferedReader(new FileReader( - assignmentFile)); + BufferedReader br = null; + if (assignmentFile.getName().endsWith(".gz")) { + br = new BufferedReader(new InputStreamReader( + new GzipCompressorInputStream(new FileInputStream( + assignmentFile)))); + } else { + br = new BufferedReader(new FileReader(assignmentFile)); + } String line, bridgePoolAssignmentLine = null; SortedSet<String> sanitizedAssignments = new TreeSet<String>(); boolean wroteLastLine = false, skipBefore20120504125947 = true; @@ -65,7 +72,11 @@ public class BridgePoolAssignmentsProcessor { } } if (skipBefore20120504125947) { - continue; + if (line == null) { + break; + } else { + continue; + } } if (line == null || line.startsWith("bridge-pool-assignment ")) {

1 0

[metrics-db/master] Adapt assignments processor to ponticum's files.
by karsten＠torproject.org 05 May '12

05 May '12

commit b6a95bb4574f4c1f53d9c702076cb2e4b222c58b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat May 5 19:10:24 2012 +0200 Adapt assignments processor to ponticum's files. --- .../ernie/db/BridgePoolAssignmentsProcessor.java | 20 ++++++++++++++++++-- 1 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java index 3f30569..1eef418 100644 --- a/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java +++ b/src/org/torproject/ernie/db/BridgePoolAssignmentsProcessor.java @@ -32,7 +32,7 @@ public class BridgePoolAssignmentsProcessor { File file = files.pop(); if (file.isDirectory()) { files.addAll(Arrays.asList(file.listFiles())); - } else { + } else if (!file.getName().endsWith(".gz")) { assignmentFiles.add(file); } } @@ -49,8 +49,24 @@ public class BridgePoolAssignmentsProcessor { assignmentFile)); String line, bridgePoolAssignmentLine = null; SortedSet<String> sanitizedAssignments = new TreeSet<String>(); - boolean wroteLastLine = false; + boolean wroteLastLine = false, skipBefore20120504125947 = true; while ((line = br.readLine()) != null || !wroteLastLine) { + if (line != null && line.startsWith("bridge-pool-assignment ")) { + String[] parts = line.split(" "); + if (parts.length != 3) { + continue; + } + /* TODO Take out this temporary hack to ignore all assignments + * coming from ponticum when byblos was still the official + * BridgeDB host. */ + if (line.compareTo( + "bridge-pool-assignment 2012-05-04 12:59:47") >= 0) { + skipBefore20120504125947 = false; + } + } + if (skipBefore20120504125947) { + continue; + } if (line == null || line.startsWith("bridge-pool-assignment ")) { if (bridgePoolAssignmentLine != null) {

1 0

[metrics-db/master] Only copy files to rsync/ that have actually changed.
by karsten＠torproject.org 05 May '12

05 May '12

commit ebc69a89e5fc7cbcf9eafc8fe6ea28ac9002e6e1 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed May 2 15:39:46 2012 +0200 Only copy files to rsync/ that have actually changed. --- src/org/torproject/ernie/db/RsyncDataProvider.java | 5 +++++ 1 files changed, 5 insertions(+), 0 deletions(-) diff --git a/src/org/torproject/ernie/db/RsyncDataProvider.java b/src/org/torproject/ernie/db/RsyncDataProvider.java index 2f2c1b4..6c0feae 100644 --- a/src/org/torproject/ernie/db/RsyncDataProvider.java +++ b/src/org/torproject/ernie/db/RsyncDataProvider.java @@ -163,6 +163,11 @@ public class RsyncDataProvider { } private void copyFile(File from, File to) { + if (from.exists() && to.exists() && + from.lastModified() == to.lastModified() && + from.length() == to.length()) { + return; + } try { to.getParentFile().mkdirs(); FileInputStream fis = new FileInputStream(from);

1 0