October 2012 - tor-commits - lists.torproject.org

[tor/release-0.2.3] Merge branch 'maint-0.2.3' into release-0.2.3
by arma＠torproject.org 13 Oct '12

13 Oct '12

1 0

[tor/release-0.2.3] Merge branch 'bug7014_023_squashed' into maint-0.2.3
by arma＠torproject.org 13 Oct '12

13 Oct '12

commit e2549c3b745313d6647c7e1d05025a84e1d33873 Merge: 0a3dfd0 721f99e Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Oct 9 23:46:56 2012 -0400 Merge branch 'bug7014_023_squashed' into maint-0.2.3 changes/bug7014 | 5 +++++ src/or/circuitbuild.c | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-)

1 0

[tor/release-0.2.3] Don't call fmt_addr() twice in a parameter list.
by arma＠torproject.org 13 Oct '12

13 Oct '12

commit 721f99e495474fc7b80250ba52351b5771f0ed36 Author: George Kadianakis <desnacked(a)riseup.net> Date: Thu Oct 4 20:54:29 2012 -0400 Don't call fmt_addr() twice in a parameter list. --- changes/bug7014 | 5 +++++ src/or/circuitbuild.c | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/changes/bug7014 b/changes/bug7014 new file mode 100644 index 0000000..1d39103 --- /dev/null +++ b/changes/bug7014 @@ -0,0 +1,5 @@ + o Minor bugfixes: + - Fix two cases in src/or/transports.c where we were calling + fmt_addr() twice in a parameter list. Bug found by David + Fifield. Fixes bug 7014; bugfix on 0.2.3.9-alpha. + diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index a7d370c..f8521c5 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -5310,19 +5310,22 @@ transport_resolve_conflicts(transport_t *t) t_tmp->marked_for_removal = 0; return 1; } else { /* same name but different addrport */ + char *new_transport_addr = tor_strdup(fmt_addr(&t->addr)); if (t_tmp->marked_for_removal) { /* marked for removal */ log_notice(LD_GENERAL, "You tried to add transport '%s' at '%s:%u' " "but there was already a transport marked for deletion at " "'%s:%u'. We deleted the old transport and registered the " - "new one.", t->name, fmt_addr(&t->addr), t->port, + "new one.", t->name, new_transport_addr, t->port, fmt_addr(&t_tmp->addr), t_tmp->port); smartlist_remove(transport_list, t_tmp); transport_free(t_tmp); + tor_free(new_transport_addr); } else { /* *not* marked for removal */ log_notice(LD_GENERAL, "You tried to add transport '%s' at '%s:%u' " "but the same transport already exists at '%s:%u'. " - "Skipping.", t->name, fmt_addr(&t->addr), t->port, + "Skipping.", t->name, new_transport_addr, t->port, fmt_addr(&t_tmp->addr), t_tmp->port); + tor_free(new_transport_addr); return -1; } }

1 0

[tor/master] fix trivial typo
by arma＠torproject.org 13 Oct '12

13 Oct '12

commit 38b79473864fb662215761f650e8e03b7c518cd7 Author: Roger Dingledine <arma(a)torproject.org> Date: Sat Oct 13 18:34:24 2012 -0400 fix trivial typo --- src/or/command.c | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/src/or/command.c b/src/or/command.c index 7ce134d..2ccd44e 100644 --- a/src/or/command.c +++ b/src/or/command.c @@ -160,7 +160,7 @@ command_process_cell(channel_t *chan, cell_t *cell) } /** Process an incoming var_cell from a channel; in the current protocol all - * the var_cells are handshake-related and handles below the channel layer, + * the var_cells are handshake-related and handled below the channel layer, * so this just logs a warning and drops the cell. */

1 0

[arm/master] Using platform.system() for procName system check
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit d83f1ad2fe62dd190f1474c100971932d1478757 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Oct 13 12:54:29 2012 -0700 Using platform.system() for procName system check As pointed out by ultramage on irc the present sys.platform checks in the procName module are overly strict. The sys.platform attribut includes the version (in his case being 'freebsd10'), causing our attempt to change the process name to fail. Only tested via a simple sanity check, running arm then looking at my ps output... atagar@morrigan:~/Desktop/arm$ ps aux | grep arm atagar 8105 6.5 0.8 46560 8420 pts/8 Sl+ 12:54 0:01 arm --- src/util/procName.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util/procName.py b/src/util/procName.py index fab0b1e..b33ff7b 100644 --- a/src/util/procName.py +++ b/src/util/procName.py @@ -11,9 +11,9 @@ argument replacement (ie, replace argv[0], argv[1], etc but with a string the same size). """ -import sys import ctypes import ctypes.util +import platform # flag for setting the process name, found in '/usr/include/linux/prctl.h' PR_SET_NAME = 15 @@ -38,9 +38,9 @@ def renameProcess(processName): """ _setArgv(processName) - if sys.platform == "linux2": + if platform.system() == "Linux": _setPrctlName(processName) - elif sys.platform == "freebsd7": + elif platform.system() in ("Darwin", "FreeBSD", "OpenBSD"): _setProcTitle(processName) def _setArgv(processName):

1 0

[stem/master] Parsing and tests for network status document v2
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 372ee9836b98af582eecf3d35844397d8935bd9b Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Oct 11 19:27:36 2012 -0700 Parsing and tests for network status document v2 Parser, unit, and integ test for version 2 network status documents. These documents are deprecated and no longer generated, however we still need a parser to read older consensuses. Unlike the v3 parser I'm cutting a few corners... - not validating parameter ordering - no validation that header/footer parameters haven't swapped places - only the bare minimum unit test, no tests for invalid content We can remedy these if necessary but with the growing irrelevance of v2 consensus parsing I doubt we ever will. Plenty of more important things to do. --- run_tests.py | 2 + stem/descriptor/networkstatus.py | 207 ++++++++++++++++++++- test/integ/descriptor/data/cached-consensus-v2 | 27 +++ test/integ/descriptor/networkstatus.py | 80 ++++++++- test/mocking.py | 34 ++++ test/unit/descriptor/networkstatus/document_v2.py | 32 ++++ 6 files changed, 371 insertions(+), 11 deletions(-) diff --git a/run_tests.py b/run_tests.py index 2475254..ef172bf 100755 --- a/run_tests.py +++ b/run_tests.py @@ -23,6 +23,7 @@ import test.unit.descriptor.extrainfo_descriptor import test.unit.descriptor.router_status_entry import test.unit.descriptor.networkstatus.directory_authority import test.unit.descriptor.networkstatus.key_certificate +import test.unit.descriptor.networkstatus.document_v2 import test.unit.descriptor.networkstatus.document_v3 import test.unit.response.control_line import test.unit.response.control_message @@ -121,6 +122,7 @@ UNIT_TESTS = ( test.unit.descriptor.router_status_entry.TestRouterStatusEntry, test.unit.descriptor.networkstatus.directory_authority.TestDirectoryAuthority, test.unit.descriptor.networkstatus.key_certificate.TestKeyCertificate, + test.unit.descriptor.networkstatus.document_v2.TestNetworkStatusDocument, test.unit.descriptor.networkstatus.document_v3.TestNetworkStatusDocument, test.unit.exit_policy.rule.TestExitPolicyRule, test.unit.exit_policy.policy.TestExitPolicy, diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 2f6972f..a028572 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -58,19 +58,39 @@ routers. Those routers refer to a 'thin' document, which doesn't have a :: parse_file - parses a network status file, providing an iterator for its routers - NetworkStatusDocumentV3 - Version 3 network status document. + + NetworkStatusDocument - Network status document. + |- NetworkStatusDocumentV2 - Version 2 network status document. + +- NetworkStatusDocumentV3 - Version 3 network status document. + DocumentSignature - Signature of a document by a directory authority. DirectoryAuthority - Directory authority as defined in a v3 network status document. """ import datetime -from StringIO import StringIO +import StringIO import stem.descriptor import stem.descriptor.router_status_entry import stem.version import stem.util.tor_tools +# Version 2 network status document fields, tuples of the form... +# (keyword, is_mandatory) + +NETWORK_STATUS_V2_FIELDS = ( + ("network-status-version", True), + ("dir-source", True), + ("fingerprint", True), + ("contact", True), + ("dir-signing-key", True), + ("client-versions", False), + ("server-versions", False), + ("published", True), + ("dir-options", False), + ("directory-signature", True), +) + # Network status document are either a 'vote' or 'consensus', with different # mandatory fields for each. Both though require that their fields appear in a # specific order. This is an ordered listing of the following... @@ -105,6 +125,7 @@ FOOTER_FIELDS = [attr[0] for attr in FOOTER_STATUS_DOCUMENT_FIELDS] AUTH_START = "dir-source" ROUTERS_START = "r" FOOTER_START = "directory-footer" +V2_FOOTER_START = "directory-signature" DEFAULT_PARAMS = { "bwweightscale": 10000, @@ -229,7 +250,179 @@ def _get_entries(document_file, validate, entry_class, entry_keyword, start_posi desc_content = "".join(stem.descriptor._read_until_keywords(entry_keyword, document_file, ignore_first = True, end_position = end_position)) yield entry_class(desc_content, validate, *extra_args) -class NetworkStatusDocumentV3(stem.descriptor.Descriptor): +class NetworkStatusDocument(stem.descriptor.Descriptor): + """ + Common parent for network status documents. + """ + + def __init__(self, raw_content): + super(NetworkStatusDocument, self).__init__(raw_content) + self._unrecognized_lines = [] + + def get_unrecognized_lines(self): + return list(self._unrecognized_lines) + +class NetworkStatusDocumentV2(NetworkStatusDocument): + """ + Version 2 network status document. These have been deprecated and are no + longer generated by Tor. + + :var tuple routers: RouterStatusEntryV2 contained in the document + + :var int version: **\*** document version + + :var str hostname: **\*** hostname of the authority + :var str address: **\*** authority's IP address + :var int dir_port: **\*** authority's DirPort + :var str fingerprint: **\*** authority's fingerprint + :var str contact: **\*** authority's contact information + :var str signing_key: **\*** authority's public signing key + + :var list client_versions: list of recommended client tor version strings + :var list server_versions: list of recommended server tor version strings + :var datetime published: **\*** time when the document was published + :var list options: **\*** list of things that this authority decides + + :var str signing_authority: **\*** name of the authority signing the document + :var str signature: **\*** authority's signature for the document + + **\*** attribute is either required when we're parsed with validation or has a default value, others are left as None if undefined + """ + + def __init__(self, raw_content, validate = True): + super(NetworkStatusDocumentV2, self).__init__(raw_content) + + self.version = None + self.hostname = None + self.address = None + self.dir_port = None + self.fingerprint = None + self.contact = None + self.signing_key = None + + self.client_versions = [] + self.server_versions = [] + self.published = None + self.options = [] + + self.signing_authority = None + self.signatures = None + + # Splitting the document from the routers. Unlike v3 documents we're not + # bending over backwards on the validation by checking the field order or + # that header/footer attributes aren't in the wrong section. This is a + # deprecated descriptor type - patches welcome if you want those checks. + + document_file = StringIO.StringIO(raw_content) + document_content = "".join(stem.descriptor._read_until_keywords((ROUTERS_START, V2_FOOTER_START), document_file)) + + self.routers = tuple(_get_entries( + document_file, + validate, + entry_class = stem.descriptor.router_status_entry.RouterStatusEntryV2, + entry_keyword = ROUTERS_START, + section_end_keywords = V2_FOOTER_START, + extra_args = (self,), + )) + + document_content += "\n" + document_file.read() + + entries = stem.descriptor._get_descriptor_components(document_content, validate) + if validate: self._check_constraints(entries) + self._parse(entries, validate) + + def _parse(self, entries, validate): + for keyword, values in entries.items(): + value, block_contents = values[0] + + line = "%s %s" % (keyword, value) # original line + if block_contents: line += "\n%s" % block_contents + + if keyword == "network-status-version": + if not value.isdigit(): + if not validate: continue + raise ValueError("Network status document has a non-numeric version: %s" % line) + + self.version = int(value) + + if validate and self.version != 2: + raise ValueError("Expected a version 2 network status document, got version '%s' instead" % self.version) + elif keyword == "dir-source": + dir_source_comp = value.split() + + if len(dir_source_comp) < 3: + if not validate: continue + raise ValueError("The 'dir-source' line of a v2 network status document must have three values: %s" % line) + + if validate: + if not dir_source_comp[0]: + # https://trac.torproject.org/7055 + raise ValueError("Authority's hostname can't be blank: %s" % line) + elif not stem.util.connection.is_valid_ip_address(dir_source_comp[1]): + raise ValueError("Authority's address isn't a valid IPv4 address: %s" % dir_source_comp[1]) + elif not stem.util.connection.is_valid_port(dir_source_comp[2], allow_zero = True): + raise ValueError("Authority's DirPort is invalid: %s" % dir_source_comp[2]) + elif not dir_source_comp[2].isdigit(): + continue + + self.hostname = dir_source_comp[0] + self.address = dir_source_comp[1] + self.dir_port = None if dir_source_comp[2] == '0' else int(dir_source_comp[2]) + elif keyword == "fingerprint": + if validate and not stem.util.tor_tools.is_valid_fingerprint(value): + raise ValueError("Authority's fingerprint in a v2 network status document is malformed: %s" % line) + + self.fingerprint = value + elif keyword == "contact": + self.contact = value + elif keyword == "dir-signing-key": + self.signing_key = block_contents + elif keyword in ("client-versions", "server-versions"): + # v2 documents existed while there were tor versions using the 'old' + # style, hence we aren't attempting to parse them + + for version_str in value.split(","): + if keyword == 'client-versions': + self.client_versions.append(version_str) + elif keyword == 'server-versions': + self.server_versions.append(version_str) + elif keyword == "published": + try: + self.published = datetime.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + except ValueError: + if validate: + raise ValueError("Versino 2 network status document's 'published' time wasn't parseable: %s" % value) + elif keyword == "dir-options": + self.options = value.split() + elif keyword == "directory-signature": + self.signing_authority = value + self.signature = block_contents + else: + self._unrecognized_lines.append(line) + + # 'client-versions' and 'server-versions' are only required if "Versions" + # is among the options + + if validate and "Versions" in self.options: + if not ('client-versions' in entries and 'server-versions' in entries): + raise ValueError("Version 2 network status documents must have a 'client-versions' and 'server-versions' when 'Versions' is listed among its dir-options:\n%s" % str(self)) + + def _check_constraints(self, entries): + required_fields = [field for (field, is_mandatory) in NETWORK_STATUS_V2_FIELDS if is_mandatory] + for keyword in required_fields: + if not keyword in entries: + raise ValueError("Network status document (v2) must have a '%s' line:\n%s" % (keyword, str(self))) + + # all recognized fields can only appear once + single_fields = [field for (field, _) in NETWORK_STATUS_V2_FIELDS] + for keyword in single_fields: + if keyword in entries and len(entries[keyword]) > 1: + raise ValueError("Network status document (v2) can only have a single '%s' line, got %i:\n%s" % (keyword, len(entries[keyword]), str(self))) + + if 'network-status-version' != entries.keys()[0]: + raise ValueError("Network status document (v2) are expected to start with a 'network-status-version' line:\n%s" % str(self)) + +class NetworkStatusDocumentV3(NetworkStatusDocument): """ Version 3 network status document. This could be either a vote or consensus. @@ -276,10 +469,9 @@ class NetworkStatusDocumentV3(stem.descriptor.Descriptor): """ super(NetworkStatusDocumentV3, self).__init__(raw_content) - document_file = StringIO(raw_content) + document_file = StringIO.StringIO(raw_content) self._header = _DocumentHeader(document_file, validate, default_params) - self._unrecognized_lines = [] # merge header attributes into us for attr, value in vars(self._header).items(): @@ -333,9 +525,6 @@ class NetworkStatusDocumentV3(stem.descriptor.Descriptor): return self._header.meets_consensus_method(method) - def get_unrecognized_lines(self): - return list(self._unrecognized_lines) - def __cmp__(self, other): if not isinstance(other, NetworkStatusDocumentV3): return 1 @@ -404,7 +593,7 @@ class _DocumentHeader(object): self.is_microdescriptor = flavor == 'microdesc' if validate and self.version != 3: - raise ValueError("Expected a version 3 network status documents, got version '%s' instead" % self.version) + raise ValueError("Expected a version 3 network status document, got version '%s' instead" % self.version) elif keyword == 'vote-status': # "vote-status" type # diff --git a/test/integ/descriptor/data/cached-consensus-v2 b/test/integ/descriptor/data/cached-consensus-v2 new file mode 100644 index 0000000..4f8cbce --- /dev/null +++ b/test/integ/descriptor/data/cached-consensus-v2 @@ -0,0 +1,27 @@ +network-status-version 2 +dir-source 18.244.0.114 18.244.0.114 80 +fingerprint 719BE45DE224B607C53707D0E2143E2D423E74CF +contact arma at mit dot edu +published 2005-12-16 00:13:46 +dir-options Names Versions +client-versions 0.0.9rc2,0.0.9rc3,0.0.9rc4-cvs,0.0.9rc4,0.0.9rc5-cvs,0.0.9rc5,0.0.9rc6-cvs,0.0.9rc6,0.0.9rc7-cvs,0.0.9rc7,0.0.9,0.0.9.1,0.0.9.2,0.0.9.3,0.0.9.4,0.0.9.5,0.0.9.6,0.0.9.7,0.0.9.8,0.0.9.9,0.0.9.10,0.1.0.0-alpha-cvs,0.1.0.1-rc,0.1.0.1-rc-cvs,0.1.0.2-rc,0.1.0.2-rc-cvs,0.1.0.3-rc,0.1.0.3-rc-cvs,0.1.0.4-rc,0.1.0.4-rc-cvs,0.1.0.5-rc,0.1.0.5-rc-cvs,0.1.0.6-rc,0.1.0.6-rc-cvs,0.1.0.7-rc,0.1.0.7-rc-cvs,0.1.0.8-rc,0.1.0.8-rc-cvs,0.1.0.9-rc,0.1.0.10,0.1.0.11,0.1.0.12,0.1.0.13,0.1.0.14,0.1.0.15,0.1.0.16,0.1.1.0-alpha-cvs,0.1.1.1-alpha,0.1.1.1-alpha-cvs,0.1.1.2-alpha,0.1.1.2-alpha-cvs,0.1.1.3-alpha,0.1.1.3-alpha-cvs,0.1.1.4-alpha,0.1.1.4-alpha-cvs,0.1.1.5-alpha,0.1.1.5-alpha-cvs,0.1.1.6-alpha,0.1.1.6-alpha-cvs,0.1.1.7-alpha,0.1.1.7-alpha-cvs,0.1.1.8-alpha,0.1.1.8-alpha-cvs,0.1.1.9-alpha,0.1.1.9-alpha-cvs,0.1.1.10-alpha,0.1.1.10-alpha-cvs +server-versions 0.0.9rc2,0.0.9rc3,0.0.9rc4-cvs,0.0.9rc4,0.0.9rc5-cvs,0.0.9rc5,0.0.9rc6-cvs,0.0.9rc6,0.0.9rc7-cvs,0.0.9rc7,0.0.9,0.0.9.1,0.0.9.2,0.0.9.3,0.0.9.4,0.0.9.5,0.0.9.6,0.0.9.7,0.0.9.8,0.0.9.9,0.0.9.10,0.1.0.0-alpha-cvs,0.1.0.1-rc,0.1.0.1-rc-cvs,0.1.0.2-rc,0.1.0.2-rc-cvs,0.1.0.3-rc,0.1.0.3-rc-cvs,0.1.0.4-rc,0.1.0.4-rc-cvs,0.1.0.5-rc,0.1.0.5-rc-cvs,0.1.0.6-rc,0.1.0.6-rc-cvs,0.1.0.7-rc,0.1.0.7-rc-cvs,0.1.0.8-rc,0.1.0.8-rc-cvs,0.1.0.9-rc,0.1.0.10,0.1.0.11,0.1.0.12,0.1.0.13,0.1.0.14,0.1.0.15,0.1.0.16,0.1.1.0-alpha-cvs,0.1.1.1-alpha,0.1.1.1-alpha-cvs,0.1.1.2-alpha,0.1.1.2-alpha-cvs,0.1.1.3-alpha,0.1.1.3-alpha-cvs,0.1.1.4-alpha,0.1.1.4-alpha-cvs,0.1.1.5-alpha,0.1.1.5-alpha-cvs,0.1.1.6-alpha,0.1.1.6-alpha-cvs,0.1.1.7-alpha,0.1.1.7-alpha-cvs,0.1.1.8-alpha,0.1.1.8-alpha-cvs,0.1.1.9-alpha,0.1.1.9-alpha-cvs,0.1.1.10-alpha,0.1.1.10-alpha-cvs +dir-signing-key +-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAOcrht/y5rkaahfX7sMe2qnpqoPibsjTSJaDvsUtaNP/Bq0MgNDGOR48 +rtwfqTRff275Edkp/UYw3G3vSgKCJr76/bqOHCmkiZrnPV1zxNfrK18gNw2Cxre0 +nTA+fD8JQqpPtb8b0SnG9kwy75eS//sRu7TErie2PzGMxrf9LH0LAgMBAAE= +-----END RSA PUBLIC KEY----- + +r moria2 cZvkXeIktgfFNwfQ4hQ+LUI+dM8 t/Pwl1uHiJ3RKF/Vehsbthf2VDI 2005-12-15 06:57:18 18.244.0.114 443 80 +s Authority Fast Named Running Valid V2Dir +r stnv CSi6RnBWxKaJ/uTvXXFIK2KJw9U ItGn7UGZvaftbEFu7NdpwY4fKlo 2005-12-15 16:24:42 84.16.236.173 9001 0 +s Named Valid +r nggrplz CehYL/Dm+F4rjkHA3AucncRuaWg swLCsByU85jj7ziTlSawZR+CTdY 2005-12-15 23:25:50 194.109.109.109 9001 0 +s Fast Stable Running Valid +directory-signature moria2 +-----BEGIN SIGNATURE----- +2nXCxVje3wzn6HrIFRNMc0nc48AhMVpHZyPwRKGXkuYfTQG55uvwQDaFgJHud4RT +27QhWltau3K1evhnzhKcpbTXwkVv1TBYJSzL6rEeAn8cQ7ZiCyqf4EJCaNcem3d2 +TpQQk3nNQF8z6UIvdlvP+DnJV4izWVkQEZgUZgIVM0E= +-----END SIGNATURE----- diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index e174f25..e0ecd00 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -108,9 +108,9 @@ class TestNetworkStatus(unittest.TestCase): self.assertEquals(80, router.or_port) self.assertEquals(None, router.dir_port) - def test_consensus(self): + def test_consensus_v3(self): """ - Checks that consensus documents are properly parsed. + Checks that version 3 consensus documents are properly parsed. """ # the document's expected client and server versions are the same @@ -193,6 +193,82 @@ I/TJmV928na7RLZe2mGHCAW3VQOvV+QkCfj05VZ8CsY= self.assertEquals("BF112F1C6D5543CFD0A32215ACABD4197B5279AD", signature.key_digest) self.assertEquals(expected_signature, signature.signature) + def test_consensus_v2(self): + """ + Checks that version 2 consensus documents are properly parsed. + """ + + expected_signing_key = """-----BEGIN RSA PUBLIC KEY----- +MIGJAoGBAOcrht/y5rkaahfX7sMe2qnpqoPibsjTSJaDvsUtaNP/Bq0MgNDGOR48 +rtwfqTRff275Edkp/UYw3G3vSgKCJr76/bqOHCmkiZrnPV1zxNfrK18gNw2Cxre0 +nTA+fD8JQqpPtb8b0SnG9kwy75eS//sRu7TErie2PzGMxrf9LH0LAgMBAAE= +-----END RSA PUBLIC KEY-----""" + + expected_signature = """-----BEGIN SIGNATURE----- +2nXCxVje3wzn6HrIFRNMc0nc48AhMVpHZyPwRKGXkuYfTQG55uvwQDaFgJHud4RT +27QhWltau3K1evhnzhKcpbTXwkVv1TBYJSzL6rEeAn8cQ7ZiCyqf4EJCaNcem3d2 +TpQQk3nNQF8z6UIvdlvP+DnJV4izWVkQEZgUZgIVM0E= +-----END SIGNATURE-----""" + + consensus_path = test.integ.descriptor.get_resource("cached-consensus-v2") + + with open(consensus_path) as descriptor_file: + document = stem.descriptor.networkstatus.NetworkStatusDocumentV2(descriptor_file.read()) + + self.assertEquals(2, document.version) + self.assertEquals("18.244.0.114", document.hostname) + self.assertEquals("18.244.0.114", document.address) + self.assertEquals(80, document.dir_port) + self.assertEquals("719BE45DE224B607C53707D0E2143E2D423E74CF", document.fingerprint) + self.assertEquals("arma at mit dot edu", document.contact) + self.assertEquals(expected_signing_key, document.signing_key) + + self.assertEquals(67, len(document.client_versions)) + self.assertEquals("0.0.9rc2", document.client_versions[0]) + self.assertEquals("0.1.1.10-alpha-cvs", document.client_versions[-1]) + + self.assertEquals(67, len(document.server_versions)) + self.assertEquals("0.0.9rc2", document.server_versions[0]) + self.assertEquals("0.1.1.10-alpha-cvs", document.server_versions[-1]) + + self.assertEquals(datetime.datetime(2005, 12, 16, 0, 13, 46), document.published) + self.assertEquals(["Names", "Versions"], document.options) + self.assertEquals("moria2", document.signing_authority) + self.assertEquals(expected_signature, document.signature) + self.assertEquals([], document.get_unrecognized_lines()) + + self.assertEqual(3, len(document.routers)) + + router1 = document.routers[0] + self.assertEquals("moria2", router1.nickname) + self.assertEquals("719BE45DE224B607C53707D0E2143E2D423E74CF", router1.fingerprint) + self.assertEquals("t/Pwl1uHiJ3RKF/Vehsbthf2VDI", router1.digest) + self.assertEquals(datetime.datetime(2005, 12, 15, 6, 57, 18), router1.published) + self.assertEquals("18.244.0.114", router1.address) + self.assertEquals(443, router1.or_port) + self.assertEquals(80, router1.dir_port) + self.assertEquals(set(["Authority", "Fast", "Named", "Running", "Valid", "V2Dir"]), set(router1.flags)) + + router2 = document.routers[1] + self.assertEquals("stnv", router2.nickname) + self.assertEquals("0928BA467056C4A689FEE4EF5D71482B6289C3D5", router2.fingerprint) + self.assertEquals("ItGn7UGZvaftbEFu7NdpwY4fKlo", router2.digest) + self.assertEquals(datetime.datetime(2005, 12, 15, 16, 24, 42), router2.published) + self.assertEquals("84.16.236.173", router2.address) + self.assertEquals(9001, router2.or_port) + self.assertEquals(None, router2.dir_port) + self.assertEquals(set(["Named", "Valid"]), set(router2.flags)) + + router3 = document.routers[2] + self.assertEquals("nggrplz", router3.nickname) + self.assertEquals("09E8582FF0E6F85E2B8E41C0DC0B9C9DC46E6968", router3.fingerprint) + self.assertEquals("swLCsByU85jj7ziTlSawZR+CTdY", router3.digest) + self.assertEquals(datetime.datetime(2005, 12, 15, 23, 25, 50), router3.published) + self.assertEquals("194.109.109.109", router3.address) + self.assertEquals(9001, router3.or_port) + self.assertEquals(None, router3.dir_port) + self.assertEquals(set(["Fast", "Stable", "Running", "Valid"]), set(router3.flags)) + def test_metrics_vote(self): """ Checks if vote documents from Metrics are parsed properly. diff --git a/test/mocking.py b/test/mocking.py index ab31eef..1a8aa41 100644 --- a/test/mocking.py +++ b/test/mocking.py @@ -36,6 +36,7 @@ calling :func:`test.mocking.revert_mocking`. stem.descriptor.networkstatus get_directory_authority - DirectoryAuthority get_key_certificate - KeyCertificate + get_network_status_document_v2 - NetworkStatusDocumentV2 get_network_status_document_v3 - NetworkStatusDocumentV3 stem.descriptor.router_status_entry @@ -152,6 +153,19 @@ KEY_CERTIFICATE_FOOTER = ( ("dir-key-certification", "\n-----BEGIN SIGNATURE-----%s-----END SIGNATURE-----" % CRYPTO_BLOB), ) +NETWORK_STATUS_DOCUMENT_HEADER_V2 = ( + ("network-status-version", "2"), + ("dir-source", "18.244.0.114 18.244.0.114 80"), + ("fingerprint", "719BE45DE224B607C53707D0E2143E2D423E74CF"), + ("contact", "arma at mit dot edu"), + ("published", "2005-12-16 00:13:46"), + ("dir-signing-key", "\n-----BEGIN RSA PUBLIC KEY-----%s-----END RSA PUBLIC KEY-----" % CRYPTO_BLOB), +) + +NETWORK_STATUS_DOCUMENT_FOOTER_V2 = ( + ("directory-signature", "moria2\n-----BEGIN SIGNATURE-----%s-----END SIGNATURE-----" % CRYPTO_BLOB), +) + NETWORK_STATUS_DOCUMENT_HEADER = ( ("network-status-version", "3"), ("vote-status", "consensus"), @@ -655,6 +669,26 @@ def get_key_certificate(attr = None, exclude = (), content = False): else: return stem.descriptor.networkstatus.KeyCertificate(desc_content, validate = True) +def get_network_status_document_v2(attr = None, exclude = (), routers = None, content = False): + """ + Provides the descriptor content for... + stem.descriptor.networkstatus.NetworkStatusDocumentV2 + + :param dict attr: keyword/value mappings to be included in the descriptor + :param list exclude: mandatory keywords to exclude from the descriptor + :param list routers: router status entries to include in the document + :param bool content: provides the str content of the descriptor rather than the class if True + + :returns: NetworkStatusDocumentV2 for the requested descriptor content + """ + + desc_content = _get_descriptor_content(attr, exclude, NETWORK_STATUS_DOCUMENT_HEADER_V2, NETWORK_STATUS_DOCUMENT_FOOTER_V2) + + if content: + return desc_content + else: + return stem.descriptor.networkstatus.NetworkStatusDocumentV2(desc_content, validate = True) + def get_network_status_document_v3(attr = None, exclude = (), authorities = None, routers = None, content = False): """ Provides the descriptor content for... diff --git a/test/unit/descriptor/networkstatus/document_v2.py b/test/unit/descriptor/networkstatus/document_v2.py new file mode 100644 index 0000000..70f904e --- /dev/null +++ b/test/unit/descriptor/networkstatus/document_v2.py @@ -0,0 +1,32 @@ +""" +Unit tests for the NetworkStatusDocumentV2 of stem.descriptor.networkstatus. +""" + +import datetime +import unittest + +from test.mocking import get_network_status_document_v2, NETWORK_STATUS_DOCUMENT_HEADER_V2, NETWORK_STATUS_DOCUMENT_FOOTER_V2 + +class TestNetworkStatusDocument(unittest.TestCase): + def test_minimal_document(self): + """ + Parses a minimal v2 network status document. + """ + + document = get_network_status_document_v2() + + self.assertEquals((), document.routers) + self.assertEquals(2, document.version) + self.assertEquals("18.244.0.114", document.hostname) + self.assertEquals("18.244.0.114", document.address) + self.assertEquals(80, document.dir_port) + self.assertEquals("719BE45DE224B607C53707D0E2143E2D423E74CF", document.fingerprint) + self.assertEquals("arma at mit dot edu", document.contact) + self.assertEquals(NETWORK_STATUS_DOCUMENT_HEADER_V2[5][1][1:], document.signing_key) + self.assertEquals([], document.client_versions) + self.assertEquals([], document.server_versions) + self.assertEquals(datetime.datetime(2005, 12, 16, 0, 13, 46), document.published) + self.assertEquals([], document.options) + self.assertEquals("moria2", document.signing_authority) + self.assertEquals(NETWORK_STATUS_DOCUMENT_FOOTER_V2[0][1][7:], document.signature) +

1 0

[stem/master] Network status document handling
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit ea356b738fff3bab4a9e468298ea7807aff37960 Merge: 5cd457b ec335d1 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Oct 13 11:33:47 2012 -0700 Network status document handling Parsing and lots 'o test for handling network status documents. This includes both v2 and v3 documents (votes, consensus, and microdescriptors). https://trac.torproject.org/6569 run_tests.py | 12 + stem/descriptor/__init__.py | 181 ++- stem/descriptor/extrainfo_descriptor.py | 11 +- stem/descriptor/networkstatus.py | 1290 ++++++++++++++++++++ stem/descriptor/reader.py | 6 +- stem/descriptor/router_status_entry.py | 537 ++++++++ stem/descriptor/server_descriptor.py | 52 +- test/integ/descriptor/data/cached-consensus | 130 ++ test/integ/descriptor/data/cached-consensus-v2 | 28 + test/integ/descriptor/data/metrics_consensus | 121 ++ test/integ/descriptor/data/metrics_vote | 85 ++ test/integ/descriptor/data/new_metrics_type | 3 + test/integ/descriptor/data/vote | 100 ++ test/integ/descriptor/networkstatus.py | 406 ++++++ test/integ/descriptor/reader.py | 29 +- test/integ/descriptor/server_descriptor.py | 18 +- test/mocking.py | 465 +++++++- test/unit/descriptor/__init__.py | 9 +- test/unit/descriptor/export.py | 17 +- test/unit/descriptor/extrainfo_descriptor.py | 139 +-- test/unit/descriptor/networkstatus/__init__.py | 6 + .../networkstatus/directory_authority.py | 250 ++++ test/unit/descriptor/networkstatus/document_v2.py | 32 + test/unit/descriptor/networkstatus/document_v3.py | 799 ++++++++++++ .../descriptor/networkstatus/key_certificate.py | 189 +++ test/unit/descriptor/router_status_entry.py | 466 +++++++ test/unit/descriptor/server_descriptor.py | 79 +- 27 files changed, 5156 insertions(+), 304 deletions(-)

1 0

[stem/master] Supporting v2 documents in the descriptor reader
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit ec335d1b5bf4368982bc9736a271f675a6a7c60d Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Oct 11 19:54:05 2012 -0700 Supporting v2 documents in the descriptor reader The v2 descriptor's integ data broke the descriptor reader tests. Turned that I had forgotten to add v2 documents to the types that it recognizes. --- stem/descriptor/__init__.py | 6 +++- stem/descriptor/networkstatus.py | 25 ++++++++++++++++------- test/integ/descriptor/data/cached-consensus-v2 | 1 + test/integ/descriptor/networkstatus.py | 1 + 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index f0ab82d..9d33ab7 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -25,7 +25,6 @@ __all__ = [ import os import re -import datetime import collections import stem.util.enum @@ -86,7 +85,7 @@ def parse_file(path, descriptor_file): elif filename == "cached-consensus": file_parser = stem.descriptor.networkstatus.parse_file elif filename == "cached-microdesc-consensus": - file_parser = lambda f: stem.descriptor.networkstatus.parse_file(f, True, True) + file_parser = lambda f: stem.descriptor.networkstatus.parse_file(f, is_microdescriptor = True) else: # Metrics descriptor handling first_line, desc = descriptor_file.readline().strip(), None @@ -128,6 +127,9 @@ def _parse_metrics_file(descriptor_type, major_version, minor_version, descripto elif descriptor_type in ("network-status-consensus-3", "network-status-vote-3") and major_version == 1: for desc in stem.descriptor.networkstatus.parse_file(descriptor_file): yield desc + elif descriptor_type == "network-status-2" and major_version == 1: + for desc in stem.descriptor.networkstatus.parse_file(descriptor_file, document_version = 2): + yield desc elif descriptor_type == "network-status-microdesc-consensus-3" and major_version == 1: for desc in stem.descriptor.networkstatus.parse_file(descriptor_file, is_microdescriptor = True): yield desc diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index a028572..21f7ee0 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -164,7 +164,7 @@ BANDWIDTH_WEIGHT_ENTRIES = ( "Wmb", "Wmd", "Wme", "Wmg", "Wmm", ) -def parse_file(document_file, validate = True, is_microdescriptor = False): +def parse_file(document_file, validate = True, is_microdescriptor = False, document_version = 3): """ Parses a network status and iterates over the RouterStatusEntry in it. The document that these instances reference have an empty 'rotuers' attribute to @@ -173,29 +173,38 @@ def parse_file(document_file, validate = True, is_microdescriptor = False): :param file document_file: file with network status document content :param bool validate: checks the validity of the document's contents if True, skips these checks otherwise :param bool is_microdescriptor: True if this is for a microdescriptor consensus, False otherwise + :param int document_version: network status document version - :returns: :class:`stem.descriptor.networkstatus.NetworkStatusDocumentV3` object + :returns: :class:`stem.descriptor.networkstatus.NetworkStatusDocument` object :raises: - * ValueError if the contents is malformed and validate is True + * ValueError if the document_version is unrecognized or the contents is malformed and validate is True * IOError if the file can't be read """ # getting the document without the routers section - header = stem.descriptor._read_until_keywords((ROUTERS_START, FOOTER_START), document_file) + header = stem.descriptor._read_until_keywords((ROUTERS_START, FOOTER_START, V2_FOOTER_START), document_file) routers_start = document_file.tell() - stem.descriptor._read_until_keywords(FOOTER_START, document_file, skip = True) + stem.descriptor._read_until_keywords((FOOTER_START, V2_FOOTER_START), document_file, skip = True) routers_end = document_file.tell() footer = document_file.readlines() document_content = "".join(header + footer) - if not is_microdescriptor: + if document_version == 2: + document_type = NetworkStatusDocumentV2 router_type = stem.descriptor.router_status_entry.RouterStatusEntryV3 + elif document_version == 3: + document_type = NetworkStatusDocumentV3 + + if not is_microdescriptor: + router_type = stem.descriptor.router_status_entry.RouterStatusEntryV3 + else: + router_type = stem.descriptor.router_status_entry.RouterStatusEntryMicroV3 else: - router_type = stem.descriptor.router_status_entry.RouterStatusEntryMicroV3 + raise ValueError("Document version %i isn't recognized (only able to parse v2 or v3)" % document_version) desc_iterator = _get_entries( document_file, @@ -204,7 +213,7 @@ def parse_file(document_file, validate = True, is_microdescriptor = False): entry_keyword = ROUTERS_START, start_position = routers_start, end_position = routers_end, - extra_args = (NetworkStatusDocumentV3(document_content, validate),), + extra_args = (document_type(document_content, validate),), ) for desc in desc_iterator: diff --git a/test/integ/descriptor/data/cached-consensus-v2 b/test/integ/descriptor/data/cached-consensus-v2 index 4f8cbce..ad64d40 100644 --- a/test/integ/descriptor/data/cached-consensus-v2 +++ b/test/integ/descriptor/data/cached-consensus-v2 @@ -1,3 +1,4 @@ +@type network-status-2 1.0 network-status-version 2 dir-source 18.244.0.114 18.244.0.114 80 fingerprint 719BE45DE224B607C53707D0E2143E2D423E74CF diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index e0ecd00..96ce12f 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -213,6 +213,7 @@ TpQQk3nNQF8z6UIvdlvP+DnJV4izWVkQEZgUZgIVM0E= consensus_path = test.integ.descriptor.get_resource("cached-consensus-v2") with open(consensus_path) as descriptor_file: + descriptor_file.readline() # strip header document = stem.descriptor.networkstatus.NetworkStatusDocumentV2(descriptor_file.read()) self.assertEquals(2, document.version)

1 0

[stem/master] Combining TestNetworkStatusDocument and TestMicrodescriptorConsensus
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit 21e5a4cd578dc81496d8355e377048c3707488b0 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Aug 18 16:30:03 2012 -0700 Combining TestNetworkStatusDocument and TestMicrodescriptorConsensus Having separate test classes was a fine idea, but resulted in our tests claiming that 'test.integ.descriptor.networkstatus' ran twice. That's confusing, so we either need to put them in separate files or merge the classes. Opting for the later for now since TestMicrodescriptorConsensus only had one test. --- run_tests.py | 1 - test/integ/descriptor/networkstatus.py | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/run_tests.py b/run_tests.py index b0550d6..8d115f1 100755 --- a/run_tests.py +++ b/run_tests.py @@ -136,7 +136,6 @@ INTEG_TESTS = ( test.integ.descriptor.server_descriptor.TestServerDescriptor, test.integ.descriptor.extrainfo_descriptor.TestExtraInfoDescriptor, test.integ.descriptor.networkstatus.TestNetworkStatusDocument, - test.integ.descriptor.networkstatus.TestMicrodescriptorConsensus, test.integ.version.TestVersion, test.integ.response.protocolinfo.TestProtocolInfo, test.integ.process.TestProcess, diff --git a/test/integ/descriptor/networkstatus.py b/test/integ/descriptor/networkstatus.py index bd326ad..4cbf42c 100644 --- a/test/integ/descriptor/networkstatus.py +++ b/test/integ/descriptor/networkstatus.py @@ -1,5 +1,5 @@ """ -Integration tests for stem.descriptor.server_descriptor. +Integration tests for stem.descriptor.networkstatus. """ from __future__ import with_statement @@ -256,8 +256,7 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEquals("27B6B5996C426270A5C95488AA5BCEB6BCC86956", desc.directory_signatures[0].identity) self.assertEquals("D5C30C15BB3F1DA27669C2D88439939E8F418FCF", desc.directory_signatures[0].key_digest) self.assertEquals(expected_signature, desc.directory_signatures[0].signature) - -class TestMicrodescriptorConsensus(unittest.TestCase): + def test_cached_microdesc_consensus(self): """ Parses the cached-microdesc-consensus file in our data directory.

1 0

[stem/master] Removing _skip_until_keywords()
by atagar＠torproject.org 13 Oct '12

13 Oct '12

commit e1ce38ebd4e49faff993e212e6959f9a46e93e50 Author: Damian Johnson <atagar(a)torproject.org> Date: Sat Aug 18 19:08:15 2012 -0700 Removing _skip_until_keywords() The _skip_until_keywords() is almost entirely a duplicate of _read_until_keywords(), its only difference being that it doesn't return the content. Adding an argument to _read_until_keywords() to do the same. --- stem/descriptor/__init__.py | 53 ++++++++----------------------------- stem/descriptor/networkstatus.py | 4 +- 2 files changed, 14 insertions(+), 43 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 814fc49..151f13e 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -253,7 +253,7 @@ def _read_keyword_line_str(keyword, lines, validate = True, optional = False): raise ValueError("Error parsing network status document: Expected %s, received: %s" % (keyword, lines[0])) else: return None -def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_first = False): +def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_first = False, skip = False): """ Reads from the descriptor file until we get to one of the given keywords or reach the end of the file. @@ -262,16 +262,19 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi :param file descriptor_file: file with the descriptor content :param bool inclusive: includes the line with the keyword if True :param bool ignore_first: doesn't check if the first line read has one of the given keywords + :param bool skip: skips buffering content, returning None :returns: list with the lines until we find one of the keywords """ - content = [] + content = None if skip else [] if type(keywords) == str: keywords = (keywords,) if ignore_first: - content.append(descriptor_file.readline()) - if content == [None]: return [] + first_line = descriptor_file.readline() + + if content != None and first_line != None: + content.append(first_line) while True: last_position = descriptor_file.tell() @@ -287,49 +290,17 @@ def _read_until_keywords(keywords, descriptor_file, inclusive = False, ignore_fi line_keyword = line_match.groups()[0] if line_keyword in keywords: - if inclusive: content.append(line) - else: descriptor_file.seek(last_position) + if not inclusive: + descriptor_file.seek(last_position) + elif content != None: + content.append(line) break - else: + elif content != None: content.append(line) return content -def _skip_until_keywords(keywords, descriptor_file, inclusive = False): - """ - Reads and discards lines of data from the descriptor file until we get to one - of the given keywords or reach the end of the file. - - :param str,list keywords: keyword(s) we want to skip until - :param file descriptor_file: file with the descriptor content - :param bool inclusive: includes the line with the keyword if True - - :returns: descriptor_file with the new offset - """ - - if type(keywords) == str: keywords = (keywords,) - - while True: - last_position = descriptor_file.tell() - line = descriptor_file.readline() - if not line: break # EOF - - line_match = KEYWORD_LINE.match(line) - - if not line_match: - # no spaces or tabs in the line - line_keyword = line.strip() - else: - line_keyword = line_match.groups()[0] - - if line_keyword in keywords: - if not inclusive: descriptor_file.seek(last_position) - - break - - return descriptor_file - def _get_pseudo_pgp_block(remaining_contents): """ Checks if given contents begins with a pseudo-Open-PGP-style block and, if diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index f9d89a8..3b592ec 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -53,7 +53,7 @@ import stem.version import stem.exit_policy import stem.util.enum -from stem.descriptor import _read_until_keywords, _skip_until_keywords, _peek_keyword, _strptime +from stem.descriptor import _read_until_keywords, _peek_keyword, _strptime from stem.descriptor import _read_keyword_line, _read_keyword_line_str, _get_pseudo_pgp_block, _peek_line _bandwidth_weights_regex = re.compile(" ".join(["W%s=\d+" % weight for weight in ["bd", @@ -101,7 +101,7 @@ def parse_file(document_file, validate = True, flavour = Flavour.NONE): # store offset r_offset = document_file.tell() # skip until end of router descriptors - _skip_until_keywords(["bandwidth-weights", "directory-footer", "directory-signature"], document_file) + _read_until_keywords(["bandwidth-weights", "directory-footer", "directory-signature"], document_file, skip = True) # parse until end document_data = document_data + document_file.read()

1 0