June 2017 - tor-commits - lists.torproject.org

[tor/master] ed25519: Add unittests for ed25519 pubkey validation.
by nickm＠torproject.org 27 Jun '17

27 Jun '17

commit e8eee3a50e3c38b6c8e97d78a637b8a331fb6560 Author: George Kadianakis <desnacked(a)riseup.net> Date: Tue Apr 25 15:20:13 2017 +0300 ed25519: Add unittests for ed25519 pubkey validation. --- src/test/test_crypto.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c index ec9d4e2..1f0c803 100644 --- a/src/test/test_crypto.c +++ b/src/test/test_crypto.c @@ -2170,6 +2170,9 @@ test_crypto_ed25519_simple(void *arg) tt_int_op(0, OP_EQ, ed25519_public_key_generate(&pub1, &sec1)); tt_int_op(0, OP_EQ, ed25519_public_key_generate(&pub2, &sec1)); + tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, 0); + tt_int_op(ed25519_validate_pubkey(&pub2), OP_EQ, 0); + tt_mem_op(pub1.pubkey, OP_EQ, pub2.pubkey, sizeof(pub1.pubkey)); tt_assert(ed25519_pubkey_eq(&pub1, &pub2)); tt_assert(ed25519_pubkey_eq(&pub1, &pub1)); @@ -2832,6 +2835,68 @@ crypto_rand_check_failure_mode_predict(void) #undef FAILURE_MODE_BUFFER_SIZE +/** Test that our ed25519 validation function rejects evil public keys and + * accepts good ones. */ +static void +test_crypto_ed25519_validation(void *arg) +{ + (void) arg; + + int retval; + ed25519_public_key_t pub1; + + /* See https://lists.torproject.org/pipermail/tor-dev/2017-April/012230.html + for a list of points with torsion components in ed25519. */ + + { /* Point with torsion component (order 8l) */ + const char badkey[] = + "300ef2e64e588e1df55b48e4da0416ffb64cc85d5b00af6463d5cc6c2b1c185e"; + retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey), + badkey, strlen(badkey)); + tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey)); + tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, -1); + } + + { /* Point with torsion component (order 4l) */ + const char badkey[] = + "f43e3a046db8749164c6e69b193f1e942c7452e7d888736f40b98093d814d5e7"; + retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey), + badkey, strlen(badkey)); + tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey)); + tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, -1); + } + + { /* Point with torsion component (order 2l) */ + const char badkey[] = + "c9fff3af0471c28e33e98c2043e44f779d0427b1e37c521a6bddc011ed1869af"; + retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey), + badkey, strlen(badkey)); + tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey)); + tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, -1); + } + + { /* This point is not even on the curve */ + const char badkey[] = + "e19c65de75c68cf3b7643ea732ba9eb1a3d20d6d57ba223c2ece1df66feb5af0"; + retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey), + badkey, strlen(badkey)); + tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey)); + tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, -1); + } + + { /* This one is a good key */ + const char goodkey[] = + "4ba2e44760dff4c559ef3c38768c1c14a8a54740c782c8d70803e9d6e3ad8794"; + retval = base16_decode((char*)pub1.pubkey, sizeof(pub1.pubkey), + goodkey, strlen(goodkey)); + tt_int_op(retval, OP_EQ, sizeof(pub1.pubkey)); + tt_int_op(ed25519_validate_pubkey(&pub1), OP_EQ, 0); + } + + + done: ; +} + static void test_crypto_failure_modes(void *arg) { @@ -2918,6 +2983,7 @@ struct testcase_t crypto_tests[] = { ED25519_TEST(convert, 0), ED25519_TEST(blinding, 0), ED25519_TEST(testvectors, 0), + ED25519_TEST(validation, 0), { "ed25519_storage", test_crypto_ed25519_storage, 0, NULL, NULL }, { "siphash", test_crypto_siphash, 0, NULL, NULL }, { "failure_modes", test_crypto_failure_modes, TT_FORK, NULL, NULL },

1 0

[stem/master] Raise NotImplementedError if signing not implemented
by atagar＠torproject.org 27 Jun '17

27 Jun '17

commit e4f1b57419f1e8ab87c54b65d9da8949e975dd95 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jun 27 09:23:30 2017 -0700 Raise NotImplementedError if signing not implemented We're documented as raising a NotImplementedError if signing isn't yet supported but we didn't yet do that. --- stem/descriptor/extrainfo_descriptor.py | 6 ++++++ stem/descriptor/hidden_service_descriptor.py | 3 +++ stem/descriptor/microdescriptor.py | 3 +++ stem/descriptor/networkstatus.py | 12 ++++++++++++ stem/descriptor/router_status_entry.py | 9 +++++++++ stem/descriptor/server_descriptor.py | 3 +++ test/unit/descriptor/extrainfo_descriptor.py | 6 ++++++ test/unit/descriptor/hidden_service_descriptor.py | 4 ++++ test/unit/descriptor/microdescriptor.py | 9 ++++++--- test/unit/descriptor/networkstatus/directory_authority.py | 6 ++++++ test/unit/descriptor/networkstatus/document_v2.py | 6 ++++++ test/unit/descriptor/networkstatus/document_v3.py | 4 ++++ test/unit/descriptor/networkstatus/key_certificate.py | 5 +++++ test/unit/descriptor/router_status_entry.py | 7 +++++++ test/unit/descriptor/server_descriptor.py | 1 + 15 files changed, 81 insertions(+), 3 deletions(-) diff --git a/stem/descriptor/extrainfo_descriptor.py b/stem/descriptor/extrainfo_descriptor.py index 847a2c7..c0484ad 100644 --- a/stem/descriptor/extrainfo_descriptor.py +++ b/stem/descriptor/extrainfo_descriptor.py @@ -974,6 +974,9 @@ class RelayExtraInfoDescriptor(ExtraInfoDescriptor): @classmethod def content(cls, attr = None, exclude = (), sign = False): + if sign: + raise NotImplementedError('Signing of %s not implemented' % cls.__name__) + return _descriptor_content(attr, exclude, sign, RELAY_EXTRAINFO_HEADER, RELAY_EXTRAINFO_FOOTER) @lru_cache() @@ -1010,6 +1013,9 @@ class BridgeExtraInfoDescriptor(ExtraInfoDescriptor): @classmethod def content(cls, attr = None, exclude = (), sign = False): + if sign: + raise NotImplementedError('Signing of %s not implemented' % cls.__name__) + return _descriptor_content(attr, exclude, sign, BRIDGE_EXTRAINFO_HEADER, BRIDGE_EXTRAINFO_FOOTER) def digest(self): diff --git a/stem/descriptor/hidden_service_descriptor.py b/stem/descriptor/hidden_service_descriptor.py index d752369..f9a5e35 100644 --- a/stem/descriptor/hidden_service_descriptor.py +++ b/stem/descriptor/hidden_service_descriptor.py @@ -252,6 +252,9 @@ class HiddenServiceDescriptor(Descriptor): @classmethod def content(cls, attr = None, exclude = (), sign = False): + if sign: + raise NotImplementedError('Signing of %s not implemented' % cls.__name__) + return _descriptor_content(attr, exclude, sign, HIDDEN_SERVICE_HEADER, HIDDEN_SERVICE_FOOTER) @classmethod diff --git a/stem/descriptor/microdescriptor.py b/stem/descriptor/microdescriptor.py index 257067e..3990720 100644 --- a/stem/descriptor/microdescriptor.py +++ b/stem/descriptor/microdescriptor.py @@ -264,6 +264,9 @@ class Microdescriptor(Descriptor): @classmethod def content(cls, attr = None, exclude = (), sign = False): + if sign: + raise NotImplementedError('Signing of %s not implemented' % cls.__name__) + return _descriptor_content(attr, exclude, sign, MICRODESCRIPTOR) def __init__(self, raw_contents, validate = False, annotations = None): diff --git a/stem/descriptor/networkstatus.py b/stem/descriptor/networkstatus.py index 4cea4eb..2fc868a 100644 --- a/stem/descriptor/networkstatus.py +++ b/stem/descriptor/networkstatus.py @@ -512,6 +512,9 @@ class NetworkStatusDocumentV2(NetworkStatusDocument): @classmethod def content(cls, attr = None, exclude = (), sign = False): + if sign: + raise NotImplementedError('Signing of %s not implemented' % cls.__name__) + return _descriptor_content(attr, exclude, sign, NETWORK_STATUS_DOCUMENT_HEADER_V2, NETWORK_STATUS_DOCUMENT_FOOTER_V2) def __init__(self, raw_content, validate = False): @@ -960,6 +963,9 @@ class NetworkStatusDocumentV3(NetworkStatusDocument): @classmethod def content(cls, attr = None, exclude = (), sign = False, authorities = None, routers = None): + if sign: + raise NotImplementedError('Signing of %s not implemented' % cls.__name__) + attr = {} if attr is None else dict(attr) is_vote = attr.get('vote-status') == 'vote' @@ -1424,6 +1430,9 @@ class DirectoryAuthority(Descriptor): @classmethod def content(cls, attr = None, exclude = (), sign = False, is_vote = False): + if sign: + raise NotImplementedError('Signing of %s not implemented' % cls.__name__) + attr = {} if attr is None else dict(attr) # include mandatory 'vote-digest' if a consensus @@ -1618,6 +1627,9 @@ class KeyCertificate(Descriptor): @classmethod def content(cls, attr = None, exclude = (), sign = False): + if sign: + raise NotImplementedError('Signing of %s not implemented' % cls.__name__) + return _descriptor_content(attr, exclude, sign, KEY_CERTIFICATE_HEADER, KEY_CERTIFICATE_FOOTER) def __init__(self, raw_content, validate = False): diff --git a/stem/descriptor/router_status_entry.py b/stem/descriptor/router_status_entry.py index 19d4d3c..c5f21bf 100644 --- a/stem/descriptor/router_status_entry.py +++ b/stem/descriptor/router_status_entry.py @@ -529,6 +529,9 @@ class RouterStatusEntryV2(RouterStatusEntry): @classmethod def content(cls, attr = None, exclude = (), sign = False): + if sign: + raise NotImplementedError('Signing of %s not implemented' % cls.__name__) + return _descriptor_content(attr, exclude, sign, ROUTER_STATUS_ENTRY_V2_HEADER) def _name(self, is_plural = False): @@ -626,6 +629,9 @@ class RouterStatusEntryV3(RouterStatusEntry): @classmethod def content(cls, attr = None, exclude = (), sign = False): + if sign: + raise NotImplementedError('Signing of %s not implemented' % cls.__name__) + return _descriptor_content(attr, exclude, sign, ROUTER_STATUS_ENTRY_V3_HEADER) def _name(self, is_plural = False): @@ -699,6 +705,9 @@ class RouterStatusEntryMicroV3(RouterStatusEntry): @classmethod def content(cls, attr = None, exclude = (), sign = False): + if sign: + raise NotImplementedError('Signing of %s not implemented' % cls.__name__) + return _descriptor_content(attr, exclude, sign, ROUTER_STATUS_ENTRY_MICRO_V3_HEADER) def _name(self, is_plural = False): diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 607908d..a66faa8 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -962,6 +962,9 @@ class BridgeDescriptor(ServerDescriptor): @classmethod def content(cls, attr = None, exclude = (), sign = False): + if sign: + raise NotImplementedError('Signing of %s not implemented' % cls.__name__) + return _descriptor_content(attr, exclude, sign, BRIDGE_SERVER_HEADER) def digest(self): diff --git a/test/unit/descriptor/extrainfo_descriptor.py b/test/unit/descriptor/extrainfo_descriptor.py index 798320e..d41f68b 100644 --- a/test/unit/descriptor/extrainfo_descriptor.py +++ b/test/unit/descriptor/extrainfo_descriptor.py @@ -8,6 +8,7 @@ import re import unittest import stem.descriptor +import test.require from stem.descriptor.extrainfo_descriptor import ( RelayExtraInfoDescriptor, @@ -133,6 +134,11 @@ k0d2aofcVbHr4fPQOSST0LXDrhFl5Fqo5um296zpJGvRUeO6S44U/EfJAGShtqWw self.assertEqual({}, desc.dir_v2_responses_unknown) self.assertEqual({}, desc.dir_v2_responses_unknown) + @test.require.cryptography + def test_descriptor_signing(self): + self.assertRaisesRegexp(NotImplementedError, 'Signing of RelayExtraInfoDescriptor not implemented', RelayExtraInfoDescriptor.create, sign = True) + self.assertRaisesRegexp(NotImplementedError, 'Signing of BridgeExtraInfoDescriptor not implemented', BridgeExtraInfoDescriptor.create, sign = True) + def test_multiple_metrics_bridge_descriptors(self): """ Check that we can read bridge descriptors when there's multiple in a file. diff --git a/test/unit/descriptor/hidden_service_descriptor.py b/test/unit/descriptor/hidden_service_descriptor.py index 2f8427f..09d8de5 100644 --- a/test/unit/descriptor/hidden_service_descriptor.py +++ b/test/unit/descriptor/hidden_service_descriptor.py @@ -274,6 +274,10 @@ class TestHiddenServiceDescriptor(unittest.TestCase): self.assertEqual([2, 3], desc.protocol_versions) @test.require.cryptography + def test_descriptor_signing(self): + self.assertRaisesRegexp(NotImplementedError, 'Signing of HiddenServiceDescriptor not implemented', HiddenServiceDescriptor.create, sign = True) + + @test.require.cryptography def test_with_basic_auth(self): """ Parse a descriptor with introduction-points encrypted with basic auth. diff --git a/test/unit/descriptor/microdescriptor.py b/test/unit/descriptor/microdescriptor.py index 35b8de0..0de947f 100644 --- a/test/unit/descriptor/microdescriptor.py +++ b/test/unit/descriptor/microdescriptor.py @@ -4,13 +4,12 @@ Unit tests for stem.descriptor.microdescriptor. import unittest -import stem.exit_policy - import stem.descriptor +import stem.exit_policy +import test.require from stem.util import str_type from stem.descriptor.microdescriptor import Microdescriptor - from test.unit.descriptor import get_resource FIRST_ONION_KEY = """\ @@ -96,6 +95,10 @@ class TestMicrodescriptor(unittest.TestCase): self.assertEqual({}, desc.protocols) self.assertEqual([], desc.get_unrecognized_lines()) + @test.require.cryptography + def test_descriptor_signing(self): + self.assertRaisesRegexp(NotImplementedError, 'Signing of Microdescriptor not implemented', Microdescriptor.create, sign = True) + def test_unrecognized_line(self): """ Includes unrecognized content in the descriptor. diff --git a/test/unit/descriptor/networkstatus/directory_authority.py b/test/unit/descriptor/networkstatus/directory_authority.py index a80cb44..f2c270e 100644 --- a/test/unit/descriptor/networkstatus/directory_authority.py +++ b/test/unit/descriptor/networkstatus/directory_authority.py @@ -4,6 +4,8 @@ Unit tests for the DirectoryAuthority of stem.descriptor.networkstatus. import unittest +import test.require + from stem.descriptor.networkstatus import ( AUTHORITY_HEADER, DirectoryAuthority, @@ -52,6 +54,10 @@ class TestDirectoryAuthority(unittest.TestCase): self.assertEqual(KeyCertificate.create(), authority.key_certificate) self.assertEqual([], authority.get_unrecognized_lines()) + @test.require.cryptography + def test_descriptor_signing(self): + self.assertRaisesRegexp(NotImplementedError, 'Signing of DirectoryAuthority not implemented', DirectoryAuthority.create, sign = True) + def test_unrecognized_line(self): """ Includes unrecognized content in the descriptor. diff --git a/test/unit/descriptor/networkstatus/document_v2.py b/test/unit/descriptor/networkstatus/document_v2.py index 7b6fdb1..2ed3177 100644 --- a/test/unit/descriptor/networkstatus/document_v2.py +++ b/test/unit/descriptor/networkstatus/document_v2.py @@ -5,6 +5,8 @@ Unit tests for the NetworkStatusDocumentV2 of stem.descriptor.networkstatus. import datetime import unittest +import test.require + from stem.descriptor.networkstatus import ( NETWORK_STATUS_DOCUMENT_HEADER_V2, NETWORK_STATUS_DOCUMENT_FOOTER_V2, @@ -111,3 +113,7 @@ TpQQk3nNQF8z6UIvdlvP+DnJV4izWVkQEZgUZgIVM0E= self.assertEqual([], document.options) self.assertEqual('moria2', document.signing_authority) self.assertEqual(NETWORK_STATUS_DOCUMENT_FOOTER_V2[0][1][7:], document.signature) + + @test.require.cryptography + def test_descriptor_signing(self): + self.assertRaisesRegexp(NotImplementedError, 'Signing of NetworkStatusDocumentV2 not implemented', NetworkStatusDocumentV2.create, sign = True) diff --git a/test/unit/descriptor/networkstatus/document_v3.py b/test/unit/descriptor/networkstatus/document_v3.py index 68d7270..3de445b 100644 --- a/test/unit/descriptor/networkstatus/document_v3.py +++ b/test/unit/descriptor/networkstatus/document_v3.py @@ -375,6 +375,10 @@ DnN5aFtYKiTc19qIC7Nmo+afPdDEf0MlJvEOP5EWl3w= self.assertEqual([DOC_SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines()) + @test.require.cryptography + def test_descriptor_signing(self): + self.assertRaisesRegexp(NotImplementedError, 'Signing of NetworkStatusDocumentV3 not implemented', NetworkStatusDocumentV3.create, sign = True) + def test_examples(self): """ Run something similar to the examples in the header pydocs. diff --git a/test/unit/descriptor/networkstatus/key_certificate.py b/test/unit/descriptor/networkstatus/key_certificate.py index 4770517..d7982ae 100644 --- a/test/unit/descriptor/networkstatus/key_certificate.py +++ b/test/unit/descriptor/networkstatus/key_certificate.py @@ -6,6 +6,7 @@ import datetime import unittest import stem.descriptor +import test.require from stem.descriptor.networkstatus import ( KEY_CERTIFICATE_HEADER, @@ -148,6 +149,10 @@ GM9hAsAMRX9Ogqhq5UjDNqEsvDKuyVeyh7unSZEOip9Zr6K/+7VsVPNb8vfBRBjo self.assertEqual(expected_key_cert, cert.certification) self.assertEqual([], cert.get_unrecognized_lines()) + @test.require.cryptography + def test_descriptor_signing(self): + self.assertRaisesRegexp(NotImplementedError, 'Signing of KeyCertificate not implemented', KeyCertificate.create, sign = True) + def test_unrecognized_line(self): """ Includes unrecognized content in the descriptor. diff --git a/test/unit/descriptor/router_status_entry.py b/test/unit/descriptor/router_status_entry.py index 37d763e..166b419 100644 --- a/test/unit/descriptor/router_status_entry.py +++ b/test/unit/descriptor/router_status_entry.py @@ -7,6 +7,7 @@ import functools import unittest import stem.descriptor +import test.require from stem import Flag from stem.exit_policy import MicroExitPolicy @@ -159,6 +160,12 @@ class TestRouterStatusEntry(unittest.TestCase): self.assertEqual('6A252497006BB9AF36A1B1B902C4D7FA2129923400DBE0101F167B1B031F63BD', entry.digest) self.assertEqual([], entry.get_unrecognized_lines()) + @test.require.cryptography + def test_descriptor_signing(self): + self.assertRaisesRegexp(NotImplementedError, 'Signing of RouterStatusEntryV2 not implemented', RouterStatusEntryV2.create, sign = True) + self.assertRaisesRegexp(NotImplementedError, 'Signing of RouterStatusEntryV3 not implemented', RouterStatusEntryV3.create, sign = True) + self.assertRaisesRegexp(NotImplementedError, 'Signing of RouterStatusEntryMicroV3 not implemented', RouterStatusEntryMicroV3.create, sign = True) + def test_without_ed25519(self): """ Parses a router status entry without a ed25519 value. diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 2c47bdf..c881751 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -258,6 +258,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= @test.require.cryptography def test_descriptor_signing(self): RelayDescriptor.create(sign = True) + self.assertRaisesRegexp(NotImplementedError, 'Signing of BridgeDescriptor not implemented', BridgeDescriptor.create, sign = True) @patch('time.time', Mock(return_value = time.mktime(datetime.date(2010, 1, 1).timetuple()))) def test_with_ed25519(self):

1 0

[stem/master] Randomize BridgeDescriptor created fields
by atagar＠torproject.org 27 Jun '17

27 Jun '17

commit e5b7cbbccec5a4a238d386d927d5b8a66aad262b Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jun 27 09:42:45 2017 -0700 Randomize BridgeDescriptor created fields These come from CollecTor. Can they even be signed? Regardless, randomizing their attributes. --- stem/descriptor/server_descriptor.py | 16 +++++++--------- test/unit/descriptor/server_descriptor.py | 3 +-- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index a66faa8..6ffe914 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -116,14 +116,6 @@ SINGLE_FIELDS = ( DEFAULT_IPV6_EXIT_POLICY = stem.exit_policy.MicroExitPolicy('reject 1-65535') REJECT_ALL_POLICY = stem.exit_policy.ExitPolicy('reject *:*') -BRIDGE_SERVER_HEADER = ( - ('router', 'Unnamed 10.45.227.253 9001 0 0'), - ('router-digest', '006FD96BA35E7785A6A3B8B75FE2E2435A13BDB4'), - ('published', '2012-03-22 17:34:38'), - ('bandwidth', '409600 819200 5120'), - ('reject', '*:*'), -) - def _parse_file(descriptor_file, is_bridge = False, validate = False, **kwargs): """ @@ -965,7 +957,13 @@ class BridgeDescriptor(ServerDescriptor): if sign: raise NotImplementedError('Signing of %s not implemented' % cls.__name__) - return _descriptor_content(attr, exclude, sign, BRIDGE_SERVER_HEADER) + return _descriptor_content(attr, exclude, sign, ( + ('router', 'Unnamed%s %s 9001 0 0' % (random.randint(0, sys.maxint), _random_ipv4_address())), + ('router-digest', '006FD96BA35E7785A6A3B8B75FE2E2435A13BDB4'), + ('published', _random_date()), + ('bandwidth', '409600 819200 5120'), + ('reject', '*:*'), + )) def digest(self): return self._digest diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index c881751..8578726 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -732,8 +732,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= desc = BridgeDescriptor.create() - self.assertEqual('Unnamed', desc.nickname) - self.assertEqual('10.45.227.253', desc.address) + self.assertTrue(desc.nickname.startswith('Unnamed')) self.assertEqual(None, desc.fingerprint) self.assertEqual('006FD96BA35E7785A6A3B8B75FE2E2435A13BDB4', desc.digest())

1 0

[stem/master] Note in comparison table that we now support descriptor creation
by atagar＠torproject.org 27 Jun '17

27 Jun '17

commit 785bbd36c187f56a9576c24c9158623f86a38ffd Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jun 27 09:49:51 2017 -0700 Note in comparison table that we now support descriptor creation --- docs/tutorials/mirror_mirror_on_the_wall.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/tutorials/mirror_mirror_on_the_wall.rst b/docs/tutorials/mirror_mirror_on_the_wall.rst index 5148c50..d215722 100644 --- a/docs/tutorials/mirror_mirror_on_the_wall.rst +++ b/docs/tutorials/mirror_mirror_on_the_wall.rst @@ -198,10 +198,6 @@ descriptor... signing-key ... -**Note:** Stem doesn't yet support signing its descriptors (:trac:`10227`). -Until it does try using `leekspin <https://pypi.python.org/pypi/leekspin>`_ -instead. - .. _validating-the-descriptors-content: Validating the descriptor's content @@ -324,7 +320,7 @@ Capability Stem Metrics-lib Zoossh =========================== ===================== =================== ============== Language :green:`Python` :green:`Java` :green:`Go` Checks signatures :green:`Mostly` :red:`No` :red:`No` -Create new descriptors :red:`No` :red:`No` :red:`No` +Create new descriptors :green:`Yes` :red:`No` :red:`No` Lazy parsing :green:`Yes` :red:`No` :green:`Yes` Type detection by @type :green:`Yes` :green:`Yes` :green:`Yes` Type detection by filename :green:`Yes` :red:`No` :red:`No`

1 0

[stem/master] Randomize created server descirptor fields
by atagar＠torproject.org 27 Jun '17

27 Jun '17

commit 25bf050edf71a7653970aa9ab41d70e4b4219b63 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jun 27 08:31:40 2017 -0700 Randomize created server descirptor fields Now that we're providing a method to create descriptors to callers (rather than just our own tests) we should randomize the default attributes. Gonna do this with all descriptor types, but starting with server descriptors since we just added signing for 'em. --- run_tests.py | 40 +++++++++++++------------------ stem/descriptor/__init__.py | 22 +++++++++++++++++ stem/descriptor/server_descriptor.py | 36 +++++++++++++++------------- test/unit/descriptor/__init__.py | 6 ++--- test/unit/descriptor/export.py | 5 +++- test/unit/descriptor/server_descriptor.py | 9 ++++--- test/unit/tutorial.py | 2 +- 7 files changed, 71 insertions(+), 49 deletions(-) diff --git a/run_tests.py b/run_tests.py index 802a6b4..c0d9cbc 100755 --- a/run_tests.py +++ b/run_tests.py @@ -310,10 +310,8 @@ def main(): _print_static_issues(static_check_issues) - runtime_label = '(%i seconds)' % (time.time() - start_time) - if error_tracker.has_errors_occured(): - println('TESTING FAILED %s' % runtime_label, ERROR, STDERR) + println('TESTING FAILED (%i seconds)' % (time.time() - start_time), ERROR, STDERR) for line in error_tracker: println(' %s' % line, ERROR, STDERR) @@ -329,7 +327,7 @@ def main(): if skipped_tests > 0: println('%i TESTS WERE SKIPPED' % skipped_tests, STATUS) - println('TESTING PASSED %s\n' % runtime_label, SUCCESS) + println('TESTING PASSED (%i seconds)\n' % (time.time() - start_time), SUCCESS) new_capabilities = test.get_new_capabilities() @@ -369,26 +367,23 @@ def _print_static_issues(static_check_issues): def _run_test(args, test_class, output_filters, logging_buffer): start_time = time.time() + # Test classes look like... + # + # test.unit.util.conf.TestConf.test_parse_enum_csv + # + # We want to strip the 'test.unit.' or 'test.integ.' prefix since it's + # redundant. We also want to drop the test class name. The individual test + # name at the end it optional (only present if we used the '--test' + # argument). + + label_comp = test_class.split('.')[2:] + del label_comp[-1 if label_comp[-1][0].isupper() else -2] + test_label = ' %-52s' % ('.'.join(label_comp) + '...') + if args.verbose: test.output.print_divider(test_class) else: - # Test classes look like... - # - # test.unit.util.conf.TestConf.test_parse_enum_csv - # - # We want to strip the 'test.unit.' or 'test.integ.' prefix since it's - # redundant. We also want to drop the test class name. The individual test - # name at the end it optional (only present if we used the '--test' - # argument). - - label_comp = test_class.split('.')[2:] - del label_comp[-1 if label_comp[-1][0].isupper() else -2] - label = '.'.join(label_comp) - - label = ' %s...' % label - label = '%-54s' % label - - println(label, STATUS, NO_NL) + println(test_label, STATUS, NO_NL) try: suite = unittest.TestLoader().loadTestsFromName(test_class) @@ -413,7 +408,7 @@ def _run_test(args, test_class, output_filters, logging_buffer): println(' success (%0.2fs)' % (time.time() - start_time), SUCCESS) else: if args.quiet: - println(label, STATUS, NO_NL, STDERR) + println(test_label, STATUS, NO_NL, STDERR) println(' failed (%0.2fs)' % (time.time() - start_time), ERROR, STDERR) println(test.output.apply_filters(test_results.getvalue(), *output_filters), STDERR) else: @@ -421,7 +416,6 @@ def _run_test(args, test_class, output_filters, logging_buffer): println(test.output.apply_filters(test_results.getvalue(), *output_filters), NO_NL) test.output.print_logging(logging_buffer) - return run_result diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index bf336fb..80901af 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -42,6 +42,7 @@ import codecs import copy import hashlib import os +import random import re import string import tarfile @@ -952,6 +953,27 @@ def _get_pseudo_pgp_block(remaining_contents): return None +def _random_ipv4_address(): + return '%i.%i.%i.%i' % (random.randint(0, 255), random.randint(0, 255), random.randint(0, 255), random.randint(0, 255)) + + +def _random_date(): + return '%i-%02i-%02i %02i:%02i:%02i' % (random.randint(2000, 2015), random.randint(1, 12), random.randint(1, 20), random.randint(0, 23), random.randint(0, 59), random.randint(0, 59)) + + +def _random_crypto_blob(block_type = None): + """ + Provides a random string that can be used for crypto blocks. + """ + + crypto_blob = stem.util.str_tools._split_by_length(base64.b64encode('%0140x' % random.randrange(16 ** 140)), 64) + + if block_type: + return '\n-----BEGIN %s-----\n%s\n-----END %s-----' % (block_type, crypto_blob, block_type) + else: + return crypto_blob + + def _descriptor_components(raw_contents, validate, extra_keywords = (), non_ascii_fields = ()): """ Initial breakup of the server descriptor contents to make parsing easier. diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 8dbd0f0..607908d 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -35,7 +35,9 @@ import base64 import binascii import functools import hashlib +import random import re +import sys import stem.descriptor.certificate import stem.descriptor.extrainfo_descriptor @@ -49,7 +51,6 @@ import stem.version from stem.util import str_type from stem.descriptor import ( - CRYPTO_BLOB, PGP_BLOCK_END, Descriptor, _descriptor_content, @@ -65,6 +66,9 @@ from stem.descriptor import ( _parse_forty_character_hex, _parse_protocol_line, _parse_key_block, + _random_ipv4_address, + _random_date, + _random_crypto_blob, ) try: @@ -112,19 +116,6 @@ SINGLE_FIELDS = ( DEFAULT_IPV6_EXIT_POLICY = stem.exit_policy.MicroExitPolicy('reject 1-65535') REJECT_ALL_POLICY = stem.exit_policy.ExitPolicy('reject *:*') -RELAY_SERVER_HEADER = ( - ('router', 'caerSidi 71.35.133.197 9001 0 0'), - ('published', '2012-03-01 17:15:27'), - ('bandwidth', '153600 256000 104590'), - ('reject', '*:*'), - ('onion-key', '\n-----BEGIN RSA PUBLIC KEY-----%s-----END RSA PUBLIC KEY-----' % CRYPTO_BLOB), - ('signing-key', '\n-----BEGIN RSA PUBLIC KEY-----%s-----END RSA PUBLIC KEY-----' % CRYPTO_BLOB), -) - -RELAY_SERVER_FOOTER = ( - ('router-signature', '\n-----BEGIN SIGNATURE-----%s-----END SIGNATURE-----' % CRYPTO_BLOB), -) - BRIDGE_SERVER_HEADER = ( ('router', 'Unnamed 10.45.227.253 9001 0 0'), ('router-digest', '006FD96BA35E7785A6A3B8B75FE2E2435A13BDB4'), @@ -818,6 +809,19 @@ class RelayDescriptor(ServerDescriptor): @classmethod def content(cls, attr = None, exclude = (), sign = False, private_signing_key = None): + base_header = ( + ('router', 'Unnamed%i %s 9001 0 0' % (random.randint(0, sys.maxint), _random_ipv4_address())), + ('published', _random_date()), + ('bandwidth', '153600 256000 104590'), + ('reject', '*:*'), + ('onion-key', _random_crypto_blob('RSA PUBLIC KEY')), + ('signing-key', _random_crypto_blob('RSA PUBLIC KEY')), + ) + + base_footer = ( + ('router-signature', _random_crypto_blob('SIGNATURE')), + ) + if sign: if not stem.prereq.is_crypto_available(): raise ImportError('Signing requires the cryptography module') @@ -861,12 +865,12 @@ class RelayDescriptor(ServerDescriptor): format = serialization.PublicFormat.PKCS1, ).strip() - content = _descriptor_content(attr, exclude, sign, RELAY_SERVER_HEADER) + b'\nrouter-signature\n' + content = _descriptor_content(attr, exclude, sign, base_header) + b'\nrouter-signature\n' signature = base64.b64encode(private_signing_key.sign(content, padding.PKCS1v15(), hashes.SHA1())) return content + b'\n'.join([b'-----BEGIN SIGNATURE-----'] + stem.util.str_tools._split_by_length(signature, 64) + [b'-----END SIGNATURE-----\n']) else: - return _descriptor_content(attr, exclude, sign, RELAY_SERVER_HEADER, RELAY_SERVER_FOOTER) + return _descriptor_content(attr, exclude, sign, base_header, base_footer) @classmethod def create(cls, attr = None, exclude = (), validate = True, sign = False, private_signing_key = None): diff --git a/test/unit/descriptor/__init__.py b/test/unit/descriptor/__init__.py index cd35ae1..bf0e861 100644 --- a/test/unit/descriptor/__init__.py +++ b/test/unit/descriptor/__init__.py @@ -29,7 +29,7 @@ def base_expect_invalid_attr(cls, default_attr, default_value, test, desc_attrs, return base_expect_invalid_attr_for_text(cls, default_attr, default_value, test, cls.content(desc_attrs), attr, expected_value) -def base_expect_invalid_attr_for_text(cls, default_attr, default_value, test, desc_text, attr = None, expected_value = None): +def base_expect_invalid_attr_for_text(cls, default_attr, default_prefix, test, desc_text, attr = None, expected_value = None): """ Asserts that construction will fail due to desc_text having a malformed attribute. If an attr is provided then we check that it matches an expected @@ -44,7 +44,7 @@ def base_expect_invalid_attr_for_text(cls, default_attr, default_value, test, de # constructed without validation test.assertEqual(expected_value, getattr(desc, attr)) - elif default_attr and default_value: - test.assertEqual(default_value, getattr(desc, default_attr)) # check a default attribute + elif default_attr and default_prefix: + test.assertTrue(getattr(desc, default_attr).startswith(default_prefix)) # check a default attribute return desc diff --git a/test/unit/descriptor/export.py b/test/unit/descriptor/export.py index cd005d1..a33c1d4 100644 --- a/test/unit/descriptor/export.py +++ b/test/unit/descriptor/export.py @@ -25,7 +25,10 @@ class TestExport(unittest.TestCase): self.skipTest('(header added in python 2.7)') return - desc = RelayDescriptor.create() + desc = RelayDescriptor.create({ + 'router': 'caerSidi 71.35.133.197 9001 0 0', + 'published': '2012-03-01 17:15:27', + }) desc_csv = export_csv(desc, included_fields = ('nickname', 'address', 'published'), header = False) expected = 'caerSidi,71.35.133.197,2012-03-01 17:15:27\n' diff --git a/test/unit/descriptor/server_descriptor.py b/test/unit/descriptor/server_descriptor.py index 227df12..2c47bdf 100644 --- a/test/unit/descriptor/server_descriptor.py +++ b/test/unit/descriptor/server_descriptor.py @@ -40,8 +40,8 @@ TARFILE_FINGERPRINTS = set([ str_type('1F43EE37A0670301AD9CB555D94AFEC2C89FDE86'), ]) -expect_invalid_attr = functools.partial(base_expect_invalid_attr, RelayDescriptor, 'nickname', 'caerSidi') -expect_invalid_attr_for_text = functools.partial(base_expect_invalid_attr_for_text, RelayDescriptor, 'nickname', 'caerSidi') +expect_invalid_attr = functools.partial(base_expect_invalid_attr, RelayDescriptor, 'nickname', 'Unnamed') +expect_invalid_attr_for_text = functools.partial(base_expect_invalid_attr_for_text, RelayDescriptor, 'nickname', 'Unnamed') class TestServerDescriptor(unittest.TestCase): @@ -440,11 +440,10 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= attributes. """ - desc = RelayDescriptor.create() + desc = RelayDescriptor.create({'router': 'caerSidi 71.35.133.197 9001 0 0'}) self.assertEqual('caerSidi', desc.nickname) self.assertEqual('71.35.133.197', desc.address) self.assertEqual(None, desc.fingerprint) - self.assertTrue(stem.descriptor.CRYPTO_BLOB in desc.onion_key) def test_with_opt(self): """ @@ -625,7 +624,7 @@ Qlx9HNCqCY877ztFRC624ja2ql6A2hBcuoYMbkHjcQ4= self.assertRaises(ValueError, list, desc_iter) desc_text = b'@pepperjack very tasty\n@mushrooms not so much\n' - desc_text += RelayDescriptor.content() + desc_text += RelayDescriptor.content({'router': 'caerSidi 71.35.133.197 9001 0 0'}) desc_iter = stem.descriptor.server_descriptor._parse_file(io.BytesIO(desc_text)) desc_entries = list(desc_iter) diff --git a/test/unit/tutorial.py b/test/unit/tutorial.py index cdfb014..b0f5b71 100644 --- a/test/unit/tutorial.py +++ b/test/unit/tutorial.py @@ -162,7 +162,7 @@ class TestTutorial(unittest.TestCase): @patch('stem.descriptor.reader.DescriptorReader', spec = DescriptorReader) def test_mirror_mirror_on_the_wall_4(self, reader_mock, stdout_mock): reader = reader_mock().__enter__() - reader.__iter__.return_value = iter([RelayDescriptor.create()]) + reader.__iter__.return_value = iter([RelayDescriptor.create({'router': 'caerSidi 71.35.133.197 9001 0 0'})]) exec_documentation_example('past_descriptors.py') self.assertEqual('found relay caerSidi (None)\n', stdout_mock.getvalue())

1 0

[metrics-web/master] Rename 'DescripTor' to 'Tor Metrics Library' as part of task-22732.
by karsten＠torproject.org 27 Jun '17

27 Jun '17

commit 1036ea84143da14adcd10b3f63b68e362d635a4e Author: iwakeh <iwakeh(a)torproject.org> Date: Thu Jun 22 13:44:11 2017 +0000 Rename 'DescripTor' to 'Tor Metrics Library' as part of task-22732. --- website/src/main/resources/web/WEB-INF/development.jsp | 2 +- website/src/main/resources/web/WEB-INF/metrics-lib.jsp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/website/src/main/resources/web/WEB-INF/development.jsp b/website/src/main/resources/web/WEB-INF/development.jsp index 445f77b..2627530 100644 --- a/website/src/main/resources/web/WEB-INF/development.jsp +++ b/website/src/main/resources/web/WEB-INF/development.jsp @@ -21,7 +21,7 @@ <h2>Parsing libraries <a href="#libraries" name="libraries" class="anchor">#</a></h2> <p>The following libraries help you with parsing Tor network data from the <a href="https://collector.torproject.org/" target="_blank">CollecTor</a> service.</p> <ul> - <li><a href="metrics-lib.html">metrics-lib</a> is a Java library to fetch and parse Tor descriptors.</li> + <li><a href="metrics-lib.html">Tor Metrics Library</a> is a Java library to fetch and parse Tor descriptors.</li> <li><a href="https://stem.torproject.org/" target="_blank">Stem</a> is a Python library that parses Tor descriptors.</li> <li><a href="https://github.com/NullHypothesis/zoossh" target="_blank">Zoossh</a> is a parser written in Go for Tor-specific data formats.</li> </ul> diff --git a/website/src/main/resources/web/WEB-INF/metrics-lib.jsp b/website/src/main/resources/web/WEB-INF/metrics-lib.jsp index e74f977..885b385 100644 --- a/website/src/main/resources/web/WEB-INF/metrics-lib.jsp +++ b/website/src/main/resources/web/WEB-INF/metrics-lib.jsp @@ -17,8 +17,8 @@ <div class="jumbotron"> <div class="text-center"> -<h2>metrics-lib</h2> -<p>metrics-lib is a Java API that facilitates processing Tor network data from the <a href="https://collector.torproject.org/">CollecTor</a> service for statistical analysis and for building services and applications.</p> +<h2>Tor Metrics Library</h2> +<p>Tor Metrics Library is a Java API that facilitates processing Tor network data from the <a href="https://collector.torproject.org/">CollecTor</a> service for statistical analysis and for building services and applications.</p> <a class="btn btn-primary btn-lg" style="margin: 10px" href="https://dist.torproject.org/descriptor/?C=M;O=D"><i class="fa fa-chevron-right" aria-hidden="true"></i> Download Release</a> <a class="btn btn-primary btn-lg" style="margin: 10px" href="https://gitweb.torproject.org/metrics-lib.git/plain/CHANGELOG.md"><i class="fa fa-chevron-right" aria-hidden="true"></i> View Change Log</a>

1 0

[metrics-lib/master] Rename 'DescripTor' to 'Tor Metrics Library' as part of task-22732.
by karsten＠torproject.org 27 Jun '17

27 Jun '17

commit 0d4898df6c74bb382253d7991621d9c08031ae8c Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Jun 26 20:12:13 2017 +0000 Rename 'DescripTor' to 'Tor Metrics Library' as part of task-22732. --- build.xml | 4 ++-- src/main/resources/overview.html | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.xml b/build.xml index 0de365c..4c7cfff 100644 --- a/build.xml +++ b/build.xml @@ -7,9 +7,9 @@ <project default="usage" name="descriptor" basedir="."> <property name="release.version" value="1.9.0-dev" /> - <property name="javadoc-title" value="DescripTor API Documentation"/> + <property name="javadoc-title" value="Tor Metrics Library API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/**" /> - <property name="implementation-title" value="DescripTor" /> + <property name="implementation-title" value="Tor Metrics Library" /> <property name="name" value="descriptor" /> <property name="jarpatternprop" value="empty" /> diff --git a/src/main/resources/overview.html b/src/main/resources/overview.html index f5c6b51..a565536 100644 --- a/src/main/resources/overview.html +++ b/src/main/resources/overview.html @@ -1,6 +1,6 @@ <html> <body> -<p>DescripTor API, which is provided and supported by Tor's Metrics +<p>Tor Metrics Library API, which is provided and supported by Tor's Metrics Team, is a library to obtain and process descriptors containing Tor network data. It is the main Java tool for processing Tor descriptors and provides a standard API consisting of interfaces and a reference @@ -54,13 +54,13 @@ the application. Stated differently, there are no descriptor sinks that would correspond to the provided descriptor sources.</li> </ul></p> -<p>Hints about using DescripTor can be found in the +<p>Hints about using Tor Metrics Library can be found in the <a href="./org/torproject/descriptor/package-summary.html#package.description">{@code org.torproject.descriptor} package description</a>. </p> <p>Contact and further information: <ul> -<li><a href="https://trac.torproject.org/projects/tor/query?status=!closed&component=Met…">DescripTor Bug Tracker</a></li> +<li><a href="https://trac.torproject.org/projects/tor/query?status=!closed&component=Met…">Tor Metrics Library Bug Tracker</a></li> <li><a href="https://lists.torproject.org/cgi-bin/mailman/listinfo/metrics-team">Metrics Team Mailing List</a></li> <li><a href="https://trac.torproject.org/projects/tor/wiki/org/teams/MetricsTeam">Metrics Team Website</a></li> </ul>

1 0

[metrics-lib/master] All examples compile and work as explained in Metrics' web-site tutorials.
by karsten＠torproject.org 27 Jun '17

27 Jun '17

commit 27f05e1d4c67d8954b5cd2975781870a0dcdb9d9 Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Jun 26 20:12:11 2017 +0000 All examples compile and work as explained in Metrics' web-site tutorials. Added a hint about where to find the examples to the README.md file. Implements task-22734. --- README.md | 6 +++ .../examples/ConsensusWeightByVersion.java | 45 +++++++++------------- .../resources/examples/PluggableTransports.java | 29 ++++++-------- 3 files changed, 36 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 0cc93d8..aa309ac 100644 --- a/README.md +++ b/README.md @@ -97,3 +97,9 @@ The Metrics website has a tutorial for getting started with metrics-lib: https://metrics.torproject.org/metrics-lib.html +The examples explained in the tutorials are available as source code in + +``` +src/main/resources/examples/ +``` + diff --git a/src/main/resources/examples/ConsensusWeightByVersion.java b/src/main/resources/examples/ConsensusWeightByVersion.java index 3b8495d..5fc7c1c 100644 --- a/src/main/resources/examples/ConsensusWeightByVersion.java +++ b/src/main/resources/examples/ConsensusWeightByVersion.java @@ -19,35 +19,26 @@ public class ConsensusWeightByVersion { // Read descriptors from disk. DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - - // Add the directory with descriptors to the descriptor reader. - descriptorReader.addDirectory(new File("descriptors/recent/relay-descriptors/consensuses")); - Iterator<DescriptorFile> descriptorFiles = descriptorReader.readDescriptors(); - while (descriptorFiles.hasNext()) { // Iterate over all descriptor files found. - DescriptorFile descriptorFile = descriptorFiles.next(); - - // Now, iterate over the descriptors contained in the file. - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (!(descriptor instanceof RelayNetworkStatusConsensus)) { - // We're only interested in consensuses. + for (Descriptor descriptor : descriptorReader.readDescriptors(new File("descriptors/recent/relay-descriptors/consensuses"))) { + if (!(descriptor instanceof RelayNetworkStatusConsensus)) { + // We're only interested in consensuses. + continue; + } + RelayNetworkStatusConsensus consensus = (RelayNetworkStatusConsensus) descriptor; + for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { + String version = entry.getVersion(); + if (!version.startsWith("Tor ") || version.length() < 9) { + // We're only interested in a.b.c type versions for this example. continue; } - RelayNetworkStatusConsensus consensus = (RelayNetworkStatusConsensus) descriptor; - for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { - String version = entry.getVersion(); - if (!version.startsWith("Tor ") || version.length() < 9) { - // We're only interested in a.b.c type versions for this example. - continue; - } - // Remove the 'Tor ' prefix and anything starting at the patch level. - version = version.substring(4, 9); - long bandwidth = entry.getBandwidth(); - totalBandwidth += bandwidth; - if (bandwidthByVersion.containsKey(version)) { - bandwidthByVersion.put(version, bandwidth + bandwidthByVersion.get(version)); - } else { - bandwidthByVersion.put(version, bandwidth); - } + // Remove the 'Tor ' prefix and anything starting at the patch level. + version = version.substring(4, 9); + long bandwidth = entry.getBandwidth(); + totalBandwidth += bandwidth; + if (bandwidthByVersion.containsKey(version)) { + bandwidthByVersion.put(version, bandwidth + bandwidthByVersion.get(version)); + } else { + bandwidthByVersion.put(version, bandwidth); } } } diff --git a/src/main/resources/examples/PluggableTransports.java b/src/main/resources/examples/PluggableTransports.java index 88e7eb6..65ce732 100644 --- a/src/main/resources/examples/PluggableTransports.java +++ b/src/main/resources/examples/PluggableTransports.java @@ -16,23 +16,18 @@ public class PluggableTransports { SortedMap<String, Integer> countedTransports = new TreeMap<>(); DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - descriptorReader.addDirectory(new File("descriptors/recent/bridge-descriptors/extra-infos")); - Iterator<DescriptorFile> descriptorFiles = descriptorReader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (!(descriptor instanceof BridgeExtraInfoDescriptor)) { - continue; - } - BridgeExtraInfoDescriptor extraInfo = (BridgeExtraInfoDescriptor) descriptor; - String fingerprint = extraInfo.getFingerprint(); - if (observedFingerprints.add(fingerprint)) { - for (String transport : extraInfo.getTransports()) { - if (countedTransports.containsKey(transport)) { - countedTransports.put(transport, 1 + countedTransports.get(transport)); - } else { - countedTransports.put(transport, 1); - } + for (Descriptor descriptor : descriptorReader.readDescriptors(new File("descriptors/recent/bridge-descriptors/extra-infos"))) { + if (!(descriptor instanceof BridgeExtraInfoDescriptor)) { + continue; + } + BridgeExtraInfoDescriptor extraInfo = (BridgeExtraInfoDescriptor) descriptor; + String fingerprint = extraInfo.getFingerprint(); + if (observedFingerprints.add(fingerprint)) { + for (String transport : extraInfo.getTransports()) { + if (countedTransports.containsKey(transport)) { + countedTransports.put(transport, 1 + countedTransports.get(transport)); + } else { + countedTransports.put(transport, 1); } } }

1 0

[tor/maint-0.3.0] Merge branch 'maint-0.2.6' into maint-0.2.7-redux
by nickm＠torproject.org 27 Jun '17

27 Jun '17

commit 9a0fd2dbb187e898f63064c483776538a90562f1 Merge: 24ee859 3de2761 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 27 11:04:44 2017 -0400 Merge branch 'maint-0.2.6' into maint-0.2.7-redux changes/bug22737 | 12 ++++++++++++ src/or/connection_or.c | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-)

1 0

[tor/maint-0.3.0] Fix an errant memset() into the middle of a struct in cell_pack().
by nickm＠torproject.org 27 Jun '17

27 Jun '17

commit 8d2978b13c7462776d8a6ba6a6df5b8d9883a633 Author: Nick Mathewson <nickm(a)torproject.org> Date: Tue Jun 27 10:45:29 2017 -0400 Fix an errant memset() into the middle of a struct in cell_pack(). This mistake causes two possible bugs. I believe they are both harmless IRL. BUG 1: memory stomping When we call the memset, we are overwriting two 0 bytes past the end of packed_cell_t.body. But I think that's harmless in practice, because the definition of packed_cell_t is: // ... typedef struct packed_cell_t { TOR_SIMPLEQ_ENTRY(packed_cell_t) next; char body[CELL_MAX_NETWORK_SIZE]; uint32_t inserted_time; } packed_cell_t; So we will overwrite either two bytes of inserted_time, or two bytes of padding, depending on how the platform handles alignment. If we're overwriting padding, that's safe. If we are overwriting the inserted_time field, that's also safe: In every case where we call cell_pack() from connection_or.c, we ignore the inserted_time field. When we call cell_pack() from relay.c, we don't set or use inserted_time until right after we have called cell_pack(). SO I believe we're safe in that case too. BUG 2: memory exposure The original reason for this memset was to avoid the possibility of accidentally leaking uninitialized ram to the network. Now remember, if wide_circ_ids is false on a connection, we shouldn't actually be sending more than 512 bytes of packed_cell_t.body, so these two bytes can only leak to the network if there is another bug somewhere else in the code that sends more data than is correct. Fortunately, in relay.c, where we allocate packed_cell_t in packed_cell_new() , we allocate it with tor_malloc_zero(), which clears the RAM, right before we call cell_pack. So those packed_cell_t.body bytes can't leak any information. That leaves the two calls to cell_pack() in connection_or.c, which use stack-alocated packed_cell_t instances. In or_handshake_state_record_cell(), we pass the cell's contents to crypto_digest_add_bytes(). When we do so, we get the number of bytes to pass using the same setting of wide_circ_ids as we passed to cell_pack(). So I believe that's safe. In connection_or_write_cell_to_buf(), we also use the same setting of wide_circ_ids in both calls. So I believe that's safe too. I introduced this bug with 1c0e87f6d8c7a0abdadf1b5cd9082c10abc7f4e2 back in 0.2.4.11-alpha; it is bug 22737 and CID 1401591 --- changes/bug22737 | 12 ++++++++++++ src/or/connection_or.c | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/changes/bug22737 b/changes/bug22737 new file mode 100644 index 0000000..f0de8e6 --- /dev/null +++ b/changes/bug22737 @@ -0,0 +1,12 @@ + o Minor bugfixes (defensive programming, undefined behavior): + + - Fix a memset() off the end of an array when packing cells. This + bug should be harmless in practice, since the corrupted bytes + are still in the same structure, and are always padding bytes, + ignored, or immediately overwritten, depending on compiler + behavior. Nevertheless, because the memset()'s purpose is to + make sure that any other cell-handling bugs can't expose bytes + to the network, we need to fix it. Fixes bug 22737; bugfix on + 0.2.4.11-alpha. Fixes CID 1401591. + + diff --git a/src/or/connection_or.c b/src/or/connection_or.c index 8e7cd9e..99ea1ef 100644 --- a/src/or/connection_or.c +++ b/src/or/connection_or.c @@ -358,9 +358,11 @@ cell_pack(packed_cell_t *dst, const cell_t *src, int wide_circ_ids) set_uint32(dest, htonl(src->circ_id)); dest += 4; } else { + /* Clear the last two bytes of dest, in case we can accidentally + * send them to the network somehow. */ + memset(dest+CELL_MAX_NETWORK_SIZE-2, 0, 2); set_uint16(dest, htons(src->circ_id)); dest += 2; - memset(dest+CELL_MAX_NETWORK_SIZE-2, 0, 2); /*make sure it's clear */ } set_uint8(dest, src->command); memcpy(dest+1, src->payload, CELL_PAYLOAD_SIZE);

1 0