- tor-commits - lists.torproject.org

[translation/vidalia] Update translations for vidalia
by translation＠torproject.org 31 Jan '12

31 Jan '12

commit 2f9bded5535dff81fe1bfafbd261d110725b2433 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Jan 31 19:15:30 2012 +0000 Update translations for vidalia --- ru/vidalia_ru.po | 36 ++++++++++++++++++------------------ 1 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ru/vidalia_ru.po b/ru/vidalia_ru.po index 2e3dbd8..cb66b53 100644 --- a/ru/vidalia_ru.po +++ b/ru/vidalia_ru.po @@ -8,7 +8,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2011-09-07 14:40+0000\n" -"PO-Revision-Date: 2012-01-31 18:44+0000\n" +"PO-Revision-Date: 2012-01-31 19:04+0000\n" "Last-Translator: liquixis <liquixis(a)gmail.com>\n" "Language-Team: Russian (http://www.transifex.net/projects/p/torproject/team/ru/)\n" "MIME-Version: 1.0\n" @@ -2020,7 +2020,7 @@ msgstr "Посмотреть информацию о версии и о лице msgctxt "MainWindow" msgid "Exit Vidalia" -msgstr "Выход Vidalia" +msgstr "Выход из Vidalia" msgctxt "MainWindow" msgid "Show this window on startup" @@ -2920,7 +2920,7 @@ msgstr "Вы должны указать как минимум имя серве msgctxt "ServerPage" msgid "Run as a client only" -msgstr "Режим работы только как клиент" +msgstr "Работать только как клиент" msgctxt "ServerPage" msgid "Relay traffic for the Tor network" @@ -2932,11 +2932,11 @@ msgstr "Порт сервера:" msgctxt "ServerPage" msgid "Enable to mirror the relay directory" -msgstr "Разрешить зеркало каталога серверов" +msgstr "Включите для зеркалирования Каталога Серверов" msgctxt "ServerPage" msgid "Attempt to automatically configure port forwarding" -msgstr "Попытка автоматически настроить переадресацию портов" +msgstr "Пытаться автоматически настроить переадресацию портов" msgctxt "ServerPage" msgid "Test" @@ -3187,7 +3187,7 @@ msgstr "Автоматически распространять адрес мо msgctxt "ServerPage" msgid "Mirror the Relay Directory" -msgstr "Зеркало каталога серверов" +msgstr "Зеркалировать Каталог Серверов" msgctxt "ServerPage" msgid "" @@ -3228,7 +3228,7 @@ msgstr "" msgctxt "ServicePage" msgid "Target may only contain address:port, address, or port." -msgstr "Цель может содержать только адрес: порт, адрес или порт." +msgstr "Цель может содержать только адрес:порт, адрес или порт." msgctxt "ServicePage" msgid "Directory already in use by another service." @@ -3533,7 +3533,7 @@ msgstr "" msgctxt "StatusEventWidget" msgid "Checking Directory Port Reachability" -msgstr "Проверяется доступность порта директории" +msgstr "Проверяется доступность порта каталога" msgctxt "StatusEventWidget" msgid "" @@ -3541,21 +3541,21 @@ msgid "" "the Tor network by connecting to itself at %1:%2. This test could take " "several minutes." msgstr "" -"Tor пытается определить доступен ли порт вашей директории ретрансляции в " -"сети Tor, путем соединения с сами собой на %1:%2. Этот тест может занять " +"Tor пытается определить доступен ли порт вашего каталога ретрансляции в сети" +" Tor, путем соединения с сами собой на %1:%2. Этот тест может занять " "несколько минут." msgctxt "StatusEventWidget" msgid "Directory Port Reachability Test Successful!" -msgstr "Проверка доступности порта директории завершена успешно!" +msgstr "Проверка доступности порта каталога завершена успешно!" msgctxt "StatusEventWidget" msgid "Your relay's directory port is reachable from the Tor network!" -msgstr "Порт вашей директории ретрансляции доступен в сети Tor!" +msgstr "Порт вашего каталога ретрансляции доступен в сети Tor!" msgctxt "StatusEventWidget" msgid "Directory Port Reachability Test Failed" -msgstr "Проверка доступности порта директории провалилась!" +msgstr "Проверка доступности порта каталога провалилась!" msgctxt "StatusEventWidget" msgid "" @@ -3564,10 +3564,10 @@ msgid "" "port forwarding. If %1:%2 is not your correct IP address and directory port," " please check your relay's configuration." msgstr "" -"Порт вашей директории ретрансляции недоступен другим клиентам Tor. Это может" -" происходить если вы находитесь за роутером или брандмауэром, что требует от" -" вас настроить проброс портов. Если %1:%2 не ваши правильные IP адрес и порт" -" директории, пожалуйста, проверьте настройки вашего ретранслятора." +"Порт вашего каталога ретрансляции недоступен другим клиентам Tor. Это может " +"происходить если вы находитесь за роутером или брандмауэром, что требует от " +"вас настроить проброс портов. Если %1:%2 не ваши правильные IP адрес и порт " +"каталога, пожалуйста, проверьте настройки вашего ретранслятора." msgctxt "StatusEventWidget" msgid "Relay Descriptor Rejected" @@ -3579,7 +3579,7 @@ msgid "" " rejected by the directory server at %1:%2. The reason given was: %3" msgstr "" "Дескриптор вашего ретранслятора, который позволяет клиентам соединяться с " -"вашим ретранслятором, был отклонен сервером директории в %1:%2. По следущей " +"вашим ретранслятором, был отклонен сервером каталога в %1:%2. По следущей " "причине: %3" msgctxt "StatusEventWidget"

1 0

[translation/vidalia] Update translations for vidalia
by translation＠torproject.org 31 Jan '12

31 Jan '12

commit b755c1c8ae04e0a8a551bd5a964a338830716079 Author: Translation commit bot <translation(a)torproject.org> Date: Tue Jan 31 18:45:16 2012 +0000 Update translations for vidalia --- ru/vidalia_ru.po | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ru/vidalia_ru.po b/ru/vidalia_ru.po index cd01f44..2e3dbd8 100644 --- a/ru/vidalia_ru.po +++ b/ru/vidalia_ru.po @@ -8,7 +8,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: https://trac.torproject.org/projects/tor\n" "POT-Creation-Date: 2011-09-07 14:40+0000\n" -"PO-Revision-Date: 2012-01-29 15:32+0000\n" +"PO-Revision-Date: 2012-01-31 18:44+0000\n" "Last-Translator: liquixis <liquixis(a)gmail.com>\n" "Language-Team: Russian (http://www.transifex.net/projects/p/torproject/team/ru/)\n" "MIME-Version: 1.0\n" @@ -1668,7 +1668,7 @@ msgstr "Помощь" msgctxt "MainWindow" msgid "New Identity" -msgstr "Сменить личину" +msgstr "Новая Личность" msgctxt "MainWindow" msgid "Ctrl+T" @@ -1772,7 +1772,7 @@ msgstr "Создание цепочки Tor" msgctxt "MainWindow" msgid "Connected to the Tor network!" -msgstr "Подключение к сети Tor!" +msgstr "Подключен к сети Tor!" msgctxt "MainWindow" msgid "Unrecognized startup status" @@ -1976,7 +1976,7 @@ msgstr "Ярлыки Vidalia" msgctxt "MainWindow" msgid "Setup Relaying" -msgstr "Настройка сервера\n" +msgstr "Настройка ретрансляции" msgctxt "MainWindow" msgid "Set up a relay and help the network grow" @@ -1992,7 +1992,7 @@ msgstr "Обзор карты сети Tor" msgctxt "MainWindow" msgid "Use a New Identity" -msgstr "Сменить личину" +msgstr "Использовать Новую Личность" msgctxt "MainWindow" msgid "Make subsequent connections appear new"

1 0

[stem/master] Previous socket use could break get_protocolinfo
by atagar＠torproject.org 31 Jan '12

31 Jan '12

commit c58177c50fcdba85e5eac7a929dbf6d87c666852 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 30 00:34:38 2012 -0800 Previous socket use could break get_protocolinfo If prior use of the control socket had caused a disconnect and already consumed the 'Authentication required.' response then future calls of get_protocolinfo would break. --- stem/connection.py | 17 ++++++++++++----- test/integ/connection/protocolinfo.py | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/stem/connection.py b/stem/connection.py index 5c1fc55..392ad9b 100644 --- a/stem/connection.py +++ b/stem/connection.py @@ -627,16 +627,23 @@ def get_protocolinfo(control_socket): socket """ - control_socket.send("PROTOCOLINFO 1") - protocolinfo_response = control_socket.recv() + try: + control_socket.send("PROTOCOLINFO 1") + protocolinfo_response = control_socket.recv() + except: + protocolinfo_response = None # Tor hangs up on sockets after receiving a PROTOCOLINFO query if it isn't # next followed by authentication. Transparently reconnect if that happens. - if str(protocolinfo_response) == "Authentication required.": + if not protocolinfo_response or str(protocolinfo_response) == "Authentication required.": control_socket.connect() - control_socket.send("PROTOCOLINFO 1") - protocolinfo_response = control_socket.recv() + + try: + control_socket.send("PROTOCOLINFO 1") + protocolinfo_response = control_socket.recv() + except stem.socket.SocketClosed, exc: + raise stem.socket.SocketError(exc) ProtocolInfoResponse.convert(protocolinfo_response) diff --git a/test/integ/connection/protocolinfo.py b/test/integ/connection/protocolinfo.py index e0e8805..a0214a2 100644 --- a/test/integ/connection/protocolinfo.py +++ b/test/integ/connection/protocolinfo.py @@ -91,6 +91,23 @@ class TestProtocolInfo(unittest.TestCase): protocolinfo_response = stem.connection.get_protocolinfo(control_socket) self.assert_matches_test_config(protocolinfo_response) + def test_pre_disconnected_query(self): + """ + Tests making a PROTOCOLINFO query when previous use of the socket had + already disconnected it. + """ + + with test.runner.get_runner().get_tor_socket(False) as control_socket: + # makes a couple protocolinfo queries outside of get_protocolinfo first + control_socket.send("PROTOCOLINFO 1") + control_socket.recv() + + control_socket.send("PROTOCOLINFO 1") + control_socket.recv() + + protocolinfo_response = stem.connection.get_protocolinfo(control_socket) + self.assert_matches_test_config(protocolinfo_response) + def assert_matches_test_config(self, protocolinfo_response): """ Makes assertions that the protocolinfo response's attributes match those of

1 0

[stem/master] Accounting for big-endian architectures in proc
by atagar＠torproject.org 31 Jan '12

31 Jan '12

commit a31aa4ad506ccf4e529182e31d0a84966e38d6e1 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Jan 30 00:02:55 2012 -0800 Accounting for big-endian architectures in proc The proc utils were assuming that encoded ip addresses were little-endian. This was fixed in the project these utils are based on (psutil) and I'm adopting the fix... https://code.google.com/p/psutil/issues/detail?id=201 https://trac.torproject.org/projects/tor/ticket/4777 This evidently occures on OpenWRT (ar71xx), thanks to swalker for the catch! This is the same as arm commit 8ec7095d79ecad9d3432193a2b1f9fdefab8d7f3 --- stem/util/proc.py | 10 +++++++++- 1 files changed, 9 insertions(+), 1 deletions(-) diff --git a/stem/util/proc.py b/stem/util/proc.py index bf91ace..cbfc3c5 100644 --- a/stem/util/proc.py +++ b/stem/util/proc.py @@ -369,7 +369,15 @@ def _decode_proc_address_encoding(addr): # The IPv4 address portion is a little-endian four-byte hexadecimal number. # That is, the least significant byte is listed first, so we need to reverse # the order of the bytes to convert it to an IP address. - ip = socket.inet_ntop(socket.AF_INET, base64.b16decode(ip)[::-1]) + # + # This needs to account for the endian ordering as per... + # http://code.google.com/p/psutil/issues/detail?id=201 + # https://trac.torproject.org/projects/tor/ticket/4777 + + if sys.byteorder == 'little': + ip = socket.inet_ntop(socket.AF_INET, base64.b16decode(ip)[::-1]) + else: + ip = socket.inet_ntop(socket.AF_INET, base64.b16decode(ip)) return (ip, port)

1 0

[stem/master] Exercising is_alive in ControlMessage integ tests
by atagar＠torproject.org 31 Jan '12

31 Jan '12

commit 2ae0298e816ccb70bf9ab52996a4c78e26f897f0 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 31 09:23:31 2012 -0800 Exercising is_alive in ControlMessage integ tests Adding a basic check for the is_alive method. The method is a little troubesome because if you aren't continually pulling from the socket then you won't detect a disconnect until the following usage. This was already documented and iirc it won't be an issue in practice for controllers since they continually pull from the socket. But still, for low level controller interaction it's kinda confusing. --- test/integ/socket/control_message.py | 6 ++++++ 1 files changed, 6 insertions(+), 0 deletions(-) diff --git a/test/integ/socket/control_message.py b/test/integ/socket/control_message.py index ce25c8f..2eb2a6e 100644 --- a/test/integ/socket/control_message.py +++ b/test/integ/socket/control_message.py @@ -36,11 +36,17 @@ class TestControlMessage(unittest.TestCase): # an error until we read from the socket. However, with a control socket # the write will cause a SocketError. + self.assertTrue(control_socket.is_alive()) + try: control_socket.send("GETINFO version") except: pass + # at this point is_alive is True with a control port and False with a + # control socket + self.assertRaises(stem.socket.SocketClosed, control_socket.recv) + self.assertFalse(control_socket.is_alive()) # Additional socket usage should fail, and pulling more responses will fail # with more closed exceptions.

1 0

[stem/master] Adding ControlLine peek_key method
by atagar＠torproject.org 31 Jan '12

31 Jan '12

commit 06498d8b61ad5f01c04c7011c70892a9846d4c27 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 31 07:46:53 2012 -0800 Adding ControlLine peek_key method Method to allow us to switch on the key/value mapping while parsing a line. This isn't needed yet (the only parsing done at this point is PROTOCOLINFO responses and that only uses positional entries). However, it's crossed my mind a few times that this will be needed later. --- stem/socket.py | 18 ++++++++++++++++++ test/unit/socket/control_line.py | 7 +++++++ 2 files changed, 25 insertions(+), 0 deletions(-) diff --git a/stem/socket.py b/stem/socket.py index 889713d..1b481ce 100644 --- a/stem/socket.py +++ b/stem/socket.py @@ -28,6 +28,7 @@ ControlLine - String subclass with methods for parsing controller responses. |- is_empty - checks if the remaining content is empty |- is_next_quoted - checks if the next entry is a quoted value |- is_next_mapping - checks if the next entry is a KEY=VALUE mapping + |- peek_key - provides the key of the next entry |- pop - removes and returns the next entry +- pop_mapping - removes and returns the next entry as a KEY=VALUE mapping @@ -487,6 +488,23 @@ class ControlLine(str): else: return False # doesn't start with a key + def peek_key(self): + """ + Provides the key of the next entry, providing None if it isn't a key/value + mapping. + + Returns: + str with the next entry's key + """ + + remainder = self._remainder + key_match = KEY_ARG.match(remainder) + + if key_match: + return key_match.groups()[0] + else: + return None + def pop(self, quoted = False, escaped = False): """ Parses the next space separated entry, removing it and the space from our diff --git a/test/unit/socket/control_line.py b/test/unit/socket/control_line.py index 6e59f43..7752147 100644 --- a/test/unit/socket/control_line.py +++ b/test/unit/socket/control_line.py @@ -52,6 +52,7 @@ class TestControlLine(unittest.TestCase): self.assertFalse(line.is_empty()) self.assertFalse(line.is_next_quoted()) self.assertFalse(line.is_next_mapping()) + self.assertEquals(None, line.peek_key()) self.assertRaises(ValueError, line.pop_mapping) self.assertEquals(line.pop(), 'PROTOCOLINFO') @@ -59,6 +60,7 @@ class TestControlLine(unittest.TestCase): self.assertFalse(line.is_empty()) self.assertFalse(line.is_next_quoted()) self.assertFalse(line.is_next_mapping()) + self.assertEquals(None, line.peek_key()) self.assertRaises(ValueError, line.pop_mapping) self.assertEquals(line.pop(), '1') @@ -66,6 +68,7 @@ class TestControlLine(unittest.TestCase): self.assertTrue(line.is_empty()) self.assertFalse(line.is_next_quoted()) self.assertFalse(line.is_next_mapping()) + self.assertEquals(None, line.peek_key()) self.assertRaises(IndexError, line.pop_mapping) self.assertRaises(IndexError, line.pop) @@ -73,6 +76,7 @@ class TestControlLine(unittest.TestCase): self.assertTrue(line.is_empty()) self.assertFalse(line.is_next_quoted()) self.assertFalse(line.is_next_mapping()) + self.assertEquals(None, line.peek_key()) def test_pop_mapping(self): """ @@ -90,6 +94,7 @@ class TestControlLine(unittest.TestCase): self.assertTrue(line.is_next_mapping(key = "Tor")) self.assertTrue(line.is_next_mapping(key = "Tor", quoted = True)) self.assertTrue(line.is_next_mapping(quoted = True)) + self.assertEquals("Tor", line.peek_key()) # try popping this as a non-quoted mapping self.assertEquals(line.pop_mapping(), ('Tor', '"0.2.1.30')) @@ -98,6 +103,7 @@ class TestControlLine(unittest.TestCase): self.assertFalse(line.is_next_quoted()) self.assertFalse(line.is_next_mapping()) self.assertRaises(ValueError, line.pop_mapping) + self.assertEquals(None, line.peek_key()) # try popping this as a quoted mapping line = stem.socket.ControlLine(version_entry) @@ -106,6 +112,7 @@ class TestControlLine(unittest.TestCase): self.assertTrue(line.is_empty()) self.assertFalse(line.is_next_quoted()) self.assertFalse(line.is_next_mapping()) + self.assertEquals(None, line.peek_key()) def test_escapes(self): """

1 0

[metrics-lib/master] Disallow reconfiguring readers after starting to read.
by karsten＠torproject.org 31 Jan '12

31 Jan '12

commit 9f13426eaefa9f1bdfca34515627ec509ae453bd Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jan 31 18:00:21 2012 +0100 Disallow reconfiguring readers after starting to read. --- .../impl/RelayOrBridgeDescriptorReaderImpl.java | 19 +++++++++++++++++++ 1 files changed, 19 insertions(+), 0 deletions(-) diff --git a/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java index d33dba5..3cfb18e 100644 --- a/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java +++ b/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java @@ -30,22 +30,41 @@ public class RelayOrBridgeDescriptorReaderImpl implements RelayDescriptorReader, BridgeDescriptorReader, BridgePoolAssignmentReader { + private boolean hasStartedReading = false; + private List<File> directories = new ArrayList<File>(); public void addDirectory(File directory) { + if (this.hasStartedReading) { + throw new IllegalStateException("Reconfiguration is not permitted " + + "after starting to read."); + } this.directories.add(directory); } private File historyFile; public void setExcludeFiles(File historyFile) { + if (this.hasStartedReading) { + throw new IllegalStateException("Reconfiguration is not permitted " + + "after starting to read."); + } this.historyFile = historyFile; } private boolean failUnrecognizedDescriptorLines = false; public void setFailUnrecognizedDescriptorLines() { + if (this.hasStartedReading) { + throw new IllegalStateException("Reconfiguration is not permitted " + + "after starting to read."); + } this.failUnrecognizedDescriptorLines = true; } public Iterator<DescriptorFile> readDescriptors() { + if (this.hasStartedReading) { + throw new IllegalStateException("Initiating reading is only " + + "permitted once."); + } + this.hasStartedReading = true; BlockingIteratorImpl<DescriptorFile> descriptorQueue = new BlockingIteratorImpl<DescriptorFile>(); DescriptorReader reader = new DescriptorReader(this.directories,

1 0

[metrics-lib/master] Remove examples.
by karsten＠torproject.org 31 Jan '12

31 Jan '12

commit e220928b10565450cb74a548ef4f2b8993b62dcb Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jan 31 11:22:07 2012 +0100 Remove examples. Example were useful when starting to design the API. Now that there are actual applications using the library, there's no need to main the examples anymore. --- TODO | 16 --- .../descriptor/example/ConsensusHealthChecker.java | 88 -------------- .../example/MetricsRelayDescriptorAggregator.java | 124 -------------------- .../example/TorStatusDatabaseUpdater.java | 57 --------- 4 files changed, 0 insertions(+), 285 deletions(-) diff --git a/TODO b/TODO index eac8965..f468893 100644 --- a/TODO +++ b/TODO @@ -12,19 +12,3 @@ which descriptors to include, and it may be useful to exclude descriptors by publication date. -- New example applications - - The metrics-web data importer could make use of this API, too. It - would read relay descriptors, bridge descriptors, Torperf data files, - and GetTor stats files from disk. - - Another example application could be a relay descriptor database - application to support searching for relays or looking up relay - descriptors in general. This application might read daily rsync'ed - relay descriptors and import them into a database. - - Another example application might be the Java version of ExoneraTor. - This application would index locally extracted relay descriptor - tarballs and parse only the relevant files. This version of - ExoneraTor could also read exit lists to provide more accurate - results. - - Another example application might be the Java version of VisiTor which - reads locally extracted exit list tarballs. - diff --git a/src/org/torproject/descriptor/example/ConsensusHealthChecker.java b/src/org/torproject/descriptor/example/ConsensusHealthChecker.java deleted file mode 100644 index b8cc12b..0000000 --- a/src/org/torproject/descriptor/example/ConsensusHealthChecker.java +++ /dev/null @@ -1,88 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.descriptor.example; - -import java.util.Iterator; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorRequest; -import org.torproject.descriptor.DescriptorSourceFactory; -import org.torproject.descriptor.RelayDescriptorDownloader; -import org.torproject.descriptor.RelayNetworkStatusConsensus; -import org.torproject.descriptor.RelayNetworkStatusVote; - -/* This is a non-functional (though syntactically correct) example for how - * the consensus-health checker could use the DescripTor API. This class - * will go away once the real consensus-health checker uses this API. */ -public class ConsensusHealthChecker { - public static void main(String[] args) { - - /* Create an instance of the descriptor downloader that contains all - * the logic to download descriptors from the directory - * authorities. */ - RelayDescriptorDownloader downloader = - DescriptorSourceFactory.createRelayDescriptorDownloader(); - - /* Make one example directory authority known to the downloader. In - * the real consensus-health checker, all directory authorities would - * be added here. (There is no list of current directory authorities - * in the DescripTor code, because it may change over time and not all - * DescripTor applications need to download descriptors from the - * directory authorities.) */ - downloader.addDirectoryAuthority("gabelmoo", "212.112.245.170", 80); - - /* Tell the descriptor that we're interested in downloading the - * current consensus from all directory authorities and all referenced - * votes. With these two preferences set, the downloader will try to - * download the consensus from gabelmoo, parse it for referenced - * votes, and try to download all of them from gabelmoo, too. */ - downloader.setIncludeCurrentConsensusFromAllDirectoryAuthorities(); - downloader.setIncludeCurrentReferencedVotes(); - - /* Set connect and read timeouts of 1 minute each and a global timeout - * of 10 minutes to avoid being blocked forever by a slow download. */ - downloader.setConnectTimeout(60L * 1000L); - downloader.setReadTimeout(60L * 1000L); - downloader.setGlobalTimeout(10L * 60L * 1000L); - - /* Run the previously configured downloads and iterate over the - * received descriptors. Don't process them right now, but add them - * to the checker class one by one and do the checking once all - * downloads are complete. */ - Iterator<DescriptorRequest> descriptorRequests = - downloader.downloadDescriptors(); - while (descriptorRequests.hasNext()) { - DescriptorRequest request = descriptorRequests.next(); - String authority = request.getDirectoryNickname(); - long fetchTime = request.getRequestEnd() - - request.getRequestStart(); - if (request.globalTimeoutHasExpired()) { - System.err.println("The global timeout for downloading " - + "descriptors has expired. That means we're missing one or " - + "more consensuses and/or votes and cannot make a good " - + "statement about the consensus health. Exiting."); - return; - } else if (request.connectTimeoutHasExpired() || - request.readTimeoutHasExpired()) { - System.out.println("The request to directory authority " - + request.getDirectoryNickname() + " to download the " - + "descriptor(s) at " + request.getRequestUrl() + " has " - + "timed out."); - } else { - for (Descriptor downloadedDescriptor : request.getDescriptors()) { - if (downloadedDescriptor instanceof - RelayNetworkStatusConsensus) { - /* Remember that we downloaded a consensus from authority and - * took fetchTime millis to do so. */ - } else if (downloadedDescriptor instanceof - RelayNetworkStatusVote) { - /* Remember that we downloaded a vote. */ - } else { - System.err.println("Did not expect a descriptor of type " - + downloadedDescriptor.getClass() + ". Ignoring."); - } - } - } - } - } -} - diff --git a/src/org/torproject/descriptor/example/MetricsRelayDescriptorAggregator.java b/src/org/torproject/descriptor/example/MetricsRelayDescriptorAggregator.java deleted file mode 100644 index bfaaa22..0000000 --- a/src/org/torproject/descriptor/example/MetricsRelayDescriptorAggregator.java +++ /dev/null @@ -1,124 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.descriptor.example; - -import java.io.File; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; -import org.torproject.descriptor.DescriptorRequest; -import org.torproject.descriptor.DescriptorSourceFactory; -import org.torproject.descriptor.RelayDescriptorDownloader; -import org.torproject.descriptor.RelayDescriptorReader; - -/* This is a non-functional (though syntactically correct) example for how - * metrics-db could use the DescripTor API to read relay descriptors from - * two sources and download only missing descriptors from the directory - * authorities. metrics-db does more than aggregating relay descriptors, - * but the other functions (sanitizing bridge descriptors, downloading - * GetTor statistics) are too specific to metrics-db to add them to the - * DescripTor API. This class will go away once a real metrics-db uses - * this API. */ -public class MetricsRelayDescriptorAggregator { - public static void main(String[] args) { - - /* Start by reading lists of previously processed descriptors from - * disk. We'll want to exclude these descriptors, plus any that we - * learn in this execution, from the descriptors we download from the - * directory authorities. We should remove descriptors that were - * published more than one week ago from the list, because they - * wouldn't be referenced in a consensus anyway. */ - long lastKnownConsensusValidAfterTime = 1234567890000L; - Map<String, Long> lastKnownVoteValidAfterTimes = - new HashMap<String, Long>(); - lastKnownVoteValidAfterTimes.put( - "1234567890ABCDEF1234567890ABCDEF12345678", 1234567890000L); - Map<String, Long> knownServerDescriptorIdentifiers = - new HashMap<String, Long>(); - Map<String, Long> knownExtraInfoDescriptorIdentifiers = - new HashMap<String, Long>(); - - /* Create a relay descriptor reader to read descriptors from cached - * descriptor files in a local Tor data directory. */ - RelayDescriptorReader reader = - DescriptorSourceFactory.createRelayDescriptorReader(); - - /* Tell the reader where to find relay descriptor files to parse. In - * this case it's a Tor data directory with cached descriptor - * files. */ - reader.addDirectory(new File("tor-data-dir")); - - /* Exclude cached descriptor files that haven't changed since we last - * ran this application. */ - //reader.setExcludeFile(new File("tor-data-dir/cached-descriptors"), - // 1234567890000L); - - /* Read descriptors and process them. */ - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - /* Do something with the read descriptors. */ - } - - /* Remember which descriptors we just processed to exclude them from - * the download. This code is independent of the API and therefore - * not shown here. */ - - /* Do the same operations as shown above for other local directories - * containing relay descriptors. For example, metrics-db rsyncs the - * directory-archive script output from tor26 once per day and imports - * them, too. The operations are very similar. We should use a new - * RelayDescriptorReader for every directory. */ - - /* Download missing descriptors from the directory authorities. - * Create an instance of the descriptor downloader that contains the - * logic to download descriptors from the directory authorities. */ - RelayDescriptorDownloader downloader = - DescriptorSourceFactory.createRelayDescriptorDownloader(); - - /* Make one or more directory authorities or directory mirrors known - * to the downloader. */ - downloader.addDirectoryAuthority("gabelmoo", "212.112.245.170", 80); - - /* Tell the descriptor that we're interested in downloading pretty - * much every descriptor type there is. */ - downloader.setIncludeCurrentConsensus(); - downloader.setIncludeCurrentReferencedVotes(); - downloader.setIncludeReferencedServerDescriptors(); - downloader.setIncludeReferencedExtraInfoDescriptors(); - - /* Exclude the descriptors that we already know. This is vital to - * avoid putting too much load on the directories. (Excluding the - * consensus and votes if they have been processed before is not shown - * here, because it requires some timestamp parsing; using the API for - * this should be trivial, though.) */ - downloader.setExcludeServerDescriptors( - knownServerDescriptorIdentifiers.keySet()); - downloader.setExcludeExtraInfoDescriptors( - knownExtraInfoDescriptorIdentifiers.keySet()); - - /* Set connect and read timeouts of 2 minutes each and a global - * timeout of 1 hour to avoid being blocked forever by a slow - * download, but also to avoid giving up too quickly. */ - downloader.setConnectTimeout(2L * 60L * 1000L); - downloader.setReadTimeout(2L * 60L * 1000L); - downloader.setGlobalTimeout(60L * 60L * 1000L); - - /* Download descriptors and process them. */ - Iterator<DescriptorRequest> descriptorRequests = - downloader.downloadDescriptors(); - while (descriptorRequests.hasNext()) { - DescriptorRequest descriptorRequest = descriptorRequests.next(); - /* Do something with the requests. */ - } - - /* Write the list of processed descriptors to disk, so that we don't - * download them in the next execution. This code is independent of - * the API and therefore not shown here. */ - - /* That's it. We're done. */ - } -} - diff --git a/src/org/torproject/descriptor/example/TorStatusDatabaseUpdater.java b/src/org/torproject/descriptor/example/TorStatusDatabaseUpdater.java deleted file mode 100644 index f3ee341..0000000 --- a/src/org/torproject/descriptor/example/TorStatusDatabaseUpdater.java +++ /dev/null @@ -1,57 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.descriptor.example; - -import java.io.File; -import java.util.Iterator; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; -import org.torproject.descriptor.DescriptorSourceFactory; -import org.torproject.descriptor.RelayDescriptorReader; -import org.torproject.descriptor.RelayNetworkStatusConsensus; -import org.torproject.descriptor.RelayNetworkStatusVote; - -/* This is a non-functional (though syntactically correct) example for how - * a TorStatus application could use the DescripTor API to read the cached - * descriptors from a local Tor data directory and update its database. - * This class will go away once a real TorStatus application uses this - * API. */ -public class TorStatusDatabaseUpdater { - public static void main(String[] args) { - - /* Create an instance of the descriptor reader that implements the - * logic to index and parse descriptor files from a local directory, - * including the logic to ignore files that have been parsed in a - * previous run. */ - RelayDescriptorReader reader = - DescriptorSourceFactory.createRelayDescriptorReader(); - - /* Tell the reader where to find relay descriptor files to parse. In - * this case it's a Tor data directory with cached descriptor - * files. */ - reader.addDirectory(new File("tor-data-dir")); - - /* Exclude cached descriptor files that haven't changed since we last - * ran this application. This may save some execution time. The - * application needs to store the information when files were last - * modified, because the API is supposed to be stateless. */ - //reader.setExcludeFile(new File("tor-data-dir/cached-descriptors"), - // 1234567890000L); - - /* Read all descriptors in the given directory and import them into - * the database. Also go through the list of parsed files and store - * their last modification times, so that we can exclude them the next - * time if they haven't changed. */ - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - for (Descriptor readDescriptor : descriptorFile.getDescriptors()) { - /* Do something with the parsed descriptor. */ - } - String fileName = descriptorFile.getFile().getName(); - long lastModified = descriptorFile.getLastModified(); - /* Do something with the file name and last modification time. */ - } - } -} -

1 0

[metrics-lib/master] Add a config option to handle unrecognized lines.
by karsten＠torproject.org 31 Jan '12

31 Jan '12

commit 26b7c97f93ea5e5ae88998085909e86641265a2a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jan 31 17:47:15 2012 +0100 Add a config option to handle unrecognized lines. Most applications don't care about unrecognized lines in descriptors as long as the lines they care about are contained. But some applications rather want to skip a descriptor with unrecognized lines, e.g., the bridge descriptor sanitizer. Add a switch to downloaders and readers to control the desired behavior, that is, to either fail parsing or ignore unrecognized lines. In the latter case, unrecognized are written to a list for later inspection by the application. Corrupt lines with known keywords always lead to failing the parse step. --- .../descriptor/BridgeDescriptorReader.java | 7 + .../descriptor/BridgePoolAssignmentReader.java | 7 + src/org/torproject/descriptor/Descriptor.java | 6 + .../descriptor/RelayDescriptorDownloader.java | 7 + .../descriptor/RelayDescriptorReader.java | 7 + .../descriptor/impl/BridgeNetworkStatusImpl.java | 9 +- .../descriptor/impl/BridgePoolAssignmentImpl.java | 25 ++-- .../torproject/descriptor/impl/DescriptorImpl.java | 33 +++- .../descriptor/impl/DirSourceEntryImpl.java | 37 ++++- .../descriptor/impl/DirectoryDownloader.java | 10 +- .../descriptor/impl/DownloadCoordinatorImpl.java | 9 +- .../descriptor/impl/ExtraInfoDescriptorImpl.java | 39 ++-- .../descriptor/impl/NetworkStatusEntryImpl.java | 25 ++- .../descriptor/impl/NetworkStatusImpl.java | 43 ++++- .../impl/RelayDescriptorDownloaderImpl.java | 12 ++- .../impl/RelayNetworkStatusConsensusImpl.java | 47 +++--- .../impl/RelayNetworkStatusVoteImpl.java | 75 ++++++--- .../impl/RelayOrBridgeDescriptorReaderImpl.java | 16 ++- .../descriptor/impl/ServerDescriptorImpl.java | 26 ++-- .../descriptor/impl/ConsensusBuilder.java | 101 +++++++++-- .../impl/RelayNetworkStatusConsensusImplTest.java | 125 ++++++++++++-- .../impl/RelayNetworkStatusVoteImplTest.java | 182 +++++++++++++++++--- .../descriptor/impl/ServerDescriptorImplTest.java | 77 ++++++--- 23 files changed, 722 insertions(+), 203 deletions(-) diff --git a/src/org/torproject/descriptor/BridgeDescriptorReader.java b/src/org/torproject/descriptor/BridgeDescriptorReader.java index 3308e82..d827124 100644 --- a/src/org/torproject/descriptor/BridgeDescriptorReader.java +++ b/src/org/torproject/descriptor/BridgeDescriptorReader.java @@ -18,6 +18,13 @@ public interface BridgeDescriptorReader { * last modified timestamp and the absolute path of a file. */ public void setExcludeFiles(File historyFile); + /* Fail descriptor parsing when encountering an unrecognized line. This + * is not set by default, because the Tor specifications allow for new + * lines to be added that shall be ignored by older Tor versions. But + * some applications may want to handle unrecognized descriptor lines + * explicitly. */ + public void setFailUnrecognizedDescriptorLines(); + /* Read the previously configured bridge descriptors and make them * available via the returned blocking iterator. Whenever the reader * runs out of descriptors and expects to provide more shortly after, it diff --git a/src/org/torproject/descriptor/BridgePoolAssignmentReader.java b/src/org/torproject/descriptor/BridgePoolAssignmentReader.java index e84f2b4..13c8ae3 100644 --- a/src/org/torproject/descriptor/BridgePoolAssignmentReader.java +++ b/src/org/torproject/descriptor/BridgePoolAssignmentReader.java @@ -18,6 +18,13 @@ public interface BridgePoolAssignmentReader { * last modified timestamp and the absolute path of a file. */ public void setExcludeFiles(File historyFile); + /* Fail descriptor parsing when encountering an unrecognized line. This + * is not set by default, because the Tor specifications allow for new + * lines to be added that shall be ignored by older Tor versions. But + * some applications may want to handle unrecognized descriptor lines + * explicitly. */ + public void setFailUnrecognizedDescriptorLines(); + /* Read the previously configured bridge pool assignments and make them * available via the returned blocking iterator. Whenever the reader * runs out of descriptors and expects to provide more shortly after, it diff --git a/src/org/torproject/descriptor/Descriptor.java b/src/org/torproject/descriptor/Descriptor.java index 76b23a0..6e02966 100644 --- a/src/org/torproject/descriptor/Descriptor.java +++ b/src/org/torproject/descriptor/Descriptor.java @@ -2,11 +2,17 @@ * See LICENSE for licensing information */ package org.torproject.descriptor; +import java.util.List; + /* Store meta-data about how a descriptor was downloaded or read from * disk. */ public interface Descriptor { /* Return the raw descriptor bytes. */ public byte[] getRawDescriptorBytes(); + + /* Return any unrecognized lines when parsing this descriptor, or an + * empty list if there were no unrecognized lines. */ + public List<String> getUnrecognizedLines(); } diff --git a/src/org/torproject/descriptor/RelayDescriptorDownloader.java b/src/org/torproject/descriptor/RelayDescriptorDownloader.java index 754c644..29171d3 100644 --- a/src/org/torproject/descriptor/RelayDescriptorDownloader.java +++ b/src/org/torproject/descriptor/RelayDescriptorDownloader.java @@ -92,6 +92,13 @@ public interface RelayDescriptorDownloader { * hour (60 * 60 * 1000). */ public void setGlobalTimeout(long globalTimeoutMillis); + /* Fail descriptor parsing when encountering an unrecognized line. This + * is not set by default, because the Tor specifications allow for new + * lines to be added that shall be ignored by older Tor versions. But + * some applications may want to handle unrecognized descriptor lines + * explicitly. */ + public void setFailUnrecognizedDescriptorLines(); + /* Download the previously configured relay descriptors and make them * available via the returned blocking iterator. Whenever the * downloader runs out of descriptors and expects to provide more diff --git a/src/org/torproject/descriptor/RelayDescriptorReader.java b/src/org/torproject/descriptor/RelayDescriptorReader.java index 021794e..142f79c 100644 --- a/src/org/torproject/descriptor/RelayDescriptorReader.java +++ b/src/org/torproject/descriptor/RelayDescriptorReader.java @@ -18,6 +18,13 @@ public interface RelayDescriptorReader { * last modified timestamp and the absolute path of a file. */ public void setExcludeFiles(File historyFile); + /* Fail descriptor parsing when encountering an unrecognized line. This + * is not set by default, because the Tor specifications allow for new + * lines to be added that shall be ignored by older Tor versions. But + * some applications may want to handle unrecognized descriptor lines + * explicitly. */ + public void setFailUnrecognizedDescriptorLines(); + /* Read the previously configured relay descriptors and make them * available via the returned blocking iterator. Whenever the reader * runs out of descriptors and expects to provide more shortly after, it diff --git a/src/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java b/src/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java index 79ee8ce..1d9818c 100644 --- a/src/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java +++ b/src/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java @@ -13,14 +13,15 @@ public class BridgeNetworkStatusImpl extends NetworkStatusImpl implements BridgeNetworkStatus { protected BridgeNetworkStatusImpl(byte[] statusBytes, - String fileName) throws DescriptorParseException { - super(statusBytes); + String fileName, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + super(statusBytes, failUnrecognizedDescriptorLines); this.setPublishedMillisFromFileName(fileName); } private void setPublishedMillisFromFileName(String fileName) throws DescriptorParseException { - if (fileName.length() == + if (fileName.length() == "20000101-000000-4A0CCD2DDC7995083D73F5D667100C8A5831F16D". length()) { String publishedString = fileName.substring(0, @@ -33,7 +34,7 @@ public class BridgeNetworkStatusImpl extends NetworkStatusImpl this.publishedMillis = fileNameFormat.parse(publishedString). getTime(); } catch (ParseException e) { - } + } } if (this.publishedMillis == 0L) { throw new DescriptorParseException("Unrecognized bridge network " diff --git a/src/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java b/src/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java index 946d897..8a6857e 100644 --- a/src/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java +++ b/src/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java @@ -20,29 +20,26 @@ public class BridgePoolAssignmentImpl extends DescriptorImpl implements BridgePoolAssignment { protected static List<BridgePoolAssignment> parseDescriptors( - byte[] descriptorsBytes) { + byte[] descriptorsBytes, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { List<BridgePoolAssignment> parsedDescriptors = new ArrayList<BridgePoolAssignment>(); List<byte[]> splitDescriptorsBytes = DescriptorImpl.splitRawDescriptorBytes(descriptorsBytes, "bridge-pool-assignment "); - try { - for (byte[] descriptorBytes : splitDescriptorsBytes) { - BridgePoolAssignment parsedDescriptor = - new BridgePoolAssignmentImpl(descriptorBytes); - parsedDescriptors.add(parsedDescriptor); - } - } catch (DescriptorParseException e) { - /* TODO Handle this error somehow. */ - System.err.println("Failed to parse descriptor. Skipping."); - e.printStackTrace(); + for (byte[] descriptorBytes : splitDescriptorsBytes) { + BridgePoolAssignment parsedDescriptor = + new BridgePoolAssignmentImpl(descriptorBytes, + failUnrecognizedDescriptorLines); + parsedDescriptors.add(parsedDescriptor); } return parsedDescriptors; } - protected BridgePoolAssignmentImpl(byte[] descriptorBytes) + protected BridgePoolAssignmentImpl(byte[] descriptorBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(descriptorBytes); + super(descriptorBytes, failUnrecognizedDescriptorLines); this.parseDescriptorBytes(); Set<String> exactlyOnceKeywords = new HashSet<String>(Arrays.asList( new String[] { "bridge-pool-assignment" })); @@ -81,7 +78,7 @@ public class BridgePoolAssignmentImpl extends DescriptorImpl parts, 1, 2); } - private void parseBridgeLine(String line) + private void parseBridgeLine(String line) throws DescriptorParseException { String[] parts = line.split(" "); if (parts.length < 2) { diff --git a/src/org/torproject/descriptor/impl/DescriptorImpl.java b/src/org/torproject/descriptor/impl/DescriptorImpl.java index 7be2fbb..d8705e2 100644 --- a/src/org/torproject/descriptor/impl/DescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/DescriptorImpl.java @@ -16,7 +16,8 @@ import org.torproject.descriptor.Descriptor; public abstract class DescriptorImpl implements Descriptor { protected static List<Descriptor> parseRelayOrBridgeDescriptors( - byte[] rawDescriptorBytes, String fileName) + byte[] rawDescriptorBytes, String fileName, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { List<Descriptor> parsedDescriptors = new ArrayList<Descriptor>(); if (rawDescriptorBytes == null) { @@ -31,10 +32,12 @@ public abstract class DescriptorImpl implements Descriptor { firstLines.contains("\nnetwork-status-version 3\n")) { if (firstLines.contains("\nvote-status consensus\n")) { parsedDescriptors.addAll(RelayNetworkStatusConsensusImpl. - parseConsensuses(rawDescriptorBytes)); + parseConsensuses(rawDescriptorBytes, + failUnrecognizedDescriptorLines)); } else if (firstLines.contains("\nvote-status vote\n")) { parsedDescriptors.addAll(RelayNetworkStatusVoteImpl. - parseVotes(rawDescriptorBytes)); + parseVotes(rawDescriptorBytes, + failUnrecognizedDescriptorLines)); } else { throw new DescriptorParseException("Could not detect relay " + "network status type in descriptor starting with '" @@ -42,19 +45,22 @@ public abstract class DescriptorImpl implements Descriptor { } } else if (firstLines.startsWith("r ")) { parsedDescriptors.add(new BridgeNetworkStatusImpl( - rawDescriptorBytes, fileName)); + rawDescriptorBytes, fileName, failUnrecognizedDescriptorLines)); } else if (firstLines.startsWith("router ") || firstLines.contains("\nrouter ")) { parsedDescriptors.addAll(ServerDescriptorImpl. - parseDescriptors(rawDescriptorBytes)); + parseDescriptors(rawDescriptorBytes, + failUnrecognizedDescriptorLines)); } else if (firstLines.startsWith("extra-info ") || firstLines.contains("\nextra-info ")) { parsedDescriptors.addAll(ExtraInfoDescriptorImpl. - parseDescriptors(rawDescriptorBytes)); + parseDescriptors(rawDescriptorBytes, + failUnrecognizedDescriptorLines)); } else if (firstLines.startsWith("bridge-pool-assignment ") || firstLines.contains("\nbridge-pool-assignment ")) { parsedDescriptors.addAll(BridgePoolAssignmentImpl. - parseDescriptors(rawDescriptorBytes)); + parseDescriptors(rawDescriptorBytes, + failUnrecognizedDescriptorLines)); } else { throw new DescriptorParseException("Could not detect descriptor " + "type in descriptor starting with '" + firstLines + "'."); @@ -90,9 +96,20 @@ public abstract class DescriptorImpl implements Descriptor { return this.rawDescriptorBytes; } - protected DescriptorImpl(byte[] rawDescriptorBytes) + protected boolean failUnrecognizedDescriptorLines = false; + + protected List<String> unrecognizedLines; + public List<String> getUnrecognizedLines() { + return this.unrecognizedLines == null ? new ArrayList<String>() : + new ArrayList<String>(this.unrecognizedLines); + } + + protected DescriptorImpl(byte[] rawDescriptorBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { this.rawDescriptorBytes = rawDescriptorBytes; + this.failUnrecognizedDescriptorLines = + failUnrecognizedDescriptorLines; this.countKeywords(rawDescriptorBytes); } diff --git a/src/org/torproject/descriptor/impl/DirSourceEntryImpl.java b/src/org/torproject/descriptor/impl/DirSourceEntryImpl.java index 2ee4556..c20ca30 100644 --- a/src/org/torproject/descriptor/impl/DirSourceEntryImpl.java +++ b/src/org/torproject/descriptor/impl/DirSourceEntryImpl.java @@ -5,6 +5,8 @@ package org.torproject.descriptor.impl; import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; import java.util.SortedSet; import java.util.TreeSet; @@ -17,9 +19,20 @@ public class DirSourceEntryImpl implements DirSourceEntry { return this.dirSourceEntryBytes; } - protected DirSourceEntryImpl(byte[] dirSourceEntryBytes) + private boolean failUnrecognizedDescriptorLines; + private List<String> unrecognizedLines; + protected List<String> getAndClearUnrecognizedLines() { + List<String> lines = this.unrecognizedLines; + this.unrecognizedLines = null; + return lines; + } + + protected DirSourceEntryImpl(byte[] dirSourceEntryBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { this.dirSourceEntryBytes = dirSourceEntryBytes; + this.failUnrecognizedDescriptorLines = + failUnrecognizedDescriptorLines; this.initializeKeywords(); this.parseDirSourceEntryBytes(); this.checkKeywords(); @@ -65,6 +78,7 @@ public class DirSourceEntryImpl implements DirSourceEntry { BufferedReader br = new BufferedReader(new StringReader( new String(this.dirSourceEntryBytes))); String line; + boolean skipCrypto = false; while ((line = br.readLine()) != null) { if (line.startsWith("dir-source")) { this.parseDirSourceLine(line); @@ -72,10 +86,20 @@ public class DirSourceEntryImpl implements DirSourceEntry { this.parseContactLine(line); } else if (line.startsWith("vote-digest")) { this.parseVoteDigestLine(line); - } else { - /* TODO Should we really throw an exception here? */ - throw new DescriptorParseException("Unknown line '" + line - + "' in dir-source entry."); + } else if (line.startsWith("-----BEGIN")) { + skipCrypto = true; + } else if (line.startsWith("-----END")) { + skipCrypto = false; + } else if (!skipCrypto) { + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in dir-source entry."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } } } } catch (IOException e) { @@ -89,6 +113,9 @@ public class DirSourceEntryImpl implements DirSourceEntry { throws DescriptorParseException { this.parsedExactlyOnceKeyword("dir-source"); String[] parts = line.split(" "); + if (parts.length != 7) { + throw new DescriptorParseException("Invalid line '" + line + "'."); + } String nickname = parts[1]; if (nickname.endsWith("-legacy")) { nickname = nickname.substring(0, nickname.length() diff --git a/src/org/torproject/descriptor/impl/DirectoryDownloader.java b/src/org/torproject/descriptor/impl/DirectoryDownloader.java index 7e9d19c..685ee60 100644 --- a/src/org/torproject/descriptor/impl/DirectoryDownloader.java +++ b/src/org/torproject/descriptor/impl/DirectoryDownloader.java @@ -37,6 +37,13 @@ public class DirectoryDownloader implements Runnable { this.readTimeout = readTimeout; } + private boolean failUnrecognizedDescriptorLines; + protected void setFailUnrecognizedDescriptorLines( + boolean failUnrecognizedDescriptorLines) { + this.failUnrecognizedDescriptorLines = + failUnrecognizedDescriptorLines; + } + public void run() { boolean keepRunning = true; do { @@ -70,7 +77,8 @@ public class DirectoryDownloader implements Runnable { request.setResponseBytes(responseBytes); request.setRequestEnd(System.currentTimeMillis()); request.setDescriptors(DescriptorImpl. - parseRelayOrBridgeDescriptors(responseBytes, null)); + parseRelayOrBridgeDescriptors(responseBytes, null, + this.failUnrecognizedDescriptorLines)); } } catch (Exception e) { request.setException(e); diff --git a/src/org/torproject/descriptor/impl/DownloadCoordinatorImpl.java b/src/org/torproject/descriptor/impl/DownloadCoordinatorImpl.java index b390b6d..4224b0e 100644 --- a/src/org/torproject/descriptor/impl/DownloadCoordinatorImpl.java +++ b/src/org/torproject/descriptor/impl/DownloadCoordinatorImpl.java @@ -33,6 +33,7 @@ public class DownloadCoordinatorImpl implements DownloadCoordinator { private long connectTimeoutMillis; private long readTimeoutMillis; private long globalTimeoutMillis; + private boolean failUnrecognizedDescriptorLines; protected DownloadCoordinatorImpl( SortedMap<String, DirectoryDownloader> directoryAuthorities, @@ -41,7 +42,7 @@ public class DownloadCoordinatorImpl implements DownloadCoordinator { boolean downloadConsensusFromAllAuthorities, Set<String> downloadVotes, boolean includeCurrentReferencedVotes, long connectTimeoutMillis, long readTimeoutMillis, - long globalTimeoutMillis) { + long globalTimeoutMillis, boolean failUnrecognizedDescriptorLines) { this.directoryAuthorities = directoryAuthorities; this.directoryMirrors = directoryMirrors; this.runningDirectories = new TreeSet<String>(); @@ -55,6 +56,8 @@ public class DownloadCoordinatorImpl implements DownloadCoordinator { this.connectTimeoutMillis = connectTimeoutMillis; this.readTimeoutMillis = readTimeoutMillis; this.globalTimeoutMillis = globalTimeoutMillis; + this.failUnrecognizedDescriptorLines = + failUnrecognizedDescriptorLines; if (this.directoryMirrors.isEmpty() && this.directoryAuthorities.isEmpty()) { this.descriptorQueue.setOutOfDescriptors(); @@ -70,6 +73,8 @@ public class DownloadCoordinatorImpl implements DownloadCoordinator { directoryMirror.setDownloadCoordinator(this); directoryMirror.setConnectTimeout(this.connectTimeoutMillis); directoryMirror.setReadTimeout(this.readTimeoutMillis); + directoryMirror.setFailUnrecognizedDescriptorLines( + this.failUnrecognizedDescriptorLines); new Thread(directoryMirror).start(); } for (DirectoryDownloader directoryAuthority : @@ -77,6 +82,8 @@ public class DownloadCoordinatorImpl implements DownloadCoordinator { directoryAuthority.setDownloadCoordinator(this); directoryAuthority.setConnectTimeout(this.connectTimeoutMillis); directoryAuthority.setReadTimeout(this.readTimeoutMillis); + directoryAuthority.setFailUnrecognizedDescriptorLines( + this.failUnrecognizedDescriptorLines); new Thread(directoryAuthority).start(); } } diff --git a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java index 60ef498..73ea040 100644 --- a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java @@ -22,29 +22,26 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl implements ExtraInfoDescriptor { protected static List<ExtraInfoDescriptor> parseDescriptors( - byte[] descriptorsBytes) { + byte[] descriptorsBytes, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { List<ExtraInfoDescriptor> parsedDescriptors = new ArrayList<ExtraInfoDescriptor>(); List<byte[]> splitDescriptorsBytes = DescriptorImpl.splitRawDescriptorBytes(descriptorsBytes, "extra-info "); - try { - for (byte[] descriptorBytes : splitDescriptorsBytes) { - ExtraInfoDescriptor parsedDescriptor = - new ExtraInfoDescriptorImpl(descriptorBytes); - parsedDescriptors.add(parsedDescriptor); - } - } catch (DescriptorParseException e) { - /* TODO Handle this error somehow. */ - System.err.println("Failed to parse descriptor. Skipping."); - e.printStackTrace(); + for (byte[] descriptorBytes : splitDescriptorsBytes) { + ExtraInfoDescriptor parsedDescriptor = + new ExtraInfoDescriptorImpl(descriptorBytes, + failUnrecognizedDescriptorLines); + parsedDescriptors.add(parsedDescriptor); } return parsedDescriptors; } - protected ExtraInfoDescriptorImpl(byte[] descriptorBytes) + protected ExtraInfoDescriptorImpl(byte[] descriptorBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(descriptorBytes); + super(descriptorBytes, failUnrecognizedDescriptorLines); this.parseDescriptorBytes(); Set<String> exactlyOnceKeywords = new HashSet<String>(Arrays.asList(( "extra-info,published").split(","))); @@ -159,13 +156,15 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl } else if (line.startsWith("-----END")) { skipCrypto = false; } else if (!skipCrypto) { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. In theory, - * dir-spec.txt says that unknown lines should be ignored. This - * also applies to the other descriptors. */ - throw new DescriptorParseException("Unrecognized line '" + line - + "'."); + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in extra-info descriptor."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } } } } catch (IOException e) { diff --git a/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java b/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java index ce235ab..2353bee 100644 --- a/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java +++ b/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java @@ -20,9 +20,20 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry { return this.statusEntryBytes; } - protected NetworkStatusEntryImpl(byte[] statusEntryBytes) + private boolean failUnrecognizedDescriptorLines; + private List<String> unrecognizedLines; + protected List<String> getAndClearUnrecognizedLines() { + List<String> lines = this.unrecognizedLines; + this.unrecognizedLines = null; + return lines; + } + + protected NetworkStatusEntryImpl(byte[] statusEntryBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { this.statusEntryBytes = statusEntryBytes; + this.failUnrecognizedDescriptorLines = + failUnrecognizedDescriptorLines; this.initializeKeywords(); this.parseStatusEntryBytes(); } @@ -73,12 +84,14 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry { this.parsePLine(line, parts); } else if (keyword.equals("m")) { this.parseMLine(line, parts); - } else { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. */ - throw new DescriptorParseException("Unknown line '" + line + } else if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + line + "' in status entry."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); } } } catch (IOException e) { diff --git a/src/org/torproject/descriptor/impl/NetworkStatusImpl.java b/src/org/torproject/descriptor/impl/NetworkStatusImpl.java index d5b73e3..fcbd8f6 100644 --- a/src/org/torproject/descriptor/impl/NetworkStatusImpl.java +++ b/src/org/torproject/descriptor/impl/NetworkStatusImpl.java @@ -18,9 +18,10 @@ import org.torproject.descriptor.NetworkStatusEntry; * delegate the specific parts to the subclasses. */ public abstract class NetworkStatusImpl extends DescriptorImpl { - protected NetworkStatusImpl(byte[] rawDescriptorBytes) + protected NetworkStatusImpl(byte[] rawDescriptorBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(rawDescriptorBytes); + super(rawDescriptorBytes, failUnrecognizedDescriptorLines); this.splitAndParseParts(rawDescriptorBytes); } @@ -161,17 +162,33 @@ public abstract class NetworkStatusImpl extends DescriptorImpl { protected void parseDirSource(byte[] dirSourceBytes) throws DescriptorParseException { - DirSourceEntry dirSourceEntry = new DirSourceEntryImpl( - dirSourceBytes); + DirSourceEntryImpl dirSourceEntry = new DirSourceEntryImpl( + dirSourceBytes, this.failUnrecognizedDescriptorLines); this.dirSourceEntries.put(dirSourceEntry.getIdentity(), dirSourceEntry); + List<String> unrecognizedDirSourceLines = dirSourceEntry. + getAndClearUnrecognizedLines(); + if (unrecognizedDirSourceLines != null) { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.addAll(unrecognizedDirSourceLines); + } } protected void parseStatusEntry(byte[] statusEntryBytes) throws DescriptorParseException { NetworkStatusEntryImpl statusEntry = new NetworkStatusEntryImpl( - statusEntryBytes); + statusEntryBytes, this.failUnrecognizedDescriptorLines); this.statusEntries.put(statusEntry.getFingerprint(), statusEntry); + List<String> unrecognizedDirSourceLines = statusEntry. + getAndClearUnrecognizedLines(); + if (unrecognizedDirSourceLines != null) { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.addAll(unrecognizedDirSourceLines); + } } protected abstract void parseFooter(byte[] footerBytes) @@ -186,6 +203,7 @@ public abstract class NetworkStatusImpl extends DescriptorImpl { BufferedReader br = new BufferedReader(new StringReader( new String(directorySignatureBytes))); String line; + boolean skipCrypto = false; while ((line = br.readLine()) != null) { if (line.startsWith("directory-signature ")) { String[] parts = line.split(" ", -1); @@ -198,7 +216,20 @@ public abstract class NetworkStatusImpl extends DescriptorImpl { String signingKeyDigest = ParseHelper.parseTwentyByteHexString( line, parts[2]); this.directorySignatures.put(identity, signingKeyDigest); - break; + } else if (line.startsWith("-----BEGIN")) { + skipCrypto = true; + } else if (line.startsWith("-----END")) { + skipCrypto = false; + } else if (!skipCrypto) { + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in dir-source entry."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } } } } catch (IOException e) { diff --git a/src/org/torproject/descriptor/impl/RelayDescriptorDownloaderImpl.java b/src/org/torproject/descriptor/impl/RelayDescriptorDownloaderImpl.java index 0888d64..6993509 100644 --- a/src/org/torproject/descriptor/impl/RelayDescriptorDownloaderImpl.java +++ b/src/org/torproject/descriptor/impl/RelayDescriptorDownloaderImpl.java @@ -235,6 +235,15 @@ public class RelayDescriptorDownloaderImpl this.globalTimeoutMillis = globalTimeoutMillis; } + private boolean failUnrecognizedDescriptorLines = false; + public void setFailUnrecognizedDescriptorLines() { + if (this.hasStartedDownloading) { + throw new IllegalStateException("Reconfiguration is not permitted " + + "after starting to download."); + } + this.failUnrecognizedDescriptorLines = true; + } + public Iterator<DescriptorRequest> downloadDescriptors() { if (this.hasStartedDownloading) { throw new IllegalStateException("Initiating downloads is only " @@ -246,7 +255,8 @@ public class RelayDescriptorDownloaderImpl this.directoryMirrors, this.downloadConsensus, this.downloadConsensusFromAllAuthorities, this.downloadVotes, this.includeCurrentReferencedVotes, this.connectTimeoutMillis, - this.readTimeoutMillis, this.globalTimeoutMillis); + this.readTimeoutMillis, this.globalTimeoutMillis, + this.failUnrecognizedDescriptorLines); Iterator<DescriptorRequest> descriptorQueue = downloadCoordinator. getDescriptorQueue(); return descriptorQueue; diff --git a/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java b/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java index bdf6b86..10f97c0 100644 --- a/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java +++ b/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java @@ -22,29 +22,26 @@ public class RelayNetworkStatusConsensusImpl extends NetworkStatusImpl implements RelayNetworkStatusConsensus { protected static List<RelayNetworkStatusConsensus> parseConsensuses( - byte[] consensusesBytes) { + byte[] consensusesBytes, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { List<RelayNetworkStatusConsensus> parsedConsensuses = new ArrayList<RelayNetworkStatusConsensus>(); List<byte[]> splitConsensusBytes = DescriptorImpl.splitRawDescriptorBytes(consensusesBytes, "network-status-version 3"); - try { - for (byte[] consensusBytes : splitConsensusBytes) { - RelayNetworkStatusConsensus parsedConsensus = - new RelayNetworkStatusConsensusImpl(consensusBytes); - parsedConsensuses.add(parsedConsensus); - } - } catch (DescriptorParseException e) { - /* TODO Handle this error somehow. */ - System.err.println("Failed to parse consensus. Skipping."); - e.printStackTrace(); + for (byte[] consensusBytes : splitConsensusBytes) { + RelayNetworkStatusConsensus parsedConsensus = + new RelayNetworkStatusConsensusImpl(consensusBytes, + failUnrecognizedDescriptorLines); + parsedConsensuses.add(parsedConsensus); } return parsedConsensuses; } - protected RelayNetworkStatusConsensusImpl(byte[] consensusBytes) + protected RelayNetworkStatusConsensusImpl(byte[] consensusBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(consensusBytes); + super(consensusBytes, failUnrecognizedDescriptorLines); Set<String> exactlyOnceKeywords = new HashSet<String>(Arrays.asList(( "vote-status,consensus-method,valid-after,fresh-until," + "valid-until,voting-delay,known-flags," @@ -88,12 +85,14 @@ public class RelayNetworkStatusConsensusImpl extends NetworkStatusImpl this.parseKnownFlagsLine(line, parts); } else if (keyword.equals("params")) { this.parseParamsLine(line, parts); - } else { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. */ + } else if (this.failUnrecognizedDescriptorLines) { throw new DescriptorParseException("Unrecognized line '" + line - + "'."); + + "' in consensus."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); } } } catch (IOException e) { @@ -115,12 +114,14 @@ public class RelayNetworkStatusConsensusImpl extends NetworkStatusImpl if (keyword.equals("directory-footer")) { } else if (keyword.equals("bandwidth-weights")) { this.parseBandwidthWeightsLine(line, parts); - } else { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. */ + } else if (this.failUnrecognizedDescriptorLines) { throw new DescriptorParseException("Unrecognized line '" + line - + "'."); + + "' in consensus."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); } } } catch (IOException e) { diff --git a/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java b/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java index 1e6f300..3729f75 100644 --- a/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java +++ b/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java @@ -22,29 +22,26 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl implements RelayNetworkStatusVote { protected static List<RelayNetworkStatusVote> parseVotes( - byte[] votesBytes) { + byte[] votesBytes, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { List<RelayNetworkStatusVote> parsedVotes = new ArrayList<RelayNetworkStatusVote>(); List<byte[]> splitVotesBytes = DescriptorImpl.splitRawDescriptorBytes(votesBytes, "network-status-version 3"); - try { - for (byte[] voteBytes : splitVotesBytes) { - RelayNetworkStatusVote parsedVote = - new RelayNetworkStatusVoteImpl(voteBytes); - parsedVotes.add(parsedVote); - } - } catch (DescriptorParseException e) { - /* TODO Handle this error somehow. */ - System.err.println("Failed to parse vote. Skipping."); - e.printStackTrace(); + for (byte[] voteBytes : splitVotesBytes) { + RelayNetworkStatusVote parsedVote = + new RelayNetworkStatusVoteImpl(voteBytes, + failUnrecognizedDescriptorLines); + parsedVotes.add(parsedVote); } return parsedVotes; } - protected RelayNetworkStatusVoteImpl(byte[] voteBytes) + protected RelayNetworkStatusVoteImpl(byte[] voteBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(voteBytes); + super(voteBytes, failUnrecognizedDescriptorLines); Set<String> exactlyOnceKeywords = new HashSet<String>(Arrays.asList(( "vote-status,consensus-methods,published,valid-after,fresh-until," + "valid-until,voting-delay,known-flags,dir-source," @@ -93,12 +90,14 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl this.parseKnownFlagsLine(line, parts); } else if (keyword.equals("params")) { this.parseParamsLine(line, parts); - } else { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. */ + } else if (this.failUnrecognizedDescriptorLines) { throw new DescriptorParseException("Unrecognized line '" + line - + "'."); + + "' in vote."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); } } } catch (IOException e) { @@ -271,11 +270,15 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl } else if (line.startsWith("-----END")) { skipCrypto = false; } else if (!skipCrypto) { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. */ - throw new DescriptorParseException("Unrecognized line '" + line - + "'."); + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in vote."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } } } } catch (IOException e) { @@ -366,8 +369,30 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl parts, 1, 2); } - protected void parseFooter(byte[] footerBytes) { - /* There is nothing in the footer that we'd want to parse. */ + protected void parseFooter(byte[] footerBytes) + throws DescriptorParseException { + try { + BufferedReader br = new BufferedReader(new StringReader( + new String(footerBytes))); + String line; + while ((line = br.readLine()) != null) { + if (!line.equals("directory-footer")) { + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in vote."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } + } + } + } catch (IOException e) { + throw new RuntimeException("Internal error: Ran into an " + + "IOException while parsing a String in memory. Something's " + + "really wrong.", e); + } } private String nickname; diff --git a/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java index 1a686f4..d33dba5 100644 --- a/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java +++ b/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java @@ -40,11 +40,17 @@ public class RelayOrBridgeDescriptorReaderImpl this.historyFile = historyFile; } + private boolean failUnrecognizedDescriptorLines = false; + public void setFailUnrecognizedDescriptorLines() { + this.failUnrecognizedDescriptorLines = true; + } + public Iterator<DescriptorFile> readDescriptors() { BlockingIteratorImpl<DescriptorFile> descriptorQueue = new BlockingIteratorImpl<DescriptorFile>(); DescriptorReader reader = new DescriptorReader(this.directories, - descriptorQueue, this.historyFile); + descriptorQueue, this.historyFile, + this.failUnrecognizedDescriptorLines); new Thread(reader).start(); return descriptorQueue; } @@ -53,12 +59,15 @@ public class RelayOrBridgeDescriptorReaderImpl private List<File> directories; private BlockingIteratorImpl<DescriptorFile> descriptorQueue; private File historyFile; + private boolean failUnrecognizedDescriptorLines; private DescriptorReader(List<File> directories, BlockingIteratorImpl<DescriptorFile> descriptorQueue, - File historyFile) { + File historyFile, boolean failUnrecognizedDescriptorLines) { this.directories = directories; this.descriptorQueue = descriptorQueue; this.historyFile = historyFile; + this.failUnrecognizedDescriptorLines = + failUnrecognizedDescriptorLines; } public void run() { this.readOldHistory(); @@ -167,7 +176,8 @@ public class RelayOrBridgeDescriptorReaderImpl bis.close(); byte[] rawDescriptorBytes = baos.toByteArray(); return DescriptorImpl.parseRelayOrBridgeDescriptors( - rawDescriptorBytes, file.getName()); + rawDescriptorBytes, file.getName(), + this.failUnrecognizedDescriptorLines); } } } diff --git a/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java b/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java index 09e9123..76873dc 100644 --- a/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java @@ -19,7 +19,7 @@ public class ServerDescriptorImpl extends DescriptorImpl implements ServerDescriptor { protected static List<ServerDescriptor> parseDescriptors( - byte[] descriptorsBytes) { + byte[] descriptorsBytes, boolean failUnrecognizedDescriptorLines) { List<ServerDescriptor> parsedDescriptors = new ArrayList<ServerDescriptor>(); List<byte[]> splitDescriptorsBytes = @@ -28,7 +28,8 @@ public class ServerDescriptorImpl extends DescriptorImpl try { for (byte[] descriptorBytes : splitDescriptorsBytes) { ServerDescriptor parsedDescriptor = - new ServerDescriptorImpl(descriptorBytes); + new ServerDescriptorImpl(descriptorBytes, + failUnrecognizedDescriptorLines); parsedDescriptors.add(parsedDescriptor); } } catch (DescriptorParseException e) { @@ -39,9 +40,10 @@ public class ServerDescriptorImpl extends DescriptorImpl return parsedDescriptors; } - protected ServerDescriptorImpl(byte[] descriptorBytes) + protected ServerDescriptorImpl(byte[] descriptorBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(descriptorBytes); + super(descriptorBytes, failUnrecognizedDescriptorLines); this.parseDescriptorBytes(); Set<String> exactlyOnceKeywords = new HashSet<String>(Arrays.asList( "router,bandwidth,published".split(","))); @@ -127,13 +129,15 @@ public class ServerDescriptorImpl extends DescriptorImpl } else if (line.startsWith("-----END")) { skipCrypto = false; } else if (!skipCrypto) { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. In theory, - * dir-spec.txt says that unknown lines should be ignored. This - * also applies to the other descriptors. */ - throw new DescriptorParseException("Unrecognized line '" + line - + "'."); + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in server descriptor."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } } } } catch (IOException e) { diff --git a/test/org/torproject/descriptor/impl/ConsensusBuilder.java b/test/org/torproject/descriptor/impl/ConsensusBuilder.java index 51fff2e..2d0291d 100644 --- a/test/org/torproject/descriptor/impl/ConsensusBuilder.java +++ b/test/org/torproject/descriptor/impl/ConsensusBuilder.java @@ -16,7 +16,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.networkStatusVersionLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String voteStatusLine = "vote-status consensus"; protected static RelayNetworkStatusConsensus @@ -24,7 +24,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.voteStatusLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String consensusMethodLine = "consensus-method 11"; protected static RelayNetworkStatusConsensus @@ -32,7 +32,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.consensusMethodLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String validAfterLine = "valid-after 2011-11-30 09:00:00"; protected static RelayNetworkStatusConsensus @@ -40,7 +40,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.validAfterLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String freshUntilLine = "fresh-until 2011-11-30 10:00:00"; protected static RelayNetworkStatusConsensus @@ -48,7 +48,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.freshUntilLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String validUntilLine = "valid-until 2011-11-30 12:00:00"; protected static RelayNetworkStatusConsensus @@ -56,7 +56,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.validUntilLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String votingDelayLine = "voting-delay 300 300"; protected static RelayNetworkStatusConsensus @@ -64,7 +64,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.votingDelayLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } String clientVersionsLine = "client-versions 0.2.1.31," + "0.2.2.34,0.2.3.6-alpha,0.2.3.7-alpha,0.2.3.8-alpha"; @@ -73,7 +73,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.clientVersionsLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } String serverVersionsLine = "server-versions 0.2.1.31," + "0.2.2.34,0.2.3.6-alpha,0.2.3.7-alpha,0.2.3.8-alpha"; @@ -82,7 +82,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.serverVersionsLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String knownFlagsLine = "known-flags Authority BadExit Exit " + "Fast Guard HSDir Named Running Stable Unnamed V2Dir Valid"; @@ -91,7 +91,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.knownFlagsLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String paramsLine = "params " + "CircuitPriorityHalflifeMsec=30000 bwauthbestratio=1 " @@ -103,7 +103,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.paramsLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } List<String> dirSources = new ArrayList<String>(); List<String> statusEntries = new ArrayList<String>(); @@ -113,7 +113,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.directoryFooterLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String bandwidthWeightsLine = "bandwidth-weights Wbd=285 " + "Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=1021 Wee=10000 " @@ -124,12 +124,62 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.bandwidthWeightsLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private List<String> directorySignatures = new ArrayList<String>(); protected void addDirectorySignature(String directorySignatureString) { this.directorySignatures.add(directorySignatureString); } + private String unrecognizedHeaderLine = null; + protected static RelayNetworkStatusConsensus + createWithUnrecognizedHeaderLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + ConsensusBuilder cb = new ConsensusBuilder(); + cb.unrecognizedHeaderLine = line; + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedDirSourceLine = null; + protected static RelayNetworkStatusConsensus + createWithUnrecognizedDirSourceLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + ConsensusBuilder cb = new ConsensusBuilder(); + cb.unrecognizedDirSourceLine = line; + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedStatusEntryLine = null; + protected static RelayNetworkStatusConsensus + createWithUnrecognizedStatusEntryLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + ConsensusBuilder cb = new ConsensusBuilder(); + cb.unrecognizedStatusEntryLine = line; + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedFooterLine = null; + protected static RelayNetworkStatusConsensus + createWithUnrecognizedFooterLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + ConsensusBuilder cb = new ConsensusBuilder(); + cb.unrecognizedFooterLine = line; + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedDirectorySignatureLine = null; + protected static RelayNetworkStatusConsensus + createWithUnrecognizedDirectorySignatureLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + ConsensusBuilder cb = new ConsensusBuilder(); + cb.unrecognizedDirectorySignatureLine = line; + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + failUnrecognizedDescriptorLines); + } protected ConsensusBuilder() { this.dirSources.add("dir-source tor26 " @@ -170,8 +220,10 @@ public class ConsensusBuilder { protected byte[] buildConsensus() { StringBuilder sb = new StringBuilder(); this.appendHeader(sb); - this.appendBody(sb); + this.appendDirSources(sb); + this.appendStatusEntries(sb); this.appendFooter(sb); + this.appendDirectorySignatures(sb); return sb.toString().getBytes(); } private void appendHeader(StringBuilder sb) { @@ -208,14 +260,25 @@ public class ConsensusBuilder { if (this.paramsLine != null) { sb.append(this.paramsLine + "\n"); } + if (this.unrecognizedHeaderLine != null) { + sb.append(this.unrecognizedHeaderLine + "\n"); + } + } + private void appendDirSources(StringBuilder sb) { for (String dirSource : this.dirSources) { sb.append(dirSource + "\n"); } + if (this.unrecognizedDirSourceLine != null) { + sb.append(this.unrecognizedDirSourceLine + "\n"); + } } - private void appendBody(StringBuilder sb) { + private void appendStatusEntries(StringBuilder sb) { for (String statusEntry : this.statusEntries) { sb.append(statusEntry + "\n"); } + if (this.unrecognizedStatusEntryLine != null) { + sb.append(this.unrecognizedStatusEntryLine + "\n"); + } } private void appendFooter(StringBuilder sb) { if (this.directoryFooterLine != null) { @@ -224,9 +287,17 @@ public class ConsensusBuilder { if (this.bandwidthWeightsLine != null) { sb.append(this.bandwidthWeightsLine + "\n"); } + if (this.unrecognizedFooterLine != null) { + sb.append(this.unrecognizedFooterLine + "\n"); + } + } + private void appendDirectorySignatures(StringBuilder sb) { for (String directorySignature : this.directorySignatures) { sb.append(directorySignature + "\n"); } + if (this.unrecognizedDirectorySignatureLine != null) { + sb.append(this.unrecognizedDirectorySignatureLine + "\n"); + } } } diff --git a/test/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java b/test/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java index 954a3d3..b37b72a 100644 --- a/test/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java +++ b/test/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java @@ -8,6 +8,9 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.util.ArrayList; +import java.util.List; + import org.junit.Test; import org.torproject.descriptor.RelayNetworkStatusConsensus; @@ -28,7 +31,8 @@ public class RelayNetworkStatusConsensusImplTest { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.dirSources.add(dirSourceString); - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + true); } private String nickname = "gabelmoo"; private static RelayNetworkStatusConsensus @@ -123,7 +127,8 @@ public class RelayNetworkStatusConsensusImplTest { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.statusEntries.add(statusEntryString); - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + true); } private String nickname = "right2privassy3"; private static RelayNetworkStatusConsensus @@ -239,7 +244,8 @@ public class RelayNetworkStatusConsensusImplTest { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.addDirectorySignature(directorySignatureString); - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + true); } private String identity = "ED03BB616EB2F60BEC80151114BB25CEF515B226"; private static RelayNetworkStatusConsensus @@ -276,7 +282,7 @@ public class RelayNetworkStatusConsensusImplTest { public void testSampleConsensus() throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); RelayNetworkStatusConsensus consensus = - new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); assertEquals(3, consensus.getNetworkStatusVersion()); assertEquals(11, consensus.getConsensusMethod()); assertEquals(1322643600000L, consensus.getValidAfterMillis()); @@ -301,6 +307,7 @@ public class RelayNetworkStatusConsensusImplTest { consensus.getDirectorySignatures().get( "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4")); assertEquals(285, (int) consensus.getBandwidthWeights().get("Wbd")); + assertTrue(consensus.getUnrecognizedLines().isEmpty()); } @Test(expected = DescriptorParseException.class) @@ -544,7 +551,7 @@ public class RelayNetworkStatusConsensusImplTest { cb.clientVersionsLine = null; cb.serverVersionsLine = null; RelayNetworkStatusConsensus consensus = - new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); assertNull(consensus.getRecommendedClientVersions()); assertNull(consensus.getRecommendedServerVersions()); } @@ -905,7 +912,7 @@ public class RelayNetworkStatusConsensusImplTest { sb.sLine = sb.sLine + "\n" + sb.sLine; ConsensusBuilder cb = new ConsensusBuilder(); cb.statusEntries.add(sb.buildStatusEntry()); - new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } @Test(expected = DescriptorParseException.class) @@ -929,7 +936,7 @@ public class RelayNetworkStatusConsensusImplTest { sb.wLine = sb.wLine + "\n" + sb.wLine; ConsensusBuilder cb = new ConsensusBuilder(); cb.statusEntries.add(sb.buildStatusEntry()); - new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } @Test(expected = DescriptorParseException.class) @@ -958,7 +965,7 @@ public class RelayNetworkStatusConsensusImplTest { sb.pLine = sb.pLine + "\n" + sb.pLine; ConsensusBuilder cb = new ConsensusBuilder(); cb.statusEntries.add(sb.buildStatusEntry()); - new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } @Test() @@ -966,7 +973,7 @@ public class RelayNetworkStatusConsensusImplTest { ConsensusBuilder cb = new ConsensusBuilder(); cb.statusEntries.clear(); RelayNetworkStatusConsensus consensus = - new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); assertFalse(consensus.containsStatusEntry( "00795A6E8D91C270FC23B30F388A495553E01894")); } @@ -1045,7 +1052,7 @@ public class RelayNetworkStatusConsensusImplTest { ConsensusBuilder cb = new ConsensusBuilder(); byte[] consensusBytes = cb.buildConsensus(); consensusBytes[20] = (byte) 200; - new RelayNetworkStatusConsensusImpl(consensusBytes); + new RelayNetworkStatusConsensusImpl(consensusBytes, true); } @Test(expected = DescriptorParseException.class) @@ -1055,7 +1062,103 @@ public class RelayNetworkStatusConsensusImplTest { cb.networkStatusVersionLine = "Xnetwork-status-version 3"; byte[] consensusBytes = cb.buildConsensus(); consensusBytes[0] = (byte) 200; - new RelayNetworkStatusConsensusImpl(consensusBytes); + new RelayNetworkStatusConsensusImpl(consensusBytes, true); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedHeaderLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + ConsensusBuilder.createWithUnrecognizedHeaderLine(unrecognizedLine, + true); + } + + @Test() + public void testUnrecognizedHeaderLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusConsensus consensus = ConsensusBuilder. + createWithUnrecognizedHeaderLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, consensus.getUnrecognizedLines()); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedDirSourceLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + ConsensusBuilder.createWithUnrecognizedDirSourceLine(unrecognizedLine, + true); + } + + @Test() + public void testUnrecognizedDirSourceLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusConsensus consensus = ConsensusBuilder. + createWithUnrecognizedDirSourceLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, consensus.getUnrecognizedLines()); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedStatusEntryLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + ConsensusBuilder.createWithUnrecognizedStatusEntryLine( + unrecognizedLine, true); + } + + @Test() + public void testUnrecognizedStatusEntryLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusConsensus consensus = ConsensusBuilder. + createWithUnrecognizedStatusEntryLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, consensus.getUnrecognizedLines()); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedDirectoryFooterLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + ConsensusBuilder.createWithUnrecognizedFooterLine(unrecognizedLine, + true); + } + + @Test() + public void testUnrecognizedDirectoryFooterLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusConsensus consensus = ConsensusBuilder. + createWithUnrecognizedFooterLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, consensus.getUnrecognizedLines()); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedDirectorySignatureLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + ConsensusBuilder.createWithUnrecognizedDirectorySignatureLine( + unrecognizedLine, true); + } + + @Test() + public void testUnrecognizedDirectorySignatureLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusConsensus consensus = ConsensusBuilder. + createWithUnrecognizedDirectorySignatureLine(unrecognizedLine, + false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, consensus.getUnrecognizedLines()); } } diff --git a/test/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java b/test/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java index 20aa2da..e907086 100644 --- a/test/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java +++ b/test/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java @@ -29,7 +29,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.networkStatusVersionLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String voteStatusLine = "vote-status vote"; private static RelayNetworkStatusVote @@ -37,7 +37,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.voteStatusLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String consensusMethodsLine = "consensus-methods 1 2 3 4 5 6 7 8 9 10 11"; @@ -46,7 +46,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.consensusMethodsLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String publishedLine = "published 2011-11-30 08:50:01"; private static RelayNetworkStatusVote @@ -54,7 +54,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.publishedLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String validAfterLine = "valid-after 2011-11-30 09:00:00"; private static RelayNetworkStatusVote @@ -62,7 +62,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.validAfterLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String freshUntilLine = "fresh-until 2011-11-30 10:00:00"; private static RelayNetworkStatusVote @@ -70,7 +70,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.freshUntilLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String validUntilLine = "valid-until 2011-11-30 12:00:00"; private static RelayNetworkStatusVote @@ -78,7 +78,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.validUntilLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String votingDelayLine = "voting-delay 300 300"; private static RelayNetworkStatusVote @@ -86,7 +86,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.votingDelayLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String clientVersionsLine = "client-versions 0.2.1.31," + "0.2.2.34,0.2.3.6-alpha,0.2.3.7-alpha,0.2.3.8-alpha"; @@ -95,7 +95,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.clientVersionsLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String serverVersionsLine = "server-versions 0.2.1.31," + "0.2.2.34,0.2.3.6-alpha,0.2.3.7-alpha,0.2.3.8-alpha"; @@ -104,7 +104,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.serverVersionsLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String knownFlagsLine = "known-flags Authority BadExit Exit " + "Fast Guard HSDir Named Running Stable Unnamed V2Dir Valid"; @@ -113,7 +113,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.knownFlagsLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String paramsLine = "params " + "CircuitPriorityHalflifeMsec=30000 bwauthbestratio=1 " @@ -125,7 +125,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.paramsLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirSourceLine = "dir-source urras " + "80550987E1D626E3EBA5E5E75A458DE0626D088C 208.83.223.34 " @@ -135,7 +135,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirSourceLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String contactLine = "contact 4096R/E012B42D Jacob Appelbaum " + "<jacob(a)appelbaum.net>"; @@ -144,7 +144,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.contactLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirKeyCertificateVersionLine = "dir-key-certificate-version 3"; @@ -153,7 +153,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirKeyCertificateVersionLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String fingerprintLine = "fingerprint " + "80550987E1D626E3EBA5E5E75A458DE0626D088C"; @@ -162,7 +162,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.fingerprintLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirKeyPublishedLine = "dir-key-published 2011-04-27 " + "05:34:37"; @@ -171,7 +171,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirKeyPublishedLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirKeyExpiresLine = "dir-key-expires 2012-04-27 " + "05:34:37"; @@ -180,7 +180,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirKeyExpiresLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirIdentityKeyLines = "dir-identity-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" @@ -199,7 +199,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirIdentityKeyLines = lines; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirSigningKeyLines = "dir-signing-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" @@ -213,7 +213,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirSigningKeyLines = lines; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirKeyCrosscertLines = "dir-key-crosscert\n" + "-----BEGIN ID SIGNATURE-----\n" @@ -226,7 +226,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirKeyCrosscertLines = lines; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirKeyCertificationLines = "dir-key-certification\n" + "-----BEGIN SIGNATURE-----\n" @@ -245,7 +245,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirKeyCertificationLines = lines; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private List<String> statusEntries = new ArrayList<String>(); private String directoryFooterLine = "directory-footer"; @@ -254,7 +254,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.directoryFooterLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String directorySignatureLines = "directory-signature " + "80550987E1D626E3EBA5E5E75A458DE0626D088C " @@ -269,8 +269,59 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.directorySignatureLines = lines; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } + private String unrecognizedHeaderLine = null; + protected static RelayNetworkStatusVote + createWithUnrecognizedHeaderLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + VoteBuilder vb = new VoteBuilder(); + vb.unrecognizedHeaderLine = line; + return new RelayNetworkStatusVoteImpl(vb.buildVote(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedDirSourceLine = null; + protected static RelayNetworkStatusVote + createWithUnrecognizedDirSourceLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + VoteBuilder vb = new VoteBuilder(); + vb.unrecognizedDirSourceLine = line; + return new RelayNetworkStatusVoteImpl(vb.buildVote(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedStatusEntryLine = null; + protected static RelayNetworkStatusVote + createWithUnrecognizedStatusEntryLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + VoteBuilder vb = new VoteBuilder(); + vb.unrecognizedStatusEntryLine = line; + return new RelayNetworkStatusVoteImpl(vb.buildVote(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedFooterLine = null; + protected static RelayNetworkStatusVote + createWithUnrecognizedFooterLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + VoteBuilder vb = new VoteBuilder(); + vb.unrecognizedFooterLine = line; + return new RelayNetworkStatusVoteImpl(vb.buildVote(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedDirectorySignatureLine = null; + protected static RelayNetworkStatusVote + createWithUnrecognizedDirectorySignatureLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + VoteBuilder vb = new VoteBuilder(); + vb.unrecognizedDirectorySignatureLine = line; + return new RelayNetworkStatusVoteImpl(vb.buildVote(), + failUnrecognizedDescriptorLines); + } + private VoteBuilder() { this.statusEntries.add("r right2privassy3 " + "ADQ6gCT3DiFHKPDFr3rODBUI8HM lJY5Vf7kXec+VdkGW2flEsfkFC8 " @@ -289,8 +340,10 @@ public class RelayNetworkStatusVoteImplTest { private byte[] buildVote() { StringBuilder sb = new StringBuilder(); this.appendHeader(sb); - this.appendBody(sb); + this.appendDirSource(sb); + this.appendStatusEntries(sb); this.appendFooter(sb); + this.appendDirectorySignature(sb); return sb.toString().getBytes(); } private void appendHeader(StringBuilder sb) { @@ -330,6 +383,11 @@ public class RelayNetworkStatusVoteImplTest { if (this.paramsLine != null) { sb.append(this.paramsLine + "\n"); } + if (this.unrecognizedHeaderLine != null) { + sb.append(this.unrecognizedHeaderLine + "\n"); + } + } + private void appendDirSource(StringBuilder sb) { if (this.dirSourceLine != null) { sb.append(this.dirSourceLine + "\n"); } @@ -360,19 +418,33 @@ public class RelayNetworkStatusVoteImplTest { if (this.dirKeyCertificationLines != null) { sb.append(this.dirKeyCertificationLines + "\n"); } + if (this.unrecognizedDirSourceLine != null) { + sb.append(this.unrecognizedDirSourceLine + "\n"); + } } - private void appendBody(StringBuilder sb) { + private void appendStatusEntries(StringBuilder sb) { for (String statusEntry : this.statusEntries) { sb.append(statusEntry + "\n"); } + if (this.unrecognizedStatusEntryLine != null) { + sb.append(this.unrecognizedStatusEntryLine + "\n"); + } } private void appendFooter(StringBuilder sb) { if (this.directoryFooterLine != null) { sb.append(this.directoryFooterLine + "\n"); } + if (this.unrecognizedFooterLine != null) { + sb.append(this.unrecognizedFooterLine + "\n"); + } + } + private void appendDirectorySignature(StringBuilder sb) { if (this.directorySignatureLines != null) { sb.append(directorySignatureLines + "\n"); } + if (this.unrecognizedDirectorySignatureLine != null) { + sb.append(this.unrecognizedDirectorySignatureLine + "\n"); + } } } @@ -380,7 +452,7 @@ public class RelayNetworkStatusVoteImplTest { public void testSampleVote() throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); RelayNetworkStatusVote vote = - new RelayNetworkStatusVoteImpl(vb.buildVote()); + new RelayNetworkStatusVoteImpl(vb.buildVote(), true); assertEquals(3, vote.getNetworkStatusVersion()); List<Integer> consensusMethods = Arrays.asList( new Integer[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}); @@ -397,6 +469,7 @@ public class RelayNetworkStatusVoteImplTest { assertEquals("Tor 0.2.1.29 (r8e9b25e6c7a2e70c)", vote.getStatusEntry("00343A8024F70E214728F0C5AF7ACE0C1508F073"). getVersion()); + assertTrue(vote.getUnrecognizedLines().isEmpty()); } @Test(expected = DescriptorParseException.class) @@ -946,5 +1019,60 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder.createWithDirectorySignatureLines(null); } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedHeaderLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + VoteBuilder.createWithUnrecognizedHeaderLine(unrecognizedLine, true); + } + + @Test() + public void testUnrecognizedHeaderLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusVote vote = VoteBuilder. + createWithUnrecognizedHeaderLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, vote.getUnrecognizedLines()); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedDirSourceLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + VoteBuilder.createWithUnrecognizedDirSourceLine(unrecognizedLine, + true); + } + + @Test() + public void testUnrecognizedDirSourceLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusVote vote = VoteBuilder. + createWithUnrecognizedDirSourceLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, vote.getUnrecognizedLines()); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedFooterLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + VoteBuilder.createWithUnrecognizedFooterLine(unrecognizedLine, true); + } + + @Test() + public void testUnrecognizedFooterLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusVote vote = VoteBuilder. + createWithUnrecognizedFooterLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, vote.getUnrecognizedLines()); + } } diff --git a/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java b/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java index eec5594..439e5bb 100644 --- a/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java +++ b/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java @@ -8,7 +8,9 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.SortedMap; import org.junit.Test; @@ -27,14 +29,14 @@ public class ServerDescriptorImplTest { String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.routerLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String bandwidthLine = "bandwidth 51200 51200 53470"; private static ServerDescriptor createWithBandwidthLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.bandwidthLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String platformLine = "platform Tor 0.2.2.35 " + "(git-b04388f9e7546a9f) on Linux i686"; @@ -42,14 +44,14 @@ public class ServerDescriptorImplTest { String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.platformLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String publishedLine = "published 2012-01-01 04:03:19"; private static ServerDescriptor createWithPublishedLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.publishedLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String fingerprintLine = "opt fingerprint D873 3048 FC8E " + "C910 2466 AD8F 3098 622B F1BF 71FD"; @@ -57,21 +59,21 @@ public class ServerDescriptorImplTest { String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.fingerprintLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String hibernatingLine = null; private static ServerDescriptor createWithHibernatingLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.hibernatingLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String uptimeLine = "uptime 48"; private static ServerDescriptor createWithUptimeLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.uptimeLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String onionKeyLines = "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" @@ -84,7 +86,7 @@ public class ServerDescriptorImplTest { String lines) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.onionKeyLines = lines; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String signingKeyLines = "signing-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" @@ -97,14 +99,14 @@ public class ServerDescriptorImplTest { String lines) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.signingKeyLines = lines; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String exitPolicyLines = "reject *:*"; private static ServerDescriptor createWithExitPolicyLines( String lines) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.exitPolicyLines = lines; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String contactLine = "contact Random Person <nobody AT " + "example dot com>"; @@ -112,42 +114,42 @@ public class ServerDescriptorImplTest { String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.contactLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String familyLine = null; private static ServerDescriptor createWithFamilyLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.familyLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String readHistoryLine = null; private static ServerDescriptor createWithReadHistoryLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.readHistoryLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String writeHistoryLine = null; private static ServerDescriptor createWithWriteHistoryLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.writeHistoryLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String eventdnsLine = null; private static ServerDescriptor createWithEventdnsLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.eventdnsLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String cachesExtraInfoLine = null; private static ServerDescriptor createWithCachesExtraInfoLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.cachesExtraInfoLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String extraInfoDigestLine = "opt extra-info-digest " + "1469D1550738A25B1E7B47CDDBCD7B2899F51B74"; @@ -155,21 +157,21 @@ public class ServerDescriptorImplTest { String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.extraInfoDigestLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String hiddenServiceDirLine = "opt hidden-service-dir"; private static ServerDescriptor createWithHiddenServiceDirLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.hiddenServiceDirLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String protocolsLine = "opt protocols Link 1 2 Circuit 1"; private static ServerDescriptor createWithProtocolsLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.protocolsLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String allowSingleHopExitsLine = null; private static ServerDescriptor @@ -177,7 +179,7 @@ public class ServerDescriptorImplTest { throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.allowSingleHopExitsLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String routerSignatureLines = "router-signature\n" + "-----BEGIN SIGNATURE-----\n" @@ -189,7 +191,16 @@ public class ServerDescriptorImplTest { String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.routerSignatureLines = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); + } + private String unrecognizedLine = null; + private static ServerDescriptor createWithUnrecognizedLine( + String line, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + DescriptorBuilder db = new DescriptorBuilder(); + db.unrecognizedLine = line; + return new ServerDescriptorImpl(db.buildDescriptor(), + failUnrecognizedDescriptorLines); } private byte[] buildDescriptor() { StringBuilder sb = new StringBuilder(); @@ -253,6 +264,9 @@ public class ServerDescriptorImplTest { if (this.allowSingleHopExitsLine != null) { sb.append(this.allowSingleHopExitsLine + "\n"); } + if (this.unrecognizedLine != null) { + sb.append(this.unrecognizedLine + "\n"); + } if (this.routerSignatureLines != null) { sb.append(this.routerSignatureLines + "\n"); } @@ -264,7 +278,7 @@ public class ServerDescriptorImplTest { public void testSampleDescriptor() throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); ServerDescriptor descriptor = - new ServerDescriptorImpl(db.buildDescriptor()); + new ServerDescriptorImpl(db.buildDescriptor(), true); assertEquals("saberrider2008", descriptor.getNickname()); assertEquals("94.134.192.243", descriptor.getAddress()); assertEquals(9001, (int) descriptor.getOrPort()); @@ -298,6 +312,7 @@ public class ServerDescriptorImplTest { assertFalse(descriptor.getUsesEnhancedDnsLogic()); assertFalse(descriptor.getCachesExtraInfo()); assertFalse(descriptor.getAllowSingleHopExits()); + assertTrue(descriptor.getUnrecognizedLines().isEmpty()); } @Test(expected = DescriptorParseException.class) @@ -1072,5 +1087,23 @@ public class ServerDescriptorImplTest { DescriptorBuilder.createWithAllowSingleHopExitsLine( "allow-single-hop-exits true"); } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + DescriptorBuilder.createWithUnrecognizedLine(unrecognizedLine, true); + } + + @Test() + public void testUnrecognizedLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + ServerDescriptor descriptor = DescriptorBuilder. + createWithUnrecognizedLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, descriptor.getUnrecognizedLines()); + } }

1 0

[tor/master] Allow 0.2.3.x clients to use 0.2.2.x bridges.
by nickm＠torproject.org 31 Jan '12

31 Jan '12

commit a0f0897795c45d22ad2d2dde98c81195a9db24dc Author: Roger Dingledine <arma(a)torproject.org> Date: Wed Jan 25 18:54:59 2012 -0500 Allow 0.2.3.x clients to use 0.2.2.x bridges. Previously the client would ask the bridge for microdescriptors, which are only supported in 0.2.3.x and later, and then fail to bootstrap when it didn't get the answers it wanted. Fixes bug 4013; bugfix on 0.2.3.2-alpha. The fix here is to revert to using normal descriptors if any of our bridges are known to not support microdescs. This is not ideal, a) because we'll start downloading a microdesc consensus as soon as we get a bridge descriptor, and that will waste time if we later get a bridge descriptor that tells us we don't like microdescriptors; and b) by changing our mind we're leaking to our other bridges that we have an old-version bridge. The alternate fix would have been to change we_use_microdescriptors_for_circuits() to ask if *any* of our bridges can support microdescriptors, and then change the directory logic that picks a bridge to only select from those that do. For people living in the future, where 0.2.2.x is obsolete, there won't be a difference. Note that in either of these potential fixes, we have risk of oscillation if our one funny-looking bridges goes away / comes back. --- changes/bug4013 | 6 ++++++ src/or/circuitbuild.c | 26 +++++++++++++++++++++++++- src/or/circuitbuild.h | 2 ++ src/or/directory.c | 14 +++++++++----- src/or/microdesc.c | 11 +++++++++-- src/or/routerparse.c | 15 ++++++++------- src/or/routerparse.h | 3 ++- 7 files changed, 61 insertions(+), 16 deletions(-) diff --git a/changes/bug4013 b/changes/bug4013 new file mode 100644 index 0000000..4a41968 --- /dev/null +++ b/changes/bug4013 @@ -0,0 +1,6 @@ + o Major bugfixes: + - Allow 0.2.3.x clients to use 0.2.2.x bridges. Previously the client + would ask the bridge for microdescriptors, which are only supported + in 0.2.3.x and later, and then fail to bootstrap when it didn't + get the answers it wanted. Fixes bug 4013; bugfix on 0.2.3.2-alpha. + diff --git a/src/or/circuitbuild.c b/src/or/circuitbuild.c index d35c08e..70e3596 100644 --- a/src/or/circuitbuild.c +++ b/src/or/circuitbuild.c @@ -3339,7 +3339,7 @@ extend_info_from_router(const routerinfo_t *r, int for_direct_connect) } /** Allocate and return a new extend_info that can be used to build a - * ircuit to or through the node node. Use the primary address + * circuit to or through the node node. Use the primary address * of the node unless for_direct_connect is true, in which case * the preferred address is used instead. May return NULL if there is * not enough info about node to extend to it--for example, if @@ -5328,6 +5328,30 @@ entries_retry_all(const or_options_t *options) entries_retry_helper(options, 1); } +/** Return true if one of our bridges is running a Tor version that can't + * provide microdescriptors to us. In that case fall back to asking for + * full descriptors. Eventually all bridges will support microdescriptors + * and we can take this check out; see bug 4013. */ +int +any_bridges_dont_support_microdescriptors(void) +{ + const node_t *node; + if (!get_options()->UseBridges || !entry_guards) + return 0; + SMARTLIST_FOREACH_BEGIN(entry_guards, entry_guard_t *, e) { + node = node_get_by_id(e->identity); + if (node && node->ri && + node_is_bridge(node) && node_is_a_configured_bridge(node) && + !tor_version_supports_microdescriptors(node->ri->platform)) { + /* This is one of our current bridges, and we know enough about + * it to know that it won't be able to answer our microdescriptor + * questions. */ + return 1; + } + } SMARTLIST_FOREACH_END(e); + return 0; +} + /** Release all storage held by the list of entry guards and related * memory structs. */ void diff --git a/src/or/circuitbuild.h b/src/or/circuitbuild.h index cded399..a7fd208 100644 --- a/src/or/circuitbuild.h +++ b/src/or/circuitbuild.h @@ -100,6 +100,8 @@ int any_pending_bridge_descriptor_fetches(void); int entries_known_but_down(const or_options_t *options); void entries_retry_all(const or_options_t *options); +int any_bridges_dont_support_microdescriptors(void); + void entry_guards_free_all(void); extern circuit_build_times_t circ_times; diff --git a/src/or/directory.c b/src/or/directory.c index 149e692..c6a527c 100644 --- a/src/or/directory.c +++ b/src/or/directory.c @@ -421,11 +421,15 @@ directory_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, if (!get_via_tor) { if (options->UseBridges && type != BRIDGE_DIRINFO) { - /* want to ask a running bridge for which we have a descriptor. */ - /* XXX023 we assume that all of our bridges can answer any - * possible directory question. This won't be true forever. -RD */ - /* It certainly is not true with conditional consensus downloading, - * so, for now, never assume the server supports that. */ + /* We want to ask a running bridge for which we have a descriptor. + * + * Be careful here: we should only ask questions that we know our + * bridges can answer. So far we're solving that by backing off to + * the behavior supported by our oldest bridge; see for example + * any_bridges_dont_support_microdescriptors(). + */ + /* XXX024 Not all bridges handle conditional consensus downloading, + * so, for now, never assume the server supports that. -PP */ const node_t *node = choose_random_entry(NULL); if (node && node->ri) { /* every bridge has a routerinfo. */ diff --git a/src/or/microdesc.c b/src/or/microdesc.c index 0a1ae91..be9b997 100644 --- a/src/or/microdesc.c +++ b/src/or/microdesc.c @@ -2,6 +2,7 @@ /* See LICENSE for licensing information */ #include "or.h" +#include "circuitbuild.h" #include "config.h" #include "directory.h" #include "dirserv.h" @@ -720,8 +721,14 @@ we_use_microdescriptors_for_circuits(const or_options_t *options) int ret = options->UseMicrodescriptors; if (ret == -1) { /* UseMicrodescriptors is "auto"; we need to decide: */ - /* So we decide that we'll use microdescriptors iff we are not a server, - * and we're not autofetching everything. */ + /* If we are configured to use bridges and one of our bridges doesn't + * know what a microdescriptor is, the answer is no. */ + if (options->UseBridges && any_bridges_dont_support_microdescriptors()) + return 0; + /* Otherwise, we decide that we'll use microdescriptors iff we are + * not a server, and we're not autofetching everything. */ + /* XXX023 what does not being a server have to do with it? also there's + * a partitioning issue here where bridges differ from clients. */ ret = !server_mode(options) && !options->FetchUselessDescriptors; } return ret; diff --git a/src/or/routerparse.c b/src/or/routerparse.c index 7a0fd17..28ec945 100644 --- a/src/or/routerparse.c +++ b/src/or/routerparse.c @@ -2125,14 +2125,8 @@ routerstatus_parse_entry_from_string(memarea_t *area, tor_version_as_new_as(tok->args[0], "0.2.0.8-alpha"); rs->version_supports_conditional_consensus = tor_version_as_new_as(tok->args[0], "0.2.1.1-alpha"); - /* XXXX023 NM microdescs: 0.2.3.1-alpha isn't widely used yet, but - * not all 0.2.3.0-alpha "versions" actually support microdesc cacheing - * right. There's a compromise here. Since this is 5 May, let's - * err on the side of having some possible caches to use. Once more - * caches are running 0.2.3.1-alpha, we can bump this version number. - */ rs->version_supports_microdesc_cache = - tor_version_as_new_as(tok->args[0], "0.2.3.0-alpha"); + tor_version_supports_microdescriptors(tok->args[0]); rs->version_supports_optimistic_data = tor_version_as_new_as(tok->args[0], "0.2.3.1-alpha"); } @@ -4431,6 +4425,13 @@ microdescs_parse_from_string(const char *s, const char *eos, return result; } +/** Return true iff this Tor version can answer directory questions + * about microdescriptors. */ +int tor_version_supports_microdescriptors(const char *platform) +{ + return tor_version_as_new_as(platform, "0.2.3.1-alpha"); +} + /** Parse the Tor version of the platform string platform, * and compare it to the version in cutoff. Return 1 if * the router is at least as new as the cutoff, else return 0. diff --git a/src/or/routerparse.h b/src/or/routerparse.h index 65f6c42..b274d21 100644 --- a/src/or/routerparse.h +++ b/src/or/routerparse.h @@ -44,8 +44,9 @@ addr_policy_t *router_parse_addr_policy_item_from_string(const char *s, int assume_action); version_status_t tor_version_is_obsolete(const char *myversion, const char *versionlist); -int tor_version_parse(const char *s, tor_version_t *out); +int tor_version_supports_microdescriptors(const char *platform); int tor_version_as_new_as(const char *platform, const char *cutoff); +int tor_version_parse(const char *s, tor_version_t *out); int tor_version_compare(tor_version_t *a, tor_version_t *b); int tor_version_same_series(tor_version_t *a, tor_version_t *b); void sort_version_list(smartlist_t *lst, int remove_duplicates);

1 0