June 2017 - tor-commits - lists.torproject.org

[metrics-lib/release] Tweak tests for removing deprecated code for #22154.
by karsten＠torproject.org 28 Jun '17

28 Jun '17

commit 1b3fd15f92d5fe5f71e62d8a76ee847134cd5e07 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jun 26 22:12:10 2017 +0200 Tweak tests for removing deprecated code for #22154. This patch makes the following changes: - Remove now unused import in DescriptorParser. - Rename methods in DescriptorParserImpl to please checkstyle. - Tweak a few things in DescriptorReaderImplTest, including accepting varargs for dirs and putting back expected excluded files. - Remove tests that were previously testing deprecated methods which are now gone. - Change the two tests ending in NoDescriptors() back so that they really don't read any descriptors. - Put back tests for unrecognized lines, but don't expect DescriptorParseExceptions anymore and instead check getUnrecognizedLines(). --- .../torproject/descriptor/DescriptorParser.java | 1 - .../descriptor/impl/DescriptorParserImpl.java | 28 +++++++------- .../descriptor/impl/DescriptorParserImplTest.java | 5 +-- .../descriptor/impl/DescriptorReaderImplTest.java | 45 ++++++++-------------- .../impl/RelayNetworkStatusVoteImplTest.java | 42 +++++++++++++++++++- .../descriptor/impl/ServerDescriptorImplTest.java | 21 ++++++++++ 6 files changed, 93 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/DescriptorParser.java b/src/main/java/org/torproject/descriptor/DescriptorParser.java index 0ad2790..259510f 100644 --- a/src/main/java/org/torproject/descriptor/DescriptorParser.java +++ b/src/main/java/org/torproject/descriptor/DescriptorParser.java @@ -4,7 +4,6 @@ package org.torproject.descriptor; import java.io.File; -import java.util.List; /** * Descriptor source that parses descriptors from raw descriptor contents. diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java index d39343d..ac7ff9c 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java @@ -52,7 +52,7 @@ public class DescriptorParserImpl implements DescriptorParser { NL + Key.NETWORK_STATUS_VERSION.keyword + SP + "3")) && firstLines.contains( NL + Key.VOTE_STATUS.keyword + SP + "consensus" + NL))) { - return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + return this.parseOneOrMoreDescriptors(rawDescriptorBytes, descriptorFile, Key.NETWORK_STATUS_VERSION, RelayNetworkStatusConsensusImpl.class); } else if (firstLines.startsWith("@type network-status-vote-3 1.") || ((firstLines.startsWith( @@ -61,7 +61,7 @@ public class DescriptorParserImpl implements DescriptorParser { NL + Key.NETWORK_STATUS_VERSION.keyword + SP + "3" + NL)) && firstLines.contains( NL + Key.VOTE_STATUS.keyword + SP + "vote" + NL))) { - return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + return this.parseOneOrMoreDescriptors(rawDescriptorBytes, descriptorFile, Key.NETWORK_STATUS_VERSION, RelayNetworkStatusVoteImpl.class); } else if (firstLines.startsWith("@type bridge-network-status 1.") || firstLines.startsWith(Key.R.keyword + SP)) { @@ -71,36 +71,36 @@ public class DescriptorParserImpl implements DescriptorParser { descriptorFile, fileName)); return parsedDescriptors; } else if (firstLines.startsWith("@type bridge-server-descriptor 1.")) { - return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + return this.parseOneOrMoreDescriptors(rawDescriptorBytes, descriptorFile, Key.ROUTER, BridgeServerDescriptorImpl.class); } else if (firstLines.startsWith("@type server-descriptor 1.") || firstLines.startsWith(Key.ROUTER.keyword + SP) || firstLines.contains(NL + Key.ROUTER.keyword + SP)) { - return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + return this.parseOneOrMoreDescriptors(rawDescriptorBytes, descriptorFile, Key.ROUTER, RelayServerDescriptorImpl.class); } else if (firstLines.startsWith("@type bridge-extra-info 1.")) { - return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + return this.parseOneOrMoreDescriptors(rawDescriptorBytes, descriptorFile, Key.EXTRA_INFO, BridgeExtraInfoDescriptorImpl.class); } else if (firstLines.startsWith("@type extra-info 1.") || firstLines.startsWith(Key.EXTRA_INFO.keyword + SP) || firstLines.contains(NL + Key.EXTRA_INFO.keyword + SP)) { - return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + return this.parseOneOrMoreDescriptors(rawDescriptorBytes, descriptorFile, Key.EXTRA_INFO, RelayExtraInfoDescriptorImpl.class); } else if (firstLines.startsWith("@type microdescriptor 1.") || firstLines.startsWith(Key.ONION_KEY.keyword + NL) || firstLines.contains(NL + Key.ONION_KEY.keyword + NL)) { - return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + return this.parseOneOrMoreDescriptors(rawDescriptorBytes, descriptorFile, Key.ONION_KEY, MicrodescriptorImpl.class); } else if (firstLines.startsWith("@type bridge-pool-assignment 1.") || firstLines.startsWith(Key.BRIDGE_POOL_ASSIGNMENT.keyword + SP) || firstLines.contains(NL + Key.BRIDGE_POOL_ASSIGNMENT.keyword + SP)) { - return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + return this.parseOneOrMoreDescriptors(rawDescriptorBytes, descriptorFile, Key.BRIDGE_POOL_ASSIGNMENT, BridgePoolAssignmentImpl.class); } else if (firstLines.startsWith("@type dir-key-certificate-3 1.") || firstLines.startsWith(Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP) || firstLines.contains( NL + Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP)) { - return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + return this.parseOneOrMoreDescriptors(rawDescriptorBytes, descriptorFile, Key.DIR_KEY_CERTIFICATE_VERSION, DirectoryKeyCertificateImpl.class); } else if (firstLines.startsWith("@type tordnsel 1.") || firstLines.startsWith("ExitNode" + SP) @@ -114,12 +114,12 @@ public class DescriptorParserImpl implements DescriptorParser { Key.NETWORK_STATUS_VERSION.keyword + SP + "2" + NL) || firstLines.contains( NL + Key.NETWORK_STATUS_VERSION.keyword + SP + "2" + NL)) { - return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + return this.parseOneOrMoreDescriptors(rawDescriptorBytes, descriptorFile, Key.NETWORK_STATUS_VERSION, RelayNetworkStatusImpl.class); } else if (firstLines.startsWith("@type directory 1.") || firstLines.startsWith(Key.SIGNED_DIRECTORY.keyword + NL) || firstLines.contains(NL + Key.SIGNED_DIRECTORY.keyword + NL)) { - return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + return this.parseOneOrMoreDescriptors(rawDescriptorBytes, descriptorFile, Key.SIGNED_DIRECTORY, RelayDirectoryImpl.class); } else if (firstLines.startsWith("@type torperf 1.")) { return TorperfResultImpl.parseTorperfResults(rawDescriptorBytes, @@ -130,7 +130,7 @@ public class DescriptorParserImpl implements DescriptorParser { } } - private List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes, + private List<Descriptor> parseOneOrMoreDescriptors(byte[] rawDescriptorBytes, File descriptorFile, Key key, Class<? extends DescriptorImpl> descriptorClass) throws DescriptorParseException { @@ -183,7 +183,7 @@ public class DescriptorParserImpl implements DescriptorParser { int[] offsetAndLength = new int[] { startAnnotations, endDescriptor - startAnnotations }; try { - parsedDescriptors.add(this.parseDescriptor(rawDescriptorBytes, + parsedDescriptors.add(this.parseOneDescriptor(rawDescriptorBytes, offsetAndLength, descriptorFile, constructor)); } catch (DescriptorParseException e) { parsedDescriptors.add(new UnparseableDescriptorImpl( @@ -194,7 +194,7 @@ public class DescriptorParserImpl implements DescriptorParser { return parsedDescriptors; } - Descriptor parseDescriptor(byte[] rawDescriptorBytes, + Descriptor parseOneDescriptor(byte[] rawDescriptorBytes, int[] offsetAndLength, File descriptorFile, Constructor<? extends DescriptorImpl> constructor) throws DescriptorParseException { diff --git a/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java b/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java index df441f3..42d0cde 100644 --- a/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java @@ -20,8 +20,7 @@ public class DescriptorParserImplTest { @Rule public ExpectedException thrown = ExpectedException.none(); - private TestDescriptor makeTestDesc(byte[] bytes) - throws Exception { + private TestDescriptor makeTestDesc(byte[] bytes) throws Exception { return new TestDescriptor(bytes, new int[]{0, bytes.length}); } @@ -61,7 +60,7 @@ public class DescriptorParserImplTest { this.thrown.expectMessage("'176x.158.53.63' in line 'router UbuntuCore169 " + "176x.158.53.63 44583 0 0' is not a valid IPv4 address."); DescriptorParserImpl dpi = new DescriptorParserImpl(); - dpi.parseDescriptor(DEFECT.getBytes(), + dpi.parseOneDescriptor(DEFECT.getBytes(), new int[]{0, DEFECT.getBytes().length}, null, constructor); } diff --git a/src/test/java/org/torproject/descriptor/impl/DescriptorReaderImplTest.java b/src/test/java/org/torproject/descriptor/impl/DescriptorReaderImplTest.java index a00e5ba..0a3791e 100644 --- a/src/test/java/org/torproject/descriptor/impl/DescriptorReaderImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/DescriptorReaderImplTest.java @@ -63,9 +63,9 @@ public class DescriptorReaderImplTest { parseHistoryContents.getBytes(StandardCharsets.UTF_8)); } - private int readAllDescriptors(File dir) { + private int readAllDescriptors(File... dirs) { Iterator<Descriptor> descriptors = this.descriptorReader - .readDescriptors(dir).iterator(); + .readDescriptors(dirs).iterator(); int count = 0; while (descriptors.hasNext()) { count++; @@ -75,8 +75,10 @@ public class DescriptorReaderImplTest { } private void assertExcludedFilesParsedFilesAndHistoryFileLines( - int expectedParsedFiles, + int expectedExcludedFiles, int expectedParsedFiles, int expectedHistoryFileLines) throws IOException { + assertEquals(expectedExcludedFiles, + this.descriptorReader.getExcludedFiles().size()); assertEquals(expectedParsedFiles, this.descriptorReader.getParsedFiles().size()); assertEquals(expectedHistoryFileLines, @@ -87,38 +89,21 @@ public class DescriptorReaderImplTest { @Test public void testDescriptors() throws IOException { this.readAllDescriptors(this.inputDirectory); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(2, 1); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(0, 2, 1); } @Test public void testNoDescriptors() throws Exception { // calling readAllDescriptors fails with IAE Iterator<Descriptor> descriptors = this.descriptorReader - .readDescriptors(null).iterator(); + .readDescriptors().iterator(); int count = 0; while (descriptors.hasNext()) { count++; descriptors.next(); } assertEquals(count, 0); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(0, 1); - } - - @Test - @SuppressWarnings("deprecation") - public void testSetExcludeFilesDescriptors() throws InterruptedException, - IOException { - this.readAllDescriptors(this.inputDirectory); - Thread.sleep(100L); /* It may take a moment to write the history file. */ - this.assertExcludedFilesParsedFilesAndHistoryFileLines(2, 1); - } - - @Test - public void testSetExcludeFilesNoDescriptors() throws InterruptedException, - IOException { - this.readAllDescriptors(this.inputDirectory); - Thread.sleep(100L); /* It may take a moment to write the history file. */ - this.assertExcludedFilesParsedFilesAndHistoryFileLines(2, 1); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(0, 0, 1); } @Test @@ -126,29 +111,29 @@ public class DescriptorReaderImplTest { this.descriptorReader.setHistoryFile(this.historyFile); this.readAllDescriptors(this.inputDirectory); descriptorReader.saveHistoryFile(this.historyFile); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(1, 2); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(1, 1, 2); } @Test public void testSetHistoryFileNoDescriptors() throws IOException { this.descriptorReader.setHistoryFile(this.historyFile); - this.readAllDescriptors(this.inputDirectory); + this.readAllDescriptors(); this.descriptorReader.saveHistoryFile(this.historyFile); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(1, 2); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(0, 0, 0); } @Test public void testSetExcludedFilesDescriptors() throws IOException { this.descriptorReader.setExcludedFiles(this.historyMap); this.readAllDescriptors(this.inputDirectory); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(1, 1); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(1, 1, 1); } @Test public void testSetExcludedFilesNoDescriptors() throws IOException { this.descriptorReader.setExcludedFiles(this.historyMap); - this.readAllDescriptors(this.inputDirectory); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(1, 1); + this.readAllDescriptors(); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(0, 0, 1); } @Test @@ -159,7 +144,7 @@ public class DescriptorReaderImplTest { int count = this.readAllDescriptors(this.inputDirectory); assertEquals("Two files should have been parsed.", 2, count); descriptorReader.saveHistoryFile(this.historyFile); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(2, 2); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(0, 2, 2); } } diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java index 8dc89bd..aba55f4 100644 --- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java @@ -962,6 +962,16 @@ public class RelayNetworkStatusVoteImplTest { } @Test + public void testConsensusMethodTwoLines() + throws DescriptorParseException { + String consensusMethodLine = "consensus-method 1"; + RelayNetworkStatusVote vote = VoteBuilder.createWithConsensusMethodsLine( + consensusMethodLine + "\n" + consensusMethodLine); + assertEquals(Arrays.asList(consensusMethodLine, consensusMethodLine), + vote.getUnrecognizedLines()); + } + + @Test public void testPublishedNoLine() throws DescriptorParseException { this.thrown.expect(DescriptorParseException.class); this.thrown.expectMessage("Keyword 'published' is contained 0 times, " @@ -1770,7 +1780,7 @@ public class RelayNetworkStatusVoteImplTest { assertEquals(signingKeyDigestSha1, secondSignature.getSigningKeyDigestSha1Hex()); assertEquals(signatureSha1 + "\n", secondSignature.getSignature()); - assertEquals(signingKeyDigestSha1, vote.getSignatures().get(0) + assertEquals(signingKeyDigestSha1, vote.getSignatures().get(1) .getSigningKeyDigestSha1Hex()); assertEquals("c0d58c8d3c3695526f6eb5c0d9f8452b2234d303", vote.getDigestSha1Hex()); @@ -1789,6 +1799,16 @@ public class RelayNetworkStatusVoteImplTest { } @Test + public void testUnrecognizedHeaderLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusVote vote = VoteBuilder.createWithUnrecognizedHeaderLine( + unrecognizedLine); + assertEquals(Arrays.asList(unrecognizedLine), + vote.getUnrecognizedLines()); + } + + @Test public void testUnrecognizedHeaderLineIgnore() throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; @@ -1800,6 +1820,16 @@ public class RelayNetworkStatusVoteImplTest { } @Test + public void testUnrecognizedDirSourceLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusVote vote = + VoteBuilder.createWithUnrecognizedDirSourceLine(unrecognizedLine); + assertEquals(Arrays.asList(unrecognizedLine), + vote.getUnrecognizedLines()); + } + + @Test public void testUnrecognizedDirSourceLineIgnore() throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; @@ -1811,6 +1841,16 @@ public class RelayNetworkStatusVoteImplTest { } @Test + public void testUnrecognizedFooterLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusVote vote = VoteBuilder.createWithUnrecognizedFooterLine( + unrecognizedLine); + assertEquals(Arrays.asList(unrecognizedLine), + vote.getUnrecognizedLines()); + } + + @Test public void testUnrecognizedFooterLineIgnore() throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; diff --git a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java index 972e207..534cb0e 100644 --- a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java @@ -1675,6 +1675,16 @@ public class ServerDescriptorImplTest { } @Test + public void testTunnelledDirServerTypo() + throws DescriptorParseException { + String tunneledDirServerLine = "tunneled-dir-server"; + ServerDescriptor descriptor = DescriptorBuilder + .createWithTunnelledDirServerLine(tunneledDirServerLine); + assertEquals(Arrays.asList(tunneledDirServerLine), + descriptor.getUnrecognizedLines()); + } + + @Test public void testTunnelledDirServerTwice() throws DescriptorParseException { this.thrown.expect(DescriptorParseException.class); @@ -1858,6 +1868,17 @@ public class ServerDescriptorImplTest { + ROUTER_SIG_ED25519_LINE); } + @Test + public void testEd25519FollowedbyUnrecognizedLine() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + ServerDescriptor serverDecriptor = DescriptorBuilder.createWithEd25519Lines( + IDENTITY_ED25519_LINES, MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE + + "\n" + unrecognizedLine); + assertEquals(Arrays.asList(unrecognizedLine), + serverDecriptor.getUnrecognizedLines()); + } + private static final String ONION_KEY_CROSSCERT_LINES = "onion-key-crosscert\n" + "-----BEGIN CROSSCERT-----\n"

1 0

[metrics-lib/release] Rename 'DescripTor' to 'Tor Metrics Library' as part of task-22732.
by karsten＠torproject.org 28 Jun '17

28 Jun '17

commit 0d4898df6c74bb382253d7991621d9c08031ae8c Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Jun 26 20:12:13 2017 +0000 Rename 'DescripTor' to 'Tor Metrics Library' as part of task-22732. --- build.xml | 4 ++-- src/main/resources/overview.html | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build.xml b/build.xml index 0de365c..4c7cfff 100644 --- a/build.xml +++ b/build.xml @@ -7,9 +7,9 @@ <project default="usage" name="descriptor" basedir="."> <property name="release.version" value="1.9.0-dev" /> - <property name="javadoc-title" value="DescripTor API Documentation"/> + <property name="javadoc-title" value="Tor Metrics Library API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/**" /> - <property name="implementation-title" value="DescripTor" /> + <property name="implementation-title" value="Tor Metrics Library" /> <property name="name" value="descriptor" /> <property name="jarpatternprop" value="empty" /> diff --git a/src/main/resources/overview.html b/src/main/resources/overview.html index f5c6b51..a565536 100644 --- a/src/main/resources/overview.html +++ b/src/main/resources/overview.html @@ -1,6 +1,6 @@ <html> <body> -DescripTor API, which is provided and supported by Tor's Metrics +Tor Metrics Library API, which is provided and supported by Tor's Metrics Team, is a library to obtain and process descriptors containing Tor network data. It is the main Java tool for processing Tor descriptors and provides a standard API consisting of interfaces and a reference @@ -54,13 +54,13 @@ the application. Stated differently, there are no descriptor sinks that would correspond to the provided descriptor sources.</li> </ul> -Hints about using DescripTor can be found in the +Hints about using Tor Metrics Library can be found in the <a href="./org/torproject/descriptor/package-summary.html#package.description">{@code org.torproject.descriptor} package description</a>. Contact and further information: <ul> -<li><a href="https://trac.torproject.org/projects/tor/query?status=!closed&component=Met…">DescripTor Bug Tracker</a></li> +<li><a href="https://trac.torproject.org/projects/tor/query?status=!closed&component=Met…">Tor Metrics Library Bug Tracker</a></li> <li><a href="https://lists.torproject.org/cgi-bin/mailman/listinfo/metrics-team">Metrics Team Mailing List</a></li> <li><a href="https://trac.torproject.org/projects/tor/wiki/org/teams/MetricsTeam">Metrics Team Website</a></li> </ul>

1 0

[metrics-lib/release] Make tests pass again.
by karsten＠torproject.org 28 Jun '17

28 Jun '17

commit c57517c7a950e5f814ed00da5753a35c00dbf574 Author: iwakeh <iwakeh(a)torproject.org> Date: Fri Jun 23 08:37:48 2017 +0000 Make tests pass again. (one still fails: testDirectorySignaturesLinesTwoAlgorithms in RelayNetworkStatusVoteImplTest) Part of task-22154. --- .../descriptor/impl/DescriptorParserImpl.java | 47 ++++------- .../descriptor/benchmark/MeasurePerformance.java | 13 ++- .../descriptor/impl/DescriptorImplTest.java | 4 +- .../descriptor/impl/DescriptorParserImplTest.java | 8 +- .../descriptor/impl/DescriptorReaderImplTest.java | 34 ++++---- .../descriptor/impl/ExitListImplTest.java | 30 ------- .../impl/ExtraInfoDescriptorImplTest.java | 20 ----- .../impl/RelayNetworkStatusConsensusImplTest.java | 96 ++-------------------- .../impl/RelayNetworkStatusImplTest.java | 3 +- .../impl/RelayNetworkStatusVoteImplTest.java | 47 +---------- .../descriptor/impl/ServerDescriptorImplTest.java | 33 +------- .../torproject/descriptor/impl/TestDescriptor.java | 5 ++ 12 files changed, 65 insertions(+), 275 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java index dbdfca9..320139d 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java @@ -60,8 +60,7 @@ public class DescriptorParserImpl implements DescriptorParser { && firstLines.contains( NL + Key.VOTE_STATUS.keyword + SP + "consensus" + NL))) { return this.parseDescriptors(rawDescriptorBytes, descriptorFile, - Key.NETWORK_STATUS_VERSION, RelayNetworkStatusConsensusImpl.class, - includeUnparseableDescriptors); + Key.NETWORK_STATUS_VERSION, RelayNetworkStatusConsensusImpl.class); } else if (firstLines.startsWith("@type network-status-vote-3 1.") || ((firstLines.startsWith( Key.NETWORK_STATUS_VERSION.keyword + SP + "3" + NL) @@ -70,8 +69,7 @@ public class DescriptorParserImpl implements DescriptorParser { && firstLines.contains( NL + Key.VOTE_STATUS.keyword + SP + "vote" + NL))) { return this.parseDescriptors(rawDescriptorBytes, descriptorFile, - Key.NETWORK_STATUS_VERSION, RelayNetworkStatusVoteImpl.class, - includeUnparseableDescriptors); + Key.NETWORK_STATUS_VERSION, RelayNetworkStatusVoteImpl.class); } else if (firstLines.startsWith("@type bridge-network-status 1.") || firstLines.startsWith(Key.R.keyword + SP)) { List<Descriptor> parsedDescriptors = new ArrayList<>(); @@ -81,43 +79,36 @@ public class DescriptorParserImpl implements DescriptorParser { return parsedDescriptors; } else if (firstLines.startsWith("@type bridge-server-descriptor 1.")) { return this.parseDescriptors(rawDescriptorBytes, descriptorFile, - Key.ROUTER, BridgeServerDescriptorImpl.class, - includeUnparseableDescriptors); + Key.ROUTER, BridgeServerDescriptorImpl.class); } else if (firstLines.startsWith("@type server-descriptor 1.") || firstLines.startsWith(Key.ROUTER.keyword + SP) || firstLines.contains(NL + Key.ROUTER.keyword + SP)) { return this.parseDescriptors(rawDescriptorBytes, descriptorFile, - Key.ROUTER, RelayServerDescriptorImpl.class, - includeUnparseableDescriptors); + Key.ROUTER, RelayServerDescriptorImpl.class); } else if (firstLines.startsWith("@type bridge-extra-info 1.")) { return this.parseDescriptors(rawDescriptorBytes, descriptorFile, - Key.EXTRA_INFO, BridgeExtraInfoDescriptorImpl.class, - includeUnparseableDescriptors); + Key.EXTRA_INFO, BridgeExtraInfoDescriptorImpl.class); } else if (firstLines.startsWith("@type extra-info 1.") || firstLines.startsWith(Key.EXTRA_INFO.keyword + SP) || firstLines.contains(NL + Key.EXTRA_INFO.keyword + SP)) { return this.parseDescriptors(rawDescriptorBytes, descriptorFile, - Key.EXTRA_INFO, RelayExtraInfoDescriptorImpl.class, - includeUnparseableDescriptors); + Key.EXTRA_INFO, RelayExtraInfoDescriptorImpl.class); } else if (firstLines.startsWith("@type microdescriptor 1.") || firstLines.startsWith(Key.ONION_KEY.keyword + NL) || firstLines.contains(NL + Key.ONION_KEY.keyword + NL)) { return this.parseDescriptors(rawDescriptorBytes, descriptorFile, - Key.ONION_KEY, MicrodescriptorImpl.class, - includeUnparseableDescriptors); + Key.ONION_KEY, MicrodescriptorImpl.class); } else if (firstLines.startsWith("@type bridge-pool-assignment 1.") || firstLines.startsWith(Key.BRIDGE_POOL_ASSIGNMENT.keyword + SP) || firstLines.contains(NL + Key.BRIDGE_POOL_ASSIGNMENT.keyword + SP)) { return this.parseDescriptors(rawDescriptorBytes, descriptorFile, - Key.BRIDGE_POOL_ASSIGNMENT, BridgePoolAssignmentImpl.class, - includeUnparseableDescriptors); + Key.BRIDGE_POOL_ASSIGNMENT, BridgePoolAssignmentImpl.class); } else if (firstLines.startsWith("@type dir-key-certificate-3 1.") || firstLines.startsWith(Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP) || firstLines.contains( NL + Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP)) { return this.parseDescriptors(rawDescriptorBytes, descriptorFile, - Key.DIR_KEY_CERTIFICATE_VERSION, DirectoryKeyCertificateImpl.class, - includeUnparseableDescriptors); + Key.DIR_KEY_CERTIFICATE_VERSION, DirectoryKeyCertificateImpl.class); } else if (firstLines.startsWith("@type tordnsel 1.") || firstLines.startsWith("ExitNode" + SP) || firstLines.contains(NL + "ExitNode" + SP)) { @@ -131,14 +122,12 @@ public class DescriptorParserImpl implements DescriptorParser { || firstLines.contains( NL + Key.NETWORK_STATUS_VERSION.keyword + SP + "2" + NL)) { return this.parseDescriptors(rawDescriptorBytes, descriptorFile, - Key.NETWORK_STATUS_VERSION, RelayNetworkStatusImpl.class, - includeUnparseableDescriptors); + Key.NETWORK_STATUS_VERSION, RelayNetworkStatusImpl.class); } else if (firstLines.startsWith("@type directory 1.") || firstLines.startsWith(Key.SIGNED_DIRECTORY.keyword + NL) || firstLines.contains(NL + Key.SIGNED_DIRECTORY.keyword + NL)) { return this.parseDescriptors(rawDescriptorBytes, descriptorFile, - Key.SIGNED_DIRECTORY, RelayDirectoryImpl.class, - includeUnparseableDescriptors); + Key.SIGNED_DIRECTORY, RelayDirectoryImpl.class); } else if (firstLines.startsWith("@type torperf 1.")) { return TorperfResultImpl.parseTorperfResults(rawDescriptorBytes, descriptorFile); @@ -150,13 +139,13 @@ public class DescriptorParserImpl implements DescriptorParser { private List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes, File descriptorFile, Key key, - Class<? extends DescriptorImpl> descriptorClass, - boolean includeUnparseableDescriptors) throws DescriptorParseException { + Class<? extends DescriptorImpl> descriptorClass) + throws DescriptorParseException { List<Descriptor> parsedDescriptors = new ArrayList<>(); Constructor<? extends DescriptorImpl> constructor; try { constructor = descriptorClass.getDeclaredConstructor(byte[].class, - int[].class, File.class, boolean.class); + int[].class, File.class); } catch (NoSuchMethodException e) { throw new RuntimeException(e); } @@ -204,12 +193,8 @@ public class DescriptorParserImpl implements DescriptorParser { parsedDescriptors.add(this.parseDescriptor(rawDescriptorBytes, offsetAndLength, descriptorFile, constructor)); } catch (DescriptorParseException e) { - if (includeUnparseableDescriptors) { - parsedDescriptors.add(new UnparseableDescriptorImpl( - rawDescriptorBytes, offsetAndLength, descriptorFile, e)); - } else { - throw e; - } + parsedDescriptors.add(new UnparseableDescriptorImpl( + rawDescriptorBytes, offsetAndLength, descriptorFile, e)); } startAnnotations = endDescriptor; } diff --git a/src/test/java/org/torproject/descriptor/benchmark/MeasurePerformance.java b/src/test/java/org/torproject/descriptor/benchmark/MeasurePerformance.java index 83faf05..09c581d 100644 --- a/src/test/java/org/torproject/descriptor/benchmark/MeasurePerformance.java +++ b/src/test/java/org/torproject/descriptor/benchmark/MeasurePerformance.java @@ -4,7 +4,6 @@ package org.torproject.descriptor.benchmark; import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorReader; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.ExtraInfoDescriptor; @@ -136,9 +135,9 @@ public class MeasurePerformance { continue; } ExtraInfoDescriptor extraInfoDescriptor = - (ExtraInfoDescriptor) descriptor; + (ExtraInfoDescriptor) descriptor; SortedMap<String, Integer> dirreqV3Reqs = - extraInfoDescriptor.getDirreqV3Reqs(); + extraInfoDescriptor.getDirreqV3Reqs(); if (dirreqV3Reqs != null) { countries.addAll(dirreqV3Reqs.keySet()); } @@ -174,9 +173,8 @@ public class MeasurePerformance { continue; } RelayNetworkStatusConsensus consensus = - (RelayNetworkStatusConsensus) descriptor; - for (NetworkStatusEntry entry : - consensus.getStatusEntries().values()) { + (RelayNetworkStatusConsensus) descriptor; + for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { if (entry.getFlags().contains("Exit")) { totalRelaysWithExitFlag++; } @@ -217,8 +215,7 @@ public class MeasurePerformance { continue; } countedMicrodescriptors++; - Microdescriptor microdescriptor = - (Microdescriptor) descriptor; + Microdescriptor microdescriptor = (Microdescriptor) descriptor; String defaultPolicy = microdescriptor.getDefaultPolicy(); if (defaultPolicy == null) { continue; diff --git a/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java index cc97d1d..09bac87 100644 --- a/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java @@ -3,11 +3,11 @@ package org.torproject.descriptor.impl; -import org.torproject.descriptor.DescriptorParseException; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import org.torproject.descriptor.DescriptorParseException; + import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java b/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java index a2a3d9e..61dc119 100644 --- a/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java @@ -19,8 +19,9 @@ public class DescriptorParserImplTest { @Rule public ExpectedException thrown = ExpectedException.none(); - private TestDescriptor makeTestDesc(byte[] bytes) throws Exception { - return new TestDescriptor(bytes, new int[]{0, bytes.length}, false); + private TestDescriptor makeTestDesc(byte[] bytes) + throws Exception { + return new TestDescriptor(bytes, new int[]{0, bytes.length}); } private static final String MICRO = @@ -47,8 +48,7 @@ public class DescriptorParserImplTest { Constructor<? extends DescriptorImpl> constructor; try { constructor = TestServerDescriptor.class - .getDeclaredConstructor(byte[].class, int[].class, File.class, - boolean.class); + .getDeclaredConstructor(byte[].class, int[].class, File.class); } catch (NoSuchMethodException e) { throw new RuntimeException(e); } diff --git a/src/test/java/org/torproject/descriptor/impl/DescriptorReaderImplTest.java b/src/test/java/org/torproject/descriptor/impl/DescriptorReaderImplTest.java index fa791a2..a00e5ba 100644 --- a/src/test/java/org/torproject/descriptor/impl/DescriptorReaderImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/DescriptorReaderImplTest.java @@ -75,10 +75,8 @@ public class DescriptorReaderImplTest { } private void assertExcludedFilesParsedFilesAndHistoryFileLines( - int expectedExcludedFiles, int expectedParsedFiles, + int expectedParsedFiles, int expectedHistoryFileLines) throws IOException { - assertEquals(expectedExcludedFiles, - this.descriptorReader.getExcludedFiles().size()); assertEquals(expectedParsedFiles, this.descriptorReader.getParsedFiles().size()); assertEquals(expectedHistoryFileLines, @@ -89,13 +87,21 @@ public class DescriptorReaderImplTest { @Test public void testDescriptors() throws IOException { this.readAllDescriptors(this.inputDirectory); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(0, 2, 1); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(2, 1); } @Test - public void testNoDescriptors() throws IOException { - this.readAllDescriptors(null); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(0, 0, 1); + public void testNoDescriptors() throws Exception { + // calling readAllDescriptors fails with IAE + Iterator<Descriptor> descriptors = this.descriptorReader + .readDescriptors(null).iterator(); + int count = 0; + while (descriptors.hasNext()) { + count++; + descriptors.next(); + } + assertEquals(count, 0); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(0, 1); } @Test @@ -104,7 +110,7 @@ public class DescriptorReaderImplTest { IOException { this.readAllDescriptors(this.inputDirectory); Thread.sleep(100L); /* It may take a moment to write the history file. */ - this.assertExcludedFilesParsedFilesAndHistoryFileLines(1, 1, 2); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(2, 1); } @Test @@ -112,7 +118,7 @@ public class DescriptorReaderImplTest { IOException { this.readAllDescriptors(this.inputDirectory); Thread.sleep(100L); /* It may take a moment to write the history file. */ - this.assertExcludedFilesParsedFilesAndHistoryFileLines(0, 0, 0); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(2, 1); } @Test @@ -120,7 +126,7 @@ public class DescriptorReaderImplTest { this.descriptorReader.setHistoryFile(this.historyFile); this.readAllDescriptors(this.inputDirectory); descriptorReader.saveHistoryFile(this.historyFile); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(1, 1, 2); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(1, 2); } @Test @@ -128,21 +134,21 @@ public class DescriptorReaderImplTest { this.descriptorReader.setHistoryFile(this.historyFile); this.readAllDescriptors(this.inputDirectory); this.descriptorReader.saveHistoryFile(this.historyFile); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(0, 0, 0); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(1, 2); } @Test public void testSetExcludedFilesDescriptors() throws IOException { this.descriptorReader.setExcludedFiles(this.historyMap); this.readAllDescriptors(this.inputDirectory); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(1, 1, 1); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(1, 1); } @Test public void testSetExcludedFilesNoDescriptors() throws IOException { this.descriptorReader.setExcludedFiles(this.historyMap); this.readAllDescriptors(this.inputDirectory); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(0, 0, 1); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(1, 1); } @Test @@ -153,7 +159,7 @@ public class DescriptorReaderImplTest { int count = this.readAllDescriptors(this.inputDirectory); assertEquals("Two files should have been parsed.", 2, count); descriptorReader.saveHistoryFile(this.historyFile); - this.assertExcludedFilesParsedFilesAndHistoryFileLines(0, 2, 2); + this.assertExcludedFilesParsedFilesAndHistoryFileLines(2, 2); } } diff --git a/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java b/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java index dabea1e..8365d4c 100644 --- a/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java @@ -7,13 +7,11 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import org.torproject.descriptor.DescriptorParseException; -import org.torproject.descriptor.ExitList; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import java.util.HashMap; import java.util.Map; public class ExitListImplTest { @@ -32,39 +30,11 @@ public class ExitListImplTest { assertEquals(1441065722000L, result.getDownloadedMillis()); assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(), result.getUnrecognizedLines().isEmpty()); - assertEquals("Found: " + result.getEntries(), 7, - result.getEntries().size()); assertEquals("Found: " + result.getEntries(), 5, result.getEntries().size()); } @Test - public void testMultipleOldExitAddresses() throws Exception { - ExitListImpl result = new ExitListImpl( - (tordnselAnnotation + multiExitAddressInput) - .getBytes("US-ASCII"), null, fileName); - assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(), - result.getUnrecognizedLines().isEmpty()); - assertEquals("Found: " + result.getEntries(), - 3, result.getEntries().size()); - Map<String, Long> testMap = new HashMap(); - testMap.put("81.7.17.171", 1441044592000L); - testMap.put("81.7.17.172", 1441044652000L); - testMap.put("81.7.17.173", 1441044712000L); - for (ExitList.Entry ele : result.getEntries()) { - Map<String, Long> map = ele.getExitAddresses(); - assertEquals("Found: " + map, 1, map.size()); - Map.Entry<String, Long> ea = map.entrySet().iterator().next(); - assertTrue("Map: " + testMap, - testMap.keySet().contains(ea.getKey())); - assertTrue("Map: " + testMap + " exitaddress: " + ea, - testMap.values().contains(ea.getValue())); - testMap.remove(ea.getKey()); - } - assertTrue("Map: " + testMap, testMap.isEmpty()); - } - - @Test public void testMultipleExitAddresses() throws Exception { ExitListImpl result = new ExitListImpl( (tordnselAnnotation + multiExitAddressInput) diff --git a/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java index 901aca8..9633f2b 100644 --- a/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java @@ -2160,15 +2160,6 @@ public class ExtraInfoDescriptorImplTest { } @Test - public void testUnrecognizedLineFail() throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage("Unrecognized line 'unrecognized-line 1' in " - + "extra-info descriptor."); - String unrecognizedLine = "unrecognized-line 1"; - DescriptorBuilder.createWithUnrecognizedLine(unrecognizedLine); - } - - @Test public void testUnrecognizedLineIgnore() throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; @@ -2292,17 +2283,6 @@ public class ExtraInfoDescriptorImplTest { } @Test - public void testEd25519FollowedbyUnrecognizedLine() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage( - "Unrecognized line 'unrecognized-line 1' in extra-info descriptor."); - DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, - MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE - + "\nunrecognized-line 1"); - } - - @Test public void testExtraInfoDigestSha256Relay() throws DescriptorParseException { byte[] descriptorBytes = ("extra-info Unnamed " diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java index 02b89e1..cafbf31 100644 --- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java @@ -436,21 +436,13 @@ public class RelayNetworkStatusConsensusImplTest { throws DescriptorParseException { this.thrown.expect(DescriptorParseException.class); this.thrown.expectMessage( - "Unrecognized line 'directory-footer' in consensus."); + "Keyword 'directory-footer' is contained 2 times, but must be " + + "contained at most once."); ConsensusBuilder.createWithNetworkStatusVersionLine( "directory-footer\nnetwork-status-version 3"); } @Test - public void testNetworkStatusVersionPrefixLinePoundChar() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage("Unrecognized line '#consensus' in consensus."); - ConsensusBuilder.createWithNetworkStatusVersionLine( - "#consensus\nnetwork-status-version 3"); - } - - @Test public void testNetworkStatusVersionNoSpace() throws DescriptorParseException { this.thrown.expect(DescriptorParseException.class); @@ -1478,19 +1470,6 @@ public class RelayNetworkStatusConsensusImplTest { } @Test - public void testDirectoryFooterNoLine() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage("Unrecognized line 'bandwidth-weights Wbd=285 " - + "Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=1021 Wee=10000 " - + "Weg=1021 Wem=10000 Wgb=10000 Wgd=8694 Wgg=10000 Wgm=10000 Wmb=10000 " - + "Wmd=285 Wme=0 Wmg=0 Wmm=10000' in status entry."); - /* This breaks, because a bandwidth-weights line without a preceding - * directory-footer line is not allowed. */ - ConsensusBuilder.createWithDirectoryFooterLine(null); - } - - @Test public void testDirectoryFooterMissing() throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); @@ -1593,8 +1572,8 @@ public class RelayNetworkStatusConsensusImplTest { @Test public void testNonAsciiByte20() throws DescriptorParseException { this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage("Unrecognized line 'network-status-versi�n 3' in " - + "consensus."); + this.thrown.expectMessage("Keyword 'network-status-version' must be " + + "contained in the first line."); ConsensusBuilder cb = new ConsensusBuilder(); byte[] consensusBytes = cb.buildConsensusBytes(); consensusBytes[20] = (byte) 200; @@ -1606,8 +1585,8 @@ public class RelayNetworkStatusConsensusImplTest { public void testNonAsciiByteMinusOne() throws DescriptorParseException { this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage( - "Unrecognized line '�network-status-version 3' in consensus."); + this.thrown.expectMessage("Keyword 'network-status-version' " + + "must be contained in the first line."); ConsensusBuilder cb = new ConsensusBuilder(); cb.networkStatusVersionLine = "Xnetwork-status-version 3"; byte[] consensusBytes = cb.buildConsensusBytes(); @@ -1617,37 +1596,6 @@ public class RelayNetworkStatusConsensusImplTest { } @Test - public void testUnrecognizedHeaderLineFail() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage( - "Unrecognized line 'unrecognized-line 1' in consensus."); - String unrecognizedLine = "unrecognized-line 1"; - ConsensusBuilder.createWithUnrecognizedHeaderLine(unrecognizedLine); - } - - @Test - public void testUnrecognizedHeaderLineIgnore() - throws DescriptorParseException { - String unrecognizedLine = "unrecognized-line 1"; - RelayNetworkStatusConsensus consensus = ConsensusBuilder - .createWithUnrecognizedHeaderLine(unrecognizedLine); - List<String> unrecognizedLines = new ArrayList<>(); - unrecognizedLines.add(unrecognizedLine); - assertEquals(unrecognizedLines, consensus.getUnrecognizedLines()); - } - - @Test - public void testUnrecognizedDirSourceLineFail() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage( - "Unrecognized line 'unrecognized-line 1' in dir-source entry."); - String unrecognizedLine = "unrecognized-line 1"; - ConsensusBuilder.createWithUnrecognizedDirSourceLine(unrecognizedLine); - } - - @Test public void testUnrecognizedDirSourceLineIgnore() throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; @@ -1659,17 +1607,6 @@ public class RelayNetworkStatusConsensusImplTest { } @Test - public void testUnrecognizedStatusEntryLineFail() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage("Unrecognized line 'unrecognized-line 1' in " - + "status entry."); - String unrecognizedLine = "unrecognized-line 1"; - ConsensusBuilder.createWithUnrecognizedStatusEntryLine( - unrecognizedLine); - } - - @Test public void testUnrecognizedStatusEntryLineIgnore() throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; @@ -1681,16 +1618,6 @@ public class RelayNetworkStatusConsensusImplTest { } @Test - public void testUnrecognizedDirectoryFooterLineFail() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage( - "Unrecognized line 'unrecognized-line 1' in consensus."); - String unrecognizedLine = "unrecognized-line 1"; - ConsensusBuilder.createWithUnrecognizedFooterLine(unrecognizedLine); - } - - @Test public void testUnrecognizedDirectoryFooterLineIgnore() throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; @@ -1702,17 +1629,6 @@ public class RelayNetworkStatusConsensusImplTest { } @Test - public void testUnrecognizedDirectorySignatureLineFail() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage( - "Unrecognized line 'unrecognized-line 1' in dir-source entry."); - String unrecognizedLine = "unrecognized-line 1"; - ConsensusBuilder.createWithUnrecognizedDirectorySignatureLine( - unrecognizedLine); - } - - @Test public void testUnrecognizedDirectorySignatureLineIgnore() throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java index 760d04c..55ce6eb 100644 --- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java @@ -44,7 +44,8 @@ public class RelayNetworkStatusImplTest { @Test public void testParseBrokenHeader() throws DescriptorParseException { this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage("Unrecognized line 'xyx' in v2 network status."); + this.thrown.expectMessage( + "Unrecognized crypto block in v2 network status."); String invalidHeader = "network-status-version 2\nxyx\nabc"; byte[] statusBytes = (validAnnotation + invalidHeader + validFooter) .getBytes(); diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java index 060cbbb..b0f53bd 100644 --- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java @@ -791,7 +791,8 @@ public class RelayNetworkStatusVoteImplTest { public void testNetworkStatusVersionPrefixLine() throws DescriptorParseException { this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage("Unrecognized line 'directory-footer' in vote."); + this.thrown.expectMessage("Keyword 'directory-footer' is contained 2 " + + "times, but must be contained at most once."); VoteBuilder.createWithNetworkStatusVersionLine( "directory-footer\nnetwork-status-version 3"); } @@ -800,7 +801,8 @@ public class RelayNetworkStatusVoteImplTest { public void testNetworkStatusVersionPrefixLinePoundChar() throws DescriptorParseException { this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage("Unrecognized line '#vote' in vote."); + this.thrown.expectMessage("Keyword 'network-status-version' " + + "must be contained in the first line."); VoteBuilder.createWithNetworkStatusVersionLine( "#vote\nnetwork-status-version 3"); } @@ -965,16 +967,6 @@ public class RelayNetworkStatusVoteImplTest { } @Test - public void testConsensusMethodTwoLines() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage( - "Unrecognized line 'consensus-method 1' in vote."); - VoteBuilder.createWithConsensusMethodsLine( - "consensus-method 1\nconsensus-method 1"); - } - - @Test public void testPublishedNoLine() throws DescriptorParseException { this.thrown.expect(DescriptorParseException.class); this.thrown.expectMessage("Keyword 'published' is contained 0 times, " @@ -1802,16 +1794,6 @@ public class RelayNetworkStatusVoteImplTest { } @Test - public void testUnrecognizedHeaderLineFail() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage( - "Unrecognized line 'unrecognized-line 1' in vote."); - String unrecognizedLine = "unrecognized-line 1"; - VoteBuilder.createWithUnrecognizedHeaderLine(unrecognizedLine, true); - } - - @Test public void testUnrecognizedHeaderLineIgnore() throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; @@ -1823,17 +1805,6 @@ public class RelayNetworkStatusVoteImplTest { } @Test - public void testUnrecognizedDirSourceLineFail() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage( - "Unrecognized line 'unrecognized-line 1' in vote."); - String unrecognizedLine = "unrecognized-line 1"; - VoteBuilder.createWithUnrecognizedDirSourceLine(unrecognizedLine, - true); - } - - @Test public void testUnrecognizedDirSourceLineIgnore() throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; @@ -1845,16 +1816,6 @@ public class RelayNetworkStatusVoteImplTest { } @Test - public void testUnrecognizedFooterLineFail() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown - .expectMessage("Unrecognized line 'unrecognized-line 1' in vote."); - String unrecognizedLine = "unrecognized-line 1"; - VoteBuilder.createWithUnrecognizedFooterLine(unrecognizedLine, true); - } - - @Test public void testUnrecognizedFooterLineIgnore() throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; diff --git a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java index 5f369ad..67145c7 100644 --- a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java @@ -1136,7 +1136,7 @@ public class ServerDescriptorImplTest { public void testExitPolicyProjectAll() throws DescriptorParseException { this.thrown.expect(DescriptorParseException.class); this.thrown.expectMessage( - "Unrecognized line 'project *:*' in server descriptor."); + "Either keyword 'accept' or 'reject' must be contained at least once."); DescriptorBuilder.createWithExitPolicyLines("project *:*"); } @@ -1677,16 +1677,6 @@ public class ServerDescriptorImplTest { } @Test - public void testTunnelledDirServerTypo() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage("Unrecognized line 'tunneled-dir-server' " - + "in server descriptor."); - DescriptorBuilder.createWithTunnelledDirServerLine( - "tunneled-dir-server"); - } - - @Test public void testTunnelledDirServerTwice() throws DescriptorParseException { this.thrown.expect(DescriptorParseException.class); @@ -1706,16 +1696,6 @@ public class ServerDescriptorImplTest { } @Test - public void testUnrecognizedLineFail() - throws DescriptorParseException { - String unrecognizedLine = "unrecognized-line 1"; - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage( - "Unrecognized line 'unrecognized-line 1' in server descriptor."); - DescriptorBuilder.createWithUnrecognizedLine(unrecognizedLine); - } - - @Test public void testUnrecognizedLineIgnore() throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; @@ -1880,17 +1860,6 @@ public class ServerDescriptorImplTest { + ROUTER_SIG_ED25519_LINE); } - @Test - public void testEd25519FollowedbyUnrecognizedLine() - throws DescriptorParseException { - this.thrown.expect(DescriptorParseException.class); - this.thrown.expectMessage("Unrecognized line 'unrecognized-line 1' " - + "in server descriptor."); - DescriptorBuilder.createWithEd25519Lines(IDENTITY_ED25519_LINES, - MASTER_KEY_ED25519_LINE, ROUTER_SIG_ED25519_LINE - + "\nunrecognized-line 1"); - } - private static final String ONION_KEY_CROSSCERT_LINES = "onion-key-crosscert\n" + "-----BEGIN CROSSCERT-----\n" diff --git a/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java b/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java index cf5efd5..48dcb98 100644 --- a/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java +++ b/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java @@ -12,6 +12,11 @@ public class TestDescriptor extends DescriptorImpl { throws DescriptorParseException { super(rawDescriptorBytes, offsetAndLength, null, blankLinesAllowed); } + + protected TestDescriptor(byte[] rawDescriptorBytes, int[] offsetAndLength) + throws DescriptorParseException { + super(rawDescriptorBytes, offsetAndLength, null); + } }

1 0

[metrics-lib/release] Remove more deprecated code for #22154.
by karsten＠torproject.org 28 Jun '17

28 Jun '17

commit 946b261e6ca9a9c5115621db7314d4073d4bcb9b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jun 26 18:12:45 2017 +0200 Remove more deprecated code for #22154. This patch makes the following changes: - Remove (non-deprecated) method DescriptorParser#parseDescriptors(byte[], String) which we clearly forgot to deprecate earlier and which we shouldn't support in the future. It returns a List (rather than Iterable) and throws DescriptorParseException rather than include UnparseableDescriptor in the result. Might deserve an extra notice in the change log. - Put back a few lines `this.unrecognizedLines.add(line);` which got lost in previous commits. - Remove `failUnrecognizedDescriptorLines` parameters everywhere. - Remove more obsolete code from DescriptorReaderImpl. - Remove now obsolete attribute oldExitListEntries from ExitListImpl. --- .../torproject/descriptor/DescriptorParser.java | 10 -- .../descriptor/impl/BridgeNetworkStatusImpl.java | 1 + .../descriptor/impl/BridgePoolAssignmentImpl.java | 4 +- .../descriptor/impl/DescriptorParserImpl.java | 17 +-- .../descriptor/impl/DescriptorReaderImpl.java | 116 +-------------------- .../impl/DirectoryKeyCertificateImpl.java | 4 +- .../torproject/descriptor/impl/ExitListImpl.java | 3 +- .../impl/RelayNetworkStatusConsensusImpl.java | 1 + .../descriptor/impl/RelayNetworkStatusImpl.java | 3 +- .../descriptor/impl/DescriptorParserImplTest.java | 9 +- .../impl/RelayNetworkStatusImplTest.java | 4 +- .../impl/RelayNetworkStatusVoteImplTest.java | 21 ++-- .../descriptor/impl/ServerDescriptorImplTest.java | 6 +- 13 files changed, 34 insertions(+), 165 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/DescriptorParser.java b/src/main/java/org/torproject/descriptor/DescriptorParser.java index 0082b5f..0ad2790 100644 --- a/src/main/java/org/torproject/descriptor/DescriptorParser.java +++ b/src/main/java/org/torproject/descriptor/DescriptorParser.java @@ -24,16 +24,6 @@ import java.util.List; public interface DescriptorParser { /** - * Parse descriptors in the given byte array, possibly parsing the - * publication time from the file name, depending on the descriptor - * type. - * - * @since 1.0.0 - */ - public List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes, - String fileName) throws DescriptorParseException; - - /** * Parse descriptors in the given byte array and return the parsed/unparseable * descriptors. * diff --git a/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java index 01cd221..66a238e 100644 --- a/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java @@ -89,6 +89,7 @@ public class BridgeNetworkStatusImpl extends NetworkStatusImpl if (this.unrecognizedLines == null) { this.unrecognizedLines = new ArrayList<>(); } + this.unrecognizedLines.add(line); } } } diff --git a/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java index 195690b..e0e5cd8 100644 --- a/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java @@ -16,8 +16,8 @@ public class BridgePoolAssignmentImpl extends DescriptorImpl implements BridgePoolAssignment { protected BridgePoolAssignmentImpl(byte[] rawDescriptorBytes, - int[] offsetAndlength, File descriptorFile, - boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { + int[] offsetAndlength, File descriptorFile) + throws DescriptorParseException { super(rawDescriptorBytes, offsetAndlength, descriptorFile, false); this.parseDescriptorBytes(); this.checkExactlyOnceKeys(EnumSet.of(Key.BRIDGE_POOL_ASSIGNMENT)); diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java index 320139d..d39343d 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java @@ -20,18 +20,11 @@ import java.util.List; public class DescriptorParserImpl implements DescriptorParser { @Override - public List<Descriptor> parseDescriptors( - byte[] rawDescriptorBytes, String fileName) - throws DescriptorParseException { - return this.parseDescriptors(rawDescriptorBytes, null, fileName, false); - } - - @Override public Iterable<Descriptor> parseDescriptors(byte[] rawDescriptorBytes, File descriptorFile, String fileName) { try { - return this.parseDescriptors(rawDescriptorBytes, descriptorFile, fileName, - true); + return this.detectTypeAndParseDescriptors(rawDescriptorBytes, + descriptorFile, fileName); } catch (DescriptorParseException e) { /* Looks like we attempted to parse the whole raw descriptor bytes at once * below and ran into a parse issue. */ @@ -42,9 +35,9 @@ public class DescriptorParserImpl implements DescriptorParser { } } - private List<Descriptor> parseDescriptors( - byte[] rawDescriptorBytes, File descriptorFile, String fileName, - boolean includeUnparseableDescriptors) throws DescriptorParseException { + private List<Descriptor> detectTypeAndParseDescriptors( + byte[] rawDescriptorBytes, File descriptorFile, String fileName) + throws DescriptorParseException { byte[] first100Chars = new byte[Math.min(100, rawDescriptorBytes.length)]; System.arraycopy(rawDescriptorBytes, 0, first100Chars, 0, diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java index e2bf987..2d58573 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java @@ -4,7 +4,6 @@ package org.torproject.descriptor.impl; import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; import org.torproject.descriptor.DescriptorReader; @@ -24,7 +23,6 @@ import java.io.FileInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -38,12 +36,6 @@ public class DescriptorReaderImpl implements DescriptorReader { DescriptorReaderImpl.class); private boolean hasStartedReading = false; - private List<File> directories = new ArrayList<>(); - - private List<File> tarballs = new ArrayList<>(); - - private File autoSaveHistoryFile; - private File manualSaveHistoryFile; @Override @@ -84,8 +76,6 @@ public class DescriptorReaderImpl implements DescriptorReader { return new TreeMap<>(this.reader.parsedFilesAfter); } - private Integer maxDescriptorFilesInQueue = null; - private int maxDescriptorsInQueue = 100; @Override @@ -108,8 +98,7 @@ public class DescriptorReaderImpl implements DescriptorReader { this.hasStartedReading = true; BlockingIteratorImpl<Descriptor> descriptorQueue = new BlockingIteratorImpl<>(this.maxDescriptorsInQueue); - this.reader = new DescriptorReaderRunnable( - descriptorFiles, descriptorQueue, this.autoSaveHistoryFile, + this.reader = new DescriptorReaderRunnable(descriptorFiles, descriptorQueue, this.manualSaveHistoryFile, this.excludedFiles); Thread readerThread = new Thread(this.reader); readerThread.setDaemon(true); @@ -130,10 +119,6 @@ public class DescriptorReaderImpl implements DescriptorReader { private File[] descriptorFiles; - private List<File> directories; - - private List<File> tarballs; - private BlockingIteratorImpl<Descriptor> descriptorQueue; private File autoSaveHistoryFile; @@ -146,16 +131,13 @@ public class DescriptorReaderImpl implements DescriptorReader { private SortedMap<String, Long> parsedFilesAfter = new TreeMap<>(); - private boolean failUnrecognizedDescriptorLines; - private DescriptorParser descriptorParser; private boolean hasFinishedReading = false; private DescriptorReaderRunnable(File[] descriptorFiles, BlockingIteratorImpl<Descriptor> descriptorQueue, - File autoSaveHistoryFile, File manualSaveHistoryFile, - SortedMap<String, Long> excludedFiles) { + File manualSaveHistoryFile, SortedMap<String, Long> excludedFiles) { this.descriptorFiles = descriptorFiles; this.descriptorQueue = descriptorQueue; this.autoSaveHistoryFile = autoSaveHistoryFile; @@ -163,7 +145,6 @@ public class DescriptorReaderImpl implements DescriptorReader { if (excludedFiles != null) { this.excludedFilesBefore = excludedFiles; } - this.failUnrecognizedDescriptorLines = failUnrecognizedDescriptorLines; this.descriptorParser = new DescriptorParserImpl(); } @@ -172,8 +153,6 @@ public class DescriptorReaderImpl implements DescriptorReader { this.readOldHistory(this.autoSaveHistoryFile); this.readOldHistory(this.manualSaveHistoryFile); this.readDescriptorFiles(); - this.readDescriptors(); - this.readTarballs(); this.hasFinishedReading = true; } catch (Throwable t) { log.error("Bug: uncaught exception or error while " @@ -322,97 +301,6 @@ public class DescriptorReaderImpl implements DescriptorReader { this.descriptorQueue.add(descriptor); } } - - private void readDescriptors() { - if (null == this.directories) { - return; - } - for (File directory : this.directories) { - if (!directory.exists() || !directory.isDirectory()) { - continue; - } - Stack<File> files = new Stack<>(); - files.add(directory); - boolean abortReading = false; - while (!abortReading && !files.isEmpty()) { - File file = files.pop(); - if (file.isDirectory()) { - files.addAll(Arrays.asList(file.listFiles())); - } else if (file.getName().endsWith(".tar") - || file.getName().endsWith(".tar.bz2") - || file.getName().endsWith(".tar.xz")) { - this.tarballs.add(file); - } else { - String absolutePath = file.getAbsolutePath(); - long lastModifiedMillis = file.lastModified(); - if (this.excludedFilesBefore.containsKey(absolutePath) - && this.excludedFilesBefore.get(absolutePath) - == lastModifiedMillis) { - this.excludedFilesAfter.put(absolutePath, - lastModifiedMillis); - continue; - } - this.parsedFilesAfter.put(absolutePath, lastModifiedMillis); - } - } - } - } - - private void readTarballs() { - if (null == this.tarballs) { - return; - } - List<File> files = new ArrayList<>(this.tarballs); - boolean abortReading = false; - while (!abortReading && !files.isEmpty()) { - File tarball = files.remove(0); - if (!tarball.getName().endsWith(".tar") - && !tarball.getName().endsWith(".tar.bz2") - && !tarball.getName().endsWith(".tar.xz")) { - continue; - } - String absolutePath = tarball.getAbsolutePath(); - long lastModifiedMillis = tarball.lastModified(); - if (this.excludedFilesBefore.containsKey(absolutePath) - && this.excludedFilesBefore.get(absolutePath) - == lastModifiedMillis) { - this.excludedFilesAfter.put(absolutePath, lastModifiedMillis); - continue; - } - this.parsedFilesAfter.put(absolutePath, lastModifiedMillis); - try { - FileInputStream in = new FileInputStream(tarball); - if (in.available() > 0) { - TarArchiveInputStream tais = null; - if (tarball.getName().endsWith(".tar.bz2")) { - tais = new TarArchiveInputStream( - new BZip2CompressorInputStream(in)); - } else if (tarball.getName().endsWith(".tar.xz")) { - tais = new TarArchiveInputStream( - new XZCompressorInputStream(in)); - } else if (tarball.getName().endsWith(".tar")) { - tais = new TarArchiveInputStream(in); - } - BufferedInputStream bis = new BufferedInputStream(tais); - TarArchiveEntry tae = null; - while ((tae = tais.getNextTarEntry()) != null) { - if (tae.isDirectory()) { - continue; - } - } - } - } catch (IOException e) { - abortReading = true; - } - } - } - - private List<Descriptor> readFile(File file) throws IOException, - DescriptorParseException { - byte[] rawDescriptorBytes = Files.readAllBytes(file.toPath()); - return this.descriptorParser.parseDescriptors(rawDescriptorBytes, - file.getName()); - } } } diff --git a/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java b/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java index 59357d5..81ece3c 100644 --- a/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java @@ -16,8 +16,8 @@ public class DirectoryKeyCertificateImpl extends DescriptorImpl implements DirectoryKeyCertificate { protected DirectoryKeyCertificateImpl(byte[] rawDescriptorBytes, - int[] offsetAndLength, File descriptorFile, - boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { + int[] offsetAndLength, File descriptorFile) + throws DescriptorParseException { super(rawDescriptorBytes, offsetAndLength, descriptorFile, false); this.parseDescriptorBytes(); this.calculateDigestSha1Hex(Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP, diff --git a/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java b/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java index d67ccd9..454efdc 100644 --- a/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java @@ -89,6 +89,7 @@ public class ExitListImpl extends DescriptorImpl implements ExitList { if (this.unrecognizedLines == null) { this.unrecognizedLines = new ArrayList<>(); } + this.unrecognizedLines.add(line); } } /* Parse the last entry. */ @@ -117,8 +118,6 @@ public class ExitListImpl extends DescriptorImpl implements ExitList { return this.downloadedMillis; } - private Set<ExitList.Entry> oldExitListEntries = new HashSet<>(); - private Set<ExitList.Entry> exitListEntries = new HashSet<>(); @Override diff --git a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java index 7c6f7e0..87ce59a 100644 --- a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java @@ -109,6 +109,7 @@ public class RelayNetworkStatusConsensusImpl extends NetworkStatusImpl if (this.unrecognizedLines == null) { this.unrecognizedLines = new ArrayList<>(); } + this.unrecognizedLines.add(line); } } } diff --git a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java index f77e903..d2be570 100644 --- a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java @@ -20,8 +20,7 @@ public class RelayNetworkStatusImpl extends NetworkStatusImpl implements RelayNetworkStatus { protected RelayNetworkStatusImpl(byte[] statusBytes, int[] offsetAndLength, - File descriptorFile, boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { + File descriptorFile) throws DescriptorParseException { super(statusBytes, offsetAndLength, descriptorFile, false, true); Set<Key> exactlyOnceKeys = EnumSet.of( Key.NETWORK_STATUS_VERSION, Key.DIR_SOURCE, Key.FINGERPRINT, diff --git a/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java b/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java index 61dc119..df441f3 100644 --- a/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java @@ -5,6 +5,7 @@ package org.torproject.descriptor.impl; import static org.junit.Assert.assertEquals; +import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import org.junit.Rule; @@ -39,8 +40,12 @@ public class DescriptorParserImplTest { public void testAnnotation() throws Exception { TestDescriptor des = makeTestDesc(MICRO.getBytes()); DescriptorParserImpl dpi = new DescriptorParserImpl(); - assertEquals(1, - dpi.parseDescriptors(des.getRawDescriptorBytes(), "dummy.file").size()); + int parsedDescriptors = 0; + for (Descriptor d : dpi.parseDescriptors(des.getRawDescriptorBytes(), null, + "dummy.file")) { + parsedDescriptors++; + } + assertEquals(1, parsedDescriptors); } @Test diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java index 55ce6eb..a2328d3 100644 --- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java @@ -50,14 +50,14 @@ public class RelayNetworkStatusImplTest { byte[] statusBytes = (validAnnotation + invalidHeader + validFooter) .getBytes(); new RelayNetworkStatusImpl(statusBytes, new int[] { 0, statusBytes.length }, - null, true); + null); } @Test public void testValidHeader() throws DescriptorParseException { byte[] statusBytes = validStatus.getBytes(); RelayNetworkStatusImpl rnsi = new RelayNetworkStatusImpl(statusBytes, - new int[] { 0, statusBytes.length }, null, true); + new int[] { 0, statusBytes.length }, null); assertEquals(rnsi.getContactLine(), "1024R/8D56913D Alex de Joode <adejoode(a)sabotage.org>"); } diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java index b0f53bd..8dc89bd 100644 --- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java @@ -470,8 +470,7 @@ public class RelayNetworkStatusVoteImplTest { private String unrecognizedHeaderLine = null; protected static RelayNetworkStatusVote - createWithUnrecognizedHeaderLine(String line, - boolean failUnrecognizedDescriptorLines) + createWithUnrecognizedHeaderLine(String line) throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.unrecognizedHeaderLine = line; @@ -481,8 +480,7 @@ public class RelayNetworkStatusVoteImplTest { private String unrecognizedDirSourceLine = null; protected static RelayNetworkStatusVote - createWithUnrecognizedDirSourceLine(String line, - boolean failUnrecognizedDescriptorLines) + createWithUnrecognizedDirSourceLine(String line) throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.unrecognizedDirSourceLine = line; @@ -492,8 +490,7 @@ public class RelayNetworkStatusVoteImplTest { private String unrecognizedStatusEntryLine = null; protected static RelayNetworkStatusVote - createWithUnrecognizedStatusEntryLine(String line, - boolean failUnrecognizedDescriptorLines) + createWithUnrecognizedStatusEntryLine(String line) throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.unrecognizedStatusEntryLine = line; @@ -503,8 +500,7 @@ public class RelayNetworkStatusVoteImplTest { private String unrecognizedFooterLine = null; protected static RelayNetworkStatusVote - createWithUnrecognizedFooterLine(String line, - boolean failUnrecognizedDescriptorLines) + createWithUnrecognizedFooterLine(String line) throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.unrecognizedFooterLine = line; @@ -514,8 +510,7 @@ public class RelayNetworkStatusVoteImplTest { private String unrecognizedDirectorySignatureLine = null; protected static RelayNetworkStatusVote - createWithUnrecognizedDirectorySignatureLine(String line, - boolean failUnrecognizedDescriptorLines) + createWithUnrecognizedDirectorySignatureLine(String line) throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.unrecognizedDirectorySignatureLine = line; @@ -1798,7 +1793,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; RelayNetworkStatusVote vote = VoteBuilder - .createWithUnrecognizedHeaderLine(unrecognizedLine, false); + .createWithUnrecognizedHeaderLine(unrecognizedLine); List<String> unrecognizedLines = new ArrayList<>(); unrecognizedLines.add(unrecognizedLine); assertEquals(unrecognizedLines, vote.getUnrecognizedLines()); @@ -1809,7 +1804,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; RelayNetworkStatusVote vote = VoteBuilder - .createWithUnrecognizedDirSourceLine(unrecognizedLine, false); + .createWithUnrecognizedDirSourceLine(unrecognizedLine); List<String> unrecognizedLines = new ArrayList<>(); unrecognizedLines.add(unrecognizedLine); assertEquals(unrecognizedLines, vote.getUnrecognizedLines()); @@ -1820,7 +1815,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { String unrecognizedLine = "unrecognized-line 1"; RelayNetworkStatusVote vote = VoteBuilder - .createWithUnrecognizedFooterLine(unrecognizedLine, false); + .createWithUnrecognizedFooterLine(unrecognizedLine); List<String> unrecognizedLines = new ArrayList<>(); unrecognizedLines.add(unrecognizedLine); assertEquals(unrecognizedLines, vote.getUnrecognizedLines()); diff --git a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java index 67145c7..972e207 100644 --- a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java @@ -315,8 +315,7 @@ public class ServerDescriptorImplTest { private byte[] nonAsciiLineBytes = null; private static ServerDescriptor createWithNonAsciiLineBytes( - byte[] lineBytes, boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { + byte[] lineBytes) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.nonAsciiLineBytes = lineBytes; return db.buildDescriptor(); @@ -1558,8 +1557,7 @@ public class ServerDescriptorImplTest { 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x2d, // "allow-" 0x73, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x2d, // "single-" 0x68, 0x6f, 0x70, 0x2d, // "hop-" - 0x65, 0x78, 0x69, 0x74, 0x73 }, // "exits" (no newline) - false); + 0x65, 0x78, 0x69, 0x74, 0x73 }); // "exits" (no newline) } @Test

1 0

[metrics-lib/release] All examples compile and work as explained in Metrics' web-site tutorials.
by karsten＠torproject.org 28 Jun '17

28 Jun '17

commit 27f05e1d4c67d8954b5cd2975781870a0dcdb9d9 Author: iwakeh <iwakeh(a)torproject.org> Date: Mon Jun 26 20:12:11 2017 +0000 All examples compile and work as explained in Metrics' web-site tutorials. Added a hint about where to find the examples to the README.md file. Implements task-22734. --- README.md | 6 +++ .../examples/ConsensusWeightByVersion.java | 45 +++++++++------------- .../resources/examples/PluggableTransports.java | 29 ++++++-------- 3 files changed, 36 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 0cc93d8..aa309ac 100644 --- a/README.md +++ b/README.md @@ -97,3 +97,9 @@ The Metrics website has a tutorial for getting started with metrics-lib: https://metrics.torproject.org/metrics-lib.html +The examples explained in the tutorials are available as source code in + +``` +src/main/resources/examples/ +``` + diff --git a/src/main/resources/examples/ConsensusWeightByVersion.java b/src/main/resources/examples/ConsensusWeightByVersion.java index 3b8495d..5fc7c1c 100644 --- a/src/main/resources/examples/ConsensusWeightByVersion.java +++ b/src/main/resources/examples/ConsensusWeightByVersion.java @@ -19,35 +19,26 @@ public class ConsensusWeightByVersion { // Read descriptors from disk. DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - - // Add the directory with descriptors to the descriptor reader. - descriptorReader.addDirectory(new File("descriptors/recent/relay-descriptors/consensuses")); - Iterator<DescriptorFile> descriptorFiles = descriptorReader.readDescriptors(); - while (descriptorFiles.hasNext()) { // Iterate over all descriptor files found. - DescriptorFile descriptorFile = descriptorFiles.next(); - - // Now, iterate over the descriptors contained in the file. - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (!(descriptor instanceof RelayNetworkStatusConsensus)) { - // We're only interested in consensuses. + for (Descriptor descriptor : descriptorReader.readDescriptors(new File("descriptors/recent/relay-descriptors/consensuses"))) { + if (!(descriptor instanceof RelayNetworkStatusConsensus)) { + // We're only interested in consensuses. + continue; + } + RelayNetworkStatusConsensus consensus = (RelayNetworkStatusConsensus) descriptor; + for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { + String version = entry.getVersion(); + if (!version.startsWith("Tor ") || version.length() < 9) { + // We're only interested in a.b.c type versions for this example. continue; } - RelayNetworkStatusConsensus consensus = (RelayNetworkStatusConsensus) descriptor; - for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { - String version = entry.getVersion(); - if (!version.startsWith("Tor ") || version.length() < 9) { - // We're only interested in a.b.c type versions for this example. - continue; - } - // Remove the 'Tor ' prefix and anything starting at the patch level. - version = version.substring(4, 9); - long bandwidth = entry.getBandwidth(); - totalBandwidth += bandwidth; - if (bandwidthByVersion.containsKey(version)) { - bandwidthByVersion.put(version, bandwidth + bandwidthByVersion.get(version)); - } else { - bandwidthByVersion.put(version, bandwidth); - } + // Remove the 'Tor ' prefix and anything starting at the patch level. + version = version.substring(4, 9); + long bandwidth = entry.getBandwidth(); + totalBandwidth += bandwidth; + if (bandwidthByVersion.containsKey(version)) { + bandwidthByVersion.put(version, bandwidth + bandwidthByVersion.get(version)); + } else { + bandwidthByVersion.put(version, bandwidth); } } } diff --git a/src/main/resources/examples/PluggableTransports.java b/src/main/resources/examples/PluggableTransports.java index 88e7eb6..65ce732 100644 --- a/src/main/resources/examples/PluggableTransports.java +++ b/src/main/resources/examples/PluggableTransports.java @@ -16,23 +16,18 @@ public class PluggableTransports { SortedMap<String, Integer> countedTransports = new TreeMap<>(); DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - descriptorReader.addDirectory(new File("descriptors/recent/bridge-descriptors/extra-infos")); - Iterator<DescriptorFile> descriptorFiles = descriptorReader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (!(descriptor instanceof BridgeExtraInfoDescriptor)) { - continue; - } - BridgeExtraInfoDescriptor extraInfo = (BridgeExtraInfoDescriptor) descriptor; - String fingerprint = extraInfo.getFingerprint(); - if (observedFingerprints.add(fingerprint)) { - for (String transport : extraInfo.getTransports()) { - if (countedTransports.containsKey(transport)) { - countedTransports.put(transport, 1 + countedTransports.get(transport)); - } else { - countedTransports.put(transport, 1); - } + for (Descriptor descriptor : descriptorReader.readDescriptors(new File("descriptors/recent/bridge-descriptors/extra-infos"))) { + if (!(descriptor instanceof BridgeExtraInfoDescriptor)) { + continue; + } + BridgeExtraInfoDescriptor extraInfo = (BridgeExtraInfoDescriptor) descriptor; + String fingerprint = extraInfo.getFingerprint(); + if (observedFingerprints.add(fingerprint)) { + for (String transport : extraInfo.getTransports()) { + if (countedTransports.containsKey(transport)) { + countedTransports.put(transport, 1 + countedTransports.get(transport)); + } else { + countedTransports.put(transport, 1); } } }

1 0

[metrics-lib/master] Prepare for 2.0.0 release.
by karsten＠torproject.org 28 Jun '17

28 Jun '17

commit 82a8d92b66dfec42918d131a89457acd33f3f660 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jun 28 11:49:27 2017 +0200 Prepare for 2.0.0 release. --- CHANGELOG.md | 3 ++- build.xml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4754dde..a0eb4ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ -# Changes in version 2.0.0 - 2017-06-?? +# Changes in version 2.0.0 - 2017-06-28 * Major changes - Always use UTF-8 as charset rather than using the platform's default charset. + - Remove all code that was deprecated in 1.x versions. * Minor changes - Replace custom ImplementationNotAccessibleException thrown by diff --git a/build.xml b/build.xml index 765d16f..8355298 100644 --- a/build.xml +++ b/build.xml @@ -6,7 +6,7 @@ <project default="usage" name="metrics-lib" basedir="."> - <property name="release.version" value="1.9.0-dev" /> + <property name="release.version" value="2.0.0" /> <property name="javadoc-title" value="Tor Metrics Library API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/**" /> <property name="implementation-title" value="Tor Metrics Library" />

1 0

[stem/master] Generated onion-key were invalid
by atagar＠torproject.org 28 Jun '17

28 Jun '17

commit eb07135e7b0f11ab784858c577c6d12e189722c2 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jun 28 10:13:13 2017 -0700 Generated onion-key were invalid Oops, our change to create random fields caused this to be rendered as a list. --- stem/descriptor/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 21e7bd7..f293c6c 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -1047,7 +1047,7 @@ def _random_crypto_blob(block_type = None): Provides a random string that can be used for crypto blocks. """ - crypto_blob = stem.util.str_tools._split_by_length(base64.b64encode('%0140x' % random.randrange(16 ** 140)), 64) + crypto_blob = '\n'.join(stem.util.str_tools._split_by_length(base64.b64encode('%0140x' % random.randrange(16 ** 140)), 64)) if block_type: return '\n-----BEGIN %s-----\n%s\n-----END %s-----' % (block_type, crypto_blob, block_type)

1 0

[stem/master] Ensure created nicknames aren't too long
by atagar＠torproject.org 28 Jun '17

28 Jun '17

commit a3194c8a344af5a613dbc528805bd379c3f41cf0 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jun 28 10:04:26 2017 -0700 Ensure created nicknames aren't too long Jenkins tests are failing because the random nicknames we make are often too long. Ensuring that isn't the case. --- stem/descriptor/server_descriptor.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 580b3d3..8228b1d 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -806,8 +806,10 @@ class RelayDescriptor(ServerDescriptor): if attr is None: attr = {} + nickname = ('Unnamed%i' % random.randint(0, sys.maxint))[:19] + base_header = ( - ('router', 'Unnamed%i %s 9001 0 0' % (random.randint(0, sys.maxint), _random_ipv4_address())), + ('router', '%s %s 9001 0 0' % (nickname, _random_ipv4_address())), ('published', _random_date()), ('bandwidth', '153600 256000 104590'), ('reject', '*:*'),

1 0

[translation/tails-misc_completed] Update translations for tails-misc_completed
by translation＠torproject.org 28 Jun '17

28 Jun '17

commit 149a316854cee892fa8bebc365c6a3774e3d6f13 Author: Translation commit bot <translation(a)torproject.org> Date: Wed Jun 28 16:48:19 2017 +0000 Update translations for tails-misc_completed --- it.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/it.po b/it.po index 0b029a1..f4b02c7 100644 --- a/it.po +++ b/it.po @@ -29,7 +29,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2017-06-05 11:57+0200\n" -"PO-Revision-Date: 2017-05-26 14:47+0000\n" +"PO-Revision-Date: 2017-06-28 16:35+0000\n" "Last-Translator: carolyn <carolyn(a)anhalt.org>\n" "Language-Team: Italian (http://www.transifex.com/otf/torproject/language/it/)\n" "MIME-Version: 1.0\n"

1 0

[translation/tails-misc] Update translations for tails-misc
by translation＠torproject.org 28 Jun '17

28 Jun '17

commit 8f630d84ca2bf3a1e64f40f3e30b1c92c00cde55 Author: Translation commit bot <translation(a)torproject.org> Date: Wed Jun 28 16:47:52 2017 +0000 Update translations for tails-misc --- it.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/it.po b/it.po index 0b029a1..f4b02c7 100644 --- a/it.po +++ b/it.po @@ -29,7 +29,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2017-06-05 11:57+0200\n" -"PO-Revision-Date: 2017-05-26 14:47+0000\n" +"PO-Revision-Date: 2017-06-28 16:35+0000\n" "Last-Translator: carolyn <carolyn(a)anhalt.org>\n" "Language-Team: Italian (http://www.transifex.com/otf/torproject/language/it/)\n" "MIME-Version: 1.0\n"

1 0