tor-commits October 2018

tor-commits@lists.torproject.org

17 participants
2350 discussions

[collector/master] Bump version to 1.8.0-dev.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 2c379bd46924053b086d7d1669b6ef3246817350 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Oct 15 16:40:15 2018 +0200 Bump version to 1.8.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index d978d73..5cea51a 100644 --- a/build.xml +++ b/build.xml @@ -8,7 +8,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.8.0" /> + <property name="release.version" value="1.8.0-dev" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.4.0" />

1 0

[collector/release] Simplify bridge descriptor tests.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 61881b9c76ff821d2f42b05a950524c7a087f8a2 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 11:01:28 2018 +0200 Simplify bridge descriptor tests. --- .../bridgedescs/SanitizedBridgesWriterTest.java | 84 +++++++++++----------- .../bridgedescs/TestDescriptorBuilder.java | 11 ++- 2 files changed, 46 insertions(+), 49 deletions(-) diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java index 91332a5..b791f88 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java @@ -224,7 +224,7 @@ public class SanitizedBridgesWriterTest { public void testServerDescriptorAdditionalAnnotation() throws Exception { this.defaultServerTestDescriptorBuilder.insertBeforeLineStartingWith( - "@purpose bridge", Arrays.asList("@source 198.50.200.131")); + "@purpose bridge", "@source 198.50.200.131"); this.runTest(); assertEquals("Expected 3 sanitized descriptors.", 3, this.parsedFiles.size()); @@ -237,7 +237,7 @@ public class SanitizedBridgesWriterTest { this.configuration.setProperty(Key.BridgeDescriptorMappingsLimit.name(), "30000"); this.defaultServerTestDescriptorBuilder.insertBeforeLineStartingWith( - "platform ", Arrays.asList("or-address [2:5:2:5:2:5:2:5]:25")); + "platform ", "or-address [2:5:2:5:2:5:2:5]:25"); Path bridgeIpSecretsFile = Paths.get(statsDirectory, "bridge-ip-secrets"); BufferedWriter writer = Files.newBufferedWriter(bridgeIpSecretsFile, StandardCharsets.UTF_8); @@ -259,7 +259,7 @@ public class SanitizedBridgesWriterTest { @Test public void testServerDescriptorRouterLineTruncated() throws Exception { this.defaultServerTestDescriptorBuilder.replaceLineStartingWith("router ", - Arrays.asList("router MeekGoogle")); + "router MeekGoogle"); this.runTest(); assertTrue("Sanitized server descriptor with invalid router line.", this.parsedServerDescriptors.isEmpty()); @@ -269,8 +269,8 @@ public class SanitizedBridgesWriterTest { public void testServerDescriptorProtoLine() throws Exception { this.defaultServerTestDescriptorBuilder .replaceLineStartingWith("protocols ", - Arrays.asList("proto Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 " - + "HSRend=1-2 Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2")); + "proto Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 " + + "HSRend=1-2 Link=1-4 LinkAuth=1 Microdesc=1-2 Relay=1-2"); this.runTest(); assertFalse("Sanitized server descriptor with valid proto line.", this.parsedServerDescriptors.isEmpty()); @@ -279,7 +279,7 @@ public class SanitizedBridgesWriterTest { @Test public void testServerDescriptorFingerprintTruncated() throws Exception { this.defaultServerTestDescriptorBuilder.replaceLineStartingWith( - "fingerprint ", Arrays.asList("fingerprint 4")); + "fingerprint ", "fingerprint 4"); this.runTest(); assertTrue("Sanitized server descriptor with invalid fingerprint " + "line.", this.parsedServerDescriptors.isEmpty()); @@ -289,7 +289,7 @@ public class SanitizedBridgesWriterTest { public void testServerDescriptorFingerprintInvalidHex() throws Exception { this.defaultServerTestDescriptorBuilder.replaceLineStartingWith( - "fingerprint ", Arrays.asList("fingerprint FUN!")); + "fingerprint ", "fingerprint FUN!"); this.runTest(); assertTrue("Sanitized server descriptor with invalid fingerprint " + "line.", this.parsedServerDescriptors.isEmpty()); @@ -299,8 +299,7 @@ public class SanitizedBridgesWriterTest { public void testServerDescriptorFingerprintOpt() throws Exception { this.defaultServerTestDescriptorBuilder .replaceLineStartingWith("fingerprint ", - Arrays.asList("opt fingerprint 46D4 A711 97B8 FA51 5A82 6C6B 017C 522F " - + "E264 655B")); + "opt fingerprint 46D4 A711 97B8 FA51 5A82 6C6B 017C 522F E264 655B"); this.runTest(); this.parsedServerDescriptors.get(0).contains("opt fingerprint 88F7 " + "4584 0F47 CE0C 6A4F E61D 8279 50B0 6F9E 4534"); @@ -310,7 +309,7 @@ public class SanitizedBridgesWriterTest { public void testServerDescriptorExtraInfoDigestInvalidHex() throws Exception { this.defaultServerTestDescriptorBuilder.replaceLineStartingWith( - "extra-info-digest ", Arrays.asList("extra-info-digest 6")); + "extra-info-digest ", "extra-info-digest 6"); this.runTest(); assertTrue("Sanitized server descriptor with invalid extra-info " + "line.", this.parsedServerDescriptors.isEmpty()); @@ -320,9 +319,9 @@ public class SanitizedBridgesWriterTest { public void testServerDescriptorExtraInfoDigestInvalidBase64() throws Exception { this.defaultServerTestDescriptorBuilder.replaceLineStartingWith( - "extra-info-digest ", Arrays.asList("extra-info-digest " + "extra-info-digest ", "extra-info-digest " + "6D03E80568DEFA102968D144CB35FFA6E3355B8A " - + "#*?$%x@nxukmmcT1+UnDg4qh0yKbjVUYKhGL8VksoJA")); + + "#*?$%x@nxukmmcT1+UnDg4qh0yKbjVUYKhGL8VksoJA"); this.runTest(); assertTrue("Invalid base64 in server descriptor accepted.", this.parsedServerDescriptors.isEmpty()); @@ -332,8 +331,8 @@ public class SanitizedBridgesWriterTest { public void testServerDescriptorExtraInfoDigestSha1Only() throws Exception { this.defaultServerTestDescriptorBuilder.replaceLineStartingWith( - "extra-info-digest ", Arrays.asList("extra-info-digest " - + "6D03E80568DEFA102968D144CB35FFA6E3355B8A")); + "extra-info-digest ", "extra-info-digest " + + "6D03E80568DEFA102968D144CB35FFA6E3355B8A"); this.runTest(); assertTrue("Expected different extra-info-digest line.", this.parsedServerDescriptors.get(0).contains( @@ -344,9 +343,9 @@ public class SanitizedBridgesWriterTest { public void testServerDescriptorExtraInfoDigestThirdArgument() throws Exception { this.defaultServerTestDescriptorBuilder.replaceLineStartingWith( - "extra-info-digest ", Arrays.asList("extra-info-digest " + "extra-info-digest ", "extra-info-digest " + "6D03E80568DEFA102968D144CB35FFA6E3355B8A " - + "cy/LwP7nxukmmcT1+UnDg4qh0yKbjVUYKhGL8VksoJA 00")); + + "cy/LwP7nxukmmcT1+UnDg4qh0yKbjVUYKhGL8VksoJA 00"); this.runTest(); assertTrue("Third argument to extra-info-digest line should not be " + "dropped silently.", this.parsedServerDescriptors.isEmpty()); @@ -355,9 +354,9 @@ public class SanitizedBridgesWriterTest { @Test public void testServerDescriptorExtraInfoDigestOpt() throws Exception { this.defaultServerTestDescriptorBuilder.replaceLineStartingWith( - "extra-info-digest ", Arrays.asList("opt extra-info-digest " + "extra-info-digest ", "opt extra-info-digest " + "6D03E80568DEFA102968D144CB35FFA6E3355B8A " - + "cy/LwP7nxukmmcT1+UnDg4qh0yKbjVUYKhGL8VksoJA")); + + "cy/LwP7nxukmmcT1+UnDg4qh0yKbjVUYKhGL8VksoJA"); this.runTest(); this.parsedServerDescriptors.get(0).contains("opt extra-info-digest " + "B026CF0F81712D94BBF1362294882688DF247887 " @@ -367,7 +366,7 @@ public class SanitizedBridgesWriterTest { @Test public void testServerDescriptorRejectOwnAddress() throws Exception { this.defaultServerTestDescriptorBuilder.insertBeforeLineStartingWith( - "reject *:*", Arrays.asList("reject 198.50.200.131:*", "accept *:80")); + "reject *:*", "reject 198.50.200.131:*", "accept *:80"); this.runTest(); List<String> parsedLines = this.parsedServerDescriptors.get(0); for (int i = 0; i < parsedLines.size(); i++) { @@ -387,8 +386,8 @@ public class SanitizedBridgesWriterTest { public void testServerDescriptorEd25519IdentityMasterKeyMismatch() throws Exception { this.defaultServerTestDescriptorBuilder.replaceLineStartingWith( - "master-key-ed25519 ", Arrays.asList("master-key-ed25519 " - + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + "master-key-ed25519 ", "master-key-ed25519 " + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"); this.runTest(); assertTrue("Mismatch between Ed25519 identity and master key.", this.parsedServerDescriptors.isEmpty()); @@ -397,12 +396,12 @@ public class SanitizedBridgesWriterTest { @Test public void testServerDescriptorEd25519IdentityA() throws Exception { this.defaultServerTestDescriptorBuilder.replaceLineStartingWith( - "identity-ed25519", Arrays.asList("identity-ed25519", + "identity-ed25519", "identity-ed25519", "-----BEGIN ED25519 CERT-----", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "-----END ED25519 CERT-----")); + "-----END ED25519 CERT-----"); this.runTest(); assertTrue("Ed25519 identity all A's conflicts with master key?", this.parsedServerDescriptors.isEmpty()); @@ -414,7 +413,7 @@ public class SanitizedBridgesWriterTest { "ZEXE7RkiEJ1l5Ij9hc9TJOpM7/9XSPZnF/PbMfE0u3n3JbOO3s82GN6BPuA0v2Cs"; this.defaultServerTestDescriptorBuilder .replaceLineStartingWith(change9sTo6s, - Arrays.asList(change9sTo6s.replaceAll("9", "6"))); + change9sTo6s.replaceAll("9", "6")); this.runTest(); assertTrue("Mismatch between identity and master key.", this.parsedServerDescriptors.isEmpty()); @@ -423,12 +422,12 @@ public class SanitizedBridgesWriterTest { @Test public void testServerDescriptorEd25519IdentitySlash() throws Exception { this.defaultServerTestDescriptorBuilder.replaceLineStartingWith( - "identity-ed25519", Arrays.asList("identity-ed25519", + "identity-ed25519", "identity-ed25519", "-----BEGIN ED25519 CERT-----", "////////////////////////////////////////////////////////////////", "////////////////////////////////////////////////////////////////", "///////////////////////////////////////////////////////////=", - "-----END ED25519 CERT-----")); + "-----END ED25519 CERT-----"); this.runTest(); assertTrue("Ed25519 identity all slashes conflicts with master key.", this.parsedServerDescriptors.isEmpty()); @@ -438,7 +437,7 @@ public class SanitizedBridgesWriterTest { public void testServerDescriptorFamilyInvalidFingerprint() throws Exception { this.defaultServerTestDescriptorBuilder.insertBeforeLineStartingWith( - "hidden-service-dir", Arrays.asList("family $0")); + "hidden-service-dir", "family $0"); this.runTest(); assertTrue("Sanitized server descriptor with invalid fingerprint in " + "family line.", this.parsedServerDescriptors.isEmpty()); @@ -453,7 +452,7 @@ public class SanitizedBridgesWriterTest { "fingerprint 46D4 A711 97B8 FA51 5A82 6C6B 017C 522F E264 655B"; this.defaultServerTestDescriptorBuilder.removeLine(fingerprintLine); this.defaultServerTestDescriptorBuilder.insertBeforeLineStartingWith( - "published ", Arrays.asList(fingerprintLine)); + "published ", fingerprintLine); this.runTest(); assertFalse(this.parsedServerDescriptors.isEmpty()); } @@ -499,7 +498,7 @@ public class SanitizedBridgesWriterTest { public void testExtraInfoDescriptorExtraInfoLineTruncated() throws Exception { this.defaultExtraInfoTestDescriptorBuilder.replaceLineStartingWith( - "extra-info ", Arrays.asList("extra-info ")); + "extra-info ", "extra-info "); this.runTest(); } @@ -507,7 +506,7 @@ public class SanitizedBridgesWriterTest { public void testExtraInfoDescriptorExtraInfoInvalidHex() throws Exception { this.defaultExtraInfoTestDescriptorBuilder.replaceLineStartingWith( - "extra-info ", Arrays.asList("extra-info MeekGoogle 4")); + "extra-info ", "extra-info MeekGoogle 4"); this.runTest(); assertTrue("Sanitized extra-info descriptor with invalid extra-info " + "line.", this.parsedExtraInfoDescriptors.isEmpty()); @@ -516,7 +515,7 @@ public class SanitizedBridgesWriterTest { @Test public void testExtraInfoDescriptorTransportSpace() throws Exception { this.defaultExtraInfoTestDescriptorBuilder.replaceLineStartingWith( - "transport ", Arrays.asList("transport ")); + "transport ", "transport "); this.runTest(); assertTrue("Sanitized extra-info descriptor with invalid transport " + "line.", this.parsedExtraInfoDescriptors.isEmpty()); @@ -525,7 +524,7 @@ public class SanitizedBridgesWriterTest { @Test public void testExtraInfoDescriptorTransportInfoRemoved() throws Exception { this.defaultExtraInfoTestDescriptorBuilder.insertBeforeLineStartingWith( - "bridge-stats-end ", Arrays.asList("transport-info secretkey")); + "bridge-stats-end ", "transport-info secretkey"); this.runTest(); for (String line : this.parsedExtraInfoDescriptors.get(0)) { assertFalse("transport-info line should not have been retained.", @@ -536,11 +535,10 @@ public class SanitizedBridgesWriterTest { @Test public void testExtraInfoDescriptorHidservRetained() throws Exception { this.defaultExtraInfoTestDescriptorBuilder.insertBeforeLineStartingWith( - "transport ", - Arrays.asList("hidserv-stats-end 2016-11-23 14:48:05 (86400 s)", + "transport ", "hidserv-stats-end 2016-11-23 14:48:05 (86400 s)", "hidserv-rend-relayed-cells 27653088 delta_f=2048 epsilon=0.30 " + "bin_size=1024", - "hidserv-dir-onions-seen 204 delta_f=8 epsilon=0.30 bin_size=8")); + "hidserv-dir-onions-seen 204 delta_f=8 epsilon=0.30 bin_size=8"); this.runTest(); int foundHidservLines = 0; if (!this.parsedExtraInfoDescriptors.isEmpty()) { @@ -558,12 +556,12 @@ public class SanitizedBridgesWriterTest { public void testExtraInfoDescriptorPaddingCountsRetained() throws Exception { this.defaultExtraInfoTestDescriptorBuilder.insertBeforeLineStartingWith( "transport ", - Arrays.asList("padding-counts 2017-05-10 01:48:43 (86400 s) " + "padding-counts 2017-05-10 01:48:43 (86400 s) " + "bin-size=10000 write-drop=10000 write-pad=10000 " + "write-total=10000 read-drop=10000 read-pad=10000 " + "read-total=70000 enabled-read-pad=0 enabled-read-total=0 " + "enabled-write-pad=0 enabled-write-total=0 " - + "max-chanpad-timers=0")); + + "max-chanpad-timers=0"); this.runTest(); int foundPaddingCountsLines = 0; if (!this.parsedExtraInfoDescriptors.isEmpty()) { @@ -581,7 +579,7 @@ public class SanitizedBridgesWriterTest { public void testExtraInfoDescriptorRouterSignatureLineSpace() throws Exception { this.defaultExtraInfoTestDescriptorBuilder.replaceLineStartingWith( - "router-signature", Arrays.asList("router-signature ")); + "router-signature", "router-signature "); this.runTest(); assertTrue("Sanitized extra-info descriptor with invalid " + "router-signature line.", @@ -639,16 +637,16 @@ public class SanitizedBridgesWriterTest { @Test public void testNetworkStatusRlineTruncated() throws Exception { this.defaultNetworkStatusTestDescriptorBuilder.replaceLineStartingWith("r ", - Arrays.asList("r MeekGoogle")); + "r MeekGoogle"); this.runTest(); } @Test public void testNetworkStatusRlineInvalidBase64() throws Exception { this.defaultNetworkStatusTestDescriptorBuilder.replaceLineStartingWith("r ", - Arrays.asList("r MeekGoogle R#SnE*e4+lFag:xr_XxSL+J;ZVs " + "r MeekGoogle R#SnE*e4+lFag:xr_XxSL+J;ZVs " + "g+M7'w+lG$mv6NW9&RmvzLO(R0Y 2016-06-30 21:43:52 " - + "198.50.200.131 8008 0")); + + "198.50.200.131 8008 0"); this.runTest(); assertTrue("Should not have accepted invalid base64.", this.parsedNetworkStatuses.isEmpty()); @@ -659,7 +657,7 @@ public class SanitizedBridgesWriterTest { this.configuration.setProperty(Key.ReplaceIpAddressesWithHashes.name(), "true"); this.defaultNetworkStatusTestDescriptorBuilder - .insertBeforeLineStartingWith("s ", Arrays.asList("a 198.50.200.132")); + .insertBeforeLineStartingWith("s ", "a 198.50.200.132"); this.runTest(); for (String line : this.parsedNetworkStatuses.get(0)) { if (line.startsWith("a ")) { @@ -671,7 +669,7 @@ public class SanitizedBridgesWriterTest { @Test public void testNetworkStatusVLineUnknown() throws Exception { this.defaultNetworkStatusTestDescriptorBuilder - .insertBeforeLineStartingWith("w ", Arrays.asList("v Tor 0.2.7.6")); + .insertBeforeLineStartingWith("w ", "v Tor 0.2.7.6"); this.runTest(); assertTrue("Should not have sanitized status with v line which is unknown " + "in this descriptor type.", this.parsedNetworkStatuses.isEmpty()); diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/TestDescriptorBuilder.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/TestDescriptorBuilder.java index 2fad06a..81df984 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/TestDescriptorBuilder.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/TestDescriptorBuilder.java @@ -10,7 +10,6 @@ import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; -import java.util.List; /** Builds a descriptor by concatenating the given lines with newlines and * writing the output to the given output stream. */ @@ -33,10 +32,10 @@ abstract class TestDescriptorBuilder extends ArrayList<String> { * given lines before it, or fails if no line can be found with that line * start. */ void insertBeforeLineStartingWith(String lineStart, - List<String> linesToInsert) { + String ... linesToInsert) { for (int i = 0; i < this.size(); i++) { if (this.get(i).startsWith(lineStart)) { - this.addAll(i, linesToInsert); + this.addAll(i, Arrays.asList(linesToInsert)); return; } } @@ -46,13 +45,13 @@ abstract class TestDescriptorBuilder extends ArrayList<String> { /** Finds the first line that starts with the given line start and replaces * that line and possibly subsequent lines, or fails if no line can be found * with that line start or there are not enough lines left to replace. */ - void replaceLineStartingWith(String lineStart, List<String> linesToReplace) { + void replaceLineStartingWith(String lineStart, String ... linesToReplace) { for (int i = 0; i < this.size(); i++) { if (this.get(i).startsWith(lineStart)) { - for (int j = 0; j < linesToReplace.size(); j++) { + for (int j = 0; j < linesToReplace.length; j++) { assertTrue("Not enough lines left to replace.", this.size() > i + j); - this.set(i + j, linesToReplace.get(j)); + this.set(i + j, linesToReplace[j]); } return; }

1 0

[collector/release] Update metrics-base.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit c2073cf672632337110c08058725bb390ff420ed Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 11:03:54 2018 +0200 Update metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 23c6e0b..08514a3 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 23c6e0be5fab9463f137615053ef412e4da2315e +Subproject commit 08514a32afefbeef848b80f9a338ee840c282604

1 0

[collector/release] Unwrap if statements that are always true.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 5223617c6edab1d193918a33651e3738f30719eb Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 10:35:17 2018 +0200 Unwrap if statements that are always true. We're already checking whether rdp is null at the beginning. No need to check this again and again. --- .../collector/relaydescs/ArchiveReader.java | 232 ++++++++++----------- .../relaydescs/CachedRelayDescriptorReader.java | 58 +++--- 2 files changed, 140 insertions(+), 150 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java index 22a3113..3976919 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java @@ -90,62 +90,17 @@ public class ArchiveReader { filesInInputDir.add(f); } } else { - if (rdp != null) { - try { - BufferedInputStream bis = null; - if (keepImportHistory - && archivesImportHistory.contains(pop.getName())) { - ignoredFiles++; - continue; - } else if (pop.getName().endsWith(".tar.bz2")) { - logger.warn("Cannot parse compressed tarball " - + pop.getAbsolutePath() + ". Skipping."); - continue; - } else if (pop.getName().endsWith(".bz2")) { - FileInputStream fis = new FileInputStream(pop); - BZip2CompressorInputStream bcis = - new BZip2CompressorInputStream(fis); - bis = new BufferedInputStream(bcis); - } else { - FileInputStream fis = new FileInputStream(pop); - bis = new BufferedInputStream(fis); - } - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] data = new byte[1024]; - while ((len = bis.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - bis.close(); - byte[] allData = baos.toByteArray(); - boolean stored = rdp.parse(allData); - if (!stored) { - filesToRetry.add(pop); - continue; - } - if (keepImportHistory) { - archivesImportHistory.add(pop.getName()); - } - parsedFiles++; - } catch (IOException e) { - problems.add(pop); - if (problems.size() > 3) { - break; - } - } - } - } - } - for (File pop : filesToRetry) { - /* TODO We need to parse microdescriptors ourselves, rather than - * RelayDescriptorParser, because only we know the valid-after - * time(s) of microdesc consensus(es) containing this - * microdescriptor. However, this breaks functional abstraction - * pretty badly. */ - if (rdp != null) { try { BufferedInputStream bis = null; - if (pop.getName().endsWith(".bz2")) { + if (keepImportHistory + && archivesImportHistory.contains(pop.getName())) { + ignoredFiles++; + continue; + } else if (pop.getName().endsWith(".tar.bz2")) { + logger.warn("Cannot parse compressed tarball " + + pop.getAbsolutePath() + ". Skipping."); + continue; + } else if (pop.getName().endsWith(".bz2")) { FileInputStream fis = new FileInputStream(pop); BZip2CompressorInputStream bcis = new BZip2CompressorInputStream(fis); @@ -162,72 +117,11 @@ public class ArchiveReader { } bis.close(); byte[] allData = baos.toByteArray(); - BufferedReader br = new BufferedReader(new StringReader( - new String(allData, "US-ASCII"))); - String line; - do { - line = br.readLine(); - } while (line != null && line.startsWith("@")); - br.close(); - if (line == null) { - logger.debug("We were given an empty descriptor for " - + "parsing. Ignoring."); + boolean stored = rdp.parse(allData); + if (!stored) { + filesToRetry.add(pop); continue; } - if (!line.equals("onion-key")) { - logger.debug("Skipping non-recognized descriptor."); - continue; - } - SimpleDateFormat parseFormat = - new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - parseFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String ascii = null; - try { - ascii = new String(allData, "US-ASCII"); - } catch (UnsupportedEncodingException e) { - /* No way that US-ASCII is not supported. */ - } - int start = -1; - int end = -1; - String startToken = "onion-key\n"; - while (end < ascii.length()) { - start = ascii.indexOf(startToken, end); - if (start < 0) { - break; - } - end = ascii.indexOf(startToken, start + 1); - if (end < 0) { - end = ascii.length(); - if (end <= start) { - break; - } - } - byte[] descBytes = new byte[end - start]; - System.arraycopy(allData, start, descBytes, 0, end - start); - String digest256Base64 = Base64.encodeBase64String( - DigestUtils.sha256(descBytes)).replaceAll("=", ""); - String digest256Hex = DigestUtils.sha256Hex(descBytes); - if (!this.microdescriptorValidAfterTimes.containsKey( - digest256Hex)) { - logger.debug("Could not store microdescriptor '" - + digest256Hex + "', which was not contained in a " - + "microdesc consensus."); - continue; - } - for (String validAfterTime : - this.microdescriptorValidAfterTimes.get(digest256Hex)) { - try { - long validAfter = - parseFormat.parse(validAfterTime).getTime(); - rdp.storeMicrodescriptor(descBytes, digest256Hex, - digest256Base64, validAfter); - } catch (ParseException e) { - logger.warn("Could not parse " - + "valid-after time '" + validAfterTime + "'. Not " - + "storing microdescriptor.", e); - } - } - } if (keepImportHistory) { archivesImportHistory.add(pop.getName()); } @@ -240,6 +134,108 @@ public class ArchiveReader { } } } + for (File pop : filesToRetry) { + /* TODO We need to parse microdescriptors ourselves, rather than + * RelayDescriptorParser, because only we know the valid-after + * time(s) of microdesc consensus(es) containing this + * microdescriptor. However, this breaks functional abstraction + * pretty badly. */ + try { + BufferedInputStream bis = null; + if (pop.getName().endsWith(".bz2")) { + FileInputStream fis = new FileInputStream(pop); + BZip2CompressorInputStream bcis = + new BZip2CompressorInputStream(fis); + bis = new BufferedInputStream(bcis); + } else { + FileInputStream fis = new FileInputStream(pop); + bis = new BufferedInputStream(fis); + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int len; + byte[] data = new byte[1024]; + while ((len = bis.read(data, 0, 1024)) >= 0) { + baos.write(data, 0, len); + } + bis.close(); + byte[] allData = baos.toByteArray(); + BufferedReader br = new BufferedReader(new StringReader( + new String(allData, "US-ASCII"))); + String line; + do { + line = br.readLine(); + } while (line != null && line.startsWith("@")); + br.close(); + if (line == null) { + logger.debug("We were given an empty descriptor for " + + "parsing. Ignoring."); + continue; + } + if (!line.equals("onion-key")) { + logger.debug("Skipping non-recognized descriptor."); + continue; + } + SimpleDateFormat parseFormat = + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + parseFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String ascii = null; + try { + ascii = new String(allData, "US-ASCII"); + } catch (UnsupportedEncodingException e) { + /* No way that US-ASCII is not supported. */ + } + int start = -1; + int end = -1; + String startToken = "onion-key\n"; + while (end < ascii.length()) { + start = ascii.indexOf(startToken, end); + if (start < 0) { + break; + } + end = ascii.indexOf(startToken, start + 1); + if (end < 0) { + end = ascii.length(); + if (end <= start) { + break; + } + } + byte[] descBytes = new byte[end - start]; + System.arraycopy(allData, start, descBytes, 0, end - start); + String digest256Base64 = Base64.encodeBase64String( + DigestUtils.sha256(descBytes)).replaceAll("=", ""); + String digest256Hex = DigestUtils.sha256Hex(descBytes); + if (!this.microdescriptorValidAfterTimes.containsKey( + digest256Hex)) { + logger.debug("Could not store microdescriptor '" + + digest256Hex + "', which was not contained in a " + + "microdesc consensus."); + continue; + } + for (String validAfterTime : + this.microdescriptorValidAfterTimes.get(digest256Hex)) { + try { + long validAfter = + parseFormat.parse(validAfterTime).getTime(); + rdp.storeMicrodescriptor(descBytes, digest256Hex, + digest256Base64, validAfter); + } catch (ParseException e) { + logger.warn("Could not parse " + + "valid-after time '" + validAfterTime + "'. Not " + + "storing microdescriptor.", e); + } + } + } + if (keepImportHistory) { + archivesImportHistory.add(pop.getName()); + } + parsedFiles++; + } catch (IOException e) { + problems.add(pop); + if (problems.size() > 3) { + break; + } + } + } if (problems.isEmpty()) { logger.debug("Finished importing files in directory " + archivesDirectory + "/."); diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java index 5b5d218..737344b 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java @@ -134,17 +134,15 @@ public class CachedRelayDescriptorReader { /* Parse the cached consensus if we haven't parsed it before * (but regardless of whether it's stale or not). */ - if (rdp != null) { - String digest = Hex.encodeHexString(DigestUtils.sha1( - allData)); - if (!lastImportHistory.contains(digest) - && !currentImportHistory.contains(digest)) { - rdp.parse(allData); - } else { - dumpStats.append(" (skipped)"); - } - currentImportHistory.add(digest); + String digest = Hex.encodeHexString(DigestUtils.sha1( + allData)); + if (!lastImportHistory.contains(digest) + && !currentImportHistory.contains(digest)) { + rdp.parse(allData); + } else { + dumpStats.append(" (skipped)"); } + currentImportHistory.add(digest); } else if (f.getName().equals("v3-status-votes")) { int parsedNum = 0; int skippedNum = 0; @@ -161,18 +159,16 @@ public class CachedRelayDescriptorReader { byte[] rawNetworkStatusBytes = new byte[next - start]; System.arraycopy(allData, start, rawNetworkStatusBytes, 0, next - start); - if (rdp != null) { - String digest = Hex.encodeHexString(DigestUtils.sha1( - rawNetworkStatusBytes)); - if (!lastImportHistory.contains(digest) - && !currentImportHistory.contains(digest)) { - rdp.parse(rawNetworkStatusBytes); - parsedNum++; - } else { - skippedNum++; - } - currentImportHistory.add(digest); + String digest = Hex.encodeHexString(DigestUtils.sha1( + rawNetworkStatusBytes)); + if (!lastImportHistory.contains(digest) + && !currentImportHistory.contains(digest)) { + rdp.parse(rawNetworkStatusBytes); + parsedNum++; + } else { + skippedNum++; } + currentImportHistory.add(digest); } start = next; } @@ -208,18 +204,16 @@ public class CachedRelayDescriptorReader { end += endToken.length(); byte[] descBytes = new byte[end - start]; System.arraycopy(allData, start, descBytes, 0, end - start); - if (rdp != null) { - String digest = Hex.encodeHexString(DigestUtils.sha1( - descBytes)); - if (!lastImportHistory.contains(digest) - && !currentImportHistory.contains(digest)) { - rdp.parse(descBytes); - parsedNum++; - } else { - skippedNum++; - } - currentImportHistory.add(digest); + String digest = Hex.encodeHexString(DigestUtils.sha1( + descBytes)); + if (!lastImportHistory.contains(digest) + && !currentImportHistory.contains(digest)) { + rdp.parse(descBytes); + parsedNum++; + } else { + skippedNum++; } + currentImportHistory.add(digest); } dumpStats.append("\n" + f.getName() + ": parsed " + parsedNum + ", skipped " + skippedNum + " "

1 0

[collector/release] Use StandardCharsets.US_ASCII rather than "US-ASCII".
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit dafd7ed24619ce2c724f752c3f53aa004970ef28 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 11:33:51 2018 +0200 Use StandardCharsets.US_ASCII rather than "US-ASCII". --- .../bridgedescs/SanitizedBridgesWriter.java | 98 +++++++++------------- .../metrics/collector/persist/VotePersistence.java | 26 +++--- .../collector/relaydescs/ArchiveReader.java | 9 +- .../relaydescs/RelayDescriptorDownloader.java | 16 +--- 4 files changed, 57 insertions(+), 92 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java index 8f28a2b..305c09a 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java @@ -28,7 +28,7 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.StringReader; -import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.security.GeneralSecurityException; import java.security.SecureRandom; @@ -981,19 +981,15 @@ public class SanitizedBridgesWriter extends CollecTorMain { /* Determine digest(s) of sanitized server descriptor. */ String descriptorDigest = null; - try { - String ascii = new String(data, "US-ASCII"); - String startToken = "router "; - String sigToken = "\nrouter-signature\n"; - int start = ascii.indexOf(startToken); - int sig = ascii.indexOf(sigToken) + sigToken.length(); - if (start >= 0 && sig >= 0 && sig > start) { - byte[] forDigest = new byte[sig - start]; - System.arraycopy(data, start, forDigest, 0, sig - start); - descriptorDigest = DigestUtils.sha1Hex(DigestUtils.sha1(forDigest)); - } - } catch (UnsupportedEncodingException e) { - /* Handle below. */ + String ascii = new String(data, StandardCharsets.US_ASCII); + String startToken = "router "; + String sigToken = "\nrouter-signature\n"; + int start = ascii.indexOf(startToken); + int sig = ascii.indexOf(sigToken) + sigToken.length(); + if (start >= 0 && sig >= 0 && sig > start) { + byte[] forDigest = new byte[sig - start]; + System.arraycopy(data, start, forDigest, 0, sig - start); + descriptorDigest = DigestUtils.sha1Hex(DigestUtils.sha1(forDigest)); } if (descriptorDigest == null) { logger.warn("Could not calculate server descriptor digest."); @@ -1001,21 +997,17 @@ public class SanitizedBridgesWriter extends CollecTorMain { } String descriptorDigestSha256Base64 = null; if (masterKeyEd25519FromIdentityEd25519 != null) { - try { - String ascii = new String(data, "US-ASCII"); - String startToken = "router "; - String sigToken = "\n-----END SIGNATURE-----\n"; - int start = ascii.indexOf(startToken); - int sig = ascii.indexOf(sigToken) + sigToken.length(); - if (start >= 0 && sig >= 0 && sig > start) { - byte[] forDigest = new byte[sig - start]; - System.arraycopy(data, start, forDigest, 0, sig - start); - descriptorDigestSha256Base64 = Base64.encodeBase64String( - DigestUtils.sha256(DigestUtils.sha256(forDigest))) - .replaceAll("=", ""); - } - } catch (UnsupportedEncodingException e) { - /* Handle below. */ + ascii = new String(data, StandardCharsets.US_ASCII); + startToken = "router "; + sigToken = "\n-----END SIGNATURE-----\n"; + start = ascii.indexOf(startToken); + sig = ascii.indexOf(sigToken) + sigToken.length(); + if (start >= 0 && sig >= 0 && sig > start) { + byte[] forDigest = new byte[sig - start]; + System.arraycopy(data, start, forDigest, 0, sig - start); + descriptorDigestSha256Base64 = Base64.encodeBase64String( + DigestUtils.sha256(DigestUtils.sha256(forDigest))) + .replaceAll("=", ""); } if (descriptorDigestSha256Base64 == null) { logger.warn("Could not calculate server descriptor SHA256 digest."); @@ -1255,19 +1247,15 @@ public class SanitizedBridgesWriter extends CollecTorMain { /* Determine filename of sanitized extra-info descriptor. */ String descriptorDigest = null; - try { - String ascii = new String(data, "US-ASCII"); - String startToken = "extra-info "; - String sigToken = "\nrouter-signature\n"; - int start = ascii.indexOf(startToken); - int sig = ascii.indexOf(sigToken) + sigToken.length(); - if (start >= 0 && sig >= 0 && sig > start) { - byte[] forDigest = new byte[sig - start]; - System.arraycopy(data, start, forDigest, 0, sig - start); - descriptorDigest = DigestUtils.sha1Hex(DigestUtils.sha1(forDigest)); - } - } catch (UnsupportedEncodingException e) { - /* Handle below. */ + String ascii = new String(data, StandardCharsets.US_ASCII); + String startToken = "extra-info "; + String sigToken = "\nrouter-signature\n"; + int start = ascii.indexOf(startToken); + int sig = ascii.indexOf(sigToken) + sigToken.length(); + if (start >= 0 && sig >= 0 && sig > start) { + byte[] forDigest = new byte[sig - start]; + System.arraycopy(data, start, forDigest, 0, sig - start); + descriptorDigest = DigestUtils.sha1Hex(DigestUtils.sha1(forDigest)); } if (descriptorDigest == null) { logger.warn("Could not calculate extra-info descriptor digest."); @@ -1275,21 +1263,17 @@ public class SanitizedBridgesWriter extends CollecTorMain { } String descriptorDigestSha256Base64 = null; if (masterKeyEd25519FromIdentityEd25519 != null) { - try { - String ascii = new String(data, "US-ASCII"); - String startToken = "extra-info "; - String sigToken = "\n-----END SIGNATURE-----\n"; - int start = ascii.indexOf(startToken); - int sig = ascii.indexOf(sigToken) + sigToken.length(); - if (start >= 0 && sig >= 0 && sig > start) { - byte[] forDigest = new byte[sig - start]; - System.arraycopy(data, start, forDigest, 0, sig - start); - descriptorDigestSha256Base64 = Base64.encodeBase64String( - DigestUtils.sha256(DigestUtils.sha256(forDigest))) - .replaceAll("=", ""); - } - } catch (UnsupportedEncodingException e) { - /* Handle below. */ + ascii = new String(data, StandardCharsets.US_ASCII); + startToken = "extra-info "; + sigToken = "\n-----END SIGNATURE-----\n"; + start = ascii.indexOf(startToken); + sig = ascii.indexOf(sigToken) + sigToken.length(); + if (start >= 0 && sig >= 0 && sig > start) { + byte[] forDigest = new byte[sig - start]; + System.arraycopy(data, start, forDigest, 0, sig - start); + descriptorDigestSha256Base64 = Base64.encodeBase64String( + DigestUtils.sha256(DigestUtils.sha256(forDigest))) + .replaceAll("=", ""); } if (descriptorDigestSha256Base64 == null) { logger.warn("Could not calculate extra-info " diff --git a/src/main/java/org/torproject/metrics/collector/persist/VotePersistence.java b/src/main/java/org/torproject/metrics/collector/persist/VotePersistence.java index a1bb032..0c1f80a 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/VotePersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/VotePersistence.java @@ -8,7 +8,7 @@ import org.torproject.metrics.collector.conf.Annotation; import org.apache.commons.codec.digest.DigestUtils; -import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; import java.nio.file.Paths; public class VotePersistence @@ -47,20 +47,16 @@ public class VotePersistence String digest = ""; String startToken = "network-status-version "; String sigToken = "directory-signature "; - try { - String ascii = new String(bytes, "US-ASCII"); - int start = ascii.indexOf(startToken); - int sig = ascii.indexOf(sigToken); - if (start >= 0 && sig >= 0 && sig > start) { - sig += sigToken.length(); - byte[] forDigest = new byte[sig - start]; - System.arraycopy(bytes, start, forDigest, 0, sig - start); - digest = DigestUtils.sha1Hex(forDigest).toUpperCase(); - } else { - log.error("No digest calculation possible. Returning empty string."); - } - } catch (UnsupportedEncodingException uee) { - log.error("Unsupported encoding. Returning empty string.", uee); + String ascii = new String(bytes, StandardCharsets.US_ASCII); + int start = ascii.indexOf(startToken); + int sig = ascii.indexOf(sigToken); + if (start >= 0 && sig >= 0 && sig > start) { + sig += sigToken.length(); + byte[] forDigest = new byte[sig - start]; + System.arraycopy(bytes, start, forDigest, 0, sig - start); + digest = DigestUtils.sha1Hex(forDigest).toUpperCase(); + } else { + log.error("No digest calculation possible. Returning empty string."); } return digest; } diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java index 3976919..84ad2b6 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java @@ -19,7 +19,7 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.StringReader; -import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -178,12 +178,7 @@ public class ArchiveReader { SimpleDateFormat parseFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); parseFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String ascii = null; - try { - ascii = new String(allData, "US-ASCII"); - } catch (UnsupportedEncodingException e) { - /* No way that US-ASCII is not supported. */ - } + String ascii = new String(allData, StandardCharsets.US_ASCII); int start = -1; int end = -1; String startToken = "onion-key\n"; diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java index a65ea9a..e1dff40 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java @@ -16,9 +16,9 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Arrays; @@ -897,12 +897,7 @@ public class RelayDescriptorDownloader { this.lastDownloadedAllDescriptors.put(authority, this.currentTimestamp); } - String ascii = null; - try { - ascii = new String(allData, "US-ASCII"); - } catch (UnsupportedEncodingException e) { - /* No way that US-ASCII is not supported. */ - } + String ascii = new String(allData, StandardCharsets.US_ASCII); int start = -1; int sig = -1; int end = -1; @@ -939,12 +934,7 @@ public class RelayDescriptorDownloader { SimpleDateFormat parseFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); parseFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String ascii = null; - try { - ascii = new String(allData, "US-ASCII"); - } catch (UnsupportedEncodingException e) { - /* No way that US-ASCII is not supported. */ - } + String ascii = new String(allData, StandardCharsets.US_ASCII); int start = -1; int end = -1; String startToken = "onion-key\n";

1 0

[collector/release] Actually fix #26193 now.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 5e1ae90ec4ff84ece0025d042d7651bc744837db Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Aug 9 08:28:06 2018 +0200 Actually fix #26193 now. The earlier attempt to fix this issue in 98244f3 did not work, because it was missing the important `-f` switch. --- CHANGELOG.md | 7 +++++++ src/main/resources/create-tarballs.sh | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aad7213..d98520c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# Changes in version 1.?.? - 2018-0?-?? + + * Minor changes + - Once more, fix the bug in the tarball-creation script where + tarballs are not compressed in a run following an aborted run. + + # Changes in version 1.7.0 - 2018-07-14 * Medium changes diff --git a/src/main/resources/create-tarballs.sh b/src/main/resources/create-tarballs.sh index b633826..d247c52 100755 --- a/src/main/resources/create-tarballs.sh +++ b/src/main/resources/create-tarballs.sh @@ -100,7 +100,7 @@ for (( i = 0 ; i < ${#TARBALLS[@]} ; i++ )); do echo `date` "Creating" ${TARBALLS[$i]}'.tar' tar chf ${TARBALLS[$i]}.tar ${TARBALLS[$i]} echo `date` "Compressing" ${TARBALLS[$i]}'.tar' - xz -9e ${TARBALLS[$i]}.tar + xz -9e -f ${TARBALLS[$i]}.tar done cd $OUTDIR/webstats/

1 0

[collector/release] Remove unnecessary boxing.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 3c89aaebfb93d2dd146a693ee104ac1fd1fc9b47 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 15:00:18 2018 +0200 Remove unnecessary boxing. --- .../collector/sync/SyncPersistenceTest.java | 52 +++++++++++----------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/test/java/org/torproject/metrics/collector/sync/SyncPersistenceTest.java b/src/test/java/org/torproject/metrics/collector/sync/SyncPersistenceTest.java index 101fa8b..a8f7095 100644 --- a/src/test/java/org/torproject/metrics/collector/sync/SyncPersistenceTest.java +++ b/src/test/java/org/torproject/metrics/collector/sync/SyncPersistenceTest.java @@ -41,14 +41,14 @@ public class SyncPersistenceTest { {"exit-lists/2016-09-20-13-02-00", // expected recent path new String[]{"exit-lists/2016/09/20/2016-09-20-13-02-00"}, // exp. out "2016-09-20-13-02-00", // test-filename in src/test/resources - Integer.valueOf(1), // expected recent count of descs files - Integer.valueOf(1)}, // expected output count of descs files + 1, // expected recent count of descs files + 1}, // expected output count of descs files {"torperf/op-nl-1048576-2017-04-11.tpf", new String[]{"torperf/2017/04/11/op-nl-1048576-2017-04-11.tpf"}, "op-nl-1048576-2017-04-11.tpf", - Integer.valueOf(1), - Integer.valueOf(1)}, + 1, + 1}, {"webstats/archive.torproject.org_" + "archeotrichon.torproject.org_access.log_20151007.xz", @@ -57,8 +57,8 @@ public class SyncPersistenceTest { + "_access.log_20151007.xz"}, "archeotrichon.torproject.org/archive.torproject.org_" + "archeotrichon.torproject.org_access.log_20151007.xz", - Integer.valueOf(1), - Integer.valueOf(1)}, + 1, + 1}, {"webstats/metrics.torproject.org_" + "meronense.torproject.org_access.log_20170531.xz", @@ -67,8 +67,8 @@ public class SyncPersistenceTest { + "_access.log_20170531.xz"}, "meronense.torproject.org/metrics.torproject.org_" + "meronense.torproject.org_access.log_20170531.gz", - Integer.valueOf(1), - Integer.valueOf(1)}, + 1, + 1}, {"relay-descriptors/server-descriptors/" + "2016-10-05-19-06-17-server-descriptors", @@ -93,23 +93,23 @@ public class SyncPersistenceTest { "relay-descriptors/server-descriptor/2016/10/a/0/" + "a0ed9227a9413f140445002ce412f8828591e7ec"}, "relay-2016-10-02-16-05-00-server-descriptors", - Integer.valueOf(1), - Integer.valueOf(10)}, + 1, + 10}, {"relay-descriptors/consensuses/2016-09-20-13-00-00-consensus", new String[]{"relay-descriptors/consensus/2016/09/20/" + "2016-09-20-13-00-00-consensus"}, "2016-09-20-13-00-00-consensus", - Integer.valueOf(1), - Integer.valueOf(1)}, + 1, + 1}, {"relay-descriptors/microdescs/consensus-microdesc/" + "2016-10-02-17-00-00-consensus-microdesc", new String[]{"relay-descriptors/microdesc/2016/10/consensus-microdesc/" + "02/2016-10-02-17-00-00-consensus-microdesc"}, "2016-10-02-17-00-00-consensus-microdesc", - Integer.valueOf(1), - Integer.valueOf(1)}, + 1, + 1}, {"relay-descriptors/votes/2016-10-01-16-00-00-vote" + "-0232AF901C31A04EE9848595AF9BB7620D4C5B2E" @@ -120,8 +120,8 @@ public class SyncPersistenceTest { + "-FEE63B4AB7CE5A6BDD09E9A5C4F01BD61EB7E4F1"}, "2016-10-01-16-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-" + "FEE63B4AB7CE5A6BDD09E9A5C4F01BD61EB7E4F1", - Integer.valueOf(1), - Integer.valueOf(1)}, + 1, + 1}, {"relay-descriptors/votes/2016-09-20-13-00-00-vote-" + "49015F787433103580E3B66A1707A00E60F2D15B" @@ -132,8 +132,8 @@ public class SyncPersistenceTest { + "-60ADC6BEC262AE921A1037D54C8A3976367DBE87"}, "2016-09-20-13-00-00-vote-49015F787433103580E3B66A1707A00E60F2D15B" + "-60ADC6BEC262AE921A1037D54C8A3976367DBE87", - Integer.valueOf(1), - Integer.valueOf(1)}, + 1, + 1}, {"relay-descriptors/extra-infos/2016-10-05-19-06-17-extra-infos", new String[]{"relay-descriptors/extra-info/2016/10/9/a/" @@ -155,8 +155,8 @@ public class SyncPersistenceTest { "relay-descriptors/extra-info/2016/10/0/4/" + "04219ada0be922fa7518d36b0d8e66afc55e8603"}, "relay-2016-10-02-08-05-00-extra-infos", - Integer.valueOf(1), - Integer.valueOf(9)}, + 1, + 9}, {"bridge-descriptors/extra-infos/2016-10-05-19-06-17-extra-infos", new String[]{"bridge-descriptors/2016/10/extra-infos/9/f/" @@ -180,8 +180,8 @@ public class SyncPersistenceTest { "bridge-descriptors/2016/10/extra-infos/f/a/" + "fabd8f614633ec2d2d405f2554e14381bc33d9cb"}, "bridge-2016-10-02-08-09-00-extra-infos", - Integer.valueOf(1), - Integer.valueOf(10) }, + 1, + 10}, {"bridge-descriptors/server-descriptors/" + "2016-10-05-19-06-17-server-descriptors", @@ -202,16 +202,16 @@ public class SyncPersistenceTest { "bridge-descriptors/2016/10/server-descriptors/0/a/" + "0a65c636a20631bd5deb2f10dc664d2032303c46"}, "bridge-2016-10-02-16-09-00-server-descriptors", - Integer.valueOf(1), - Integer.valueOf(8)}, + 1, + 8}, {"bridge-descriptors/statuses/" + "20160920-063816-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1", new String[]{"bridge-descriptors/2016/09/statuses/20/" + "20160920-063816-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1"}, "20160920-063816-1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1", - Integer.valueOf(1), - Integer.valueOf(1)} + 1, + 1} }); }

1 0

[collector/release] Use Java 5 and 7 idioms in more places.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 3ba8bdcc16b3bf9ec24eed8a44b981a069092879 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 12:21:11 2018 +0200 Use Java 5 and 7 idioms in more places. --- .../bridgedescs/SanitizedBridgesWriter.java | 3 +-- .../metrics/collector/conf/Configuration.java | 4 ++-- .../metrics/collector/relaydescs/ArchiveReader.java | 2 +- .../metrics/collector/relaydescs/ArchiveWriter.java | 21 +++++++-------------- .../collector/relaydescs/ReferenceChecker.java | 7 ++----- .../relaydescs/RelayDescriptorDownloader.java | 4 ++-- .../collector/relaydescs/RelayDescriptorParser.java | 5 +---- .../metrics/collector/webstats/LogFileMap.java | 4 ++-- 8 files changed, 18 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java index 305c09a..6511e66 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java @@ -307,8 +307,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { /* Invalid IPv6 address. */ return null; } - for (int i = 0; i < parts.length; i++) { - String part = parts[i]; + for (String part : parts) { if (part.contains(".")) { String[] ipParts = part.split("\\."); byte[] ipv4Bytes = new byte[4]; diff --git a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java index 503ebbf..6f38c45 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java +++ b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java @@ -263,8 +263,8 @@ public class Configuration extends Observable implements Cloneable { try { checkClass(key, SourceType[].class); String[] interim = props.getProperty(key.name()).split(FIELDSEP); - for (int i = 0; i < interim.length; i++) { - SourceType st = SourceType.valueOf(interim[i].trim()); + for (String anInterim : interim) { + SourceType st = SourceType.valueOf(anInterim.trim()); if (null == res) { res = EnumSet.of(st); } else { diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java index 84ad2b6..0119aef 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java @@ -276,7 +276,7 @@ public class ArchiveReader { if (!this.microdescriptorValidAfterTimes.containsKey( microdescriptor)) { this.microdescriptorValidAfterTimes.put(microdescriptor, - new HashSet<String>()); + new HashSet<>()); } this.microdescriptorValidAfterTimes.get(microdescriptor).add( validAfterTime); diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java index 11388f2..31dfdef 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java @@ -230,8 +230,7 @@ public class ArchiveWriter extends CollecTorMain { continue; } if (!this.storedServerDescriptors.containsKey(published)) { - this.storedServerDescriptors.put(published, - new HashMap<String, String>()); + this.storedServerDescriptors.put(published, new HashMap<>()); } String serverDescriptorDigest = parts[1]; String extraInfoDescriptorDigest = parts[2].equals("NA") ? null @@ -259,8 +258,7 @@ public class ArchiveWriter extends CollecTorMain { continue; } if (!this.storedExtraInfoDescriptors.containsKey(published)) { - this.storedExtraInfoDescriptors.put(published, - new HashSet<String>()); + this.storedExtraInfoDescriptors.put(published, new HashSet<>()); } String extraInfoDescriptorDigest = parts[1]; this.storedExtraInfoDescriptors.get(published).add( @@ -286,8 +284,7 @@ public class ArchiveWriter extends CollecTorMain { continue; } if (!this.storedMicrodescriptors.containsKey(validAfter)) { - this.storedMicrodescriptors.put(validAfter, - new HashSet<String>()); + this.storedMicrodescriptors.put(validAfter, new HashSet<>()); } String microdescriptorDigest = parts[1]; this.storedMicrodescriptors.get(validAfter).add( @@ -706,8 +703,7 @@ public class ArchiveWriter extends CollecTorMain { if (!tarballFileExistedBefore && this.now - validAfter < 3L * 60L * 60L * 1000L) { if (!this.storedVotes.containsKey(validAfter)) { - this.storedVotes.put(validAfter, - new TreeMap<String, SortedSet<String>>()); + this.storedVotes.put(validAfter, new TreeMap<>()); } this.storedVotes.get(validAfter).put(fingerprint, serverDescriptorDigests); @@ -750,8 +746,7 @@ public class ArchiveWriter extends CollecTorMain { if (!tarballFileExistedBefore && this.now - published < 48L * 60L * 60L * 1000L) { if (!this.storedServerDescriptors.containsKey(published)) { - this.storedServerDescriptors.put(published, - new HashMap<String, String>()); + this.storedServerDescriptors.put(published, new HashMap<>()); } this.storedServerDescriptors.get(published).put(digest, extraInfoDigest); @@ -779,8 +774,7 @@ public class ArchiveWriter extends CollecTorMain { if (!tarballFileExistedBefore && this.now - published < 48L * 60L * 60L * 1000L) { if (!this.storedExtraInfoDescriptors.containsKey(published)) { - this.storedExtraInfoDescriptors.put(published, - new HashSet<String>()); + this.storedExtraInfoDescriptors.put(published, new HashSet<>()); } this.storedExtraInfoDescriptors.get(published).add(extraInfoDigest); } @@ -815,8 +809,7 @@ public class ArchiveWriter extends CollecTorMain { if (!tarballFileExistedBefore && this.now - validAfter < 40L * 24L * 60L * 60L * 1000L) { if (!this.storedMicrodescriptors.containsKey(validAfter)) { - this.storedMicrodescriptors.put(validAfter, - new HashSet<String>()); + this.storedMicrodescriptors.put(validAfter, new HashSet<>()); } this.storedMicrodescriptors.get(validAfter).add( microdescriptorDigest); diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ReferenceChecker.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ReferenceChecker.java index 83b3ae9..0984de0 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ReferenceChecker.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ReferenceChecker.java @@ -28,7 +28,6 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.HashSet; -import java.util.Iterator; import java.util.Locale; import java.util.Set; import java.util.SortedSet; @@ -170,10 +169,8 @@ public class ReferenceChecker { DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); descriptorReader.setHistoryFile(this.historyFile); - Iterator<Descriptor> descriptors - = descriptorReader.readDescriptors(this.descriptorsDir).iterator(); - while (descriptors.hasNext()) { - Descriptor descriptor = descriptors.next(); + for (Descriptor descriptor + : descriptorReader.readDescriptors(this.descriptorsDir)) { if (descriptor instanceof RelayNetworkStatusConsensus) { RelayNetworkStatusConsensus consensus = (RelayNetworkStatusConsensus) descriptor; diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java index e1dff40..cf9ebc4 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java @@ -383,7 +383,7 @@ public class RelayDescriptorDownloader { if (!this.microdescriptorKeys.containsKey( microdescriptorDigest)) { this.microdescriptorKeys.put( - microdescriptorDigest, new HashSet<String>()); + microdescriptorDigest, new HashSet<>()); } this.microdescriptorKeys.get(microdescriptorDigest).add( microdescriptorKey); @@ -552,7 +552,7 @@ public class RelayDescriptorDownloader { } } else { this.microdescriptorKeys.put( - microdescriptorDigest, new HashSet<String>()); + microdescriptorDigest, new HashSet<>()); } this.microdescriptorKeys.get(microdescriptorDigest).add( microdescriptorKey); diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java index 74e0ce9..65e2e2e 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java @@ -316,10 +316,7 @@ public class RelayDescriptorParser { * storeMicrodescriptor below. */ } br.close(); - } catch (IOException e) { - logger.warn("Could not parse descriptor. " - + "Skipping.", e); - } catch (ParseException e) { + } catch (IOException | ParseException e) { logger.warn("Could not parse descriptor. " + "Skipping.", e); } diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java index 0d23f9a..7fa3970 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java @@ -75,13 +75,13 @@ public class LogFileMap TreeMap<String, TreeMap<LocalDate, LogMetadata>> virtualHosts = this.get(metadata.virtualHost); if (null == virtualHosts) { - virtualHosts = new TreeMap<String, TreeMap<LocalDate, LogMetadata>>(); + virtualHosts = new TreeMap<>(); this.put(metadata.virtualHost, virtualHosts); } TreeMap<LocalDate, LogMetadata> physicalHosts = virtualHosts.get(metadata.physicalHost); if (null == physicalHosts) { - physicalHosts = new TreeMap<LocalDate, LogMetadata>(); + physicalHosts = new TreeMap<>(); virtualHosts.put(metadata.physicalHost, physicalHosts); } physicalHosts.put(metadata.date, metadata);

1 0

[collector/release] Replace anonymous types with lambdas.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 20742f46907e0085592aebe917acabe8f77ed0c7 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 15:11:13 2018 +0200 Replace anonymous types with lambdas. --- .../metrics/collector/conf/Configuration.java | 30 ++++++++++------------ .../metrics/collector/conf/ConfigurationTest.java | 14 ++-------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java index 6f38c45..f797947 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java +++ b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java @@ -60,23 +60,21 @@ public class Configuration extends Observable implements Cloneable { if (this.getBool(Key.RunOnce)) { // no need to watch return; } - this.scheduler.scheduleAtFixedRate(new Runnable() { - public void run() { - logger.trace("Check configuration file."); - try { - FileTime ftNow = Files.getLastModifiedTime(confPath); - if (ft.compareTo(ftNow) < 0) { - logger.info("Configuration file was changed."); - reload(); - setChanged(); - notifyObservers(null); - } - ft = ftNow; - } catch (Throwable th) { // Catch all and keep running. - logger.error("Cannot reload configuration file.", th); - } + this.scheduler.scheduleAtFixedRate(() -> { + logger.trace("Check configuration file."); + try { + FileTime ftNow = Files.getLastModifiedTime(confPath); + if (ft.compareTo(ftNow) < 0) { + logger.info("Configuration file was changed."); + reload(); + setChanged(); + notifyObservers(null); } - }, 5, 5, TimeUnit.SECONDS); + ft = ftNow; + } catch (Throwable th) { // Catch all and keep running. + logger.error("Cannot reload configuration file.", th); + } + }, 5, 5, TimeUnit.SECONDS); } private final void reload() throws IOException { diff --git a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java index f6172bf..4ac623e 100644 --- a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java @@ -23,8 +23,6 @@ import java.net.URL; import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; -import java.util.Observable; -import java.util.Observer; import java.util.Random; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; @@ -186,11 +184,7 @@ public class ConfigurationTest { public void testConfigChange() throws Exception { Configuration conf = new Configuration(); final AtomicBoolean called = new AtomicBoolean(false); - conf.addObserver(new Observer() { - public void update(Observable obs, Object obj) { - called.set(true); - } - }); + conf.addObserver((obs, obj) -> called.set(true)); File confFile = tmpf.newFile("empty"); Files.write(confFile.toPath(), (Key.RelaydescsActivated.name() + "=true") .getBytes()); @@ -208,11 +202,7 @@ public class ConfigurationTest { public void testConfigUnreadable() throws Exception { Configuration conf = new Configuration(); final AtomicBoolean called = new AtomicBoolean(false); - conf.addObserver(new Observer() { - public void update(Observable obs, Object obj) { - called.set(true); - } - }); + conf.addObserver((obs, obj) -> called.set(true)); File confFile = tmpf.newFile("empty"); Files.write(confFile.toPath(), (Key.RelaydescsActivated.name() + "=true") .getBytes());

1 0

[collector/release] Remove unthrown exceptions.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit cfd3aecaf7456432841cffe07b90ed0070429d6e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 14:47:34 2018 +0200 Remove unthrown exceptions. --- src/main/java/org/torproject/metrics/collector/Main.java | 2 +- .../metrics/collector/bridgedescs/BridgeDescriptorParser.java | 4 +--- .../metrics/collector/bridgedescs/SanitizedBridgesWriter.java | 3 +-- .../metrics/collector/exitlists/ExitListDownloader.java | 2 +- .../torproject/metrics/collector/indexer/CreateIndexJson.java | 3 +-- .../torproject/metrics/collector/relaydescs/ArchiveWriter.java | 2 +- .../org/torproject/metrics/collector/webstats/LogFileMap.java | 9 +++------ .../torproject/metrics/collector/webstats/SanitizeWeblogs.java | 3 +-- src/test/java/org/torproject/metrics/collector/MainTest.java | 6 +++--- .../collector/bridgedescs/BridgeDescriptorParserTest.java | 2 +- .../metrics/collector/bridgedescs/TarballTestBuilder.java | 2 +- .../org/torproject/metrics/collector/conf/ConfigurationTest.java | 4 ++-- src/test/java/org/torproject/metrics/collector/cron/Broken.java | 3 +-- src/test/java/org/torproject/metrics/collector/cron/Counter.java | 3 +-- src/test/java/org/torproject/metrics/collector/cron/Dummy.java | 3 +-- .../org/torproject/metrics/collector/sync/FileCollector.java | 4 +--- .../torproject/metrics/collector/sync/SyncPersistenceTest.java | 4 ++-- .../torproject/metrics/collector/webstats/LogMetadataTest.java | 2 +- 18 files changed, 24 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/Main.java b/src/main/java/org/torproject/metrics/collector/Main.java index 6233dd2..05aead7 100644 --- a/src/main/java/org/torproject/metrics/collector/Main.java +++ b/src/main/java/org/torproject/metrics/collector/Main.java @@ -61,7 +61,7 @@ public class Main { * At most one argument. * See class description {@link Main}. */ - public static void main(String[] args) throws Exception { + public static void main(String[] args) { try { Path confPath = null; if (args == null || args.length == 0) { diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java index 5134f0f..c993d89 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java @@ -3,8 +3,6 @@ package org.torproject.metrics.collector.bridgedescs; -import org.torproject.metrics.collector.conf.ConfigurationException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,7 +30,7 @@ public class BridgeDescriptorParser { /** Parses the first line of the given descriptor data to determine the * descriptor type and passes it to the sanitized bridges writer. */ public void parse(byte[] allData, String dateTime, - String authorityFingerprint) throws ConfigurationException { + String authorityFingerprint) { try { BufferedReader br = new BufferedReader(new StringReader( new String(allData, "US-ASCII"))); diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java index 6511e66..7c335c9 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java @@ -440,8 +440,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { * Sanitizes a network status and writes it to disk. */ public void sanitizeAndStoreNetworkStatus(byte[] data, - String publicationTime, String authorityFingerprint) - throws ConfigurationException { + String publicationTime, String authorityFingerprint) { if (this.persistenceProblemWithSecrets) { /* There's a persistence problem, so we shouldn't scrub more IP diff --git a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java index 4ef8974..2531cb7 100644 --- a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java @@ -194,7 +194,7 @@ public class ExitListDownloader extends CollecTorMain { /** Delete all files from the rsync directory that have not been modified * in the last three days. */ - public void cleanUpRsyncDirectory() throws ConfigurationException { + public void cleanUpRsyncDirectory() { long cutOffMillis = System.currentTimeMillis() - 3L * 24L * 60L * 60L * 1000L; Stack<File> allFiles = new Stack<>(); diff --git a/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java b/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java index 55ebd15..5f3a649 100644 --- a/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java +++ b/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java @@ -8,7 +8,6 @@ import org.torproject.descriptor.index.FileNode; import org.torproject.descriptor.index.IndexNode; import org.torproject.descriptor.internal.FileType; import org.torproject.metrics.collector.conf.Configuration; -import org.torproject.metrics.collector.conf.ConfigurationException; import org.torproject.metrics.collector.conf.Key; import org.torproject.metrics.collector.cron.CollecTorMain; @@ -82,7 +81,7 @@ public class CreateIndexJson extends CollecTorMain { } @Override - protected void startProcessing() throws ConfigurationException { + protected void startProcessing() { try { indexJsonFile = new File(config.getPath(Key.IndexPath).toFile(), "index.json"); diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java index 31dfdef..98d2864 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java @@ -90,7 +90,7 @@ public class ArchiveWriter extends CollecTorMain { private static final String MICRODESCS = "microdescs"; /** Initialize an archive writer with a given configuration. */ - public ArchiveWriter(Configuration config) throws ConfigurationException { + public ArchiveWriter(Configuration config) { super(config); this.mapPathDescriptors.put("recent/relay-descriptors/votes", RelayNetworkStatusVote.class); diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java index 7fa3970..b265e4b 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java @@ -37,8 +37,7 @@ public class LogFileMap try { Files.walkFileTree(startDir, new SimpleFileVisitor<Path>() { @Override - public FileVisitResult visitFile(Path path, BasicFileAttributes att) - throws IOException { + public FileVisitResult visitFile(Path path, BasicFileAttributes att) { Optional<LogMetadata> optionalMetadata = LogMetadata.create(path); if (optionalMetadata.isPresent()) { logFileMap.add(optionalMetadata.get()); @@ -47,14 +46,12 @@ public class LogFileMap } @Override - public FileVisitResult visitFileFailed(Path path, IOException ex) - throws IOException { + public FileVisitResult visitFileFailed(Path path, IOException ex) { return logIfError(path, ex); } @Override - public FileVisitResult postVisitDirectory(Path path, IOException ex) - throws IOException { + public FileVisitResult postVisitDirectory(Path path, IOException ex) { return logIfError(path, ex); } diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java index 3d25cea..32d4702 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java @@ -18,7 +18,6 @@ import org.torproject.descriptor.log.InternalWebServerAccessLog; import org.torproject.descriptor.log.WebServerAccessLogImpl; import org.torproject.descriptor.log.WebServerAccessLogLine; import org.torproject.metrics.collector.conf.Configuration; -import org.torproject.metrics.collector.conf.ConfigurationException; import org.torproject.metrics.collector.conf.Key; import org.torproject.metrics.collector.conf.SourceType; import org.torproject.metrics.collector.cron.CollecTorMain; @@ -86,7 +85,7 @@ public class SanitizeWeblogs extends CollecTorMain { } @Override - protected void startProcessing() throws ConfigurationException { + protected void startProcessing() { try { Files.createDirectories(this.config.getPath(Key.OutputPath)); Files.createDirectories(this.config.getPath(Key.RecentPath)); diff --git a/src/test/java/org/torproject/metrics/collector/MainTest.java b/src/test/java/org/torproject/metrics/collector/MainTest.java index 6459040..820cfb6 100644 --- a/src/test/java/org/torproject/metrics/collector/MainTest.java +++ b/src/test/java/org/torproject/metrics/collector/MainTest.java @@ -55,7 +55,7 @@ public class MainTest { } @Test() - public void testInitializationNullArgs() throws Exception { + public void testInitializationNullArgs() { File conf = new File(Main.CONF_FILE); checkCleanEnv(conf); Main.main(null); @@ -77,7 +77,7 @@ public class MainTest { } @Test() - public void testInitializationEmptyArgs() throws Exception { + public void testInitializationEmptyArgs() { File conf = new File(Main.CONF_FILE); checkCleanEnv(conf); Main.main(new String[] { }); @@ -86,7 +86,7 @@ public class MainTest { } @Test() - public void testInitializationTooManyArgs() throws Exception { + public void testInitializationTooManyArgs() { File conf = new File(Main.CONF_FILE); checkCleanEnv(conf); Main.main(new String[] { "x", "y" }); diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParserTest.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParserTest.java index 040d161..43fad5e 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParserTest.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParserTest.java @@ -10,7 +10,7 @@ import org.junit.Test; public class BridgeDescriptorParserTest { @Test(expected = IllegalArgumentException.class) - public void testNullArgForConstructor() throws Exception { + public void testNullArgForConstructor() { new BridgeDescriptorParser(null); } diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/TarballTestBuilder.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/TarballTestBuilder.java index 44f4490..e2f4c40 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/TarballTestBuilder.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/TarballTestBuilder.java @@ -63,7 +63,7 @@ class TarballTestBuilder { /** Adds a new file to the tarball with given name, last-modified time, and * descriptor builders to generate the file content. */ TarballTestBuilder add(String fileName, long modifiedMillis, - List<TestDescriptorBuilder> descriptorBuilders) throws IOException { + List<TestDescriptorBuilder> descriptorBuilders) { TarballFile file = new TarballFile(); file.modifiedMillis = modifiedMillis; file.descriptorBuilders = descriptorBuilders; diff --git a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java index d91013b..f6172bf 100644 --- a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java @@ -37,14 +37,14 @@ public class ConfigurationTest { public TemporaryFolder tmpf = new TemporaryFolder(); @Test() - public void testKeyCount() throws Exception { + public void testKeyCount() { assertEquals("The number of properties keys in enum Key changed." + "\n This test class should be adapted.", 53, Key.values().length); } @Test() - public void testConfiguration() throws Exception { + public void testConfiguration() { Configuration conf = new Configuration(); String val = "xyz"; conf.setProperty(Key.OutputPath.name(), val); diff --git a/src/test/java/org/torproject/metrics/collector/cron/Broken.java b/src/test/java/org/torproject/metrics/collector/cron/Broken.java index ef896bf..44aa88b 100644 --- a/src/test/java/org/torproject/metrics/collector/cron/Broken.java +++ b/src/test/java/org/torproject/metrics/collector/cron/Broken.java @@ -4,7 +4,6 @@ package org.torproject.metrics.collector.cron; import org.torproject.metrics.collector.conf.Configuration; -import org.torproject.metrics.collector.conf.ConfigurationException; import java.util.concurrent.atomic.AtomicInteger; @@ -17,7 +16,7 @@ public class Broken extends CollecTorMain { } @Override - public void startProcessing() throws ConfigurationException { + public void startProcessing() { count.getAndIncrement(); try { Thread.sleep(10); diff --git a/src/test/java/org/torproject/metrics/collector/cron/Counter.java b/src/test/java/org/torproject/metrics/collector/cron/Counter.java index d48b62c..f783d7b 100644 --- a/src/test/java/org/torproject/metrics/collector/cron/Counter.java +++ b/src/test/java/org/torproject/metrics/collector/cron/Counter.java @@ -1,7 +1,6 @@ package org.torproject.metrics.collector.cron; import org.torproject.metrics.collector.conf.Configuration; -import org.torproject.metrics.collector.conf.ConfigurationException; import java.util.concurrent.atomic.AtomicInteger; @@ -14,7 +13,7 @@ public class Counter extends CollecTorMain { } @Override - public void startProcessing() throws ConfigurationException { + public void startProcessing() { count.getAndIncrement(); } diff --git a/src/test/java/org/torproject/metrics/collector/cron/Dummy.java b/src/test/java/org/torproject/metrics/collector/cron/Dummy.java index 1110676..c26da7d 100644 --- a/src/test/java/org/torproject/metrics/collector/cron/Dummy.java +++ b/src/test/java/org/torproject/metrics/collector/cron/Dummy.java @@ -1,7 +1,6 @@ package org.torproject.metrics.collector.cron; import org.torproject.metrics.collector.conf.Configuration; -import org.torproject.metrics.collector.conf.ConfigurationException; public class Dummy extends CollecTorMain { @@ -10,7 +9,7 @@ public class Dummy extends CollecTorMain { } @Override - public void startProcessing() throws ConfigurationException { + public void startProcessing() { // dummy doesn't do anything. } diff --git a/src/test/java/org/torproject/metrics/collector/sync/FileCollector.java b/src/test/java/org/torproject/metrics/collector/sync/FileCollector.java index 57d9216..e84a62f 100644 --- a/src/test/java/org/torproject/metrics/collector/sync/FileCollector.java +++ b/src/test/java/org/torproject/metrics/collector/sync/FileCollector.java @@ -4,7 +4,6 @@ package org.torproject.metrics.collector.sync; import java.io.File; -import java.io.IOException; import java.nio.file.FileVisitResult; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; @@ -20,8 +19,7 @@ public class FileCollector extends SimpleFileVisitor<Path> { } @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { this.list.add(file.toFile()); return FileVisitResult.CONTINUE; } diff --git a/src/test/java/org/torproject/metrics/collector/sync/SyncPersistenceTest.java b/src/test/java/org/torproject/metrics/collector/sync/SyncPersistenceTest.java index 0a20338..101fa8b 100644 --- a/src/test/java/org/torproject/metrics/collector/sync/SyncPersistenceTest.java +++ b/src/test/java/org/torproject/metrics/collector/sync/SyncPersistenceTest.java @@ -355,7 +355,7 @@ public class SyncPersistenceTest { .getClassLoader().getResource(filename).toURI())).toPath()); } - private String stringFromResource() throws Exception { + private String stringFromResource() { BufferedReader br = new BufferedReader(new InputStreamReader(getClass() .getClassLoader().getResourceAsStream(filename))); return br.lines().collect(Collectors.joining("\n", "", "\n")); @@ -366,7 +366,7 @@ public class SyncPersistenceTest { .collect(Collectors.joining("\n", "", "\n")); } - private List<String> linesFromResource(String filename) throws Exception { + private List<String> linesFromResource(String filename) { BufferedReader br = new BufferedReader(new InputStreamReader(getClass() .getClassLoader().getResourceAsStream(filename))); return br.lines().collect(Collectors.toList()); diff --git a/src/test/java/org/torproject/metrics/collector/webstats/LogMetadataTest.java b/src/test/java/org/torproject/metrics/collector/webstats/LogMetadataTest.java index 9a58cb9..24629ab 100644 --- a/src/test/java/org/torproject/metrics/collector/webstats/LogMetadataTest.java +++ b/src/test/java/org/torproject/metrics/collector/webstats/LogMetadataTest.java @@ -67,7 +67,7 @@ public class LogMetadataTest { } @Test - public void testCreate() throws Exception { + public void testCreate() { Optional<LogMetadata> element = LogMetadata.create(this.path); assertEquals(this.failureMessage, this.valid, element.isPresent()); if (!this.valid) {

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

tor-commits October 2018