August 2018 - tor-commits - lists.torproject.org

[onionoo/master] Replace Collections.sort with List.sort.
by karsten＠torproject.org 27 Aug '18

27 Aug '18

commit 17f2c134fb199dcda021a61f754512ec8b06a353 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 15:24:54 2018 +0200 Replace Collections.sort with List.sort. --- src/main/java/org/torproject/onionoo/server/RequestHandler.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java index 33c35e1..81cd8dc 100644 --- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java +++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java @@ -9,7 +9,6 @@ import org.torproject.onionoo.docs.SummaryDocument; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -661,8 +660,8 @@ public class RequestHandler { } Comparator<SummaryDocument> comparator = new SummaryDocumentComparator(this.order); - Collections.sort(uniqueRelays, comparator); - Collections.sort(uniqueBridges, comparator); + uniqueRelays.sort(comparator); + uniqueBridges.sort(comparator); this.orderedRelays.addAll(uniqueRelays); this.orderedBridges.addAll(uniqueBridges); }

1 0

[onionoo/master] Simplify compareTo methods.
by karsten＠torproject.org 27 Aug '18

27 Aug '18

commit f8ee7cf0ece963f35c311ed9e057271cdd5992bf Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 15:22:37 2018 +0200 Simplify compareTo methods. --- src/main/java/org/torproject/onionoo/docs/ClientsHistory.java | 3 +-- src/main/java/org/torproject/onionoo/docs/UptimeHistory.java | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java b/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java index ba21458..15e70c5 100644 --- a/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java +++ b/src/main/java/org/torproject/onionoo/docs/ClientsHistory.java @@ -194,8 +194,7 @@ public class ClientsHistory implements Comparable<ClientsHistory> { @Override public int compareTo(ClientsHistory other) { - return this.startMillis < other.startMillis ? -1 - : this.startMillis > other.startMillis ? 1 : 0; + return Long.compare(this.startMillis, other.startMillis); } @Override diff --git a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java index f8cc116..d549f92 100644 --- a/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java +++ b/src/main/java/org/torproject/onionoo/docs/UptimeHistory.java @@ -128,8 +128,7 @@ public class UptimeHistory implements Comparable<UptimeHistory> { } else if (!this.relay && other.relay) { return 1; } - return this.startMillis < other.startMillis ? -1 - : this.startMillis > other.startMillis ? 1 : 0; + return Long.compare(this.startMillis, other.startMillis); } @Override

1 0

[collector/master] Simplify expression.
by karsten＠torproject.org 27 Aug '18

27 Aug '18

commit ca4ec0c161c6fb9a6a78a96f9b44438938af190c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 16:05:16 2018 +0200 Simplify expression. --- .../java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java index 1e13d6d..ac3f5e3 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java @@ -834,7 +834,7 @@ public class ArchiveWriter extends CollecTorMain { try { for (int i = 0; i < outputFiles.length; i++) { File outputFile = outputFiles[i]; - boolean appendToFile = append == null ? false : append[i]; + boolean appendToFile = append != null && append[i]; outputFile.getParentFile().mkdirs(); BufferedOutputStream bos = new BufferedOutputStream( new FileOutputStream(outputFile, appendToFile));

1 0

[collector/master] Remove some more unthrown exceptions.
by karsten＠torproject.org 27 Aug '18

27 Aug '18

commit bb8c53a64dc6ac774f6ba24b8d75dc66ccbd844c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 20:30:30 2018 +0200 Remove some more unthrown exceptions. --- .../metrics/collector/bridgedescs/BridgeSnapshotReader.java | 5 +---- .../metrics/collector/bridgedescs/BridgeDescriptorParserTest.java | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java index abd30ab..fccee5e 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java @@ -3,8 +3,6 @@ package org.torproject.metrics.collector.bridgedescs; -import org.torproject.metrics.collector.conf.ConfigurationException; - import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; @@ -38,8 +36,7 @@ public class BridgeSnapshotReader { * Reads the half-hourly snapshots of bridge descriptors from Bifroest. */ public BridgeSnapshotReader(BridgeDescriptorParser bdp, - File bridgeDirectoriesDir, File statsDirectory) - throws ConfigurationException { + File bridgeDirectoriesDir, File statsDirectory) { if (bdp == null || bridgeDirectoriesDir == null || statsDirectory == null) { diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParserTest.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParserTest.java index 43fad5e..4ea2e97 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParserTest.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParserTest.java @@ -15,7 +15,7 @@ public class BridgeDescriptorParserTest { } @Test(expected = NullPointerException.class) - public void testNullData() throws Exception { + public void testNullData() { BridgeDescriptorParser bdp = new BridgeDescriptorParser( new SanitizedBridgesWriter(new Configuration())); bdp.parse(null, "", ""); @@ -24,7 +24,7 @@ public class BridgeDescriptorParserTest { @Test /* Empty data is not passed down to the sanitized writer. * This test passes when there is no exception. */ - public void testDataEmpty() throws Exception { + public void testDataEmpty() { BridgeDescriptorParser bdp = new BridgeDescriptorParser( new SanitizedBridgesWriter(new Configuration())); bdp.parse(new byte[]{}, null, null); @@ -34,7 +34,7 @@ public class BridgeDescriptorParserTest { /* The SanitizedBridgesWriter wasn't initialized sufficiently. * Actually that should be corrected in SanitizedBridgesWriter * at some point, but that's a bigger rewrite. */ - public void testMinimalData() throws Exception { + public void testMinimalData() { BridgeDescriptorParser bdp = new BridgeDescriptorParser( new SanitizedBridgesWriter(new Configuration())); bdp.parse(new byte[]{0}, "2010-10-10 10:10:10", null);

1 0

[collector/master] Replace addAll with parameterized constructor call.
by karsten＠torproject.org 27 Aug '18

27 Aug '18

commit 1c74a356ad02ff79846fe53d8a74567e4d15fa52 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 16:27:18 2018 +0200 Replace addAll with parameterized constructor call. --- .../org/torproject/metrics/collector/webstats/SanitizeWeblogs.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java index f3fbf43..7096832 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java @@ -218,8 +218,7 @@ public class SanitizeWeblogs extends CollecTorMain { if (dates.isEmpty()) { // return the empty interval return new LocalDate[]{LocalDate.MAX, LocalDate.MIN}; } - SortedSet<LocalDate> sorted = new TreeSet<>(); - sorted.addAll(dates); + SortedSet<LocalDate> sorted = new TreeSet<>(dates); if (this.limits) { for (int i = 0; i < LIMIT - 1; i++) { sorted.remove(sorted.last());

1 0

[collector/master] Use Arrays.asList with varargs.
by karsten＠torproject.org 27 Aug '18

27 Aug '18

commit 1c376afd3b5120e0dda4ca1b140fb9b3a9f4b902 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 16:20:13 2018 +0200 Use Arrays.asList with varargs. --- .../collector/bridgedescs/SanitizedBridgesWriterTest.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java index b791f88..819c830 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java @@ -112,16 +112,14 @@ public class SanitizedBridgesWriterTest { this.defaultTarballTestBuilder = new TarballTestBuilder( "from-tonga-2016-07-01T000702Z.tar.gz", 1467331624000L); this.defaultTarballTestBuilder.add("bridge-descriptors", 1467331622000L, - Arrays.asList(new TestDescriptorBuilder[] { - this.defaultServerTestDescriptorBuilder })); + Arrays.asList(this.defaultServerTestDescriptorBuilder)); this.defaultTarballTestBuilder.add("cached-extrainfo", 1467327972000L, - Arrays.asList(new TestDescriptorBuilder[] { - this.defaultExtraInfoTestDescriptorBuilder })); + Arrays.asList(this.defaultExtraInfoTestDescriptorBuilder)); this.defaultTarballTestBuilder.add("cached-extrainfo.new", 1467331623000L, Arrays.asList(new TestDescriptorBuilder[] { })); this.defaultTarballTestBuilder.add("networkstatus-bridges", - 1467330028000L, Arrays.asList(new TestDescriptorBuilder[] { - this.defaultNetworkStatusTestDescriptorBuilder })); + 1467330028000L, + Arrays.asList(this.defaultNetworkStatusTestDescriptorBuilder)); this.tarballBuilders = new ArrayList<>( Arrays.asList(this.defaultTarballTestBuilder)); } @@ -713,8 +711,7 @@ public class SanitizedBridgesWriterTest { @Test public void testTarballContainsSameFileTwice() throws Exception { this.defaultTarballTestBuilder.add("cached-extrainfo.new", 1467331623000L, - Arrays.asList(new TestDescriptorBuilder[] { - this.defaultExtraInfoTestDescriptorBuilder })); + Arrays.asList(this.defaultExtraInfoTestDescriptorBuilder)); this.runTest(); assertEquals("There should only be one.", 1, this.parsedExtraInfoDescriptors.size());

1 0

[collector/master] Remove redundant initializers.
by karsten＠torproject.org 27 Aug '18

27 Aug '18

commit cc692b7e2f26766eb977641586bd62b3c30e9767 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 16:29:32 2018 +0200 Remove redundant initializers. --- src/main/java/org/torproject/metrics/collector/Main.java | 2 +- .../metrics/collector/bridgedescs/BridgeSnapshotReader.java | 10 +++++----- .../metrics/collector/bridgedescs/SanitizedBridgesWriter.java | 10 +++++----- .../metrics/collector/exitlists/ExitListDownloader.java | 2 +- .../metrics/collector/onionperf/OnionPerfDownloader.java | 4 ++-- .../torproject/metrics/collector/relaydescs/ArchiveReader.java | 8 ++++---- .../collector/relaydescs/CachedRelayDescriptorReader.java | 6 +++--- .../collector/relaydescs/RelayDescriptorDownloader.java | 6 +++--- 8 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/Main.java b/src/main/java/org/torproject/metrics/collector/Main.java index 05aead7..1e186d4 100644 --- a/src/main/java/org/torproject/metrics/collector/Main.java +++ b/src/main/java/org/torproject/metrics/collector/Main.java @@ -63,7 +63,7 @@ public class Main { */ public static void main(String[] args) { try { - Path confPath = null; + Path confPath; if (args == null || args.length == 0) { confPath = Paths.get(CONF_FILE); } else if (args.length == 1) { diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java index 9df94d9..abd30ab 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java @@ -55,7 +55,7 @@ public class BridgeSnapshotReader { logger.debug("Reading file " + pbdFile.getAbsolutePath() + "..."); try { BufferedReader br = new BufferedReader(new FileReader(pbdFile)); - String line = null; + String line; while ((line = br.readLine()) != null) { parsed.add(line); } @@ -88,7 +88,7 @@ public class BridgeSnapshotReader { try { FileInputStream in = new FileInputStream(pop); if (in.available() > 0) { - TarArchiveInputStream tais = null; + TarArchiveInputStream tais; if (pop.getName().endsWith(".tar.gz")) { GzipCompressorInputStream gcis = new GzipCompressorInputStream(in); @@ -149,7 +149,7 @@ public class BridgeSnapshotReader { String ascii = new String(allData, "US-ASCII"); BufferedReader br3 = new BufferedReader(new StringReader( ascii)); - String firstLine = null; + String firstLine; while ((firstLine = br3.readLine()) != null) { if (firstLine.startsWith("@")) { continue; @@ -171,8 +171,8 @@ public class BridgeSnapshotReader { skippedFiles++; continue; } else { - int start = -1; - int sig = -1; + int start; + int sig; int end = -1; String startToken = firstLine.startsWith("router ") ? "router " : "extra-info "; diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java index 7c335c9..2bd85ef 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java @@ -242,7 +242,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { String addressPart = orAddress.substring(0, orAddress.lastIndexOf(":")); String portPart = orAddress.substring(orAddress.lastIndexOf(":") + 1); - String scrubbedAddressPart = null; + String scrubbedAddressPart; if (addressPart.startsWith("[")) { scrubbedAddressPart = this.scrubIpv6Address(addressPart, fingerprintBytes, published); @@ -340,7 +340,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { } hex.append(hexParts.get(1)); } - byte[] ipBytes = null; + byte[] ipBytes; try { ipBytes = Hex.decodeHex(hex.toString().toCharArray()); } catch (DecoderException e) { @@ -473,7 +473,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { DescriptorBuilder scrubbed = new DescriptorBuilder(); BufferedReader br = new BufferedReader(new StringReader(new String( data, "US-ASCII"))); - String line = null; + String line; String mostRecentDescPublished = null; byte[] fingerprintBytes = null; String descPublicationTime = null; @@ -1120,9 +1120,9 @@ public class SanitizedBridgesWriter extends CollecTorMain { try { BufferedReader br = new BufferedReader(new StringReader(new String( data, "US-ASCII"))); - String line = null; + String line; DescriptorBuilder scrubbed = null; - String hashedBridgeIdentity = null; + String hashedBridgeIdentity; String masterKeyEd25519 = null; while ((line = br.readLine()) != null) { diff --git a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java index a346d02..0b7dfad 100644 --- a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java @@ -70,7 +70,7 @@ public class ExitListDownloader extends CollecTorMain { recentPathName = Paths.get(config.getPath(Key.RecentPath).toString(), EXITLISTS).toString(); Date downloadedDate = new Date(); - String downloadedExitList = null; + String downloadedExitList; logger.debug("Downloading exit list..."); StringBuilder sb = new StringBuilder(); sb.append(Annotation.ExitList.toString()); diff --git a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java index ab7c4f9..f97a4f8 100644 --- a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java @@ -184,8 +184,8 @@ public class OnionPerfDownloader extends CollecTorMain { tpfFileName); return; } - int fileSize = 0; - String date = null; + int fileSize; + String date; try { fileSize = Integer.parseInt( tpfFileNameParts[tpfFileNameParts.length - 4]); diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java index aea169f..74700f7 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java @@ -67,7 +67,7 @@ public class ArchiveReader { try { BufferedReader br = new BufferedReader(new FileReader( archivesImportHistoryFile)); - String line = null; + String line; while ((line = br.readLine()) != null) { archivesImportHistory.add(line); } @@ -90,7 +90,7 @@ public class ArchiveReader { Collections.addAll(filesInInputDir, pop.listFiles()); } else { try { - BufferedInputStream bis = null; + BufferedInputStream bis; if (keepImportHistory && archivesImportHistory.contains(pop.getName())) { ignoredFiles++; @@ -140,7 +140,7 @@ public class ArchiveReader { * microdescriptor. However, this breaks functional abstraction * pretty badly. */ try { - BufferedInputStream bis = null; + BufferedInputStream bis; if (pop.getName().endsWith(".bz2")) { FileInputStream fis = new FileInputStream(pop); BZip2CompressorInputStream bcis = @@ -178,7 +178,7 @@ public class ArchiveReader { new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); parseFormat.setTimeZone(TimeZone.getTimeZone("UTC")); String ascii = new String(allData, StandardCharsets.US_ASCII); - int start = -1; + int start; int end = -1; String startToken = "onion-key\n"; while (end < ascii.length()) { diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java index ad0f6a5..f64d019 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java @@ -110,7 +110,7 @@ public class CachedRelayDescriptorReader { /* Check if directory information is stale. */ BufferedReader br = new BufferedReader(new StringReader( new String(allData, "US-ASCII"))); - String line = null; + String line; while ((line = br.readLine()) != null) { if (line.startsWith("valid-after ")) { dumpStats.append("\n").append(f.getName()).append(": ") @@ -178,8 +178,8 @@ public class CachedRelayDescriptorReader { } else if (f.getName().startsWith("cached-descriptors") || f.getName().startsWith("cached-extrainfo")) { String ascii = new String(allData, "US-ASCII"); - int start = -1; - int sig = -1; + int start; + int sig; int end = -1; String startToken = f.getName().startsWith("cached-descriptors") diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java index 8becbe1..f876319 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java @@ -898,8 +898,8 @@ public class RelayDescriptorDownloader { this.currentTimestamp); } String ascii = new String(allData, StandardCharsets.US_ASCII); - int start = -1; - int sig = -1; + int start; + int sig; int end = -1; String startToken = resource.startsWith("/tor/server/") ? "router " : "extra-info "; @@ -935,7 +935,7 @@ public class RelayDescriptorDownloader { new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); parseFormat.setTimeZone(TimeZone.getTimeZone("UTC")); String ascii = new String(allData, StandardCharsets.US_ASCII); - int start = -1; + int start; int end = -1; String startToken = "onion-key\n"; while (end < ascii.length()) {

1 0

[collector/master] Refactor ArchiveReader.
by karsten＠torproject.org 27 Aug '18

27 Aug '18

commit 1caca7c1f4786ef31207b42ed8298998c989487b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 21:08:19 2018 +0200 Refactor ArchiveReader. --- .../collector/relaydescs/ArchiveReader.java | 100 +++++++++++++-------- .../collector/relaydescs/ArchiveWriter.java | 3 +- 2 files changed, 66 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java index 74700f7..7c59054 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java @@ -46,30 +46,52 @@ public class ArchiveReader { private Map<String, Set<String>> microdescriptorValidAfterTimes = new HashMap<>(); - /** Reads all descriptors from the given directory, possibly using a - * parse history file, and passes them to the given descriptor - * parser. */ - public ArchiveReader(RelayDescriptorParser rdp, File archivesDirectory, - File statsDirectory, boolean keepImportHistory) { + private RelayDescriptorParser rdp; + + private File archivesDirectory; + + private boolean keepImportHistory; + private int parsedFiles = 0; + + private int ignoredFiles = 0; + + private SortedSet<String> archivesImportHistory = new TreeSet<>(); + + private File archivesImportHistoryFile; + + /** Initializes an archive reader but without reading any descriptors yet. */ + ArchiveReader(RelayDescriptorParser rdp, File archivesDirectory, + File statsDirectory, boolean keepImportHistory) { if (rdp == null || archivesDirectory == null || statsDirectory == null) { throw new IllegalArgumentException(); } - - rdp.setArchiveReader(this); - int parsedFiles = 0; - int ignoredFiles = 0; - SortedSet<String> archivesImportHistory = new TreeSet<>(); - File archivesImportHistoryFile = new File(statsDirectory, + this.rdp = rdp; + this.rdp.setArchiveReader(this); + this.archivesDirectory = archivesDirectory; + this.keepImportHistory = keepImportHistory; + this.archivesImportHistoryFile = new File(statsDirectory, "archives-import-history"); - if (keepImportHistory && archivesImportHistoryFile.exists()) { + } + + /** Reads all descriptors from the given directory, possibly using a + * parse history file, and passes them to the given descriptor + * parser. */ + public void readDescriptors() { + this.readHistoryFile(); + this.readDescriptorFiles(); + this.writeHistoryFile(); + } + + private void readHistoryFile() { + if (this.keepImportHistory && this.archivesImportHistoryFile.exists()) { try { BufferedReader br = new BufferedReader(new FileReader( - archivesImportHistoryFile)); + this.archivesImportHistoryFile)); String line; while ((line = br.readLine()) != null) { - archivesImportHistory.add(line); + this.archivesImportHistory.add(line); } br.close(); } catch (IOException e) { @@ -77,11 +99,14 @@ public class ArchiveReader { + "history file. Skipping.", e); } } - if (archivesDirectory.exists()) { - logger.debug("Importing files in directory " + archivesDirectory + } + + private void readDescriptorFiles() { + if (this.archivesDirectory.exists()) { + logger.debug("Importing files in directory " + this.archivesDirectory + "/..."); Stack<File> filesInInputDir = new Stack<>(); - filesInInputDir.add(archivesDirectory); + filesInInputDir.add(this.archivesDirectory); List<File> problems = new ArrayList<>(); Set<File> filesToRetry = new HashSet<>(); while (!filesInInputDir.isEmpty()) { @@ -91,9 +116,9 @@ public class ArchiveReader { } else { try { BufferedInputStream bis; - if (keepImportHistory - && archivesImportHistory.contains(pop.getName())) { - ignoredFiles++; + if (this.keepImportHistory + && this.archivesImportHistory.contains(pop.getName())) { + this.ignoredFiles++; continue; } else if (pop.getName().endsWith(".tar.bz2")) { logger.warn("Cannot parse compressed tarball " @@ -116,15 +141,15 @@ public class ArchiveReader { } bis.close(); byte[] allData = baos.toByteArray(); - boolean stored = rdp.parse(allData); + boolean stored = this.rdp.parse(allData); if (!stored) { filesToRetry.add(pop); continue; } - if (keepImportHistory) { - archivesImportHistory.add(pop.getName()); + if (this.keepImportHistory) { + this.archivesImportHistory.add(pop.getName()); } - parsedFiles++; + this.parsedFiles++; } catch (IOException e) { problems.add(pop); if (problems.size() > 3) { @@ -219,10 +244,10 @@ public class ArchiveReader { } } } - if (keepImportHistory) { - archivesImportHistory.add(pop.getName()); + if (this.keepImportHistory) { + this.archivesImportHistory.add(pop.getName()); } - parsedFiles++; + this.parsedFiles++; } catch (IOException e) { problems.add(pop); if (problems.size() > 3) { @@ -232,10 +257,10 @@ public class ArchiveReader { } if (problems.isEmpty()) { logger.debug("Finished importing files in directory " - + archivesDirectory + "/."); + + this.archivesDirectory + "/."); } else { StringBuilder sb = new StringBuilder("Failed importing files in " - + "directory " + archivesDirectory + "/:"); + + "directory " + this.archivesDirectory + "/:"); int printed = 0; for (File f : problems) { sb.append("\n ").append(f.getAbsolutePath()); @@ -246,12 +271,15 @@ public class ArchiveReader { } } } - if (keepImportHistory) { + } + + private void writeHistoryFile() { + if (this.keepImportHistory) { try { - archivesImportHistoryFile.getParentFile().mkdirs(); + this.archivesImportHistoryFile.getParentFile().mkdirs(); BufferedWriter bw = new BufferedWriter(new FileWriter( - archivesImportHistoryFile)); - for (String line : archivesImportHistory) { + this.archivesImportHistoryFile)); + for (String line : this.archivesImportHistory) { bw.write(line + "\n"); } bw.close(); @@ -261,15 +289,15 @@ public class ArchiveReader { } } logger.info("Finished importing relay descriptors from local " - + "directory:\nParsed " + parsedFiles + ", ignored " - + ignoredFiles + " files."); + + "directory:\nParsed " + this.parsedFiles + ", ignored " + + this.ignoredFiles + " files."); } /** Stores the valid-after time and microdescriptor digests of a given * microdesc consensus, so that microdescriptors (which don't contain a * publication time) can later be sorted into the correct month * folders. */ - public void haveParsedMicrodescConsensus(String validAfterTime, + void haveParsedMicrodescConsensus(String validAfterTime, SortedSet<String> microdescriptorDigests) { for (String microdescriptor : microdescriptorDigests) { if (!this.microdescriptorValidAfterTimes.containsKey( diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java index ac3f5e3..8679439 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java @@ -169,7 +169,8 @@ public class ArchiveWriter extends CollecTorMain { new ArchiveReader(rdp, config.getPath(Key.RelayLocalOrigins).toFile(), statsDirectory, - config.getBool(Key.KeepDirectoryArchiveImportHistory)); + config.getBool(Key.KeepDirectoryArchiveImportHistory)) + .readDescriptors(); this.intermediateStats("importing relay descriptors from local " + "directory"); }

1 0

[collector/master] Remove unnecessary modifiers and semicolons.
by karsten＠torproject.org 27 Aug '18

27 Aug '18

commit 941a18353ae98b1b4eefe6c3d621e81978e85ae9 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Aug 21 09:23:45 2018 +0200 Remove unnecessary modifiers and semicolons. --- src/main/java/org/torproject/metrics/collector/conf/Annotation.java | 2 +- src/main/java/org/torproject/metrics/collector/conf/SourceType.java | 2 +- src/main/java/org/torproject/metrics/collector/sync/Criterium.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java index a796f4d..f90516b 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java +++ b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java @@ -22,7 +22,7 @@ public enum Annotation { private final String annotation; private final byte[] bytes; - private Annotation(String annotation) { + Annotation(String annotation) { this.annotation = annotation; this.bytes = annotation.getBytes(); } diff --git a/src/main/java/org/torproject/metrics/collector/conf/SourceType.java b/src/main/java/org/torproject/metrics/collector/conf/SourceType.java index 984ff21..ed5df4d 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/SourceType.java +++ b/src/main/java/org/torproject/metrics/collector/conf/SourceType.java @@ -7,5 +7,5 @@ public enum SourceType { Cache, Local, Remote, - Sync; + Sync } diff --git a/src/main/java/org/torproject/metrics/collector/sync/Criterium.java b/src/main/java/org/torproject/metrics/collector/sync/Criterium.java index c9d6212..a90de8e 100644 --- a/src/main/java/org/torproject/metrics/collector/sync/Criterium.java +++ b/src/main/java/org/torproject/metrics/collector/sync/Criterium.java @@ -7,7 +7,7 @@ package org.torproject.metrics.collector.sync; public interface Criterium<T> { /** Determine, if the given object of type T fulfills the Criterium. */ - public boolean applies(T object); + boolean applies(T object); }

1 0

[collector/master] Refactor CachedRelayDescriptorReader.
by karsten＠torproject.org 27 Aug '18

27 Aug '18

commit 698d15d94f52fe1192fd2afb82439abace27066c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 21:17:03 2018 +0200 Refactor CachedRelayDescriptorReader. --- .../collector/relaydescs/ArchiveWriter.java | 3 +- .../relaydescs/CachedRelayDescriptorReader.java | 96 ++++++++++++++-------- 2 files changed, 62 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java index 8679439..3429a0a 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java @@ -161,7 +161,8 @@ public class ArchiveWriter extends CollecTorMain { } if (sources.contains(SourceType.Cache)) { new CachedRelayDescriptorReader(rdp, - config.getStringArray(Key.RelayCacheOrigins), statsDirectory); + config.getStringArray(Key.RelayCacheOrigins), statsDirectory) + .readDescriptors(); this.intermediateStats("importing relay descriptors from local " + "Tor data directories"); } diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java index f64d019..4f0d012 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java @@ -37,32 +37,52 @@ public class CachedRelayDescriptorReader { private static final Logger logger = LoggerFactory.getLogger( CachedRelayDescriptorReader.class); - /** Reads cached-descriptor files from one or more directories and - * passes them to the given descriptor parser. */ - public CachedRelayDescriptorReader(RelayDescriptorParser rdp, - String[] inputDirectories, File statsDirectory) { + private RelayDescriptorParser rdp; + + private String[] inputDirectories; + + private File importHistoryFile; + private StringBuilder dumpStats; + + private Set<String> lastImportHistory = new HashSet<>(); + + private Set<String> currentImportHistory = new HashSet<>(); + + /** Initializes this reader but without starting to read yet. */ + CachedRelayDescriptorReader(RelayDescriptorParser rdp, + String[] inputDirectories, File statsDirectory) { if (rdp == null || inputDirectories == null || inputDirectories.length == 0 || statsDirectory == null) { throw new IllegalArgumentException(); } + this.rdp = rdp; + this.inputDirectories = inputDirectories; + this.importHistoryFile = new File(statsDirectory, + "cacheddesc-import-history"); - StringBuilder dumpStats = new StringBuilder("Finished importing " + this.dumpStats = new StringBuilder("Finished importing " + "relay descriptors from local Tor data directories:"); + } - /* Read import history containing SHA-1 digests of previously parsed - * statuses and descriptors, so that we can skip them in this run. */ - Set<String> lastImportHistory = new HashSet<>(); - Set<String> currentImportHistory = new HashSet<>(); - File importHistoryFile = new File(statsDirectory, - "cacheddesc-import-history"); + /** Reads cached-descriptor files from one or more directories and + * passes them to the given descriptor parser. */ + public void readDescriptors() { + this.readHistoryFile(); + this.readDescriptorFiles(); + this.writeHistoryFile(); + } + + /** Read import history containing SHA-1 digests of previously parsed + * statuses and descriptors, so that we can skip them in this run. */ + private void readHistoryFile() { if (importHistoryFile.exists()) { try { BufferedReader br = new BufferedReader(new FileReader( importHistoryFile)); String line; while ((line = br.readLine()) != null) { - lastImportHistory.add(line); + this.lastImportHistory.add(line); } br.close(); } catch (IOException e) { @@ -70,9 +90,11 @@ public class CachedRelayDescriptorReader { + importHistoryFile.getAbsolutePath() + ".", e); } } + } - /* Read cached descriptors directories. */ - for (String inputDirectory : inputDirectories) { + /** Read cached descriptors directories. */ + private void readDescriptorFiles() { + for (String inputDirectory : this.inputDirectories) { File cachedDescDir = new File(inputDirectory); if (!cachedDescDir.exists()) { logger.warn("Directory " + cachedDescDir.getAbsolutePath() @@ -113,7 +135,7 @@ public class CachedRelayDescriptorReader { String line; while ((line = br.readLine()) != null) { if (line.startsWith("valid-after ")) { - dumpStats.append("\n").append(f.getName()).append(": ") + this.dumpStats.append("\n").append(f.getName()).append(": ") .append(line.substring("valid-after ".length())); SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); @@ -125,7 +147,7 @@ public class CachedRelayDescriptorReader { + cachedDescDir.getAbsolutePath() + " are stale. " + "The valid-after line in cached-consensus is '" + line + "'."); - dumpStats.append(" (stale!)"); + this.dumpStats.append(" (stale!)"); } break; } @@ -136,13 +158,13 @@ public class CachedRelayDescriptorReader { * (but regardless of whether it's stale or not). */ String digest = Hex.encodeHexString(DigestUtils.sha1( allData)); - if (!lastImportHistory.contains(digest) - && !currentImportHistory.contains(digest)) { - rdp.parse(allData); + if (!this.lastImportHistory.contains(digest) + && !this.currentImportHistory.contains(digest)) { + this.rdp.parse(allData); } else { - dumpStats.append(" (skipped)"); + this.dumpStats.append(" (skipped)"); } - currentImportHistory.add(digest); + this.currentImportHistory.add(digest); } else if (f.getName().equals("v3-status-votes")) { int parsedNum = 0; int skippedNum = 0; @@ -161,18 +183,18 @@ public class CachedRelayDescriptorReader { next - start); String digest = Hex.encodeHexString(DigestUtils.sha1( rawNetworkStatusBytes)); - if (!lastImportHistory.contains(digest) - && !currentImportHistory.contains(digest)) { - rdp.parse(rawNetworkStatusBytes); + if (!this.lastImportHistory.contains(digest) + && !this.currentImportHistory.contains(digest)) { + this.rdp.parse(rawNetworkStatusBytes); parsedNum++; } else { skippedNum++; } - currentImportHistory.add(digest); + this.currentImportHistory.add(digest); } start = next; } - dumpStats.append("\n").append(f.getName()).append(": parsed ") + this.dumpStats.append("\n").append(f.getName()).append(": parsed ") .append(parsedNum).append(", skipped ").append(skippedNum) .append(" votes"); } else if (f.getName().startsWith("cached-descriptors") @@ -183,7 +205,7 @@ public class CachedRelayDescriptorReader { int end = -1; String startToken = f.getName().startsWith("cached-descriptors") - ? "router " : "extra-info "; + ? "router " : "extra-info "; String sigToken = "\nrouter-signature\n"; String endToken = "\n-----END SIGNATURE-----\n"; int parsedNum = 0; @@ -207,16 +229,16 @@ public class CachedRelayDescriptorReader { System.arraycopy(allData, start, descBytes, 0, end - start); String digest = Hex.encodeHexString(DigestUtils.sha1( descBytes)); - if (!lastImportHistory.contains(digest) - && !currentImportHistory.contains(digest)) { - rdp.parse(descBytes); + if (!this.lastImportHistory.contains(digest) + && !this.currentImportHistory.contains(digest)) { + this.rdp.parse(descBytes); parsedNum++; } else { skippedNum++; } - currentImportHistory.add(digest); + this.currentImportHistory.add(digest); } - dumpStats.append("\n").append(f.getName()).append(": parsed ") + this.dumpStats.append("\n").append(f.getName()).append(": parsed ") .append(parsedNum).append(", skipped ").append(skippedNum) .append(" ").append(f.getName().startsWith("cached-descriptors") ? "server" : "extra-info").append(" descriptors"); @@ -229,19 +251,21 @@ public class CachedRelayDescriptorReader { logger.debug("Finished reading " + cachedDescDir.getAbsolutePath() + " directory."); } + } - /* Write import history containing SHA-1 digests to disk. */ + /** Write import history containing SHA-1 digests to disk. */ + private void writeHistoryFile() { try { - importHistoryFile.getParentFile().mkdirs(); + this.importHistoryFile.getParentFile().mkdirs(); BufferedWriter bw = new BufferedWriter(new FileWriter( - importHistoryFile)); + this.importHistoryFile)); for (String digest : currentImportHistory) { bw.write(digest + "\n"); } bw.close(); } catch (IOException e) { logger.warn("Could not write import history to " - + importHistoryFile.getAbsolutePath() + ".", e); + + this.importHistoryFile.getAbsolutePath() + ".", e); } logger.info(dumpStats.toString());

1 0