October 2018 - tor-commits - lists.torproject.org

[collector/release] Refactor ArchiveReader.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 1caca7c1f4786ef31207b42ed8298998c989487b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 20 21:08:19 2018 +0200 Refactor ArchiveReader. --- .../collector/relaydescs/ArchiveReader.java | 100 +++++++++++++-------- .../collector/relaydescs/ArchiveWriter.java | 3 +- 2 files changed, 66 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java index 74700f7..7c59054 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java @@ -46,30 +46,52 @@ public class ArchiveReader { private Map<String, Set<String>> microdescriptorValidAfterTimes = new HashMap<>(); - /** Reads all descriptors from the given directory, possibly using a - * parse history file, and passes them to the given descriptor - * parser. */ - public ArchiveReader(RelayDescriptorParser rdp, File archivesDirectory, - File statsDirectory, boolean keepImportHistory) { + private RelayDescriptorParser rdp; + + private File archivesDirectory; + + private boolean keepImportHistory; + private int parsedFiles = 0; + + private int ignoredFiles = 0; + + private SortedSet<String> archivesImportHistory = new TreeSet<>(); + + private File archivesImportHistoryFile; + + /** Initializes an archive reader but without reading any descriptors yet. */ + ArchiveReader(RelayDescriptorParser rdp, File archivesDirectory, + File statsDirectory, boolean keepImportHistory) { if (rdp == null || archivesDirectory == null || statsDirectory == null) { throw new IllegalArgumentException(); } - - rdp.setArchiveReader(this); - int parsedFiles = 0; - int ignoredFiles = 0; - SortedSet<String> archivesImportHistory = new TreeSet<>(); - File archivesImportHistoryFile = new File(statsDirectory, + this.rdp = rdp; + this.rdp.setArchiveReader(this); + this.archivesDirectory = archivesDirectory; + this.keepImportHistory = keepImportHistory; + this.archivesImportHistoryFile = new File(statsDirectory, "archives-import-history"); - if (keepImportHistory && archivesImportHistoryFile.exists()) { + } + + /** Reads all descriptors from the given directory, possibly using a + * parse history file, and passes them to the given descriptor + * parser. */ + public void readDescriptors() { + this.readHistoryFile(); + this.readDescriptorFiles(); + this.writeHistoryFile(); + } + + private void readHistoryFile() { + if (this.keepImportHistory && this.archivesImportHistoryFile.exists()) { try { BufferedReader br = new BufferedReader(new FileReader( - archivesImportHistoryFile)); + this.archivesImportHistoryFile)); String line; while ((line = br.readLine()) != null) { - archivesImportHistory.add(line); + this.archivesImportHistory.add(line); } br.close(); } catch (IOException e) { @@ -77,11 +99,14 @@ public class ArchiveReader { + "history file. Skipping.", e); } } - if (archivesDirectory.exists()) { - logger.debug("Importing files in directory " + archivesDirectory + } + + private void readDescriptorFiles() { + if (this.archivesDirectory.exists()) { + logger.debug("Importing files in directory " + this.archivesDirectory + "/..."); Stack<File> filesInInputDir = new Stack<>(); - filesInInputDir.add(archivesDirectory); + filesInInputDir.add(this.archivesDirectory); List<File> problems = new ArrayList<>(); Set<File> filesToRetry = new HashSet<>(); while (!filesInInputDir.isEmpty()) { @@ -91,9 +116,9 @@ public class ArchiveReader { } else { try { BufferedInputStream bis; - if (keepImportHistory - && archivesImportHistory.contains(pop.getName())) { - ignoredFiles++; + if (this.keepImportHistory + && this.archivesImportHistory.contains(pop.getName())) { + this.ignoredFiles++; continue; } else if (pop.getName().endsWith(".tar.bz2")) { logger.warn("Cannot parse compressed tarball " @@ -116,15 +141,15 @@ public class ArchiveReader { } bis.close(); byte[] allData = baos.toByteArray(); - boolean stored = rdp.parse(allData); + boolean stored = this.rdp.parse(allData); if (!stored) { filesToRetry.add(pop); continue; } - if (keepImportHistory) { - archivesImportHistory.add(pop.getName()); + if (this.keepImportHistory) { + this.archivesImportHistory.add(pop.getName()); } - parsedFiles++; + this.parsedFiles++; } catch (IOException e) { problems.add(pop); if (problems.size() > 3) { @@ -219,10 +244,10 @@ public class ArchiveReader { } } } - if (keepImportHistory) { - archivesImportHistory.add(pop.getName()); + if (this.keepImportHistory) { + this.archivesImportHistory.add(pop.getName()); } - parsedFiles++; + this.parsedFiles++; } catch (IOException e) { problems.add(pop); if (problems.size() > 3) { @@ -232,10 +257,10 @@ public class ArchiveReader { } if (problems.isEmpty()) { logger.debug("Finished importing files in directory " - + archivesDirectory + "/."); + + this.archivesDirectory + "/."); } else { StringBuilder sb = new StringBuilder("Failed importing files in " - + "directory " + archivesDirectory + "/:"); + + "directory " + this.archivesDirectory + "/:"); int printed = 0; for (File f : problems) { sb.append("\n ").append(f.getAbsolutePath()); @@ -246,12 +271,15 @@ public class ArchiveReader { } } } - if (keepImportHistory) { + } + + private void writeHistoryFile() { + if (this.keepImportHistory) { try { - archivesImportHistoryFile.getParentFile().mkdirs(); + this.archivesImportHistoryFile.getParentFile().mkdirs(); BufferedWriter bw = new BufferedWriter(new FileWriter( - archivesImportHistoryFile)); - for (String line : archivesImportHistory) { + this.archivesImportHistoryFile)); + for (String line : this.archivesImportHistory) { bw.write(line + "\n"); } bw.close(); @@ -261,15 +289,15 @@ public class ArchiveReader { } } logger.info("Finished importing relay descriptors from local " - + "directory:\nParsed " + parsedFiles + ", ignored " - + ignoredFiles + " files."); + + "directory:\nParsed " + this.parsedFiles + ", ignored " + + this.ignoredFiles + " files."); } /** Stores the valid-after time and microdescriptor digests of a given * microdesc consensus, so that microdescriptors (which don't contain a * publication time) can later be sorted into the correct month * folders. */ - public void haveParsedMicrodescConsensus(String validAfterTime, + void haveParsedMicrodescConsensus(String validAfterTime, SortedSet<String> microdescriptorDigests) { for (String microdescriptor : microdescriptorDigests) { if (!this.microdescriptorValidAfterTimes.containsKey( diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java index ac3f5e3..8679439 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java @@ -169,7 +169,8 @@ public class ArchiveWriter extends CollecTorMain { new ArchiveReader(rdp, config.getPath(Key.RelayLocalOrigins).toFile(), statsDirectory, - config.getBool(Key.KeepDirectoryArchiveImportHistory)); + config.getBool(Key.KeepDirectoryArchiveImportHistory)) + .readDescriptors(); this.intermediateStats("importing relay descriptors from local " + "directory"); }

1 0

[collector/release] Remove a couple redundant string operations.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 00eb709d43499da76bc12fec5702197e1fc7e04a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Aug 21 12:07:28 2018 +0200 Remove a couple redundant string operations. --- .../metrics/collector/bridgedescs/SanitizedBridgesWriter.java | 3 +-- .../torproject/metrics/collector/persist/DescriptorPersistence.java | 2 +- .../metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java index c5a28d6..d9a0322 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java @@ -358,8 +358,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { byte[] secret = this.getSecretForMonth(month); System.arraycopy(secret, 31, hashInput, 36, 19); String hashOutput = DigestUtils.sha256Hex(hashInput); - sb.append(hashOutput.substring(hashOutput.length() - 6, - hashOutput.length() - 4)); + sb.append(hashOutput, hashOutput.length() - 6, hashOutput.length() - 4); sb.append(":"); sb.append(hashOutput.substring(hashOutput.length() - 4)); } diff --git a/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java index bd24c81..fed4839 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java @@ -43,7 +43,7 @@ public abstract class DescriptorPersistence<T extends Descriptor> { if (annotations.isEmpty()) { this.annotation = defaultAnnotation; } else { - StringBuilder sb = new StringBuilder(""); + StringBuilder sb = new StringBuilder(); for (String annotation : annotations) { sb.append(annotation).append("\n"); } diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java index 819c830..1267d4a 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java @@ -136,7 +136,7 @@ public class SanitizedBridgesWriterTest { this.configuration.setProperty(Key.BridgeLocalOrigins.name(), bridgeDirectoriesDir); this.configuration.setProperty(Key.OutputPath.name(), - sanitizedBridgesDirectory.toFile().getParent().toString()); + sanitizedBridgesDirectory.toFile().getParent()); } /** Runs this test by executing all builders, performing the sanitizing

1 0

[collector/release] Remove unused assignment.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit adb442aaa0421c9691ff9c2bfecf40d19a5050f5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Aug 21 12:00:54 2018 +0200 Remove unused assignment. --- .../java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java index edd03d7..aa4c65b 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java @@ -178,7 +178,6 @@ public class ArchiveWriter extends CollecTorMain { if (rdd != null) { rdd.downloadDescriptors(); rdd.writeFile(); - rdd = null; this.intermediateStats("downloading relay descriptors from the " + "directory authorities"); }

1 0

[collector/release] Remove unused assignment.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 1ea6a9dc1d80f1050788194a12297d2dd56d3e3b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Aug 21 12:05:50 2018 +0200 Remove unused assignment. --- .../torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java index 1cab69f..c5a28d6 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java @@ -597,7 +597,6 @@ public class SanitizedBridgesWriter extends CollecTorMain { if (scrubbed.hasContent()) { String scrubbedLine = scrubbed.toString(); scrubbedLines.put(hashedBridgeIdentityHex, scrubbedLine); - scrubbed = new DescriptorBuilder(); } if (!includesFingerprintLine) { header.append("fingerprint ").append(authorityFingerprint).newLine();

1 0

[collector/release] Replace if with switch.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 5080bf9907d3b8e3e3238b9abcf3786a1742ee32 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Aug 21 12:22:57 2018 +0200 Replace if with switch. --- .../relaydescs/RelayDescriptorDownloader.java | 37 ++++--- .../relaydescs/RelayDescriptorParser.java | 106 +++++++++++---------- 2 files changed, 77 insertions(+), 66 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java index 6e470d8..564d1bc 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java @@ -812,21 +812,28 @@ public class RelayDescriptorDownloader { this.downloadResourceFromAuthority(authority, combinedResource.toString()); } - if (type.equals("server")) { - this.requestedMissingServerDescriptors += - requestedDescriptors; - this.downloadedMissingServerDescriptors += - downloadedDescriptors; - } else if (type.equals("extra")) { - this.requestedMissingExtraInfoDescriptors += - requestedDescriptors; - this.downloadedMissingExtraInfoDescriptors += - downloadedDescriptors; - } else if (type.equals("micro")) { - this.requestedMissingMicrodescriptors += - requestedDescriptors; - this.downloadedMissingMicrodescriptors += - downloadedDescriptors; + switch (type) { + case "server": + this.requestedMissingServerDescriptors += + requestedDescriptors; + this.downloadedMissingServerDescriptors += + downloadedDescriptors; + break; + case "extra": + this.requestedMissingExtraInfoDescriptors += + requestedDescriptors; + this.downloadedMissingExtraInfoDescriptors += + downloadedDescriptors; + break; + case "micro": + this.requestedMissingMicrodescriptors += + requestedDescriptors; + this.downloadedMissingMicrodescriptors += + downloadedDescriptors; + break; + default: + /* Unknown type, nothing to do. */ + break; } } } diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java index 59b9fb0..5224a61 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java @@ -175,60 +175,64 @@ public class RelayDescriptorParser { } } } - if (statusType.equals("consensus")) { - if (this.rdd != null) { - this.rdd.haveParsedConsensus(validAfterTime, dirSources, - serverDescriptors); - } - if (this.aw != null) { - this.aw.storeConsensus(data, validAfter, dirSources, - serverDescriptorDigests); - stored = true; - } - } else if (statusType.equals("consensus-microdesc")) { - if (this.rdd != null) { - this.rdd.haveParsedMicrodescConsensus(validAfterTime, - microdescriptorKeys); - } - if (this.ar != null) { - this.ar.haveParsedMicrodescConsensus(validAfterTime, - microdescriptorDigests); - } - if (this.aw != null) { - this.aw.storeMicrodescConsensus(data, validAfter, - microdescriptorDigests); - stored = true; - } - } else { - if (this.aw != null || this.rdd != null) { - String ascii = new String(data, "US-ASCII"); - String startToken = "network-status-version "; - String sigToken = "directory-signature "; - int start = ascii.indexOf(startToken); - int sig = ascii.indexOf(sigToken); - if (start >= 0 && sig >= 0 && sig > start) { - sig += sigToken.length(); - byte[] forDigest = new byte[sig - start]; - System.arraycopy(data, start, forDigest, 0, sig - start); - String digest = DigestUtils.sha1Hex(forDigest).toUpperCase(); - if (this.aw != null) { - this.aw.storeVote(data, validAfter, dirSource, digest, - serverDescriptorDigests); - stored = true; - } - if (this.rdd != null) { - this.rdd.haveParsedVote(validAfterTime, fingerprint, - serverDescriptors); - } + switch (statusType) { + case "consensus": + if (this.rdd != null) { + this.rdd.haveParsedConsensus(validAfterTime, dirSources, + serverDescriptors); } - if (certificateString != null) { - if (this.aw != null) { - this.aw.storeCertificate(certificateString.getBytes(), - dirSource, dirKeyPublished); - stored = true; + if (this.aw != null) { + this.aw.storeConsensus(data, validAfter, dirSources, + serverDescriptorDigests); + stored = true; + } + break; + case "consensus-microdesc": + if (this.rdd != null) { + this.rdd.haveParsedMicrodescConsensus(validAfterTime, + microdescriptorKeys); + } + if (this.ar != null) { + this.ar.haveParsedMicrodescConsensus(validAfterTime, + microdescriptorDigests); + } + if (this.aw != null) { + this.aw.storeMicrodescConsensus(data, validAfter, + microdescriptorDigests); + stored = true; + } + break; + default: + if (this.aw != null || this.rdd != null) { + String ascii = new String(data, "US-ASCII"); + String startToken = "network-status-version "; + String sigToken = "directory-signature "; + int start = ascii.indexOf(startToken); + int sig = ascii.indexOf(sigToken); + if (start >= 0 && sig >= 0 && sig > start) { + sig += sigToken.length(); + byte[] forDigest = new byte[sig - start]; + System.arraycopy(data, start, forDigest, 0, sig - start); + String digest = DigestUtils.sha1Hex(forDigest).toUpperCase(); + if (this.aw != null) { + this.aw.storeVote(data, validAfter, dirSource, digest, + serverDescriptorDigests); + stored = true; + } + if (this.rdd != null) { + this.rdd.haveParsedVote(validAfterTime, fingerprint, + serverDescriptors); + } + } + if (certificateString != null) { + if (this.aw != null) { + this.aw.storeCertificate(certificateString.getBytes(), + dirSource, dirKeyPublished); + stored = true; + } } } - } + break; } } else if (line.startsWith("router ")) { String publishedTime = null;

1 0

[collector/release] Use Map.putIfAbsent and Map.getOrDefault where possible.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 4361bd5e5f1d68ba122f01ee0221dc7ab0f9712c Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Aug 21 14:58:17 2018 +0200 Use Map.putIfAbsent and Map.getOrDefault where possible. --- .../collector/relaydescs/ArchiveReader.java | 7 ++---- .../collector/relaydescs/ArchiveWriter.java | 29 ++++++---------------- .../relaydescs/RelayDescriptorDownloader.java | 7 ++---- 3 files changed, 12 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java index 4e0e28d..27010a9 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java @@ -298,11 +298,8 @@ public class ArchiveReader { void haveParsedMicrodescConsensus(String validAfterTime, SortedSet<String> microdescriptorDigests) { for (String microdescriptor : microdescriptorDigests) { - if (!this.microdescriptorValidAfterTimes.containsKey( - microdescriptor)) { - this.microdescriptorValidAfterTimes.put(microdescriptor, - new HashSet<>()); - } + this.microdescriptorValidAfterTimes.putIfAbsent(microdescriptor, + new HashSet<>()); this.microdescriptorValidAfterTimes.get(microdescriptor).add( validAfterTime); } diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java index aa4c65b..a79f0a6 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java @@ -229,9 +229,7 @@ public class ArchiveWriter extends CollecTorMain { if (published < this.now - 48L * 60L * 60L * 1000L) { continue; } - if (!this.storedServerDescriptors.containsKey(published)) { - this.storedServerDescriptors.put(published, new HashMap<>()); - } + this.storedServerDescriptors.putIfAbsent(published, new HashMap<>()); String serverDescriptorDigest = parts[1]; String extraInfoDescriptorDigest = parts[2].equals("NA") ? null : parts[2]; @@ -256,9 +254,8 @@ public class ArchiveWriter extends CollecTorMain { if (published < this.now - 48L * 60L * 60L * 1000L) { continue; } - if (!this.storedExtraInfoDescriptors.containsKey(published)) { - this.storedExtraInfoDescriptors.put(published, new HashSet<>()); - } + this.storedExtraInfoDescriptors.putIfAbsent(published, + new HashSet<>()); String extraInfoDescriptorDigest = parts[1]; this.storedExtraInfoDescriptors.get(published).add( extraInfoDescriptorDigest); @@ -281,9 +278,7 @@ public class ArchiveWriter extends CollecTorMain { if (validAfter < this.now - 40L * 24L * 60L * 60L * 1000L) { continue; } - if (!this.storedMicrodescriptors.containsKey(validAfter)) { - this.storedMicrodescriptors.put(validAfter, new HashSet<>()); - } + this.storedMicrodescriptors.putIfAbsent(validAfter, new HashSet<>()); String microdescriptorDigest = parts[1]; this.storedMicrodescriptors.get(validAfter).add( microdescriptorDigest); @@ -695,9 +690,7 @@ public class ArchiveWriter extends CollecTorMain { } if (!tarballFileExistedBefore && this.now - validAfter < 3L * 60L * 60L * 1000L) { - if (!this.storedVotes.containsKey(validAfter)) { - this.storedVotes.put(validAfter, new TreeMap<>()); - } + this.storedVotes.putIfAbsent(validAfter, new TreeMap<>()); this.storedVotes.get(validAfter).put(fingerprint, serverDescriptorDigests); } @@ -738,9 +731,7 @@ public class ArchiveWriter extends CollecTorMain { } if (!tarballFileExistedBefore && this.now - published < 48L * 60L * 60L * 1000L) { - if (!this.storedServerDescriptors.containsKey(published)) { - this.storedServerDescriptors.put(published, new HashMap<>()); - } + this.storedServerDescriptors.putIfAbsent(published, new HashMap<>()); this.storedServerDescriptors.get(published).put(digest, extraInfoDigest); } @@ -766,9 +757,7 @@ public class ArchiveWriter extends CollecTorMain { } if (!tarballFileExistedBefore && this.now - published < 48L * 60L * 60L * 1000L) { - if (!this.storedExtraInfoDescriptors.containsKey(published)) { - this.storedExtraInfoDescriptors.put(published, new HashSet<>()); - } + this.storedExtraInfoDescriptors.putIfAbsent(published, new HashSet<>()); this.storedExtraInfoDescriptors.get(published).add(extraInfoDigest); } } @@ -801,9 +790,7 @@ public class ArchiveWriter extends CollecTorMain { } if (!tarballFileExistedBefore && this.now - validAfter < 40L * 24L * 60L * 60L * 1000L) { - if (!this.storedMicrodescriptors.containsKey(validAfter)) { - this.storedMicrodescriptors.put(validAfter, new HashSet<>()); - } + this.storedMicrodescriptors.putIfAbsent(validAfter, new HashSet<>()); this.storedMicrodescriptors.get(validAfter).add( microdescriptorDigest); } diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java index 564d1bc..4764a4b 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java @@ -380,11 +380,8 @@ public class RelayDescriptorDownloader { String microdescriptorDigest = line.split(",")[3]; String microdescriptorKey = line.substring(0, line.lastIndexOf(",")); - if (!this.microdescriptorKeys.containsKey( - microdescriptorDigest)) { - this.microdescriptorKeys.put( - microdescriptorDigest, new HashSet<>()); - } + this.microdescriptorKeys.putIfAbsent(microdescriptorDigest, + new HashSet<>()); this.microdescriptorKeys.get(microdescriptorDigest).add( microdescriptorKey); if (line.endsWith("NA") && !this.missingMicrodescriptors

1 0

[collector/release] Inline redundant local variables.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 7c44c114e47603df5e643c67975568821ab6f7cb Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Aug 21 14:10:41 2018 +0200 Inline redundant local variables. --- .../metrics/collector/bridgedescs/BridgeSnapshotReader.java | 5 ++--- .../metrics/collector/bridgedescs/SanitizedBridgesWriter.java | 7 ++----- .../org/torproject/metrics/collector/indexer/CreateIndexJson.java | 6 ++---- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java index 4f72b5a..f16122a 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java @@ -44,10 +44,9 @@ public class BridgeSnapshotReader { } SortedSet<String> parsed = new TreeSet<>(); - File bdDir = bridgeDirectoriesDir; File pbdFile = new File(statsDirectory, "parsed-bridge-directories"); boolean modified = false; - if (bdDir.exists()) { + if (bridgeDirectoriesDir.exists()) { if (pbdFile.exists()) { logger.debug("Reading file {}...", pbdFile.getAbsolutePath()); try { @@ -73,7 +72,7 @@ public class BridgeSnapshotReader { int parsedExtraInfoDescriptors = 0; int skippedExtraInfoDescriptors = 0; Stack<File> filesInInputDir = new Stack<>(); - filesInInputDir.add(bdDir); + filesInInputDir.add(bridgeDirectoriesDir); while (!filesInInputDir.isEmpty()) { File pop = filesInInputDir.pop(); if (pop.isDirectory()) { diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java index d9a0322..921396c 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java @@ -274,11 +274,10 @@ public class SanitizedBridgesWriter extends CollecTorMain { byte[] secret = this.getSecretForMonth(month); System.arraycopy(secret, 0, hashInput, 24, 31); byte[] hashOutput = DigestUtils.sha256(hashInput); - String hashedAddress = "10." + return "10." + (((int) hashOutput[0] + 256) % 256) + "." + (((int) hashOutput[1] + 256) % 256) + "." + (((int) hashOutput[2] + 256) % 256); - return hashedAddress; } else { return "127.0.0.1"; } @@ -1090,9 +1089,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { masterKeyEd25519, 0, masterKeyEd25519.length); String masterKeyEd25519Base64 = Base64.encodeBase64String( masterKeyEd25519); - String masterKeyEd25519Base64NoTrailingEqualSigns = - masterKeyEd25519Base64.replaceAll("=", ""); - return masterKeyEd25519Base64NoTrailingEqualSigns; + return masterKeyEd25519Base64.replaceAll("=", ""); } extensionStart += 4 + extensionLength; } diff --git a/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java b/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java index a018f8b..a40798e 100644 --- a/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java +++ b/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java @@ -146,16 +146,14 @@ public class CreateIndexJson extends CollecTorMain { } } } - DirectoryNode directoryNode = new DirectoryNode( + return new DirectoryNode( directory.getName(), fileNodes.isEmpty() ? null : fileNodes, directoryNodes.isEmpty() ? null : directoryNodes); - return directoryNode; } private FileNode indexFile(File file) { - FileNode fileNode = new FileNode(file.getName(), file.length(), + return new FileNode(file.getName(), file.length(), dateTimeFormat.format(file.lastModified())); - return fileNode; } private void writeIndex(IndexNode indexNode) throws Exception {

1 0

[collector/release] Avoid unnecessary continue statements.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 6714bde1ced764d70b0ad1f3b85bd7f7319d25fb Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Aug 21 15:19:29 2018 +0200 Avoid unnecessary continue statements. --- .../metrics/collector/bridgedescs/BridgeSnapshotReader.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java index f16122a..68ee28b 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java @@ -143,13 +143,9 @@ public class BridgeSnapshotReader { BufferedReader br3 = new BufferedReader(new StringReader( ascii)); String firstLine; - while ((firstLine = br3.readLine()) != null) { - if (firstLine.startsWith("@")) { - continue; - } else { - break; - } - } + do { + firstLine = br3.readLine(); + } while (firstLine != null && firstLine.startsWith("@")); if (firstLine == null) { continue; } @@ -224,7 +220,6 @@ public class BridgeSnapshotReader { } catch (IOException e) { logger.warn("Could not parse bridge snapshot {}!", pop.getName(), e); - continue; } } }

1 0

[collector/release] Properly clean up web server logs in recent/.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 1f438871787eef7554e2e518d968611c3940e9ab Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Aug 30 10:51:45 2018 +0200 Properly clean up web server logs in recent/. Fixes #27390. --- CHANGELOG.md | 4 ++++ .../metrics/collector/persist/PersistenceUtils.java | 13 ++++++++++++- .../metrics/collector/webstats/SanitizeWeblogs.java | 5 ++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d98520c..fc24633 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changes in version 1.?.? - 2018-0?-?? + * Medium changes + - Properly clean up sanitized web server logs in the recent/ + directory when they turn older than three days. + * Minor changes - Once more, fix the bug in the tarball-creation script where tarballs are not compressed in a run following an aborted run. diff --git a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java index 7d30cb1..78eac72 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java +++ b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java @@ -82,12 +82,23 @@ public class PersistenceUtils { /** Move temporary files to their final location. */ public static void cleanDirectory(Path pathToClean) throws IOException { + PersistenceUtils.cleanDirectory(pathToClean, -1L); + } + + /** Clean up the given directory by deleting files that are older than the + * given cut-off timestamp, and by moving temporary files to their final + * location. */ + public static void cleanDirectory(Path pathToClean, long cutOffMillis) + throws IOException { SimpleFileVisitor<Path> sfv = new SimpleFileVisitor<Path>() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { String tempName = file.toString(); - if (tempName.endsWith(TEMPFIX)) { + if (cutOffMillis >= 0L + && attrs.lastModifiedTime().toMillis() < cutOffMillis) { + file.toFile().delete(); + } else if (tempName.endsWith(TEMPFIX)) { Path outputPath = Paths .get(tempName.substring(0, tempName.length() - TEMPFIX.length())); Files.deleteIfExists(outputPath); diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java index 0e83598..027cfde 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java @@ -97,7 +97,10 @@ public class SanitizeWeblogs extends CollecTorMain { if (sources.contains(SourceType.Local)) { log.info("Processing logs using batch value {}.", BATCH); findCleanWrite(this.config.getPath(Key.WebstatsLocalOrigins)); - PersistenceUtils.cleanDirectory(this.config.getPath(Key.RecentPath)); + long cutOffMillis = System.currentTimeMillis() + - 3L * 24L * 60L * 60L * 1000L; + PersistenceUtils.cleanDirectory(this.config.getPath(Key.RecentPath), + cutOffMillis); } } catch (Exception e) { log.error("Cannot sanitize web-logs: {}", e.getMessage(), e);

1 0

[collector/release] Prepare for 1.8.0 release.
by karsten＠torproject.org 15 Oct '18

15 Oct '18

commit 7900e9df643c01122ab0f420014b0ac18f6e682d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Oct 11 14:36:32 2018 +0200 Prepare for 1.8.0 release. --- CHANGELOG.md | 6 +++++- build.xml | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc24633..0307748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.?.? - 2018-0?-?? +# Changes in version 1.8.0 - 2018-10-11 * Medium changes - Properly clean up sanitized web server logs in the recent/ @@ -7,6 +7,10 @@ * Minor changes - Once more, fix the bug in the tarball-creation script where tarballs are not compressed in a run following an aborted run. + - Improve logging to find possible issue with missing server + descriptors. + - Update directory authority IP addresses in default properties + file. # Changes in version 1.7.0 - 2018-07-14 diff --git a/build.xml b/build.xml index ea88b8a..d978d73 100644 --- a/build.xml +++ b/build.xml @@ -8,7 +8,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.7.0-dev" /> + <property name="release.version" value="1.8.0" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.4.0" />

1 0