[tor-commits] [metrics-web/master] Update to metrics-lib 1.9.0.
karsten at torproject.org
karsten at torproject.org
Thu Jun 29 13:47:53 UTC 2017
commit 9ee4fe35e08428ce3aae7946d01da843e8004f5b
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Sat Jun 17 17:42:00 2017 +0200
Update to metrics-lib 1.9.0.
Implements #22683.
---
.../org/torproject/metrics/advbwdist/Main.java | 170 ++++++++----------
.../java/org/torproject/metrics/clients/Main.java | 54 ++----
.../org/torproject/metrics/connbidirect/Main.java | 27 +--
.../java/org/torproject/metrics/hidserv/Main.java | 8 +-
.../org/torproject/metrics/hidserv/Parser.java | 33 ++--
.../cron/RelayDescriptorDatabaseImporter.java | 34 ++--
.../cron/network/ConsensusStatsFileHandler.java | 30 ++--
.../ernie/cron/performance/TorperfProcessor.java | 97 +++++-----
.../org/torproject/metrics/onionperf/Main.java | 197 ++++++++++-----------
shared/build-base.xml | 2 +-
10 files changed, 287 insertions(+), 365 deletions(-)
diff --git a/modules/advbwdist/src/main/java/org/torproject/metrics/advbwdist/Main.java b/modules/advbwdist/src/main/java/org/torproject/metrics/advbwdist/Main.java
index 7193108..8928be2 100644
--- a/modules/advbwdist/src/main/java/org/torproject/metrics/advbwdist/Main.java
+++ b/modules/advbwdist/src/main/java/org/torproject/metrics/advbwdist/Main.java
@@ -4,7 +4,6 @@
package org.torproject.metrics.advbwdist;
import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
import org.torproject.descriptor.DescriptorReader;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.NetworkStatusEntry;
@@ -19,7 +18,6 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
@@ -31,37 +29,27 @@ public class Main {
/* Parse server descriptors, not keeping a parse history, and memorize
* the advertised bandwidth for every server descriptor. */
+ Map<String, Long> serverDescriptors = new HashMap<>();
DescriptorReader descriptorReader =
DescriptorSourceFactory.createDescriptorReader();
- descriptorReader.addDirectory(
- new File("../../shared/in/recent/relay-descriptors/"
- + "server-descriptors"));
- Iterator<DescriptorFile> descriptorFiles =
- descriptorReader.readDescriptors();
- Map<String, Long> serverDescriptors = new HashMap<>();
- while (descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = descriptorFiles.next();
- for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- if (!(descriptor instanceof ServerDescriptor)) {
- continue;
- }
- ServerDescriptor serverDescriptor = (ServerDescriptor) descriptor;
- String digest = serverDescriptor.getDigestSha1Hex();
- long advertisedBandwidth = Math.min(Math.min(
- serverDescriptor.getBandwidthRate(),
- serverDescriptor.getBandwidthBurst()),
- serverDescriptor.getBandwidthObserved());
- serverDescriptors.put(digest.toUpperCase(), advertisedBandwidth);
+ for (Descriptor descriptor : descriptorReader.readDescriptors(new File(
+ "../../shared/in/recent/relay-descriptors/server-descriptors"))) {
+ if (!(descriptor instanceof ServerDescriptor)) {
+ continue;
}
+ ServerDescriptor serverDescriptor = (ServerDescriptor) descriptor;
+ String digest = serverDescriptor.getDigestSha1Hex();
+ long advertisedBandwidth = Math.min(Math.min(
+ serverDescriptor.getBandwidthRate(),
+ serverDescriptor.getBandwidthBurst()),
+ serverDescriptor.getBandwidthObserved());
+ serverDescriptors.put(digest.toUpperCase(), advertisedBandwidth);
}
/* Parse consensuses, keeping a parse history. */
descriptorReader = DescriptorSourceFactory.createDescriptorReader();
- descriptorReader.addDirectory(
- new File("../../shared/in/recent/relay-descriptors/consensuses"));
File historyFile = new File("status/parsed-consensuses");
descriptorReader.setHistoryFile(historyFile);
- descriptorFiles = descriptorReader.readDescriptors();
File resultsFile = new File("stats/advbwdist-validafter.csv");
resultsFile.getParentFile().mkdirs();
boolean writeHeader = !resultsFile.exists();
@@ -73,82 +61,80 @@ public class Main {
SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
"yyyy-MM-dd HH:mm:ss");
dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- while (descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = descriptorFiles.next();
- for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- if (!(descriptor instanceof RelayNetworkStatusConsensus)) {
+ for (Descriptor descriptor : descriptorReader.readDescriptors(new File(
+ "../../shared/in/recent/relay-descriptors/consensuses"))) {
+ if (!(descriptor instanceof RelayNetworkStatusConsensus)) {
+ continue;
+ }
+
+ /* Parse server descriptor digests from consensus and look up
+ * advertised bandwidths. */
+ RelayNetworkStatusConsensus consensus =
+ (RelayNetworkStatusConsensus) descriptor;
+ String validAfter = dateTimeFormat.format(
+ consensus.getValidAfterMillis());
+ List<Long> advertisedBandwidthsAllRelays = new ArrayList<>();
+ List<Long> advertisedBandwidthsExitsOnly = new ArrayList<>();
+ for (NetworkStatusEntry relay
+ : consensus.getStatusEntries().values()) {
+ if (!relay.getFlags().contains("Running")) {
continue;
}
-
- /* Parse server descriptor digests from consensus and look up
- * advertised bandwidths. */
- RelayNetworkStatusConsensus consensus =
- (RelayNetworkStatusConsensus) descriptor;
- String validAfter = dateTimeFormat.format(
- consensus.getValidAfterMillis());
- List<Long> advertisedBandwidthsAllRelays = new ArrayList<>();
- List<Long> advertisedBandwidthsExitsOnly = new ArrayList<>();
- for (NetworkStatusEntry relay
- : consensus.getStatusEntries().values()) {
- if (!relay.getFlags().contains("Running")) {
- continue;
- }
- String serverDescriptorDigest = relay.getDescriptor()
- .toUpperCase();
- if (!serverDescriptors.containsKey(serverDescriptorDigest)) {
- continue;
- }
- long advertisedBandwidth = serverDescriptors.get(
- serverDescriptorDigest);
- advertisedBandwidthsAllRelays.add(advertisedBandwidth);
- if (relay.getFlags().contains("Exit")
- && !relay.getFlags().contains("BadExit")) {
- advertisedBandwidthsExitsOnly.add(advertisedBandwidth);
- }
+ String serverDescriptorDigest = relay.getDescriptor()
+ .toUpperCase();
+ if (!serverDescriptors.containsKey(serverDescriptorDigest)) {
+ continue;
+ }
+ long advertisedBandwidth = serverDescriptors.get(
+ serverDescriptorDigest);
+ advertisedBandwidthsAllRelays.add(advertisedBandwidth);
+ if (relay.getFlags().contains("Exit")
+ && !relay.getFlags().contains("BadExit")) {
+ advertisedBandwidthsExitsOnly.add(advertisedBandwidth);
}
+ }
- /* Write advertised bandwidths of n-th fastest relays/exits. */
- Collections.sort(advertisedBandwidthsAllRelays,
- Collections.reverseOrder());
- Collections.sort(advertisedBandwidthsExitsOnly,
- Collections.reverseOrder());
- int[] fastestRelays = new int[] { 1, 2, 3, 5, 10, 20, 30, 50, 100,
- 200, 300, 500, 1000, 2000, 3000, 5000 };
- for (int fastestRelay : fastestRelays) {
- if (advertisedBandwidthsAllRelays.size() >= fastestRelay) {
- bw.write(String.format("%s,,%d,,%d%n", validAfter,
- fastestRelay,
- advertisedBandwidthsAllRelays.get(fastestRelay - 1)));
- }
+ /* Write advertised bandwidths of n-th fastest relays/exits. */
+ Collections.sort(advertisedBandwidthsAllRelays,
+ Collections.reverseOrder());
+ Collections.sort(advertisedBandwidthsExitsOnly,
+ Collections.reverseOrder());
+ int[] fastestRelays = new int[] { 1, 2, 3, 5, 10, 20, 30, 50, 100,
+ 200, 300, 500, 1000, 2000, 3000, 5000 };
+ for (int fastestRelay : fastestRelays) {
+ if (advertisedBandwidthsAllRelays.size() >= fastestRelay) {
+ bw.write(String.format("%s,,%d,,%d%n", validAfter,
+ fastestRelay,
+ advertisedBandwidthsAllRelays.get(fastestRelay - 1)));
}
- for (int fastestRelay : fastestRelays) {
- if (advertisedBandwidthsExitsOnly.size() >= fastestRelay) {
- bw.write(String.format("%s,TRUE,%d,,%d%n", validAfter,
- fastestRelay,
- advertisedBandwidthsExitsOnly.get(fastestRelay - 1)));
- }
+ }
+ for (int fastestRelay : fastestRelays) {
+ if (advertisedBandwidthsExitsOnly.size() >= fastestRelay) {
+ bw.write(String.format("%s,TRUE,%d,,%d%n", validAfter,
+ fastestRelay,
+ advertisedBandwidthsExitsOnly.get(fastestRelay - 1)));
}
+ }
- /* Write advertised bandwidth percentiles of relays/exits. */
- Collections.sort(advertisedBandwidthsAllRelays);
- Collections.sort(advertisedBandwidthsExitsOnly);
- int[] percentiles = new int[] { 0, 1, 2, 3, 5, 9, 10, 20, 25, 30,
- 40, 50, 60, 70, 75, 80, 90, 91, 95, 97, 98, 99, 100 };
- if (!advertisedBandwidthsAllRelays.isEmpty()) {
- for (int percentile : percentiles) {
- bw.write(String.format("%s,,,%d,%d%n", validAfter,
- percentile, advertisedBandwidthsAllRelays.get(
- ((advertisedBandwidthsAllRelays.size() - 1)
- * percentile) / 100)));
- }
+ /* Write advertised bandwidth percentiles of relays/exits. */
+ Collections.sort(advertisedBandwidthsAllRelays);
+ Collections.sort(advertisedBandwidthsExitsOnly);
+ int[] percentiles = new int[] { 0, 1, 2, 3, 5, 9, 10, 20, 25, 30,
+ 40, 50, 60, 70, 75, 80, 90, 91, 95, 97, 98, 99, 100 };
+ if (!advertisedBandwidthsAllRelays.isEmpty()) {
+ for (int percentile : percentiles) {
+ bw.write(String.format("%s,,,%d,%d%n", validAfter,
+ percentile, advertisedBandwidthsAllRelays.get(
+ ((advertisedBandwidthsAllRelays.size() - 1)
+ * percentile) / 100)));
}
- if (!advertisedBandwidthsExitsOnly.isEmpty()) {
- for (int percentile : percentiles) {
- bw.write(String.format("%s,TRUE,,%d,%d%n", validAfter,
- percentile, advertisedBandwidthsExitsOnly.get(
- ((advertisedBandwidthsExitsOnly.size() - 1)
- * percentile) / 100)));
- }
+ }
+ if (!advertisedBandwidthsExitsOnly.isEmpty()) {
+ for (int percentile : percentiles) {
+ bw.write(String.format("%s,TRUE,,%d,%d%n", validAfter,
+ percentile, advertisedBandwidthsExitsOnly.get(
+ ((advertisedBandwidthsExitsOnly.size() - 1)
+ * percentile) / 100)));
}
}
}
diff --git a/modules/clients/src/main/java/org/torproject/metrics/clients/Main.java b/modules/clients/src/main/java/org/torproject/metrics/clients/Main.java
index dc34d37..384c621 100644
--- a/modules/clients/src/main/java/org/torproject/metrics/clients/Main.java
+++ b/modules/clients/src/main/java/org/torproject/metrics/clients/Main.java
@@ -6,7 +6,6 @@ package org.torproject.metrics.clients;
import org.torproject.descriptor.BandwidthHistory;
import org.torproject.descriptor.BridgeNetworkStatus;
import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
import org.torproject.descriptor.DescriptorReader;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.ExtraInfoDescriptor;
@@ -19,7 +18,6 @@ import java.io.FileWriter;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
import java.util.TimeZone;
@@ -65,25 +63,16 @@ public class Main {
DescriptorSourceFactory.createDescriptorReader();
File historyFile = new File("status/relay-descriptors");
descriptorReader.setHistoryFile(historyFile);
- descriptorReader.addDirectory(new File(
- "../../shared/in/recent/relay-descriptors/consensuses"));
- descriptorReader.addDirectory(new File(
- "../../shared/in/recent/relay-descriptors/extra-infos"));
- descriptorReader.addDirectory(new File(
- "../../shared/in/archive/relay-descriptors/consensuses"));
- descriptorReader.addDirectory(new File(
- "../../shared/in/archive/relay-descriptors/extra-infos"));
- Iterator<DescriptorFile> descriptorFiles =
- descriptorReader.readDescriptors();
- while (descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = descriptorFiles.next();
- for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- if (descriptor instanceof ExtraInfoDescriptor) {
- parseRelayExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
- } else if (descriptor instanceof RelayNetworkStatusConsensus) {
- parseRelayNetworkStatusConsensus(
- (RelayNetworkStatusConsensus) descriptor);
- }
+ for (Descriptor descriptor : descriptorReader.readDescriptors(
+ new File("../../shared/in/recent/relay-descriptors/consensuses"),
+ new File("../../shared/in/recent/relay-descriptors/extra-infos"),
+ new File("../../shared/in/archive/relay-descriptors/consensuses"),
+ new File("../../shared/in/archive/relay-descriptors/extra-infos"))) {
+ if (descriptor instanceof ExtraInfoDescriptor) {
+ parseRelayExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
+ } else if (descriptor instanceof RelayNetworkStatusConsensus) {
+ parseRelayNetworkStatusConsensus(
+ (RelayNetworkStatusConsensus) descriptor);
}
}
descriptorReader.saveHistoryFile(historyFile);
@@ -209,21 +198,14 @@ public class Main {
DescriptorSourceFactory.createDescriptorReader();
File historyFile = new File("status/bridge-descriptors");
descriptorReader.setHistoryFile(historyFile);
- descriptorReader.addDirectory(new File(
- "../../shared/in/recent/bridge-descriptors"));
- descriptorReader.addDirectory(new File(
- "../../shared/in/archive/bridge-descriptors"));
- Iterator<DescriptorFile> descriptorFiles =
- descriptorReader.readDescriptors();
- while (descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = descriptorFiles.next();
- for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- if (descriptor instanceof ExtraInfoDescriptor) {
- parseBridgeExtraInfoDescriptor(
- (ExtraInfoDescriptor) descriptor);
- } else if (descriptor instanceof BridgeNetworkStatus) {
- parseBridgeNetworkStatus((BridgeNetworkStatus) descriptor);
- }
+ for (Descriptor descriptor : descriptorReader.readDescriptors(
+ new File("../../shared/in/recent/bridge-descriptors"),
+ new File("../../shared/in/archive/bridge-descriptors"))) {
+ if (descriptor instanceof ExtraInfoDescriptor) {
+ parseBridgeExtraInfoDescriptor(
+ (ExtraInfoDescriptor) descriptor);
+ } else if (descriptor instanceof BridgeNetworkStatus) {
+ parseBridgeNetworkStatus((BridgeNetworkStatus) descriptor);
}
}
descriptorReader.saveHistoryFile(historyFile);
diff --git a/modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java b/modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java
index 63dc6be..1de49d3 100644
--- a/modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java
+++ b/modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java
@@ -4,7 +4,6 @@
package org.torproject.metrics.connbidirect;
import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
import org.torproject.descriptor.DescriptorReader;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.ExtraInfoDescriptor;
@@ -21,7 +20,6 @@ import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.SortedMap;
@@ -358,23 +356,16 @@ public class Main {
SortedMap<String, Long> parseHistory) {
DescriptorReader descriptorReader =
DescriptorSourceFactory.createDescriptorReader();
- for (File descriptorsDirectory : descriptorsDirectories) {
- descriptorReader.addDirectory(descriptorsDirectory);
- }
descriptorReader.setExcludedFiles(parseHistory);
- Iterator<DescriptorFile> descriptorFiles =
- descriptorReader.readDescriptors();
- while (descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = descriptorFiles.next();
- for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- if (!(descriptor instanceof ExtraInfoDescriptor)) {
- continue;
- }
- RawStat rawStat = parseRawStatFromDescriptor(
- (ExtraInfoDescriptor) descriptor);
- if (rawStat != null) {
- rawStats.add(rawStat);
- }
+ for (Descriptor descriptor : descriptorReader.readDescriptors(
+ descriptorsDirectories)) {
+ if (!(descriptor instanceof ExtraInfoDescriptor)) {
+ continue;
+ }
+ RawStat rawStat = parseRawStatFromDescriptor(
+ (ExtraInfoDescriptor) descriptor);
+ if (rawStat != null) {
+ rawStats.add(rawStat);
}
}
parseHistory.clear();
diff --git a/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Main.java b/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Main.java
index ad0b415..62a06b0 100644
--- a/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Main.java
+++ b/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Main.java
@@ -19,11 +19,9 @@ public class Main {
public static void main(String[] args) {
/* Initialize directories and file paths. */
- Set<File> inDirectories = new HashSet<>();
- inDirectories.add(
- new File("../../shared/in/recent/relay-descriptors/consensuses"));
- inDirectories.add(
- new File("../../shared/in/recent/relay-descriptors/extra-infos"));
+ File[] inDirectories = new File[] {
+ new File("../../shared/in/recent/relay-descriptors/consensuses"),
+ new File("../../shared/in/recent/relay-descriptors/extra-infos") };
File statusDirectory = new File("status");
/* Initialize parser and read parse history to avoid parsing
diff --git a/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Parser.java b/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Parser.java
index eccb0c0..ebfe869 100644
--- a/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Parser.java
+++ b/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Parser.java
@@ -4,7 +4,6 @@
package org.torproject.metrics.hidserv;
import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
import org.torproject.descriptor.DescriptorReader;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.ExtraInfoDescriptor;
@@ -21,7 +20,6 @@ import java.math.BigInteger;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.SortedMap;
@@ -42,6 +40,9 @@ public class Parser {
* consensuses. */
private DescriptorReader descriptorReader;
+ /** Input directories containing descriptors to parse. */
+ private File[] inDirectories;
+
/** Document file containing previously parsed reported hidden-service
* statistics. */
private File reportedHidServStatsFile;
@@ -61,7 +62,7 @@ public class Parser {
/** Initializes a new parser object using the given directories and
* document stores. */
- public Parser(Set<File> inDirectories, File statusDirectory,
+ public Parser(File[] inDirectories, File statusDirectory,
DocumentStore<ReportedHidServStats> reportedHidServStatsStore,
DocumentStore<ComputedNetworkFractions>
computedNetworkFractionsStore) {
@@ -73,10 +74,8 @@ public class Parser {
* rather small extra-info descriptors. */
this.descriptorReader =
DescriptorSourceFactory.createDescriptorReader();
- for (File inDirectory : inDirectories) {
- this.descriptorReader.addDirectory(inDirectory);
- }
- this.descriptorReader.setMaxDescriptorFilesInQueue(5);
+ this.inDirectories = inDirectories;
+ this.descriptorReader.setMaxDescriptorsInQueue(5);
/* Create File instances for the files and directories in the provided
* status directory. */
@@ -168,18 +167,14 @@ public class Parser {
* handles the resulting parsed descriptors if they are either
* extra-info descriptors or consensuses. */
public boolean parseDescriptors() {
- Iterator<DescriptorFile> descriptorFiles =
- this.descriptorReader.readDescriptors();
- while (descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = descriptorFiles.next();
- for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- if (descriptor instanceof ExtraInfoDescriptor) {
- this.parseExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
- } else if (descriptor instanceof RelayNetworkStatusConsensus) {
- if (!this.parseRelayNetworkStatusConsensus(
- (RelayNetworkStatusConsensus) descriptor)) {
- return false;
- }
+ for (Descriptor descriptor : descriptorReader.readDescriptors(
+ this.inDirectories)) {
+ if (descriptor instanceof ExtraInfoDescriptor) {
+ this.parseExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
+ } else if (descriptor instanceof RelayNetworkStatusConsensus) {
+ if (!this.parseRelayNetworkStatusConsensus(
+ (RelayNetworkStatusConsensus) descriptor)) {
+ return false;
}
}
}
diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
index a04f95f..82fb70f 100644
--- a/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
+++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
@@ -4,7 +4,6 @@
package org.torproject.ernie.cron;
import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
import org.torproject.descriptor.DescriptorReader;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.ExtraInfoDescriptor;
@@ -836,33 +835,22 @@ public final class RelayDescriptorDatabaseImporter {
if (!this.archivesDirectories.isEmpty()) {
DescriptorReader reader =
DescriptorSourceFactory.createDescriptorReader();
- reader.setMaxDescriptorFilesInQueue(10);
- for (String archivesPath : this.archivesDirectories) {
- File archivesDirectory = new File(archivesPath);
- if (archivesDirectory.exists()) {
- reader.addDirectory(archivesDirectory);
- }
- }
+ reader.setMaxDescriptorsInQueue(10);
File historyFile = new File(statsDirectory,
"database-importer-relay-descriptor-history");
if (keepImportHistory) {
reader.setHistoryFile(historyFile);
}
- Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors();
- while (descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = descriptorFiles.next();
- if (descriptorFile.getDescriptors() != null) {
- for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- if (descriptor instanceof RelayNetworkStatusConsensus) {
- this.addRelayNetworkStatusConsensus(
- (RelayNetworkStatusConsensus) descriptor);
- } else if (descriptor instanceof ServerDescriptor) {
- this.addServerDescriptor((ServerDescriptor) descriptor);
- } else if (descriptor instanceof ExtraInfoDescriptor) {
- this.addExtraInfoDescriptor(
- (ExtraInfoDescriptor) descriptor);
- }
- }
+ for (Descriptor descriptor : reader.readDescriptors(
+ this.archivesDirectories.toArray(
+ new File[this.archivesDirectories.size()]))) {
+ if (descriptor instanceof RelayNetworkStatusConsensus) {
+ this.addRelayNetworkStatusConsensus(
+ (RelayNetworkStatusConsensus) descriptor);
+ } else if (descriptor instanceof ServerDescriptor) {
+ this.addServerDescriptor((ServerDescriptor) descriptor);
+ } else if (descriptor instanceof ExtraInfoDescriptor) {
+ this.addExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
}
}
if (keepImportHistory) {
diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java
index 801794f..c06acc3 100644
--- a/modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java
+++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java
@@ -5,7 +5,6 @@ package org.torproject.ernie.cron.network;
import org.torproject.descriptor.BridgeNetworkStatus;
import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
import org.torproject.descriptor.DescriptorReader;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.NetworkStatusEntry;
@@ -191,36 +190,29 @@ public class ConsensusStatsFileHandler {
logger.fine("Importing files in directory " + bridgesDir + "/...");
DescriptorReader reader =
DescriptorSourceFactory.createDescriptorReader();
- reader.addDirectory(bridgesDir);
File historyFile = new File(statsDirectory,
"consensus-stats-bridge-descriptor-history");
if (keepImportHistory) {
reader.setHistoryFile(historyFile);
}
- Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors();
- while (descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = descriptorFiles.next();
- if (descriptorFile.getDescriptors() != null) {
+ for (Descriptor descriptor : reader.readDescriptors(bridgesDir)) {
+ if (descriptor instanceof BridgeNetworkStatus) {
+ String descriptorFileName = descriptor.getDescriptorFile().getName();
String authority = null;
- if (descriptorFile.getFileName().contains(
+ if (descriptorFileName.contains(
"4A0CCD2DDC7995083D73F5D667100C8A5831F16D")) {
authority = "Tonga";
- } else if (descriptorFile.getFileName().contains(
+ } else if (descriptorFileName.contains(
"1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1")) {
authority = "Bifroest";
}
- for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- if (descriptor instanceof BridgeNetworkStatus) {
- if (authority == null) {
- this.logger.warning("Did not recognize the bridge authority "
- + "that generated " + descriptorFile.getFileName()
- + ". Skipping.");
- continue;
- }
- this.addBridgeNetworkStatus(
- (BridgeNetworkStatus) descriptor, authority);
- }
+ if (authority == null) {
+ this.logger.warning("Did not recognize the bridge authority "
+ + "that generated " + descriptorFileName + ". Skipping.");
+ continue;
}
+ this.addBridgeNetworkStatus(
+ (BridgeNetworkStatus) descriptor, authority);
}
}
if (keepImportHistory) {
diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java
index 0c2be9d..b5dae12 100644
--- a/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java
+++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java
@@ -4,10 +4,10 @@
package org.torproject.ernie.cron.performance;
import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
import org.torproject.descriptor.DescriptorReader;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.TorperfResult;
+import org.torproject.descriptor.UnparseableDescriptor;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -83,60 +83,55 @@ public class TorperfProcessor {
logger.fine("Importing files in " + torperfDirectory + "/...");
DescriptorReader descriptorReader =
DescriptorSourceFactory.createDescriptorReader();
- descriptorReader.addDirectory(torperfDirectory);
File historyFile = new File(statsDirectory, "torperf-history");
descriptorReader.setHistoryFile(historyFile);
- Iterator<DescriptorFile> descriptorFiles =
- descriptorReader.readDescriptors();
- while (descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = descriptorFiles.next();
- if (descriptorFile.getException() != null) {
- logger.log(Level.FINE, "Error parsing file.",
- descriptorFile.getException());
+ for (Descriptor descriptor : descriptorReader.readDescriptors(
+ torperfDirectory)) {
+ if (descriptor instanceof UnparseableDescriptor) {
+ logger.log(Level.FINE, "Error parsing descriptor.",
+ ((UnparseableDescriptor) descriptor)
+ .getDescriptorParseException());
+ continue;
+ } else if (!(descriptor instanceof TorperfResult)) {
continue;
}
- for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- if (!(descriptor instanceof TorperfResult)) {
- continue;
- }
- TorperfResult result = (TorperfResult) descriptor;
- if (null != result.getUnrecognizedKeys()
- && result.getUnrecognizedKeys().containsKey("ENDPOINTREMOTE")
- && result.getUnrecognizedKeys().get("ENDPOINTREMOTE")
- .contains(".onion")) {
- continue;
- }
- String source = result.getSource();
- long fileSize = result.getFileSize();
- if (fileSize == 51200) {
- source += "-50kb";
- } else if (fileSize == 1048576) {
- source += "-1mb";
- } else if (fileSize == 5242880) {
- source += "-5mb";
- } else {
- logger.fine("Unexpected file size '" + fileSize
- + "'. Skipping.");
- continue;
- }
- String dateTime = formatter.format(result.getStartMillis());
- long completeMillis = result.getDataCompleteMillis()
- - result.getStartMillis();
- String key = source + "," + dateTime;
- String value = key;
- if ((result.didTimeout() == null
- && result.getDataCompleteMillis() < 1)
- || (result.didTimeout() != null && result.didTimeout())) {
- value += ",-2"; // -2 for timeout
- } else if (result.getReadBytes() < fileSize) {
- value += ",-1"; // -1 for failure
- } else {
- value += "," + completeMillis;
- }
- if (!rawObs.containsKey(key)) {
- rawObs.put(key, value);
- addedRawObs++;
- }
+ TorperfResult result = (TorperfResult) descriptor;
+ if (null != result.getUnrecognizedKeys()
+ && result.getUnrecognizedKeys().containsKey("ENDPOINTREMOTE")
+ && result.getUnrecognizedKeys().get("ENDPOINTREMOTE")
+ .contains(".onion")) {
+ continue;
+ }
+ String source = result.getSource();
+ long fileSize = result.getFileSize();
+ if (fileSize == 51200) {
+ source += "-50kb";
+ } else if (fileSize == 1048576) {
+ source += "-1mb";
+ } else if (fileSize == 5242880) {
+ source += "-5mb";
+ } else {
+ logger.fine("Unexpected file size '" + fileSize
+ + "'. Skipping.");
+ continue;
+ }
+ String dateTime = formatter.format(result.getStartMillis());
+ long completeMillis = result.getDataCompleteMillis()
+ - result.getStartMillis();
+ String key = source + "," + dateTime;
+ String value = key;
+ if ((result.didTimeout() == null
+ && result.getDataCompleteMillis() < 1)
+ || (result.didTimeout() != null && result.didTimeout())) {
+ value += ",-2"; // -2 for timeout
+ } else if (result.getReadBytes() < fileSize) {
+ value += ",-1"; // -1 for failure
+ } else {
+ value += "," + completeMillis;
+ }
+ if (!rawObs.containsKey(key)) {
+ rawObs.put(key, value);
+ addedRawObs++;
}
}
descriptorReader.saveHistoryFile(historyFile);
diff --git a/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java b/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java
index 2af3100..802e4ef 100644
--- a/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java
+++ b/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java
@@ -1,7 +1,6 @@
package org.torproject.metrics.onionperf;
import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
import org.torproject.descriptor.DescriptorReader;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.TorperfResult;
@@ -79,118 +78,114 @@ public class Main {
+ "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, "
+ "?, ?, ?, ?, ?, ?)", Statement.RETURN_GENERATED_KEYS);
- DescriptorReader dr = DescriptorSourceFactory.createDescriptorReader();
- dr.addDirectory(new File("../../shared/in/archive/torperf"));
- dr.addDirectory(new File("../../shared/in/recent/torperf"));
- Iterator<DescriptorFile> dfs = dr.readDescriptors();
Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
- while (dfs.hasNext()) {
- DescriptorFile df = dfs.next();
- for (Descriptor d : df.getDescriptors()) {
- if (!(d instanceof TorperfResult)) {
- continue;
+ DescriptorReader dr = DescriptorSourceFactory.createDescriptorReader();
+ for (Descriptor d : dr.readDescriptors(
+ new File("../../shared/in/archive/torperf"),
+ new File("../../shared/in/recent/torperf"))) {
+ if (!(d instanceof TorperfResult)) {
+ continue;
+ }
+ TorperfResult tr = (TorperfResult) d;
+ int measurementId = -1;
+ String truncatedSource = truncateString(tr.getSource(), 32);
+ psMeasurementsSelect.clearParameters();
+ psMeasurementsSelect.setString(1, truncatedSource);
+ psMeasurementsSelect.setInt(2, tr.getFileSize());
+ psMeasurementsSelect.setTimestamp(3,
+ new Timestamp(tr.getStartMillis()), calendar);
+ try (ResultSet rs = psMeasurementsSelect.executeQuery()) {
+ if (rs.next()) {
+ measurementId = rs.getInt(1);
}
- TorperfResult tr = (TorperfResult) d;
- int measurementId = -1;
- String truncatedSource = truncateString(tr.getSource(), 32);
- psMeasurementsSelect.clearParameters();
- psMeasurementsSelect.setString(1, truncatedSource);
- psMeasurementsSelect.setInt(2, tr.getFileSize());
- psMeasurementsSelect.setTimestamp(3,
+ }
+ if (measurementId < 0) {
+ psMeasurementsInsert.clearParameters();
+ psMeasurementsInsert.setString(1, truncatedSource);
+ psMeasurementsInsert.setInt(2, tr.getFileSize());
+ psMeasurementsInsert.setTimestamp(3,
new Timestamp(tr.getStartMillis()), calendar);
- try (ResultSet rs = psMeasurementsSelect.executeQuery()) {
- if (rs.next()) {
- measurementId = rs.getInt(1);
- }
- }
- if (measurementId < 0) {
- psMeasurementsInsert.clearParameters();
- psMeasurementsInsert.setString(1, truncatedSource);
- psMeasurementsInsert.setInt(2, tr.getFileSize());
- psMeasurementsInsert.setTimestamp(3,
- new Timestamp(tr.getStartMillis()), calendar);
- long[] timestamps = new long[] { tr.getSocketMillis(),
- tr.getConnectMillis(), tr.getNegotiateMillis(),
- tr.getRequestMillis(), tr.getResponseMillis(),
- tr.getDataRequestMillis(), tr.getDataResponseMillis(),
- tr.getDataCompleteMillis() };
- for (int i = 4, j = 0; j < timestamps.length; i++, j++) {
- if (timestamps[j] == 0L) {
- psMeasurementsInsert.setNull(i, Types.INTEGER);
- } else {
- psMeasurementsInsert.setInt(i,
- (int) (timestamps[j] - tr.getStartMillis()));
- }
- }
- psMeasurementsInsert.setInt(12, tr.getWriteBytes());
- psMeasurementsInsert.setInt(13, tr.getReadBytes());
- if (null == tr.didTimeout()) {
- psMeasurementsInsert.setNull(14, Types.BOOLEAN);
- } else {
- psMeasurementsInsert.setBoolean(14, tr.didTimeout());
- }
- for (int i = 15, p = 0; i <= 25 && p <= 100; i++, p += 10) {
- if (null == tr.getDataPercentiles()
- || !tr.getDataPercentiles().containsKey(p)) {
- psMeasurementsInsert.setNull(i, Types.INTEGER);
- } else {
- psMeasurementsInsert.setInt(i,
- (int) (tr.getDataPercentiles().get(p) - tr.getStartMillis()));
- }
- }
- if (tr.getLaunchMillis() < 0L) {
- psMeasurementsInsert.setNull(26, Types.TIMESTAMP);
- } else {
- psMeasurementsInsert.setTimestamp(26,
- new Timestamp(tr.getLaunchMillis()), calendar);
- }
- if (tr.getUsedAtMillis() < 0L) {
- psMeasurementsInsert.setNull(27, Types.TIMESTAMP);
- } else {
- psMeasurementsInsert.setTimestamp(27,
- new Timestamp(tr.getUsedAtMillis()), calendar);
- }
- if (tr.getTimeout() < 0L) {
- psMeasurementsInsert.setNull(28, Types.INTEGER);
+ long[] timestamps = new long[] { tr.getSocketMillis(),
+ tr.getConnectMillis(), tr.getNegotiateMillis(),
+ tr.getRequestMillis(), tr.getResponseMillis(),
+ tr.getDataRequestMillis(), tr.getDataResponseMillis(),
+ tr.getDataCompleteMillis() };
+ for (int i = 4, j = 0; j < timestamps.length; i++, j++) {
+ if (timestamps[j] == 0L) {
+ psMeasurementsInsert.setNull(i, Types.INTEGER);
} else {
- psMeasurementsInsert.setInt(28, (int) tr.getTimeout());
+ psMeasurementsInsert.setInt(i,
+ (int) (timestamps[j] - tr.getStartMillis()));
}
- if (tr.getQuantile() < 0.0) {
- psMeasurementsInsert.setNull(29, Types.REAL);
- } else {
- psMeasurementsInsert.setDouble(29, tr.getQuantile());
- }
- if (tr.getCircId() < 0L) {
- psMeasurementsInsert.setNull(30, Types.INTEGER);
+ }
+ psMeasurementsInsert.setInt(12, tr.getWriteBytes());
+ psMeasurementsInsert.setInt(13, tr.getReadBytes());
+ if (null == tr.didTimeout()) {
+ psMeasurementsInsert.setNull(14, Types.BOOLEAN);
+ } else {
+ psMeasurementsInsert.setBoolean(14, tr.didTimeout());
+ }
+ for (int i = 15, p = 0; i <= 25 && p <= 100; i++, p += 10) {
+ if (null == tr.getDataPercentiles()
+ || !tr.getDataPercentiles().containsKey(p)) {
+ psMeasurementsInsert.setNull(i, Types.INTEGER);
} else {
- psMeasurementsInsert.setInt(30, tr.getCircId());
+ psMeasurementsInsert.setInt(i,
+ (int) (tr.getDataPercentiles().get(p) - tr.getStartMillis()));
}
- if (tr.getUsedBy() < 0L) {
- psMeasurementsInsert.setNull(31, Types.INTEGER);
+ }
+ if (tr.getLaunchMillis() < 0L) {
+ psMeasurementsInsert.setNull(26, Types.TIMESTAMP);
+ } else {
+ psMeasurementsInsert.setTimestamp(26,
+ new Timestamp(tr.getLaunchMillis()), calendar);
+ }
+ if (tr.getUsedAtMillis() < 0L) {
+ psMeasurementsInsert.setNull(27, Types.TIMESTAMP);
+ } else {
+ psMeasurementsInsert.setTimestamp(27,
+ new Timestamp(tr.getUsedAtMillis()), calendar);
+ }
+ if (tr.getTimeout() < 0L) {
+ psMeasurementsInsert.setNull(28, Types.INTEGER);
+ } else {
+ psMeasurementsInsert.setInt(28, (int) tr.getTimeout());
+ }
+ if (tr.getQuantile() < 0.0) {
+ psMeasurementsInsert.setNull(29, Types.REAL);
+ } else {
+ psMeasurementsInsert.setDouble(29, tr.getQuantile());
+ }
+ if (tr.getCircId() < 0L) {
+ psMeasurementsInsert.setNull(30, Types.INTEGER);
+ } else {
+ psMeasurementsInsert.setInt(30, tr.getCircId());
+ }
+ if (tr.getUsedBy() < 0L) {
+ psMeasurementsInsert.setNull(31, Types.INTEGER);
+ } else {
+ psMeasurementsInsert.setInt(31, tr.getUsedBy());
+ }
+ String[] unrecognizedKeys = new String[] { "ENDPOINTLOCAL",
+ "ENDPOINTPROXY", "ENDPOINTREMOTE", "HOSTNAMELOCAL",
+ "HOSTNAMEREMOTE", "SOURCEADDRESS" };
+ for (int i = 32, j = 0; j < unrecognizedKeys.length; i++, j++) {
+ if (null == tr.getUnrecognizedKeys()
+ || !tr.getUnrecognizedKeys().containsKey(unrecognizedKeys[j])) {
+ psMeasurementsInsert.setNull(i, Types.VARCHAR);
} else {
- psMeasurementsInsert.setInt(31, tr.getUsedBy());
- }
- String[] unrecognizedKeys = new String[] { "ENDPOINTLOCAL",
- "ENDPOINTPROXY", "ENDPOINTREMOTE", "HOSTNAMELOCAL",
- "HOSTNAMEREMOTE", "SOURCEADDRESS" };
- for (int i = 32, j = 0; j < unrecognizedKeys.length; i++, j++) {
- if (null == tr.getUnrecognizedKeys()
- || !tr.getUnrecognizedKeys().containsKey(unrecognizedKeys[j])) {
- psMeasurementsInsert.setNull(i, Types.VARCHAR);
- } else {
- psMeasurementsInsert.setString(i, truncateString(
- tr.getUnrecognizedKeys().get(unrecognizedKeys[j]), 64));
- }
+ psMeasurementsInsert.setString(i, truncateString(
+ tr.getUnrecognizedKeys().get(unrecognizedKeys[j]), 64));
}
- psMeasurementsInsert.execute();
- try (ResultSet rs = psMeasurementsInsert.getGeneratedKeys()) {
- if (rs.next()) {
- measurementId = rs.getInt(1);
- }
+ }
+ psMeasurementsInsert.execute();
+ try (ResultSet rs = psMeasurementsInsert.getGeneratedKeys()) {
+ if (rs.next()) {
+ measurementId = rs.getInt(1);
}
}
- /* Could use measurementId to insert path. */
}
+ /* Could use measurementId to insert path. */
connection.commit();
}
}
diff --git a/shared/build-base.xml b/shared/build-base.xml
index f473b77..c446aa4 100644
--- a/shared/build-base.xml
+++ b/shared/build-base.xml
@@ -8,7 +8,7 @@
<property name="classes" value="${generated}/classes/"/>
<property name="testclasses" value="${generated}/test-classes/"/>
<property name="source-and-target-java-version" value="1.7" />
- <property name="descriptorversion" value="1.8.2" />
+ <property name="descriptorversion" value="1.9.0" />
<path id="base.classpath">
<pathelement path="${classes}"/>
More information about the tor-commits
mailing list