commit 9ee4fe35e08428ce3aae7946d01da843e8004f5b Author: Karsten Loesing karsten.loesing@gmx.net Date: Sat Jun 17 17:42:00 2017 +0200

Update to metrics-lib 1.9.0.

Implements #22683. --- .../org/torproject/metrics/advbwdist/Main.java | 170 ++++++++---------- .../java/org/torproject/metrics/clients/Main.java | 54 ++---- .../org/torproject/metrics/connbidirect/Main.java | 27 +-- .../java/org/torproject/metrics/hidserv/Main.java | 8 +- .../org/torproject/metrics/hidserv/Parser.java | 33 ++-- .../cron/RelayDescriptorDatabaseImporter.java | 34 ++-- .../cron/network/ConsensusStatsFileHandler.java | 30 ++-- .../ernie/cron/performance/TorperfProcessor.java | 97 +++++----- .../org/torproject/metrics/onionperf/Main.java | 197 ++++++++++----------- shared/build-base.xml | 2 +- 10 files changed, 287 insertions(+), 365 deletions(-)

diff --git a/modules/advbwdist/src/main/java/org/torproject/metrics/advbwdist/Main.java b/modules/advbwdist/src/main/java/org/torproject/metrics/advbwdist/Main.java index 7193108..8928be2 100644 --- a/modules/advbwdist/src/main/java/org/torproject/metrics/advbwdist/Main.java +++ b/modules/advbwdist/src/main/java/org/torproject/metrics/advbwdist/Main.java @@ -4,7 +4,6 @@ package org.torproject.metrics.advbwdist;

import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; import org.torproject.descriptor.DescriptorReader; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.NetworkStatusEntry; @@ -19,7 +18,6 @@ import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TimeZone; @@ -31,37 +29,27 @@ public class Main {

/* Parse server descriptors, not keeping a parse history, and memorize * the advertised bandwidth for every server descriptor. */ + Map<String, Long> serverDescriptors = new HashMap<>(); DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - descriptorReader.addDirectory( - new File("../../shared/in/recent/relay-descriptors/" - + "server-descriptors")); - Iterator<DescriptorFile> descriptorFiles = - descriptorReader.readDescriptors(); - Map<String, Long> serverDescriptors = new HashMap<>(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (!(descriptor instanceof ServerDescriptor)) { - continue; - } - ServerDescriptor serverDescriptor = (ServerDescriptor) descriptor; - String digest = serverDescriptor.getDigestSha1Hex(); - long advertisedBandwidth = Math.min(Math.min( - serverDescriptor.getBandwidthRate(), - serverDescriptor.getBandwidthBurst()), - serverDescriptor.getBandwidthObserved()); - serverDescriptors.put(digest.toUpperCase(), advertisedBandwidth); + for (Descriptor descriptor : descriptorReader.readDescriptors(new File( + "../../shared/in/recent/relay-descriptors/server-descriptors"))) { + if (!(descriptor instanceof ServerDescriptor)) { + continue; } + ServerDescriptor serverDescriptor = (ServerDescriptor) descriptor; + String digest = serverDescriptor.getDigestSha1Hex(); + long advertisedBandwidth = Math.min(Math.min( + serverDescriptor.getBandwidthRate(), + serverDescriptor.getBandwidthBurst()), + serverDescriptor.getBandwidthObserved()); + serverDescriptors.put(digest.toUpperCase(), advertisedBandwidth); }

/* Parse consensuses, keeping a parse history. */ descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - descriptorReader.addDirectory( - new File("../../shared/in/recent/relay-descriptors/consensuses")); File historyFile = new File("status/parsed-consensuses"); descriptorReader.setHistoryFile(historyFile); - descriptorFiles = descriptorReader.readDescriptors(); File resultsFile = new File("stats/advbwdist-validafter.csv"); resultsFile.getParentFile().mkdirs(); boolean writeHeader = !resultsFile.exists(); @@ -73,82 +61,80 @@ public class Main { SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (!(descriptor instanceof RelayNetworkStatusConsensus)) { + for (Descriptor descriptor : descriptorReader.readDescriptors(new File( + "../../shared/in/recent/relay-descriptors/consensuses"))) { + if (!(descriptor instanceof RelayNetworkStatusConsensus)) { + continue; + } + + /* Parse server descriptor digests from consensus and look up + * advertised bandwidths. */ + RelayNetworkStatusConsensus consensus = + (RelayNetworkStatusConsensus) descriptor; + String validAfter = dateTimeFormat.format( + consensus.getValidAfterMillis()); + List<Long> advertisedBandwidthsAllRelays = new ArrayList<>(); + List<Long> advertisedBandwidthsExitsOnly = new ArrayList<>(); + for (NetworkStatusEntry relay + : consensus.getStatusEntries().values()) { + if (!relay.getFlags().contains("Running")) { continue; } - - /* Parse server descriptor digests from consensus and look up - * advertised bandwidths. */ - RelayNetworkStatusConsensus consensus = - (RelayNetworkStatusConsensus) descriptor; - String validAfter = dateTimeFormat.format( - consensus.getValidAfterMillis()); - List<Long> advertisedBandwidthsAllRelays = new ArrayList<>(); - List<Long> advertisedBandwidthsExitsOnly = new ArrayList<>(); - for (NetworkStatusEntry relay - : consensus.getStatusEntries().values()) { - if (!relay.getFlags().contains("Running")) { - continue; - } - String serverDescriptorDigest = relay.getDescriptor() - .toUpperCase(); - if (!serverDescriptors.containsKey(serverDescriptorDigest)) { - continue; - } - long advertisedBandwidth = serverDescriptors.get( - serverDescriptorDigest); - advertisedBandwidthsAllRelays.add(advertisedBandwidth); - if (relay.getFlags().contains("Exit") - && !relay.getFlags().contains("BadExit")) { - advertisedBandwidthsExitsOnly.add(advertisedBandwidth); - } + String serverDescriptorDigest = relay.getDescriptor() + .toUpperCase(); + if (!serverDescriptors.containsKey(serverDescriptorDigest)) { + continue; + } + long advertisedBandwidth = serverDescriptors.get( + serverDescriptorDigest); + advertisedBandwidthsAllRelays.add(advertisedBandwidth); + if (relay.getFlags().contains("Exit") + && !relay.getFlags().contains("BadExit")) { + advertisedBandwidthsExitsOnly.add(advertisedBandwidth); } + }

- /* Write advertised bandwidths of n-th fastest relays/exits. */ - Collections.sort(advertisedBandwidthsAllRelays, - Collections.reverseOrder()); - Collections.sort(advertisedBandwidthsExitsOnly, - Collections.reverseOrder()); - int[] fastestRelays = new int[] { 1, 2, 3, 5, 10, 20, 30, 50, 100, - 200, 300, 500, 1000, 2000, 3000, 5000 }; - for (int fastestRelay : fastestRelays) { - if (advertisedBandwidthsAllRelays.size() >= fastestRelay) { - bw.write(String.format("%s,,%d,,%d%n", validAfter, - fastestRelay, - advertisedBandwidthsAllRelays.get(fastestRelay - 1))); - } + /* Write advertised bandwidths of n-th fastest relays/exits. */ + Collections.sort(advertisedBandwidthsAllRelays, + Collections.reverseOrder()); + Collections.sort(advertisedBandwidthsExitsOnly, + Collections.reverseOrder()); + int[] fastestRelays = new int[] { 1, 2, 3, 5, 10, 20, 30, 50, 100, + 200, 300, 500, 1000, 2000, 3000, 5000 }; + for (int fastestRelay : fastestRelays) { + if (advertisedBandwidthsAllRelays.size() >= fastestRelay) { + bw.write(String.format("%s,,%d,,%d%n", validAfter, + fastestRelay, + advertisedBandwidthsAllRelays.get(fastestRelay - 1))); } - for (int fastestRelay : fastestRelays) { - if (advertisedBandwidthsExitsOnly.size() >= fastestRelay) { - bw.write(String.format("%s,TRUE,%d,,%d%n", validAfter, - fastestRelay, - advertisedBandwidthsExitsOnly.get(fastestRelay - 1))); - } + } + for (int fastestRelay : fastestRelays) { + if (advertisedBandwidthsExitsOnly.size() >= fastestRelay) { + bw.write(String.format("%s,TRUE,%d,,%d%n", validAfter, + fastestRelay, + advertisedBandwidthsExitsOnly.get(fastestRelay - 1))); } + }

- /* Write advertised bandwidth percentiles of relays/exits. */ - Collections.sort(advertisedBandwidthsAllRelays); - Collections.sort(advertisedBandwidthsExitsOnly); - int[] percentiles = new int[] { 0, 1, 2, 3, 5, 9, 10, 20, 25, 30, - 40, 50, 60, 70, 75, 80, 90, 91, 95, 97, 98, 99, 100 }; - if (!advertisedBandwidthsAllRelays.isEmpty()) { - for (int percentile : percentiles) { - bw.write(String.format("%s,,,%d,%d%n", validAfter, - percentile, advertisedBandwidthsAllRelays.get( - ((advertisedBandwidthsAllRelays.size() - 1) - * percentile) / 100))); - } + /* Write advertised bandwidth percentiles of relays/exits. */ + Collections.sort(advertisedBandwidthsAllRelays); + Collections.sort(advertisedBandwidthsExitsOnly); + int[] percentiles = new int[] { 0, 1, 2, 3, 5, 9, 10, 20, 25, 30, + 40, 50, 60, 70, 75, 80, 90, 91, 95, 97, 98, 99, 100 }; + if (!advertisedBandwidthsAllRelays.isEmpty()) { + for (int percentile : percentiles) { + bw.write(String.format("%s,,,%d,%d%n", validAfter, + percentile, advertisedBandwidthsAllRelays.get( + ((advertisedBandwidthsAllRelays.size() - 1) + * percentile) / 100))); } - if (!advertisedBandwidthsExitsOnly.isEmpty()) { - for (int percentile : percentiles) { - bw.write(String.format("%s,TRUE,,%d,%d%n", validAfter, - percentile, advertisedBandwidthsExitsOnly.get( - ((advertisedBandwidthsExitsOnly.size() - 1) - * percentile) / 100))); - } + } + if (!advertisedBandwidthsExitsOnly.isEmpty()) { + for (int percentile : percentiles) { + bw.write(String.format("%s,TRUE,,%d,%d%n", validAfter, + percentile, advertisedBandwidthsExitsOnly.get( + ((advertisedBandwidthsExitsOnly.size() - 1) + * percentile) / 100))); } } } diff --git a/modules/clients/src/main/java/org/torproject/metrics/clients/Main.java b/modules/clients/src/main/java/org/torproject/metrics/clients/Main.java index dc34d37..384c621 100644 --- a/modules/clients/src/main/java/org/torproject/metrics/clients/Main.java +++ b/modules/clients/src/main/java/org/torproject/metrics/clients/Main.java @@ -6,7 +6,6 @@ package org.torproject.metrics.clients; import org.torproject.descriptor.BandwidthHistory; import org.torproject.descriptor.BridgeNetworkStatus; import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; import org.torproject.descriptor.DescriptorReader; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.ExtraInfoDescriptor; @@ -19,7 +18,6 @@ import java.io.FileWriter; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.HashMap; -import java.util.Iterator; import java.util.Map; import java.util.SortedMap; import java.util.TimeZone; @@ -65,25 +63,16 @@ public class Main { DescriptorSourceFactory.createDescriptorReader(); File historyFile = new File("status/relay-descriptors"); descriptorReader.setHistoryFile(historyFile); - descriptorReader.addDirectory(new File( - "../../shared/in/recent/relay-descriptors/consensuses")); - descriptorReader.addDirectory(new File( - "../../shared/in/recent/relay-descriptors/extra-infos")); - descriptorReader.addDirectory(new File( - "../../shared/in/archive/relay-descriptors/consensuses")); - descriptorReader.addDirectory(new File( - "../../shared/in/archive/relay-descriptors/extra-infos")); - Iterator<DescriptorFile> descriptorFiles = - descriptorReader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof ExtraInfoDescriptor) { - parseRelayExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); - } else if (descriptor instanceof RelayNetworkStatusConsensus) { - parseRelayNetworkStatusConsensus( - (RelayNetworkStatusConsensus) descriptor); - } + for (Descriptor descriptor : descriptorReader.readDescriptors( + new File("../../shared/in/recent/relay-descriptors/consensuses"), + new File("../../shared/in/recent/relay-descriptors/extra-infos"), + new File("../../shared/in/archive/relay-descriptors/consensuses"), + new File("../../shared/in/archive/relay-descriptors/extra-infos"))) { + if (descriptor instanceof ExtraInfoDescriptor) { + parseRelayExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); + } else if (descriptor instanceof RelayNetworkStatusConsensus) { + parseRelayNetworkStatusConsensus( + (RelayNetworkStatusConsensus) descriptor); } } descriptorReader.saveHistoryFile(historyFile); @@ -209,21 +198,14 @@ public class Main { DescriptorSourceFactory.createDescriptorReader(); File historyFile = new File("status/bridge-descriptors"); descriptorReader.setHistoryFile(historyFile); - descriptorReader.addDirectory(new File( - "../../shared/in/recent/bridge-descriptors")); - descriptorReader.addDirectory(new File( - "../../shared/in/archive/bridge-descriptors")); - Iterator<DescriptorFile> descriptorFiles = - descriptorReader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof ExtraInfoDescriptor) { - parseBridgeExtraInfoDescriptor( - (ExtraInfoDescriptor) descriptor); - } else if (descriptor instanceof BridgeNetworkStatus) { - parseBridgeNetworkStatus((BridgeNetworkStatus) descriptor); - } + for (Descriptor descriptor : descriptorReader.readDescriptors( + new File("../../shared/in/recent/bridge-descriptors"), + new File("../../shared/in/archive/bridge-descriptors"))) { + if (descriptor instanceof ExtraInfoDescriptor) { + parseBridgeExtraInfoDescriptor( + (ExtraInfoDescriptor) descriptor); + } else if (descriptor instanceof BridgeNetworkStatus) { + parseBridgeNetworkStatus((BridgeNetworkStatus) descriptor); } } descriptorReader.saveHistoryFile(historyFile); diff --git a/modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java b/modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java index 63dc6be..1de49d3 100644 --- a/modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java +++ b/modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java @@ -4,7 +4,6 @@ package org.torproject.metrics.connbidirect;

import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; import org.torproject.descriptor.DescriptorReader; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.ExtraInfoDescriptor; @@ -21,7 +20,6 @@ import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.SortedMap; @@ -358,23 +356,16 @@ public class Main { SortedMap<String, Long> parseHistory) { DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - for (File descriptorsDirectory : descriptorsDirectories) { - descriptorReader.addDirectory(descriptorsDirectory); - } descriptorReader.setExcludedFiles(parseHistory); - Iterator<DescriptorFile> descriptorFiles = - descriptorReader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (!(descriptor instanceof ExtraInfoDescriptor)) { - continue; - } - RawStat rawStat = parseRawStatFromDescriptor( - (ExtraInfoDescriptor) descriptor); - if (rawStat != null) { - rawStats.add(rawStat); - } + for (Descriptor descriptor : descriptorReader.readDescriptors( + descriptorsDirectories)) { + if (!(descriptor instanceof ExtraInfoDescriptor)) { + continue; + } + RawStat rawStat = parseRawStatFromDescriptor( + (ExtraInfoDescriptor) descriptor); + if (rawStat != null) { + rawStats.add(rawStat); } } parseHistory.clear(); diff --git a/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Main.java b/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Main.java index ad0b415..62a06b0 100644 --- a/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Main.java +++ b/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Main.java @@ -19,11 +19,9 @@ public class Main { public static void main(String[] args) {

/* Initialize directories and file paths. */ - Set<File> inDirectories = new HashSet<>(); - inDirectories.add( - new File("../../shared/in/recent/relay-descriptors/consensuses")); - inDirectories.add( - new File("../../shared/in/recent/relay-descriptors/extra-infos")); + File[] inDirectories = new File[] { + new File("../../shared/in/recent/relay-descriptors/consensuses"), + new File("../../shared/in/recent/relay-descriptors/extra-infos") }; File statusDirectory = new File("status");

/* Initialize parser and read parse history to avoid parsing diff --git a/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Parser.java b/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Parser.java index eccb0c0..ebfe869 100644 --- a/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Parser.java +++ b/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Parser.java @@ -4,7 +4,6 @@ package org.torproject.metrics.hidserv;

import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; import org.torproject.descriptor.DescriptorReader; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.ExtraInfoDescriptor; @@ -21,7 +20,6 @@ import java.math.BigInteger; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; -import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.SortedMap; @@ -42,6 +40,9 @@ public class Parser { * consensuses. */ private DescriptorReader descriptorReader;

+ /** Input directories containing descriptors to parse. */ + private File[] inDirectories; + /** Document file containing previously parsed reported hidden-service * statistics. */ private File reportedHidServStatsFile; @@ -61,7 +62,7 @@ public class Parser {

/** Initializes a new parser object using the given directories and * document stores. */ - public Parser(Set<File> inDirectories, File statusDirectory, + public Parser(File[] inDirectories, File statusDirectory, DocumentStore<ReportedHidServStats> reportedHidServStatsStore, DocumentStore<ComputedNetworkFractions> computedNetworkFractionsStore) { @@ -73,10 +74,8 @@ public class Parser { * rather small extra-info descriptors. */ this.descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - for (File inDirectory : inDirectories) { - this.descriptorReader.addDirectory(inDirectory); - } - this.descriptorReader.setMaxDescriptorFilesInQueue(5); + this.inDirectories = inDirectories; + this.descriptorReader.setMaxDescriptorsInQueue(5);

/* Create File instances for the files and directories in the provided * status directory. */ @@ -168,18 +167,14 @@ public class Parser { * handles the resulting parsed descriptors if they are either * extra-info descriptors or consensuses. */ public boolean parseDescriptors() { - Iterator<DescriptorFile> descriptorFiles = - this.descriptorReader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof ExtraInfoDescriptor) { - this.parseExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); - } else if (descriptor instanceof RelayNetworkStatusConsensus) { - if (!this.parseRelayNetworkStatusConsensus( - (RelayNetworkStatusConsensus) descriptor)) { - return false; - } + for (Descriptor descriptor : descriptorReader.readDescriptors( + this.inDirectories)) { + if (descriptor instanceof ExtraInfoDescriptor) { + this.parseExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); + } else if (descriptor instanceof RelayNetworkStatusConsensus) { + if (!this.parseRelayNetworkStatusConsensus( + (RelayNetworkStatusConsensus) descriptor)) { + return false; } } } diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java index a04f95f..82fb70f 100644 --- a/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java +++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java @@ -4,7 +4,6 @@ package org.torproject.ernie.cron;

import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; import org.torproject.descriptor.DescriptorReader; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.ExtraInfoDescriptor; @@ -836,33 +835,22 @@ public final class RelayDescriptorDatabaseImporter { if (!this.archivesDirectories.isEmpty()) { DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader(); - reader.setMaxDescriptorFilesInQueue(10); - for (String archivesPath : this.archivesDirectories) { - File archivesDirectory = new File(archivesPath); - if (archivesDirectory.exists()) { - reader.addDirectory(archivesDirectory); - } - } + reader.setMaxDescriptorsInQueue(10); File historyFile = new File(statsDirectory, "database-importer-relay-descriptor-history"); if (keepImportHistory) { reader.setHistoryFile(historyFile); } - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getDescriptors() != null) { - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof RelayNetworkStatusConsensus) { - this.addRelayNetworkStatusConsensus( - (RelayNetworkStatusConsensus) descriptor); - } else if (descriptor instanceof ServerDescriptor) { - this.addServerDescriptor((ServerDescriptor) descriptor); - } else if (descriptor instanceof ExtraInfoDescriptor) { - this.addExtraInfoDescriptor( - (ExtraInfoDescriptor) descriptor); - } - } + for (Descriptor descriptor : reader.readDescriptors( + this.archivesDirectories.toArray( + new File[this.archivesDirectories.size()]))) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + this.addRelayNetworkStatusConsensus( + (RelayNetworkStatusConsensus) descriptor); + } else if (descriptor instanceof ServerDescriptor) { + this.addServerDescriptor((ServerDescriptor) descriptor); + } else if (descriptor instanceof ExtraInfoDescriptor) { + this.addExtraInfoDescriptor((ExtraInfoDescriptor) descriptor); } } if (keepImportHistory) { diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java index 801794f..c06acc3 100644 --- a/modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java +++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java @@ -5,7 +5,6 @@ package org.torproject.ernie.cron.network;

import org.torproject.descriptor.BridgeNetworkStatus; import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; import org.torproject.descriptor.DescriptorReader; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.NetworkStatusEntry; @@ -191,36 +190,29 @@ public class ConsensusStatsFileHandler { logger.fine("Importing files in directory " + bridgesDir + "/..."); DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(bridgesDir); File historyFile = new File(statsDirectory, "consensus-stats-bridge-descriptor-history"); if (keepImportHistory) { reader.setHistoryFile(historyFile); } - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getDescriptors() != null) { + for (Descriptor descriptor : reader.readDescriptors(bridgesDir)) { + if (descriptor instanceof BridgeNetworkStatus) { + String descriptorFileName = descriptor.getDescriptorFile().getName(); String authority = null; - if (descriptorFile.getFileName().contains( + if (descriptorFileName.contains( "4A0CCD2DDC7995083D73F5D667100C8A5831F16D")) { authority = "Tonga"; - } else if (descriptorFile.getFileName().contains( + } else if (descriptorFileName.contains( "1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1")) { authority = "Bifroest"; } - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof BridgeNetworkStatus) { - if (authority == null) { - this.logger.warning("Did not recognize the bridge authority " - + "that generated " + descriptorFile.getFileName() - + ". Skipping."); - continue; - } - this.addBridgeNetworkStatus( - (BridgeNetworkStatus) descriptor, authority); - } + if (authority == null) { + this.logger.warning("Did not recognize the bridge authority " + + "that generated " + descriptorFileName + ". Skipping."); + continue; } + this.addBridgeNetworkStatus( + (BridgeNetworkStatus) descriptor, authority); } } if (keepImportHistory) { diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java index 0c2be9d..b5dae12 100644 --- a/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java +++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java @@ -4,10 +4,10 @@ package org.torproject.ernie.cron.performance;

import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; import org.torproject.descriptor.DescriptorReader; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.TorperfResult; +import org.torproject.descriptor.UnparseableDescriptor;

import java.io.BufferedReader; import java.io.BufferedWriter; @@ -83,60 +83,55 @@ public class TorperfProcessor { logger.fine("Importing files in " + torperfDirectory + "/..."); DescriptorReader descriptorReader = DescriptorSourceFactory.createDescriptorReader(); - descriptorReader.addDirectory(torperfDirectory); File historyFile = new File(statsDirectory, "torperf-history"); descriptorReader.setHistoryFile(historyFile); - Iterator<DescriptorFile> descriptorFiles = - descriptorReader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getException() != null) { - logger.log(Level.FINE, "Error parsing file.", - descriptorFile.getException()); + for (Descriptor descriptor : descriptorReader.readDescriptors( + torperfDirectory)) { + if (descriptor instanceof UnparseableDescriptor) { + logger.log(Level.FINE, "Error parsing descriptor.", + ((UnparseableDescriptor) descriptor) + .getDescriptorParseException()); + continue; + } else if (!(descriptor instanceof TorperfResult)) { continue; } - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (!(descriptor instanceof TorperfResult)) { - continue; - } - TorperfResult result = (TorperfResult) descriptor; - if (null != result.getUnrecognizedKeys() - && result.getUnrecognizedKeys().containsKey("ENDPOINTREMOTE") - && result.getUnrecognizedKeys().get("ENDPOINTREMOTE") - .contains(".onion")) { - continue; - } - String source = result.getSource(); - long fileSize = result.getFileSize(); - if (fileSize == 51200) { - source += "-50kb"; - } else if (fileSize == 1048576) { - source += "-1mb"; - } else if (fileSize == 5242880) { - source += "-5mb"; - } else { - logger.fine("Unexpected file size '" + fileSize - + "'. Skipping."); - continue; - } - String dateTime = formatter.format(result.getStartMillis()); - long completeMillis = result.getDataCompleteMillis() - - result.getStartMillis(); - String key = source + "," + dateTime; - String value = key; - if ((result.didTimeout() == null - && result.getDataCompleteMillis() < 1) - || (result.didTimeout() != null && result.didTimeout())) { - value += ",-2"; // -2 for timeout - } else if (result.getReadBytes() < fileSize) { - value += ",-1"; // -1 for failure - } else { - value += "," + completeMillis; - } - if (!rawObs.containsKey(key)) { - rawObs.put(key, value); - addedRawObs++; - } + TorperfResult result = (TorperfResult) descriptor; + if (null != result.getUnrecognizedKeys() + && result.getUnrecognizedKeys().containsKey("ENDPOINTREMOTE") + && result.getUnrecognizedKeys().get("ENDPOINTREMOTE") + .contains(".onion")) { + continue; + } + String source = result.getSource(); + long fileSize = result.getFileSize(); + if (fileSize == 51200) { + source += "-50kb"; + } else if (fileSize == 1048576) { + source += "-1mb"; + } else if (fileSize == 5242880) { + source += "-5mb"; + } else { + logger.fine("Unexpected file size '" + fileSize + + "'. Skipping."); + continue; + } + String dateTime = formatter.format(result.getStartMillis()); + long completeMillis = result.getDataCompleteMillis() + - result.getStartMillis(); + String key = source + "," + dateTime; + String value = key; + if ((result.didTimeout() == null + && result.getDataCompleteMillis() < 1) + || (result.didTimeout() != null && result.didTimeout())) { + value += ",-2"; // -2 for timeout + } else if (result.getReadBytes() < fileSize) { + value += ",-1"; // -1 for failure + } else { + value += "," + completeMillis; + } + if (!rawObs.containsKey(key)) { + rawObs.put(key, value); + addedRawObs++; } } descriptorReader.saveHistoryFile(historyFile); diff --git a/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java b/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java index 2af3100..802e4ef 100644 --- a/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java +++ b/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java @@ -1,7 +1,6 @@ package org.torproject.metrics.onionperf;

import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; import org.torproject.descriptor.DescriptorReader; import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.TorperfResult; @@ -79,118 +78,114 @@ public class Main { + "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, " + "?, ?, ?, ?, ?, ?)", Statement.RETURN_GENERATED_KEYS);

- DescriptorReader dr = DescriptorSourceFactory.createDescriptorReader(); - dr.addDirectory(new File("../../shared/in/archive/torperf")); - dr.addDirectory(new File("../../shared/in/recent/torperf")); - Iterator<DescriptorFile> dfs = dr.readDescriptors(); Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - while (dfs.hasNext()) { - DescriptorFile df = dfs.next(); - for (Descriptor d : df.getDescriptors()) { - if (!(d instanceof TorperfResult)) { - continue; + DescriptorReader dr = DescriptorSourceFactory.createDescriptorReader(); + for (Descriptor d : dr.readDescriptors( + new File("../../shared/in/archive/torperf"), + new File("../../shared/in/recent/torperf"))) { + if (!(d instanceof TorperfResult)) { + continue; + } + TorperfResult tr = (TorperfResult) d; + int measurementId = -1; + String truncatedSource = truncateString(tr.getSource(), 32); + psMeasurementsSelect.clearParameters(); + psMeasurementsSelect.setString(1, truncatedSource); + psMeasurementsSelect.setInt(2, tr.getFileSize()); + psMeasurementsSelect.setTimestamp(3, + new Timestamp(tr.getStartMillis()), calendar); + try (ResultSet rs = psMeasurementsSelect.executeQuery()) { + if (rs.next()) { + measurementId = rs.getInt(1); } - TorperfResult tr = (TorperfResult) d; - int measurementId = -1; - String truncatedSource = truncateString(tr.getSource(), 32); - psMeasurementsSelect.clearParameters(); - psMeasurementsSelect.setString(1, truncatedSource); - psMeasurementsSelect.setInt(2, tr.getFileSize()); - psMeasurementsSelect.setTimestamp(3, + } + if (measurementId < 0) { + psMeasurementsInsert.clearParameters(); + psMeasurementsInsert.setString(1, truncatedSource); + psMeasurementsInsert.setInt(2, tr.getFileSize()); + psMeasurementsInsert.setTimestamp(3, new Timestamp(tr.getStartMillis()), calendar); - try (ResultSet rs = psMeasurementsSelect.executeQuery()) { - if (rs.next()) { - measurementId = rs.getInt(1); - } - } - if (measurementId < 0) { - psMeasurementsInsert.clearParameters(); - psMeasurementsInsert.setString(1, truncatedSource); - psMeasurementsInsert.setInt(2, tr.getFileSize()); - psMeasurementsInsert.setTimestamp(3, - new Timestamp(tr.getStartMillis()), calendar); - long[] timestamps = new long[] { tr.getSocketMillis(), - tr.getConnectMillis(), tr.getNegotiateMillis(), - tr.getRequestMillis(), tr.getResponseMillis(), - tr.getDataRequestMillis(), tr.getDataResponseMillis(), - tr.getDataCompleteMillis() }; - for (int i = 4, j = 0; j < timestamps.length; i++, j++) { - if (timestamps[j] == 0L) { - psMeasurementsInsert.setNull(i, Types.INTEGER); - } else { - psMeasurementsInsert.setInt(i, - (int) (timestamps[j] - tr.getStartMillis())); - } - } - psMeasurementsInsert.setInt(12, tr.getWriteBytes()); - psMeasurementsInsert.setInt(13, tr.getReadBytes()); - if (null == tr.didTimeout()) { - psMeasurementsInsert.setNull(14, Types.BOOLEAN); - } else { - psMeasurementsInsert.setBoolean(14, tr.didTimeout()); - } - for (int i = 15, p = 0; i <= 25 && p <= 100; i++, p += 10) { - if (null == tr.getDataPercentiles() - || !tr.getDataPercentiles().containsKey(p)) { - psMeasurementsInsert.setNull(i, Types.INTEGER); - } else { - psMeasurementsInsert.setInt(i, - (int) (tr.getDataPercentiles().get(p) - tr.getStartMillis())); - } - } - if (tr.getLaunchMillis() < 0L) { - psMeasurementsInsert.setNull(26, Types.TIMESTAMP); - } else { - psMeasurementsInsert.setTimestamp(26, - new Timestamp(tr.getLaunchMillis()), calendar); - } - if (tr.getUsedAtMillis() < 0L) { - psMeasurementsInsert.setNull(27, Types.TIMESTAMP); - } else { - psMeasurementsInsert.setTimestamp(27, - new Timestamp(tr.getUsedAtMillis()), calendar); - } - if (tr.getTimeout() < 0L) { - psMeasurementsInsert.setNull(28, Types.INTEGER); + long[] timestamps = new long[] { tr.getSocketMillis(), + tr.getConnectMillis(), tr.getNegotiateMillis(), + tr.getRequestMillis(), tr.getResponseMillis(), + tr.getDataRequestMillis(), tr.getDataResponseMillis(), + tr.getDataCompleteMillis() }; + for (int i = 4, j = 0; j < timestamps.length; i++, j++) { + if (timestamps[j] == 0L) { + psMeasurementsInsert.setNull(i, Types.INTEGER); } else { - psMeasurementsInsert.setInt(28, (int) tr.getTimeout()); + psMeasurementsInsert.setInt(i, + (int) (timestamps[j] - tr.getStartMillis())); } - if (tr.getQuantile() < 0.0) { - psMeasurementsInsert.setNull(29, Types.REAL); - } else { - psMeasurementsInsert.setDouble(29, tr.getQuantile()); - } - if (tr.getCircId() < 0L) { - psMeasurementsInsert.setNull(30, Types.INTEGER); + } + psMeasurementsInsert.setInt(12, tr.getWriteBytes()); + psMeasurementsInsert.setInt(13, tr.getReadBytes()); + if (null == tr.didTimeout()) { + psMeasurementsInsert.setNull(14, Types.BOOLEAN); + } else { + psMeasurementsInsert.setBoolean(14, tr.didTimeout()); + } + for (int i = 15, p = 0; i <= 25 && p <= 100; i++, p += 10) { + if (null == tr.getDataPercentiles() + || !tr.getDataPercentiles().containsKey(p)) { + psMeasurementsInsert.setNull(i, Types.INTEGER); } else { - psMeasurementsInsert.setInt(30, tr.getCircId()); + psMeasurementsInsert.setInt(i, + (int) (tr.getDataPercentiles().get(p) - tr.getStartMillis())); } - if (tr.getUsedBy() < 0L) { - psMeasurementsInsert.setNull(31, Types.INTEGER); + } + if (tr.getLaunchMillis() < 0L) { + psMeasurementsInsert.setNull(26, Types.TIMESTAMP); + } else { + psMeasurementsInsert.setTimestamp(26, + new Timestamp(tr.getLaunchMillis()), calendar); + } + if (tr.getUsedAtMillis() < 0L) { + psMeasurementsInsert.setNull(27, Types.TIMESTAMP); + } else { + psMeasurementsInsert.setTimestamp(27, + new Timestamp(tr.getUsedAtMillis()), calendar); + } + if (tr.getTimeout() < 0L) { + psMeasurementsInsert.setNull(28, Types.INTEGER); + } else { + psMeasurementsInsert.setInt(28, (int) tr.getTimeout()); + } + if (tr.getQuantile() < 0.0) { + psMeasurementsInsert.setNull(29, Types.REAL); + } else { + psMeasurementsInsert.setDouble(29, tr.getQuantile()); + } + if (tr.getCircId() < 0L) { + psMeasurementsInsert.setNull(30, Types.INTEGER); + } else { + psMeasurementsInsert.setInt(30, tr.getCircId()); + } + if (tr.getUsedBy() < 0L) { + psMeasurementsInsert.setNull(31, Types.INTEGER); + } else { + psMeasurementsInsert.setInt(31, tr.getUsedBy()); + } + String[] unrecognizedKeys = new String[] { "ENDPOINTLOCAL", + "ENDPOINTPROXY", "ENDPOINTREMOTE", "HOSTNAMELOCAL", + "HOSTNAMEREMOTE", "SOURCEADDRESS" }; + for (int i = 32, j = 0; j < unrecognizedKeys.length; i++, j++) { + if (null == tr.getUnrecognizedKeys() + || !tr.getUnrecognizedKeys().containsKey(unrecognizedKeys[j])) { + psMeasurementsInsert.setNull(i, Types.VARCHAR); } else { - psMeasurementsInsert.setInt(31, tr.getUsedBy()); - } - String[] unrecognizedKeys = new String[] { "ENDPOINTLOCAL", - "ENDPOINTPROXY", "ENDPOINTREMOTE", "HOSTNAMELOCAL", - "HOSTNAMEREMOTE", "SOURCEADDRESS" }; - for (int i = 32, j = 0; j < unrecognizedKeys.length; i++, j++) { - if (null == tr.getUnrecognizedKeys() - || !tr.getUnrecognizedKeys().containsKey(unrecognizedKeys[j])) { - psMeasurementsInsert.setNull(i, Types.VARCHAR); - } else { - psMeasurementsInsert.setString(i, truncateString( - tr.getUnrecognizedKeys().get(unrecognizedKeys[j]), 64)); - } + psMeasurementsInsert.setString(i, truncateString( + tr.getUnrecognizedKeys().get(unrecognizedKeys[j]), 64)); } - psMeasurementsInsert.execute(); - try (ResultSet rs = psMeasurementsInsert.getGeneratedKeys()) { - if (rs.next()) { - measurementId = rs.getInt(1); - } + } + psMeasurementsInsert.execute(); + try (ResultSet rs = psMeasurementsInsert.getGeneratedKeys()) { + if (rs.next()) { + measurementId = rs.getInt(1); } } - /* Could use measurementId to insert path. */ } + /* Could use measurementId to insert path. */ connection.commit(); } } diff --git a/shared/build-base.xml b/shared/build-base.xml index f473b77..c446aa4 100644 --- a/shared/build-base.xml +++ b/shared/build-base.xml @@ -8,7 +8,7 @@ <property name="classes" value="${generated}/classes/"/> <property name="testclasses" value="${generated}/test-classes/"/> <property name="source-and-target-java-version" value="1.7" /> - <property name="descriptorversion" value="1.8.2" /> + <property name="descriptorversion" value="1.9.0" />

<path id="base.classpath"> <pathelement path="${classes}"/>