[tor-commits] [collector/master] Move all Java sources to src/main/java/.
karsten at torproject.org
karsten at torproject.org
Wed May 18 07:10:21 UTC 2016
commit 1c0661e99eeed206f6c0c0beee72ae82afdda131
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Wed May 18 09:09:11 2016 +0200
Move all Java sources to src/main/java/.
---
build.xml | 4 +-
src/main/java/org/torproject/collector/Main.java | 83 ++
.../bridgedescs/BridgeDescriptorParser.java | 51 +
.../bridgedescs/BridgeSnapshotReader.java | 228 ++++
.../bridgedescs/SanitizedBridgesWriter.java | 1330 ++++++++++++++++++++
.../collector/exitlists/ExitListDownloader.java | 212 ++++
.../collector/index/CreateIndexJson.java | 168 +++
.../torproject/collector/main/Configuration.java | 318 +++++
.../org/torproject/collector/main/LockFile.java | 56 +
.../collector/relaydescs/ArchiveReader.java | 286 +++++
.../collector/relaydescs/ArchiveWriter.java | 845 +++++++++++++
.../relaydescs/CachedRelayDescriptorReader.java | 255 ++++
.../collector/relaydescs/ReferenceChecker.java | 319 +++++
.../relaydescs/RelayDescriptorDownloader.java | 1134 +++++++++++++++++
.../relaydescs/RelayDescriptorParser.java | 337 +++++
.../collector/torperf/TorperfDownloader.java | 643 ++++++++++
src/org/torproject/collector/Main.java | 83 --
.../bridgedescs/BridgeDescriptorParser.java | 51 -
.../bridgedescs/BridgeSnapshotReader.java | 228 ----
.../bridgedescs/SanitizedBridgesWriter.java | 1330 --------------------
.../collector/exitlists/ExitListDownloader.java | 212 ----
.../collector/index/CreateIndexJson.java | 168 ---
.../torproject/collector/main/Configuration.java | 318 -----
src/org/torproject/collector/main/LockFile.java | 56 -
.../collector/relaydescs/ArchiveReader.java | 286 -----
.../collector/relaydescs/ArchiveWriter.java | 845 -------------
.../relaydescs/CachedRelayDescriptorReader.java | 255 ----
.../collector/relaydescs/ReferenceChecker.java | 319 -----
.../relaydescs/RelayDescriptorDownloader.java | 1134 -----------------
.../relaydescs/RelayDescriptorParser.java | 337 -----
.../collector/torperf/TorperfDownloader.java | 643 ----------
31 files changed, 6267 insertions(+), 6267 deletions(-)
diff --git a/build.xml b/build.xml
index f07c67b..bbd83af 100644
--- a/build.xml
+++ b/build.xml
@@ -3,11 +3,11 @@
<property name="name" value="CollecTor"/>
<property name="descriptorversion" value="1.1.0-dev" />
<property name="source-and-target-java-version" value="1.7" />
- <property name="sources" value="src/"/>
+ <property name="sources" value="src/main/java"/>
<property name="resources" value="resources/"/>
<property name="generated" value="generated/"/>
<property name="testresult" value="${generated}/test-results"/>
- <property name="testsources" value="test"/>
+ <property name="testsources" value="src/test/java"/>
<property name="coverageresult" value="${generated}/coverage-report/"/>
<property name="instrument" value="${generated}/instrument/"/>
<property name="classes" value="${generated}/classes/"/>
diff --git a/src/main/java/org/torproject/collector/Main.java b/src/main/java/org/torproject/collector/Main.java
new file mode 100644
index 0000000..9c64696
--- /dev/null
+++ b/src/main/java/org/torproject/collector/Main.java
@@ -0,0 +1,83 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector;
+
+import org.torproject.collector.bridgedescs.SanitizedBridgesWriter;
+import org.torproject.collector.exitlists.ExitListDownloader;
+import org.torproject.collector.index.CreateIndexJson;
+import org.torproject.collector.relaydescs.ArchiveWriter;
+import org.torproject.collector.torperf.TorperfDownloader;
+
+import java.lang.reflect.InvocationTargetException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.logging.Logger;
+
+/**
+ * Main class for starting a CollecTor instance.
+ * <br>
+ * Run without arguments in order to read the usage information, i.e.
+ * <br>
+ * <code>java -jar collector.jar</code>
+ */
+public class Main {
+
+ private static Logger log = Logger.getLogger(Main.class.getName());
+
+ /** All possible main classes.
+ * If a new CollecTorMain class is available, just add it to this map.
+ */
+ private static final Map<String, Class> collecTorMains = new HashMap<>();
+
+ static { // add a new main class here
+ collecTorMains.put("bridgedescs", SanitizedBridgesWriter.class);
+ collecTorMains.put("exitlists", ExitListDownloader.class);
+ collecTorMains.put("updateindex", CreateIndexJson.class);
+ collecTorMains.put("relaydescs", ArchiveWriter.class);
+ collecTorMains.put("torperf", TorperfDownloader.class);
+ }
+
+ private static final String modules = collecTorMains.keySet().toString()
+ .replace("[", "").replace("]", "").replaceAll(", ", "|");
+
+ /**
+ * One argument is necessary.
+ * See class description {@link Main}.
+ */
+ public static void main(String[] args) {
+ if (null == args || args.length != 1) {
+ printUsageAndExit("CollecTor needs exactly one argument.");
+ } else {
+ invokeGivenMainAndExit(args[0]);
+ }
+ }
+
+ private static void printUsageAndExit(String msg) {
+ final String usage = "Usage:\njava -jar collector.jar "
+ + "<" + modules + ">";
+ System.out.println(msg + "\n" + usage);
+ System.exit(0);
+ }
+
+ private static void invokeGivenMainAndExit(String mainId) {
+ Class clazz = collecTorMains.get(mainId);
+ if (null == clazz) {
+ printUsageAndExit("Unknown argument: " + mainId);
+ }
+ invokeMainOnClassAndExit(clazz);
+ }
+
+ private static void invokeMainOnClassAndExit(Class clazz) {
+ try {
+ clazz.getMethod("main", new Class[] { String[].class })
+ .invoke(null, (Object) new String[]{});
+ System.exit(0);
+ } catch (NoSuchMethodException | IllegalAccessException
+ | InvocationTargetException e) {
+ log.severe("Cannot invoke 'main' method on "
+ + clazz.getName() + ". " + e);
+ }
+ }
+}
+
diff --git a/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java b/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java
new file mode 100644
index 0000000..f683ea0
--- /dev/null
+++ b/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java
@@ -0,0 +1,51 @@
+/* Copyright 2010--2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class BridgeDescriptorParser {
+
+ private SanitizedBridgesWriter sbw;
+
+ private Logger logger;
+
+ public BridgeDescriptorParser(SanitizedBridgesWriter sbw) {
+ this.sbw = sbw;
+ this.logger =
+ Logger.getLogger(BridgeDescriptorParser.class.getName());
+ }
+
+ public void parse(byte[] allData, String dateTime) {
+ try {
+ BufferedReader br = new BufferedReader(new StringReader(
+ new String(allData, "US-ASCII")));
+ String line = br.readLine();
+ if (line == null) {
+ return;
+ } else if (line.startsWith("router ")) {
+ if (this.sbw != null) {
+ this.sbw.sanitizeAndStoreServerDescriptor(allData);
+ }
+ } else if (line.startsWith("extra-info ")) {
+ if (this.sbw != null) {
+ this.sbw.sanitizeAndStoreExtraInfoDescriptor(allData);
+ }
+ } else {
+ if (this.sbw != null) {
+ this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime);
+ }
+ }
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not parse bridge descriptor.",
+ e);
+ return;
+ }
+ }
+}
+
diff --git a/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java b/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java
new file mode 100644
index 0000000..2d41d18
--- /dev/null
+++ b/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java
@@ -0,0 +1,228 @@
+/* Copyright 2010--2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.Stack;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Reads the half-hourly snapshots of bridge descriptors from Tonga.
+ */
+public class BridgeSnapshotReader {
+ public BridgeSnapshotReader(BridgeDescriptorParser bdp,
+ File bridgeDirectoriesDir, File statsDirectory) {
+
+ if (bdp == null || bridgeDirectoriesDir == null
+ || statsDirectory == null) {
+ throw new IllegalArgumentException();
+ }
+
+ Logger logger =
+ Logger.getLogger(BridgeSnapshotReader.class.getName());
+ SortedSet<String> parsed = new TreeSet<String>();
+ File bdDir = bridgeDirectoriesDir;
+ File pbdFile = new File(statsDirectory, "parsed-bridge-directories");
+ boolean modified = false;
+ if (bdDir.exists()) {
+ if (pbdFile.exists()) {
+ logger.fine("Reading file " + pbdFile.getAbsolutePath() + "...");
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(pbdFile));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ parsed.add(line);
+ }
+ br.close();
+ logger.fine("Finished reading file "
+ + pbdFile.getAbsolutePath() + ".");
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed reading file "
+ + pbdFile.getAbsolutePath() + "!", e);
+ return;
+ }
+ }
+ logger.fine("Importing files in directory " + bridgeDirectoriesDir
+ + "/...");
+ Set<String> descriptorImportHistory = new HashSet<String>();
+ int parsedFiles = 0;
+ int skippedFiles = 0;
+ int parsedStatuses = 0;
+ int parsedServerDescriptors = 0;
+ int skippedServerDescriptors = 0;
+ int parsedExtraInfoDescriptors = 0;
+ int skippedExtraInfoDescriptors = 0;
+ Stack<File> filesInInputDir = new Stack<File>();
+ filesInInputDir.add(bdDir);
+ while (!filesInInputDir.isEmpty()) {
+ File pop = filesInInputDir.pop();
+ if (pop.isDirectory()) {
+ for (File f : pop.listFiles()) {
+ filesInInputDir.add(f);
+ }
+ } else if (!parsed.contains(pop.getName())) {
+ try {
+ FileInputStream in = new FileInputStream(pop);
+ if (in.available() > 0) {
+ TarArchiveInputStream tais = null;
+ if (pop.getName().endsWith(".tar.gz")) {
+ GzipCompressorInputStream gcis =
+ new GzipCompressorInputStream(in);
+ tais = new TarArchiveInputStream(gcis);
+ } else if (pop.getName().endsWith(".tar")) {
+ tais = new TarArchiveInputStream(in);
+ } else {
+ continue;
+ }
+ BufferedInputStream bis = new BufferedInputStream(tais);
+ String fn = pop.getName();
+ String dateTime = fn.substring(11, 21) + " "
+ + fn.substring(22, 24) + ":" + fn.substring(24, 26)
+ + ":" + fn.substring(26, 28);
+ while ((tais.getNextTarEntry()) != null) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ byte[] allData = baos.toByteArray();
+ if (allData.length == 0) {
+ continue;
+ }
+ String fileDigest = Hex.encodeHexString(DigestUtils.sha(
+ allData));
+ String ascii = new String(allData, "US-ASCII");
+ BufferedReader br3 = new BufferedReader(new StringReader(
+ ascii));
+ String firstLine = null;
+ while ((firstLine = br3.readLine()) != null) {
+ if (firstLine.startsWith("@")) {
+ continue;
+ } else {
+ break;
+ }
+ }
+ if (firstLine.startsWith("published ")
+ || firstLine.startsWith("flag-thresholds ")
+ || firstLine.startsWith("r ")) {
+ bdp.parse(allData, dateTime);
+ parsedStatuses++;
+ } else if (descriptorImportHistory.contains(fileDigest)) {
+ /* Skip server descriptors or extra-info descriptors if
+ * we parsed them before. */
+ skippedFiles++;
+ continue;
+ } else {
+ int start = -1;
+ int sig = -1;
+ int end = -1;
+ String startToken = firstLine.startsWith("router ")
+ ? "router " : "extra-info ";
+ String sigToken = "\nrouter-signature\n";
+ String endToken = "\n-----END SIGNATURE-----\n";
+ while (end < ascii.length()) {
+ start = ascii.indexOf(startToken, end);
+ if (start < 0) {
+ break;
+ }
+ sig = ascii.indexOf(sigToken, start);
+ if (sig < 0) {
+ break;
+ }
+ sig += sigToken.length();
+ end = ascii.indexOf(endToken, sig);
+ if (end < 0) {
+ break;
+ }
+ end += endToken.length();
+ byte[] descBytes = new byte[end - start];
+ System.arraycopy(allData, start, descBytes, 0,
+ end - start);
+ String descriptorDigest = Hex.encodeHexString(
+ DigestUtils.sha(descBytes));
+ if (!descriptorImportHistory.contains(
+ descriptorDigest)) {
+ bdp.parse(descBytes, dateTime);
+ descriptorImportHistory.add(descriptorDigest);
+ if (firstLine.startsWith("router ")) {
+ parsedServerDescriptors++;
+ } else {
+ parsedExtraInfoDescriptors++;
+ }
+ } else {
+ if (firstLine.startsWith("router ")) {
+ skippedServerDescriptors++;
+ } else {
+ skippedExtraInfoDescriptors++;
+ }
+ }
+ }
+ }
+ descriptorImportHistory.add(fileDigest);
+ parsedFiles++;
+ }
+ bis.close();
+ }
+ in.close();
+
+ /* Let's give some memory back, or we'll run out of it. */
+ System.gc();
+
+ parsed.add(pop.getName());
+ modified = true;
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not parse bridge snapshot "
+ + pop.getName() + "!", e);
+ continue;
+ }
+ }
+ }
+ logger.fine("Finished importing files in directory "
+ + bridgeDirectoriesDir + "/. In total, we parsed "
+ + parsedFiles + " files (skipped " + skippedFiles
+ + ") containing " + parsedStatuses + " statuses, "
+ + parsedServerDescriptors + " server descriptors (skipped "
+ + skippedServerDescriptors + "), and "
+ + parsedExtraInfoDescriptors + " extra-info descriptors "
+ + "(skipped " + skippedExtraInfoDescriptors + ").");
+ if (!parsed.isEmpty() && modified) {
+ logger.fine("Writing file " + pbdFile.getAbsolutePath() + "...");
+ try {
+ pbdFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(pbdFile));
+ for (String f : parsed) {
+ bw.append(f + "\n");
+ }
+ bw.close();
+ logger.fine("Finished writing file " + pbdFile.getAbsolutePath()
+ + ".");
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed writing file "
+ + pbdFile.getAbsolutePath() + "!", e);
+ }
+ }
+ }
+ }
+}
+
diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
new file mode 100644
index 0000000..3214715
--- /dev/null
+++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -0,0 +1,1330 @@
+/* Copyright 2010--2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import org.torproject.collector.main.Configuration;
+import org.torproject.collector.main.LockFile;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.UnsupportedEncodingException;
+import java.security.GeneralSecurityException;
+import java.security.SecureRandom;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.Stack;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Sanitizes bridge descriptors, i.e., removes all possibly sensitive
+ * information from them, and writes them to a local directory structure.
+ * During the sanitizing process, all information about the bridge
+ * identity or IP address are removed or replaced. The goal is to keep the
+ * sanitized bridge descriptors useful for statistical analysis while not
+ * making it easier for an adversary to enumerate bridges.
+ *
+ * There are three types of bridge descriptors: bridge network statuses
+ * (lists of all bridges at a given time), server descriptors (published
+ * by the bridge to advertise their capabilities), and extra-info
+ * descriptors (published by the bridge, mainly for statistical analysis).
+ */
+public class SanitizedBridgesWriter extends Thread {
+
+ public static void main(String[] args) {
+
+ Logger logger = Logger.getLogger(
+ SanitizedBridgesWriter.class.getName());
+ logger.info("Starting bridge-descriptors module of CollecTor.");
+
+ // Initialize configuration
+ Configuration config = new Configuration();
+
+ // Use lock file to avoid overlapping runs
+ LockFile lf = new LockFile("bridge-descriptors");
+ if (!lf.acquireLock()) {
+ logger.severe("Warning: CollecTor is already running or has not exited "
+ + "cleanly! Exiting!");
+ System.exit(1);
+ }
+
+ // Sanitize bridge descriptors
+ new SanitizedBridgesWriter(config).run();
+
+ // Remove lock file
+ lf.releaseLock();
+
+ logger.info("Terminating bridge-descriptors module of CollecTor.");
+ }
+
+ private Configuration config;
+
+ /**
+ * Initializes this class.
+ */
+ public SanitizedBridgesWriter(Configuration config) {
+ this.config = config;
+ }
+
+ /**
+ * Logger for this class.
+ */
+ private Logger logger;
+
+ private String rsyncCatString;
+
+ private File bridgeDirectoriesDirectory;
+
+ /**
+ * Output directory for writing sanitized bridge descriptors.
+ */
+ private File sanitizedBridgesDirectory;
+
+ private boolean replaceIPAddressesWithHashes;
+
+ private boolean persistenceProblemWithSecrets;
+
+ private SortedMap<String, byte[]> secretsForHashingIPAddresses;
+
+ private String bridgeSanitizingCutOffTimestamp;
+
+ private boolean haveWarnedAboutInterval;
+
+ private File bridgeIpSecretsFile;
+
+ private SecureRandom secureRandom;
+
+ public void run() {
+
+ File bridgeDirectoriesDirectory =
+ new File(config.getBridgeSnapshotsDirectory());
+ File sanitizedBridgesDirectory =
+ new File(config.getSanitizedBridgesWriteDirectory());
+ boolean replaceIPAddressesWithHashes =
+ config.getReplaceIPAddressesWithHashes();
+ long limitBridgeSanitizingInterval =
+ config.getLimitBridgeDescriptorMappings();
+ File statsDirectory = new File("stats");
+
+ if (bridgeDirectoriesDirectory == null
+ || sanitizedBridgesDirectory == null || statsDirectory == null) {
+ throw new IllegalArgumentException();
+ }
+
+ /* Memorize argument values. */
+ this.bridgeDirectoriesDirectory = bridgeDirectoriesDirectory;
+ this.sanitizedBridgesDirectory = sanitizedBridgesDirectory;
+ this.replaceIPAddressesWithHashes = replaceIPAddressesWithHashes;
+
+ /* Initialize logger. */
+ this.logger = Logger.getLogger(
+ SanitizedBridgesWriter.class.getName());
+
+ SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
+ "yyyy-MM-dd-HH-mm-ss");
+ rsyncCatFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ this.rsyncCatString = rsyncCatFormat.format(
+ System.currentTimeMillis());
+
+ /* Initialize secure random number generator if we need it. */
+ if (this.replaceIPAddressesWithHashes) {
+ try {
+ this.secureRandom = SecureRandom.getInstance("SHA1PRNG", "SUN");
+ } catch (GeneralSecurityException e) {
+ this.logger.log(Level.WARNING, "Could not initialize secure "
+ + "random number generator! Not calculating any IP address "
+ + "hashes in this execution!", e);
+ this.persistenceProblemWithSecrets = true;
+ }
+ }
+
+ /* Read hex-encoded secrets for replacing IP addresses with hashes
+ * from disk. */
+ this.secretsForHashingIPAddresses = new TreeMap<String, byte[]>();
+ this.bridgeIpSecretsFile = new File(statsDirectory,
+ "bridge-ip-secrets");
+ if (this.bridgeIpSecretsFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.bridgeIpSecretsFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(",");
+ if ((line.length() != ("yyyy-MM,".length() + 31 * 2)
+ && line.length() != ("yyyy-MM,".length() + 50 * 2))
+ || parts.length != 2) {
+ this.logger.warning("Invalid line in bridge-ip-secrets file "
+ + "starting with '" + line.substring(0, 7) + "'! "
+ + "Not calculating any IP address hashes in this "
+ + "execution!");
+ this.persistenceProblemWithSecrets = true;
+ break;
+ }
+ String month = parts[0];
+ byte[] secret = Hex.decodeHex(parts[1].toCharArray());
+ this.secretsForHashingIPAddresses.put(month, secret);
+ }
+ br.close();
+ if (!this.persistenceProblemWithSecrets) {
+ this.logger.fine("Read "
+ + this.secretsForHashingIPAddresses.size() + " secrets for "
+ + "hashing bridge IP addresses.");
+ }
+ } catch (DecoderException e) {
+ this.logger.log(Level.WARNING, "Failed to decode hex string in "
+ + this.bridgeIpSecretsFile + "! Not calculating any IP "
+ + "address hashes in this execution!", e);
+ this.persistenceProblemWithSecrets = true;
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed to read "
+ + this.bridgeIpSecretsFile + "! Not calculating any IP "
+ + "address hashes in this execution!", e);
+ this.persistenceProblemWithSecrets = true;
+ }
+ }
+
+ /* If we're configured to keep secrets only for a limited time, define
+ * the cut-off day and time. */
+ if (limitBridgeSanitizingInterval >= 0L) {
+ SimpleDateFormat formatter = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
+ this.bridgeSanitizingCutOffTimestamp = formatter.format(
+ System.currentTimeMillis() - 24L * 60L * 60L * 1000L
+ * limitBridgeSanitizingInterval);
+ } else {
+ this.bridgeSanitizingCutOffTimestamp = "1999-12-31 23:59:59";
+ }
+
+ // Prepare bridge descriptor parser
+ BridgeDescriptorParser bdp = new BridgeDescriptorParser(this);
+
+ // Import bridge descriptors
+ new BridgeSnapshotReader(bdp, this.bridgeDirectoriesDirectory,
+ statsDirectory);
+
+ // Finish writing sanitized bridge descriptors to disk
+ this.finishWriting();
+
+ this.checkStaleDescriptors();
+
+ this.cleanUpRsyncDirectory();
+ }
+
+ private String scrubOrAddress(String orAddress, byte[] fingerprintBytes,
+ String published) throws IOException {
+ if (!orAddress.contains(":")) {
+ /* Malformed or-address or a line. */
+ return null;
+ }
+ String addressPart = orAddress.substring(0,
+ orAddress.lastIndexOf(":"));
+ String portPart = orAddress.substring(orAddress.lastIndexOf(":") + 1);
+ String scrubbedAddressPart = null;
+ if (addressPart.startsWith("[")) {
+ scrubbedAddressPart = this.scrubIpv6Address(addressPart,
+ fingerprintBytes, published);
+ } else {
+ scrubbedAddressPart = this.scrubIpv4Address(addressPart,
+ fingerprintBytes, published);
+ }
+ return (scrubbedAddressPart == null ? null :
+ scrubbedAddressPart + ":" + portPart);
+ }
+
+ private String scrubIpv4Address(String address, byte[] fingerprintBytes,
+ String published) throws IOException {
+ if (this.replaceIPAddressesWithHashes) {
+ if (this.persistenceProblemWithSecrets) {
+ /* There's a persistence problem, so we shouldn't scrub more IP
+ * addresses in this execution. */
+ return null;
+ }
+ byte[] hashInput = new byte[4 + 20 + 31];
+ String[] ipParts = address.split("\\.");
+ for (int i = 0; i < 4; i++) {
+ hashInput[i] = (byte) Integer.parseInt(ipParts[i]);
+ }
+ System.arraycopy(fingerprintBytes, 0, hashInput, 4, 20);
+ String month = published.substring(0, "yyyy-MM".length());
+ byte[] secret = this.getSecretForMonth(month);
+ System.arraycopy(secret, 0, hashInput, 24, 31);
+ byte[] hashOutput = DigestUtils.sha256(hashInput);
+ String hashedAddress = "10."
+ + (((int) hashOutput[0] + 256) % 256) + "."
+ + (((int) hashOutput[1] + 256) % 256) + "."
+ + (((int) hashOutput[2] + 256) % 256);
+ return hashedAddress;
+ } else {
+ return "127.0.0.1";
+ }
+ }
+
+ private String scrubIpv6Address(String address, byte[] fingerprintBytes,
+ String published) throws IOException {
+ StringBuilder sb = new StringBuilder("[fd9f:2e19:3bcf::");
+ if (this.replaceIPAddressesWithHashes) {
+ if (this.persistenceProblemWithSecrets) {
+ /* There's a persistence problem, so we shouldn't scrub more IP
+ * addresses in this execution. */
+ return null;
+ }
+ byte[] hashInput = new byte[16 + 20 + 19];
+ String[] doubleColonSeparatedParts = address.substring(1,
+ address.length() - 1).split("::", -1);
+ if (doubleColonSeparatedParts.length > 2) {
+ /* Invalid IPv6 address. */
+ return null;
+ }
+ List<String> hexParts = new ArrayList<String>();
+ for (String doubleColonSeparatedPart : doubleColonSeparatedParts) {
+ StringBuilder hexPart = new StringBuilder();
+ String[] parts = doubleColonSeparatedPart.split(":", -1);
+ if (parts.length < 1 || parts.length > 8) {
+ /* Invalid IPv6 address. */
+ return null;
+ }
+ for (int i = 0; i < parts.length; i++) {
+ String part = parts[i];
+ if (part.contains(".")) {
+ String[] ipParts = part.split("\\.");
+ byte[] ipv4Bytes = new byte[4];
+ if (ipParts.length != 4) {
+ /* Invalid IPv4 part in IPv6 address. */
+ return null;
+ }
+ for (int m = 0; m < 4; m++) {
+ ipv4Bytes[m] = (byte) Integer.parseInt(ipParts[m]);
+ }
+ hexPart.append(Hex.encodeHexString(ipv4Bytes));
+ } else if (part.length() > 4) {
+ /* Invalid IPv6 address. */
+ return null;
+ } else {
+ for (int k = part.length(); k < 4; k++) {
+ hexPart.append("0");
+ }
+ hexPart.append(part);
+ }
+ }
+ hexParts.add(hexPart.toString());
+ }
+ StringBuilder hex = new StringBuilder();
+ hex.append(hexParts.get(0));
+ if (hexParts.size() == 2) {
+ for (int i = 32 - hexParts.get(0).length()
+ - hexParts.get(1).length(); i > 0; i--) {
+ hex.append("0");
+ }
+ hex.append(hexParts.get(1));
+ }
+ byte[] ipBytes = null;
+ try {
+ ipBytes = Hex.decodeHex(hex.toString().toCharArray());
+ } catch (DecoderException e) {
+ /* TODO Invalid IPv6 address. */
+ return null;
+ }
+ if (ipBytes.length != 16) {
+ /* TODO Invalid IPv6 address. */
+ return null;
+ }
+ System.arraycopy(ipBytes, 0, hashInput, 0, 16);
+ System.arraycopy(fingerprintBytes, 0, hashInput, 16, 20);
+ String month = published.substring(0, "yyyy-MM".length());
+ byte[] secret = this.getSecretForMonth(month);
+ System.arraycopy(secret, 31, hashInput, 36, 19);
+ String hashOutput = DigestUtils.sha256Hex(hashInput);
+ sb.append(hashOutput.substring(hashOutput.length() - 6,
+ hashOutput.length() - 4));
+ sb.append(":");
+ sb.append(hashOutput.substring(hashOutput.length() - 4));
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
+ private byte[] getSecretForMonth(String month) throws IOException {
+ if (!this.secretsForHashingIPAddresses.containsKey(month)
+ || this.secretsForHashingIPAddresses.get(month).length == 31) {
+ byte[] secret = new byte[50];
+ this.secureRandom.nextBytes(secret);
+ if (this.secretsForHashingIPAddresses.containsKey(month)) {
+ System.arraycopy(this.secretsForHashingIPAddresses.get(month), 0,
+ secret, 0, 31);
+ }
+ if (month.compareTo(
+ this.bridgeSanitizingCutOffTimestamp) < 0) {
+ this.logger.warning("Generated a secret that we won't make "
+ + "persistent, because it's outside our bridge descriptor "
+ + "sanitizing interval.");
+ } else {
+ /* Append secret to file on disk immediately before using it, or
+ * we might end with inconsistently sanitized bridges. */
+ try {
+ if (!this.bridgeIpSecretsFile.exists()) {
+ this.bridgeIpSecretsFile.getParentFile().mkdirs();
+ }
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.bridgeIpSecretsFile,
+ this.bridgeIpSecretsFile.exists()));
+ bw.write(month + "," + Hex.encodeHexString(secret) + "\n");
+ bw.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not store new secret "
+ + "to disk! Not calculating any IP address hashes in "
+ + "this execution!", e);
+ this.persistenceProblemWithSecrets = true;
+ throw new IOException(e);
+ }
+ }
+ this.secretsForHashingIPAddresses.put(month, secret);
+ }
+ return this.secretsForHashingIPAddresses.get(month);
+ }
+
+ private String maxNetworkStatusPublishedTime = "1970-01-01 00:00:00";
+
+ /**
+ * Sanitizes a network status and writes it to disk.
+ */
+ public void sanitizeAndStoreNetworkStatus(byte[] data,
+ String publicationTime) {
+
+ if (this.persistenceProblemWithSecrets) {
+ /* There's a persistence problem, so we shouldn't scrub more IP
+ * addresses in this execution. */
+ return;
+ }
+
+ if (publicationTime.compareTo(maxNetworkStatusPublishedTime) > 0) {
+ maxNetworkStatusPublishedTime = publicationTime;
+ }
+
+ if (this.bridgeSanitizingCutOffTimestamp
+ .compareTo(publicationTime) > 0) {
+ this.logger.log(!this.haveWarnedAboutInterval ? Level.WARNING
+ : Level.FINE, "Sanitizing and storing network status with "
+ + "publication time outside our descriptor sanitizing "
+ + "interval.");
+ this.haveWarnedAboutInterval = true;
+ }
+
+ /* Parse the given network status line by line. */
+ StringBuilder header = new StringBuilder();
+ SortedMap<String, String> scrubbedLines =
+ new TreeMap<String, String>();
+ try {
+ StringBuilder scrubbed = new StringBuilder();
+ BufferedReader br = new BufferedReader(new StringReader(new String(
+ data, "US-ASCII")));
+ String line = null;
+ String mostRecentDescPublished = null;
+ byte[] fingerprintBytes = null;
+ String descPublicationTime = null;
+ String hashedBridgeIdentityHex = null;
+ while ((line = br.readLine()) != null) {
+
+ /* Use publication time from "published" line instead of the
+ * file's last-modified time. Don't copy over the line, because
+ * we're going to write a "published" line below. */
+ if (line.startsWith("published ")) {
+ publicationTime = line.substring("published ".length());
+
+ /* Additional header lines don't have to be cleaned up. */
+ } else if (line.startsWith("flag-thresholds ")) {
+ header.append(line + "\n");
+
+ /* r lines contain sensitive information that needs to be removed
+ * or replaced. */
+ } else if (line.startsWith("r ")) {
+
+ /* Clear buffer from previously scrubbed lines. */
+ if (scrubbed.length() > 0) {
+ String scrubbedLine = scrubbed.toString();
+ scrubbedLines.put(hashedBridgeIdentityHex, scrubbedLine);
+ scrubbed = new StringBuilder();
+ }
+
+ /* Parse the relevant parts of this r line. */
+ String[] parts = line.split(" ");
+ String nickname = parts[1];
+ fingerprintBytes = Base64.decodeBase64(parts[2] + "==");
+ String descriptorIdentifier = parts[3];
+ descPublicationTime = parts[4] + " " + parts[5];
+ String address = parts[6];
+ String orPort = parts[7];
+ String dirPort = parts[8];
+
+ /* Determine most recent descriptor publication time. */
+ if (descPublicationTime.compareTo(publicationTime) <= 0
+ && (mostRecentDescPublished == null
+ || descPublicationTime.compareTo(
+ mostRecentDescPublished) > 0)) {
+ mostRecentDescPublished = descPublicationTime;
+ }
+
+ /* Write scrubbed r line to buffer. */
+ byte[] hashedBridgeIdentity = DigestUtils.sha(fingerprintBytes);
+ String hashedBridgeIdentityBase64 = Base64.encodeBase64String(
+ hashedBridgeIdentity).substring(0, 27);
+ hashedBridgeIdentityHex = Hex.encodeHexString(
+ hashedBridgeIdentity);
+ String hashedDescriptorIdentifier = Base64.encodeBase64String(
+ DigestUtils.sha(Base64.decodeBase64(descriptorIdentifier
+ + "=="))).substring(0, 27);
+ String scrubbedAddress = scrubIpv4Address(address,
+ fingerprintBytes,
+ descPublicationTime);
+ scrubbed.append("r " + nickname + " "
+ + hashedBridgeIdentityBase64 + " "
+ + hashedDescriptorIdentifier + " " + descPublicationTime
+ + " " + scrubbedAddress + " " + orPort + " " + dirPort
+ + "\n");
+
+ /* Sanitize any addresses in a lines using the fingerprint and
+ * descriptor publication time from the previous r line. */
+ } else if (line.startsWith("a ")) {
+ String scrubbedOrAddress = scrubOrAddress(
+ line.substring("a ".length()), fingerprintBytes,
+ descPublicationTime);
+ if (scrubbedOrAddress != null) {
+ scrubbed.append("a " + scrubbedOrAddress + "\n");
+ } else {
+ this.logger.warning("Invalid address in line '" + line
+ + "' in bridge network status. Skipping line!");
+ }
+
+ /* Nothing special about s, w, and p lines; just copy them. */
+ } else if (line.startsWith("s ") || line.equals("s")
+ || line.startsWith("w ") || line.equals("w")
+ || line.startsWith("p ") || line.equals("p")) {
+ scrubbed.append(line + "\n");
+
+ /* There should be nothing else but r, w, p, and s lines in the
+ * network status. If there is, we should probably learn before
+ * writing anything to the sanitized descriptors. */
+ } else {
+ this.logger.fine("Unknown line '" + line + "' in bridge "
+ + "network status. Not writing to disk!");
+ return;
+ }
+ }
+ br.close();
+ if (scrubbed.length() > 0) {
+ String scrubbedLine = scrubbed.toString();
+ scrubbedLines.put(hashedBridgeIdentityHex, scrubbedLine);
+ scrubbed = new StringBuilder();
+ }
+
+ /* Check if we can tell from the descriptor publication times
+ * whether this status is possibly stale. */
+ SimpleDateFormat formatter = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
+ if (formatter.parse(publicationTime).getTime()
+ - formatter.parse(mostRecentDescPublished).getTime()
+ > 60L * 60L * 1000L) {
+ this.logger.warning("The most recent descriptor in the bridge "
+ + "network status published at " + publicationTime + " was "
+ + "published at " + mostRecentDescPublished + " which is "
+ + "more than 1 hour before the status. This is a sign for "
+ + "the status being stale. Please check!");
+ }
+ } catch (ParseException e) {
+ this.logger.log(Level.WARNING, "Could not parse timestamp in "
+ + "bridge network status.", e);
+ return;
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not parse bridge network "
+ + "status.", e);
+ return;
+ }
+
+ /* Write the sanitized network status to disk. */
+ try {
+ String syear = publicationTime.substring(0, 4);
+ String smonth = publicationTime.substring(5, 7);
+ String sday = publicationTime.substring(8, 10);
+ String stime = publicationTime.substring(11, 13)
+ + publicationTime.substring(14, 16)
+ + publicationTime.substring(17, 19);
+ File tarballFile = new File(
+ this.sanitizedBridgesDirectory.getAbsolutePath() + "/" + syear
+ + "/" + smonth + "/statuses/" + sday + "/" + syear + smonth
+ + sday + "-" + stime + "-"
+ + "4A0CCD2DDC7995083D73F5D667100C8A5831F16D");
+ File rsyncFile = new File("recent/bridge-descriptors/statuses/"
+ + tarballFile.getName());
+ File[] outputFiles = new File[] { tarballFile, rsyncFile };
+ for (File outputFile : outputFiles) {
+ outputFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ outputFile));
+ bw.write("@type bridge-network-status 1.0\n");
+ bw.write("published " + publicationTime + "\n");
+ bw.write(header.toString());
+ for (String scrubbed : scrubbedLines.values()) {
+ bw.write(scrubbed);
+ }
+ bw.close();
+ }
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not write sanitized bridge "
+ + "network status to disk.", e);
+ return;
+ }
+ }
+
+ private String maxServerDescriptorPublishedTime = "1970-01-01 00:00:00";
+
+ /**
+ * Sanitizes a bridge server descriptor and writes it to disk.
+ */
+ public void sanitizeAndStoreServerDescriptor(byte[] data) {
+
+ if (this.persistenceProblemWithSecrets) {
+ /* There's a persistence problem, so we shouldn't scrub more IP
+ * addresses in this execution. */
+ return;
+ }
+
+ /* Parse descriptor to generate a sanitized version. */
+ String scrubbedDesc = null;
+ String published = null;
+ String masterKeyEd25519FromIdentityEd25519 = null;
+ try {
+ BufferedReader br = new BufferedReader(new StringReader(
+ new String(data, "US-ASCII")));
+ StringBuilder scrubbed = new StringBuilder();
+ String line = null;
+ String hashedBridgeIdentity = null;
+ String address = null;
+ String routerLine = null;
+ String scrubbedAddress = null;
+ String masterKeyEd25519 = null;
+ List<String> orAddresses = null;
+ List<String> scrubbedOrAddresses = null;
+ boolean skipCrypto = false;
+ while ((line = br.readLine()) != null) {
+
+ /* Skip all crypto parts that might be used to derive the bridge's
+ * identity fingerprint. */
+ if (skipCrypto && !line.startsWith("-----END ")) {
+ continue;
+
+ /* Store the router line for later processing, because we may need
+ * the bridge identity fingerprint for replacing the IP address in
+ * the scrubbed version. */
+ } else if (line.startsWith("router ")) {
+ address = line.split(" ")[2];
+ routerLine = line;
+
+ /* Store or-address parts in a list and sanitize them when we have
+ * read the fingerprint. */
+ } else if (line.startsWith("or-address ")) {
+ if (orAddresses == null) {
+ orAddresses = new ArrayList<String>();
+ }
+ orAddresses.add(line.substring("or-address ".length()));
+
+ /* Parse the publication time to see if we're still inside the
+ * sanitizing interval. */
+ } else if (line.startsWith("published ")) {
+ published = line.substring("published ".length());
+ if (published.compareTo(maxServerDescriptorPublishedTime) > 0) {
+ maxServerDescriptorPublishedTime = published;
+ }
+ if (this.bridgeSanitizingCutOffTimestamp
+ .compareTo(published) > 0) {
+ this.logger.log(!this.haveWarnedAboutInterval
+ ? Level.WARNING : Level.FINE, "Sanitizing and storing "
+ + "server descriptor with publication time outside our "
+ + "descriptor sanitizing interval.");
+ this.haveWarnedAboutInterval = true;
+ }
+ scrubbed.append(line + "\n");
+
+ /* Parse the fingerprint to determine the hashed bridge
+ * identity. */
+ } else if (line.startsWith("opt fingerprint ")
+ || line.startsWith("fingerprint ")) {
+ String fingerprint = line.substring(line.startsWith("opt ")
+ ? "opt fingerprint".length() : "fingerprint".length())
+ .replaceAll(" ", "").toLowerCase();
+ byte[] fingerprintBytes = Hex.decodeHex(
+ fingerprint.toCharArray());
+ hashedBridgeIdentity = DigestUtils.shaHex(fingerprintBytes)
+ .toLowerCase();
+ try {
+ scrubbedAddress = scrubIpv4Address(address, fingerprintBytes,
+ published);
+ if (orAddresses != null) {
+ scrubbedOrAddresses = new ArrayList<String>();
+ for (String orAddress : orAddresses) {
+ String scrubbedOrAddress = scrubOrAddress(orAddress,
+ fingerprintBytes, published);
+ if (scrubbedOrAddress != null) {
+ scrubbedOrAddresses.add(scrubbedOrAddress);
+ } else {
+ this.logger.warning("Invalid address in line "
+ + "'or-address " + orAddress + "' in bridge server "
+ + "descriptor. Skipping line!");
+ }
+ }
+ }
+ } catch (IOException e) {
+ /* There's a persistence problem, so we shouldn't scrub more
+ * IP addresses in this execution. */
+ this.persistenceProblemWithSecrets = true;
+ return;
+ }
+ scrubbed.append((line.startsWith("opt ") ? "opt " : "")
+ + "fingerprint");
+ for (int i = 0; i < hashedBridgeIdentity.length() / 4; i++) {
+ scrubbed.append(" " + hashedBridgeIdentity.substring(4 * i,
+ 4 * (i + 1)).toUpperCase());
+ }
+ scrubbed.append("\n");
+
+ /* Replace the contact line (if present) with a generic one. */
+ } else if (line.startsWith("contact ")) {
+ scrubbed.append("contact somebody\n");
+
+ /* When we reach the signature, we're done. Write the sanitized
+ * descriptor to disk below. */
+ } else if (line.startsWith("router-signature")) {
+ String[] routerLineParts = routerLine.split(" ");
+ scrubbedDesc = "router " + routerLineParts[1] + " "
+ + scrubbedAddress + " " + routerLineParts[3] + " "
+ + routerLineParts[4] + " " + routerLineParts[5] + "\n";
+ if (scrubbedOrAddresses != null) {
+ for (String scrubbedOrAddress : scrubbedOrAddresses) {
+ scrubbedDesc = scrubbedDesc += "or-address "
+ + scrubbedOrAddress + "\n";
+ }
+ }
+ scrubbedDesc += scrubbed.toString();
+ break;
+
+ /* Replace extra-info digest with the hashed digest of the
+ * non-scrubbed descriptor. */
+ } else if (line.startsWith("opt extra-info-digest ")
+ || line.startsWith("extra-info-digest ")) {
+ String[] parts = line.split(" ");
+ if (line.startsWith("opt ")) {
+ scrubbed.append("opt ");
+ parts = line.substring(4).split(" ");
+ }
+ scrubbed.append("extra-info-digest " + DigestUtils.shaHex(
+ Hex.decodeHex(parts[1].toCharArray())).toUpperCase());
+ if (parts.length > 2) {
+ scrubbed.append(" " + Base64.encodeBase64String(
+ DigestUtils.sha256(Base64.decodeBase64(parts[2])))
+ .replaceAll("=", ""));
+ }
+ scrubbed.append("\n");
+
+ /* Possibly sanitize reject lines if they contain the bridge's own
+ * IP address. */
+ } else if (line.startsWith("reject ")) {
+ if (address != null && line.startsWith("reject " + address)) {
+ scrubbed.append("reject " + scrubbedAddress
+ + line.substring("reject ".length() + address.length())
+ + "\n");
+ } else {
+ scrubbed.append(line + "\n");
+ }
+
+ /* Extract master-key-ed25519 from identity-ed25519. */
+ } else if (line.equals("identity-ed25519")) {
+ StringBuilder sb = new StringBuilder();
+ while ((line = br.readLine()) != null
+ && !line.equals("-----END ED25519 CERT-----")) {
+ if (line.equals("-----BEGIN ED25519 CERT-----")) {
+ continue;
+ }
+ sb.append(line);
+ }
+ masterKeyEd25519FromIdentityEd25519 =
+ this.parseMasterKeyEd25519FromIdentityEd25519(
+ sb.toString());
+ String sha256MasterKeyEd25519 = Base64.encodeBase64String(
+ DigestUtils.sha256(Base64.decodeBase64(
+ masterKeyEd25519FromIdentityEd25519 + "=")))
+ .replaceAll("=", "");
+ scrubbed.append("master-key-ed25519 " + sha256MasterKeyEd25519
+ + "\n");
+ if (masterKeyEd25519 != null && !masterKeyEd25519.equals(
+ masterKeyEd25519FromIdentityEd25519)) {
+ this.logger.warning("Mismatch between identity-ed25519 and "
+ + "master-key-ed25519. Skipping.");
+ return;
+ }
+
+ /* Verify that identity-ed25519 and master-key-ed25519 match. */
+ } else if (line.startsWith("master-key-ed25519 ")) {
+ masterKeyEd25519 = line.substring(line.indexOf(" ") + 1);
+ if (masterKeyEd25519FromIdentityEd25519 != null
+ && !masterKeyEd25519FromIdentityEd25519.equals(
+ masterKeyEd25519)) {
+ this.logger.warning("Mismatch between identity-ed25519 and "
+ + "master-key-ed25519. Skipping.");
+ return;
+ }
+
+ /* Write the following lines unmodified to the sanitized
+ * descriptor. */
+ } else if (line.startsWith("accept ")
+ || line.startsWith("platform ")
+ || line.startsWith("opt protocols ")
+ || line.startsWith("protocols ")
+ || line.startsWith("uptime ")
+ || line.startsWith("bandwidth ")
+ || line.startsWith("opt hibernating ")
+ || line.startsWith("hibernating ")
+ || line.startsWith("ntor-onion-key ")
+ || line.equals("opt hidden-service-dir")
+ || line.equals("hidden-service-dir")
+ || line.equals("opt caches-extra-info")
+ || line.equals("caches-extra-info")
+ || line.equals("opt allow-single-hop-exits")
+ || line.equals("allow-single-hop-exits")
+ || line.startsWith("ipv6-policy ")
+ || line.equals("tunnelled-dir-server")) {
+ scrubbed.append(line + "\n");
+
+ /* Replace node fingerprints in the family line with their hashes
+ * and leave nicknames unchanged. */
+ } else if (line.startsWith("family ")) {
+ StringBuilder familyLine = new StringBuilder("family");
+ for (String s : line.substring(7).split(" ")) {
+ if (s.startsWith("$")) {
+ familyLine.append(" $" + DigestUtils.shaHex(Hex.decodeHex(
+ s.substring(1).toCharArray())).toUpperCase());
+ } else {
+ familyLine.append(" " + s);
+ }
+ }
+ scrubbed.append(familyLine.toString() + "\n");
+
+ /* Skip the purpose line that the bridge authority adds to its
+ * cached-descriptors file. */
+ } else if (line.startsWith("@purpose ")) {
+ continue;
+
+ /* Skip all crypto parts that might leak the bridge's identity
+ * fingerprint. */
+ } else if (line.startsWith("-----BEGIN ")
+ || line.equals("onion-key") || line.equals("signing-key")
+ || line.equals("onion-key-crosscert")
+ || line.startsWith("ntor-onion-key-crosscert ")) {
+ skipCrypto = true;
+
+ /* Stop skipping lines when the crypto parts are over. */
+ } else if (line.startsWith("-----END ")) {
+ skipCrypto = false;
+
+ /* Skip the ed25519 signature; we'll include a SHA256 digest of
+ * the SHA256 descriptor digest in router-digest-sha256. */
+ } else if (line.startsWith("router-sig-ed25519 ")) {
+ continue;
+
+ /* If we encounter an unrecognized line, stop parsing and print
+ * out a warning. We might have overlooked sensitive information
+ * that we need to remove or replace for the sanitized descriptor
+ * version. */
+ } else {
+ this.logger.warning("Unrecognized line '" + line
+ + "'. Skipping.");
+ return;
+ }
+ }
+ br.close();
+ } catch (Exception e) {
+ this.logger.log(Level.WARNING, "Could not parse server "
+ + "descriptor.", e);
+ return;
+ }
+
+ /* Determine filename of sanitized server descriptor. */
+ String descriptorDigest = null;
+ try {
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "router ";
+ String sigToken = "\nrouter-signature\n";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ if (start >= 0 && sig >= 0 && sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ descriptorDigest = DigestUtils.shaHex(DigestUtils.sha(forDigest));
+ }
+ } catch (UnsupportedEncodingException e) {
+ /* Handle below. */
+ }
+ if (descriptorDigest == null) {
+ this.logger.log(Level.WARNING, "Could not calculate server "
+ + "descriptor digest.");
+ return;
+ }
+ String descriptorDigestSha256Base64 = null;
+ if (masterKeyEd25519FromIdentityEd25519 != null) {
+ try {
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "router ";
+ String sigToken = "\n-----END SIGNATURE-----\n";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ if (start >= 0 && sig >= 0 && sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ descriptorDigestSha256Base64 = Base64.encodeBase64String(
+ DigestUtils.sha256(DigestUtils.sha256(forDigest)))
+ .replaceAll("=", "");
+ }
+ } catch (UnsupportedEncodingException e) {
+ /* Handle below. */
+ }
+ if (descriptorDigestSha256Base64 == null) {
+ this.logger.log(Level.WARNING, "Could not calculate server "
+ + "descriptor SHA256 digest.");
+ return;
+ }
+ }
+ String dyear = published.substring(0, 4);
+ String dmonth = published.substring(5, 7);
+ File tarballFile = new File(
+ this.sanitizedBridgesDirectory.getAbsolutePath() + "/"
+ + dyear + "/" + dmonth + "/server-descriptors/"
+ + "/" + descriptorDigest.charAt(0) + "/"
+ + descriptorDigest.charAt(1) + "/"
+ + descriptorDigest);
+ File rsyncCatFile = new File("recent/bridge-descriptors/"
+ + "server-descriptors/" + this.rsyncCatString
+ + "-server-descriptors.tmp");
+ File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
+ boolean[] append = new boolean[] { false, true };
+ try {
+ for (int i = 0; i < outputFiles.length; i++) {
+ File outputFile = outputFiles[i];
+ boolean appendToFile = append[i];
+ if (outputFile.exists() && !appendToFile) {
+ /* We already stored this descriptor to disk before, so let's
+ * not store it yet another time. */
+ break;
+ }
+ outputFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ outputFile, appendToFile));
+ bw.write("@type bridge-server-descriptor 1.1\n");
+ bw.write(scrubbedDesc);
+ if (descriptorDigestSha256Base64 != null) {
+ bw.write("router-digest-sha256 " + descriptorDigestSha256Base64
+ + "\n");
+ }
+ bw.write("router-digest " + descriptorDigest.toUpperCase()
+ + "\n");
+ bw.close();
+ }
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not write sanitized server "
+ + "descriptor to disk.", e);
+ return;
+ }
+ }
+
+ private String parseMasterKeyEd25519FromIdentityEd25519(
+ String identityEd25519Base64) {
+ byte[] identityEd25519 = Base64.decodeBase64(identityEd25519Base64);
+ if (identityEd25519.length < 40) {
+ this.logger.warning("Invalid length of identity-ed25519 (in "
+ + "bytes): " + identityEd25519.length);
+ } else if (identityEd25519[0] != 0x01) {
+ this.logger.warning("Unknown version in identity-ed25519: "
+ + identityEd25519[0]);
+ } else if (identityEd25519[1] != 0x04) {
+ this.logger.warning("Unknown cert type in identity-ed25519: "
+ + identityEd25519[1]);
+ } else if (identityEd25519[6] != 0x01) {
+ this.logger.warning("Unknown certified key type in "
+ + "identity-ed25519: " + identityEd25519[1]);
+ } else if (identityEd25519[39] == 0x00) {
+ this.logger.warning("No extensions in identity-ed25519 (which "
+ + "would contain the encoded master-key-ed25519): "
+ + identityEd25519[39]);
+ } else {
+ int extensionStart = 40;
+ for (int i = 0; i < (int) identityEd25519[39]; i++) {
+ if (identityEd25519.length < extensionStart + 4) {
+ this.logger.warning("Invalid extension with id " + i
+ + " in identity-ed25519.");
+ break;
+ }
+ int extensionLength = identityEd25519[extensionStart];
+ extensionLength <<= 8;
+ extensionLength += identityEd25519[extensionStart + 1];
+ int extensionType = identityEd25519[extensionStart + 2];
+ if (extensionLength == 32 && extensionType == 4) {
+ if (identityEd25519.length < extensionStart + 4 + 32) {
+ this.logger.warning("Invalid extension with id " + i
+ + " in identity-ed25519.");
+ break;
+ }
+ byte[] masterKeyEd25519 = new byte[32];
+ System.arraycopy(identityEd25519, extensionStart + 4,
+ masterKeyEd25519, 0, masterKeyEd25519.length);
+ String masterKeyEd25519Base64 = Base64.encodeBase64String(
+ masterKeyEd25519);
+ String masterKeyEd25519Base64NoTrailingEqualSigns =
+ masterKeyEd25519Base64.replaceAll("=", "");
+ return masterKeyEd25519Base64NoTrailingEqualSigns;
+ }
+ extensionStart += 4 + extensionLength;
+ }
+ }
+ this.logger.warning("Unable to locate master-key-ed25519 in "
+ + "identity-ed25519.");
+ return null;
+ }
+
+ private String maxExtraInfoDescriptorPublishedTime =
+ "1970-01-01 00:00:00";
+
+ /**
+ * Sanitizes an extra-info descriptor and writes it to disk.
+ */
+ public void sanitizeAndStoreExtraInfoDescriptor(byte[] data) {
+
+ /* Parse descriptor to generate a sanitized version. */
+ String scrubbedDesc = null;
+ String published = null;
+ String masterKeyEd25519FromIdentityEd25519 = null;
+ try {
+ BufferedReader br = new BufferedReader(new StringReader(new String(
+ data, "US-ASCII")));
+ String line = null;
+ StringBuilder scrubbed = null;
+ String hashedBridgeIdentity = null;
+ String masterKeyEd25519 = null;
+ while ((line = br.readLine()) != null) {
+
+ /* Parse bridge identity from extra-info line and replace it with
+ * its hash in the sanitized descriptor. */
+ String[] parts = line.split(" ");
+ if (line.startsWith("extra-info ")) {
+ hashedBridgeIdentity = DigestUtils.shaHex(Hex.decodeHex(
+ parts[2].toCharArray())).toLowerCase();
+ scrubbed = new StringBuilder("extra-info " + parts[1] + " "
+ + hashedBridgeIdentity.toUpperCase() + "\n");
+
+ /* Parse the publication time to determine the file name. */
+ } else if (line.startsWith("published ")) {
+ scrubbed.append(line + "\n");
+ published = line.substring("published ".length());
+ if (published.compareTo(maxExtraInfoDescriptorPublishedTime)
+ > 0) {
+ maxExtraInfoDescriptorPublishedTime = published;
+ }
+
+ /* Remove everything from transport lines except the transport
+ * name. */
+ } else if (line.startsWith("transport ")) {
+ if (parts.length < 3) {
+ this.logger.fine("Illegal line in extra-info descriptor: '"
+ + line + "'. Skipping descriptor.");
+ return;
+ }
+ scrubbed.append("transport " + parts[1] + "\n");
+
+ /* Skip transport-info lines entirely. */
+ } else if (line.startsWith("transport-info ")) {
+
+ /* Extract master-key-ed25519 from identity-ed25519. */
+ } else if (line.equals("identity-ed25519")) {
+ StringBuilder sb = new StringBuilder();
+ while ((line = br.readLine()) != null
+ && !line.equals("-----END ED25519 CERT-----")) {
+ if (line.equals("-----BEGIN ED25519 CERT-----")) {
+ continue;
+ }
+ sb.append(line);
+ }
+ masterKeyEd25519FromIdentityEd25519 =
+ this.parseMasterKeyEd25519FromIdentityEd25519(
+ sb.toString());
+ String sha256MasterKeyEd25519 = Base64.encodeBase64String(
+ DigestUtils.sha256(Base64.decodeBase64(
+ masterKeyEd25519FromIdentityEd25519 + "=")))
+ .replaceAll("=", "");
+ scrubbed.append("master-key-ed25519 " + sha256MasterKeyEd25519
+ + "\n");
+ if (masterKeyEd25519 != null && !masterKeyEd25519.equals(
+ masterKeyEd25519FromIdentityEd25519)) {
+ this.logger.warning("Mismatch between identity-ed25519 and "
+ + "master-key-ed25519. Skipping.");
+ return;
+ }
+
+ /* Verify that identity-ed25519 and master-key-ed25519 match. */
+ } else if (line.startsWith("master-key-ed25519 ")) {
+ masterKeyEd25519 = line.substring(line.indexOf(" ") + 1);
+ if (masterKeyEd25519FromIdentityEd25519 != null
+ && !masterKeyEd25519FromIdentityEd25519.equals(
+ masterKeyEd25519)) {
+ this.logger.warning("Mismatch between identity-ed25519 and "
+ + "master-key-ed25519. Skipping.");
+ return;
+ }
+
+ /* Write the following lines unmodified to the sanitized
+ * descriptor. */
+ } else if (line.startsWith("write-history ")
+ || line.startsWith("read-history ")
+ || line.startsWith("geoip-start-time ")
+ || line.startsWith("geoip-client-origins ")
+ || line.startsWith("geoip-db-digest ")
+ || line.startsWith("geoip6-db-digest ")
+ || line.startsWith("conn-bi-direct ")
+ || line.startsWith("bridge-")
+ || line.startsWith("dirreq-")
+ || line.startsWith("cell-")
+ || line.startsWith("entry-")
+ || line.startsWith("exit-")) {
+ scrubbed.append(line + "\n");
+
+ /* When we reach the signature, we're done. Write the sanitized
+ * descriptor to disk below. */
+ } else if (line.startsWith("router-signature")) {
+ scrubbedDesc = scrubbed.toString();
+ break;
+
+ /* Skip the ed25519 signature; we'll include a SHA256 digest of
+ * the SHA256 descriptor digest in router-digest-sha256. */
+ } else if (line.startsWith("router-sig-ed25519 ")) {
+ continue;
+
+ /* If we encounter an unrecognized line, stop parsing and print
+ * out a warning. We might have overlooked sensitive information
+ * that we need to remove or replace for the sanitized descriptor
+ * version. */
+ } else {
+ this.logger.warning("Unrecognized line '" + line
+ + "'. Skipping.");
+ return;
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not parse extra-info "
+ + "descriptor.", e);
+ return;
+ } catch (DecoderException e) {
+ this.logger.log(Level.WARNING, "Could not parse extra-info "
+ + "descriptor.", e);
+ return;
+ }
+
+ /* Determine filename of sanitized extra-info descriptor. */
+ String descriptorDigest = null;
+ try {
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "extra-info ";
+ String sigToken = "\nrouter-signature\n";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ if (start >= 0 && sig >= 0 && sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ descriptorDigest = DigestUtils.shaHex(DigestUtils.sha(forDigest));
+ }
+ } catch (UnsupportedEncodingException e) {
+ /* Handle below. */
+ }
+ if (descriptorDigest == null) {
+ this.logger.log(Level.WARNING, "Could not calculate extra-info "
+ + "descriptor digest.");
+ return;
+ }
+ String descriptorDigestSha256Base64 = null;
+ if (masterKeyEd25519FromIdentityEd25519 != null) {
+ try {
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "extra-info ";
+ String sigToken = "\n-----END SIGNATURE-----\n";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ if (start >= 0 && sig >= 0 && sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ descriptorDigestSha256Base64 = Base64.encodeBase64String(
+ DigestUtils.sha256(DigestUtils.sha256(forDigest)))
+ .replaceAll("=", "");
+ }
+ } catch (UnsupportedEncodingException e) {
+ /* Handle below. */
+ }
+ if (descriptorDigestSha256Base64 == null) {
+ this.logger.log(Level.WARNING, "Could not calculate extra-info "
+ + "descriptor SHA256 digest.");
+ return;
+ }
+ }
+ String dyear = published.substring(0, 4);
+ String dmonth = published.substring(5, 7);
+ File tarballFile = new File(
+ this.sanitizedBridgesDirectory.getAbsolutePath() + "/"
+ + dyear + "/" + dmonth + "/extra-infos/"
+ + descriptorDigest.charAt(0) + "/"
+ + descriptorDigest.charAt(1) + "/"
+ + descriptorDigest);
+ File rsyncCatFile = new File("recent/bridge-descriptors/"
+ + "extra-infos/" + this.rsyncCatString + "-extra-infos.tmp");
+ File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
+ boolean[] append = new boolean[] { false, true };
+ try {
+ for (int i = 0; i < outputFiles.length; i++) {
+ File outputFile = outputFiles[i];
+ boolean appendToFile = append[i];
+ if (outputFile.exists() && !appendToFile) {
+ /* We already stored this descriptor to disk before, so let's
+ * not store it yet another time. */
+ break;
+ }
+ outputFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ outputFile, appendToFile));
+ bw.write("@type bridge-extra-info 1.3\n");
+ bw.write(scrubbedDesc);
+ if (descriptorDigestSha256Base64 != null) {
+ bw.write("router-digest-sha256 " + descriptorDigestSha256Base64
+ + "\n");
+ }
+ bw.write("router-digest " + descriptorDigest.toUpperCase()
+ + "\n");
+ bw.close();
+ }
+ } catch (Exception e) {
+ this.logger.log(Level.WARNING, "Could not write sanitized "
+ + "extra-info descriptor to disk.", e);
+ }
+ }
+
+ /**
+ * Rewrite all network statuses that might contain references to server
+ * descriptors we added or updated in this execution. This applies to
+ * all statuses that have been published up to 24 hours after any added
+ * or updated server descriptor.
+ */
+ public void finishWriting() {
+
+ /* Delete secrets that we don't need anymore. */
+ if (!this.secretsForHashingIPAddresses.isEmpty()
+ && this.secretsForHashingIPAddresses.firstKey().compareTo(
+ this.bridgeSanitizingCutOffTimestamp) < 0) {
+ try {
+ int kept = 0;
+ int deleted = 0;
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.bridgeIpSecretsFile));
+ for (Map.Entry<String, byte[]> e :
+ this.secretsForHashingIPAddresses.entrySet()) {
+ if (e.getKey().compareTo(
+ this.bridgeSanitizingCutOffTimestamp) < 0) {
+ deleted++;
+ } else {
+ bw.write(e.getKey() + "," + Hex.encodeHexString(e.getValue())
+ + "\n");
+ kept++;
+ }
+ }
+ bw.close();
+ this.logger.info("Deleted " + deleted + " secrets that we don't "
+ + "need anymore and kept " + kept + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not store reduced set of "
+ + "secrets to disk! This is a bad sign, better check what's "
+ + "going on!", e);
+ }
+ }
+ }
+
+ private void checkStaleDescriptors() {
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ long tooOldMillis = System.currentTimeMillis() - 330L * 60L * 1000L;
+ try {
+ long maxNetworkStatusPublishedMillis =
+ dateTimeFormat.parse(maxNetworkStatusPublishedTime).getTime();
+ if (maxNetworkStatusPublishedMillis > 0L
+ && maxNetworkStatusPublishedMillis < tooOldMillis) {
+ this.logger.warning("The last known bridge network status was "
+ + "published " + maxNetworkStatusPublishedTime + ", which is "
+ + "more than 5:30 hours in the past.");
+ }
+ long maxServerDescriptorPublishedMillis =
+ dateTimeFormat.parse(maxServerDescriptorPublishedTime)
+ .getTime();
+ if (maxServerDescriptorPublishedMillis > 0L
+ && maxServerDescriptorPublishedMillis < tooOldMillis) {
+ this.logger.warning("The last known bridge server descriptor was "
+ + "published " + maxServerDescriptorPublishedTime + ", which "
+ + "is more than 5:30 hours in the past.");
+ }
+ long maxExtraInfoDescriptorPublishedMillis =
+ dateTimeFormat.parse(maxExtraInfoDescriptorPublishedTime)
+ .getTime();
+ if (maxExtraInfoDescriptorPublishedMillis > 0L
+ && maxExtraInfoDescriptorPublishedMillis < tooOldMillis) {
+ this.logger.warning("The last known bridge extra-info descriptor "
+ + "was published " + maxExtraInfoDescriptorPublishedTime
+ + ", which is more than 5:30 hours in the past.");
+ }
+ } catch (ParseException e) {
+ this.logger.log(Level.WARNING, "Unable to parse timestamp for "
+ + "stale check.", e);
+ }
+ }
+
+ /* Delete all files from the rsync directory that have not been modified
+ * in the last three days, and remove the .tmp extension from newly
+ * written files. */
+ public void cleanUpRsyncDirectory() {
+ long cutOffMillis = System.currentTimeMillis()
+ - 3L * 24L * 60L * 60L * 1000L;
+ Stack<File> allFiles = new Stack<File>();
+ allFiles.add(new File("recent/bridge-descriptors"));
+ while (!allFiles.isEmpty()) {
+ File file = allFiles.pop();
+ if (file.isDirectory()) {
+ allFiles.addAll(Arrays.asList(file.listFiles()));
+ } else if (file.lastModified() < cutOffMillis) {
+ file.delete();
+ } else if (file.getName().endsWith(".tmp")) {
+ file.renameTo(new File(file.getParentFile(),
+ file.getName().substring(0,
+ file.getName().lastIndexOf(".tmp"))));
+ }
+ }
+ }
+}
+
diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
new file mode 100644
index 0000000..54fd50f
--- /dev/null
+++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
@@ -0,0 +1,212 @@
+/* Copyright 2010--2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.exitlists;
+
+import org.torproject.collector.main.Configuration;
+import org.torproject.collector.main.LockFile;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorParseException;
+import org.torproject.descriptor.DescriptorParser;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.ExitList;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.SortedSet;
+import java.util.Stack;
+import java.util.TimeZone;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class ExitListDownloader extends Thread {
+
+ public static void main(String[] args) {
+
+ Logger logger = Logger.getLogger(ExitListDownloader.class.getName());
+ logger.info("Starting exit-lists module of CollecTor.");
+
+ // Initialize configuration
+ Configuration config = new Configuration();
+
+ // Use lock file to avoid overlapping runs
+ LockFile lf = new LockFile("exit-lists");
+ if (!lf.acquireLock()) {
+ logger.severe("Warning: CollecTor is already running or has not exited "
+ + "cleanly! Exiting!");
+ System.exit(1);
+ }
+
+ // Download exit list and store it to disk
+ new ExitListDownloader(config).run();
+
+ // Remove lock file
+ lf.releaseLock();
+
+ logger.info("Terminating exit-lists module of CollecTor.");
+ }
+
+ public ExitListDownloader(Configuration config) {
+ }
+
+ public void run() {
+
+ Logger logger = Logger.getLogger(ExitListDownloader.class.getName());
+
+ SimpleDateFormat dateTimeFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+ Date downloadedDate = new Date();
+ String downloadedExitList = null;
+ try {
+ logger.fine("Downloading exit list...");
+ StringBuilder sb = new StringBuilder();
+ sb.append("@type tordnsel 1.0\n");
+ sb.append("Downloaded " + dateTimeFormat.format(downloadedDate)
+ + "\n");
+ String exitAddressesUrl =
+ "http://exitlist.torproject.org/exit-addresses";
+ URL u = new URL(exitAddressesUrl);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.connect();
+ int response = huc.getResponseCode();
+ if (response != 200) {
+ logger.warning("Could not download exit list. Response code "
+ + response);
+ return;
+ }
+ BufferedInputStream in = new BufferedInputStream(
+ huc.getInputStream());
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = in.read(data, 0, 1024)) >= 0) {
+ sb.append(new String(data, 0, len));
+ }
+ in.close();
+ downloadedExitList = sb.toString();
+ logger.fine("Finished downloading exit list.");
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed downloading exit list", e);
+ return;
+ }
+ if (downloadedExitList == null) {
+ logger.warning("Failed downloading exit list");
+ return;
+ }
+
+ SimpleDateFormat tarballFormat =
+ new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
+ tarballFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ File tarballFile = new File("out/exit-lists/" + tarballFormat.format(
+ downloadedDate));
+
+ long maxScanMillis = 0L;
+ try {
+ DescriptorParser descriptorParser =
+ DescriptorSourceFactory.createDescriptorParser();
+ List<Descriptor> parsedDescriptors =
+ descriptorParser.parseDescriptors(downloadedExitList.getBytes(),
+ tarballFile.getName());
+ if (parsedDescriptors.size() != 1
+ || !(parsedDescriptors.get(0) instanceof ExitList)) {
+ logger.warning("Could not parse downloaded exit list");
+ return;
+ }
+ ExitList parsedExitList = (ExitList) parsedDescriptors.get(0);
+ for (ExitList.Entry entry : parsedExitList.getEntries()) {
+ for (long scanMillis : entry.getExitAddresses().values()) {
+ maxScanMillis = Math.max(maxScanMillis, scanMillis);
+ }
+ }
+ } catch (DescriptorParseException e) {
+ logger.log(Level.WARNING, "Could not parse downloaded exit list",
+ e);
+ }
+ if (maxScanMillis > 0L
+ && maxScanMillis + 330L * 60L * 1000L < System.currentTimeMillis()) {
+ logger.warning("The last reported scan in the downloaded exit list "
+ + "took place at " + dateTimeFormat.format(maxScanMillis)
+ + ", which is more than 5:30 hours in the past.");
+ }
+
+ /* Write to disk. */
+ File rsyncFile = new File("recent/exit-lists/"
+ + tarballFile.getName());
+ File[] outputFiles = new File[] { tarballFile, rsyncFile };
+ for (File outputFile : outputFiles) {
+ try {
+ outputFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ outputFile));
+ bw.write(downloadedExitList);
+ bw.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not write downloaded exit list "
+ + "to " + outputFile.getAbsolutePath(), e);
+ }
+ }
+
+ /* Write stats. */
+ StringBuilder dumpStats = new StringBuilder("Finished downloading "
+ + "exit list.\nLast three exit lists are:");
+ Stack<File> filesInInputDir = new Stack<File>();
+ filesInInputDir.add(new File("out/exit-lists"));
+ SortedSet<File> lastThreeExitLists = new TreeSet<File>();
+ while (!filesInInputDir.isEmpty()) {
+ File pop = filesInInputDir.pop();
+ if (pop.isDirectory()) {
+ SortedSet<File> lastThreeElements = new TreeSet<File>();
+ for (File f : pop.listFiles()) {
+ lastThreeElements.add(f);
+ }
+ while (lastThreeElements.size() > 3) {
+ lastThreeElements.remove(lastThreeElements.first());
+ }
+ for (File f : lastThreeElements) {
+ filesInInputDir.add(f);
+ }
+ } else {
+ lastThreeExitLists.add(pop);
+ while (lastThreeExitLists.size() > 3) {
+ lastThreeExitLists.remove(lastThreeExitLists.first());
+ }
+ }
+ }
+ for (File f : lastThreeExitLists) {
+ dumpStats.append("\n" + f.getName());
+ }
+ logger.info(dumpStats.toString());
+
+ this.cleanUpRsyncDirectory();
+ }
+
+ /* Delete all files from the rsync directory that have not been modified
+ * in the last three days. */
+ public void cleanUpRsyncDirectory() {
+ long cutOffMillis = System.currentTimeMillis()
+ - 3L * 24L * 60L * 60L * 1000L;
+ Stack<File> allFiles = new Stack<File>();
+ allFiles.add(new File("recent/exit-lists"));
+ while (!allFiles.isEmpty()) {
+ File file = allFiles.pop();
+ if (file.isDirectory()) {
+ allFiles.addAll(Arrays.asList(file.listFiles()));
+ } else if (file.lastModified() < cutOffMillis) {
+ file.delete();
+ }
+ }
+ }
+}
+
diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/CreateIndexJson.java
new file mode 100644
index 0000000..ac5adf5
--- /dev/null
+++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java
@@ -0,0 +1,168 @@
+/* Copyright 2015--2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.index;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Locale;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeSet;
+import java.util.zip.GZIPOutputStream;
+
+/* Create a fresh index.json containing all directories and files in the
+ * archive/ and recent/ directories.
+ *
+ * Note that if this ever takes longer than a few seconds, we'll have to
+ * cache index parts of directories or files that haven't changed.
+ * Example: if we parse include cryptographic hashes or @type information,
+ * we'll likely have to do that. */
+public class CreateIndexJson {
+
+ static final File indexJsonFile = new File("index.json");
+
+ static final String basePath = "https://collector.torproject.org";
+
+ static final File[] indexedDirectories = new File[] {
+ new File("archive"), new File("recent") };
+
+ static final String dateTimePattern = "yyyy-MM-dd HH:mm";
+
+ static final Locale dateTimeLocale = Locale.US;
+
+ static final TimeZone dateTimezone = TimeZone.getTimeZone("UTC");
+
+ public static void main(String[] args) throws IOException {
+ writeIndex(indexDirectories());
+ }
+
+ static class DirectoryNode implements Comparable<DirectoryNode> {
+ String path;
+ SortedSet<FileNode> files;
+ SortedSet<DirectoryNode> directories;
+
+ DirectoryNode(String path, SortedSet<FileNode> files,
+ SortedSet<DirectoryNode> directories) {
+ this.path = path;
+ this.files = files;
+ this.directories = directories;
+ }
+
+ public int compareTo(DirectoryNode o) {
+ return this.path.compareTo(o.path);
+ }
+ }
+
+ static class IndexNode {
+ String index_created;
+ String path;
+ SortedSet<FileNode> files;
+ SortedSet<DirectoryNode> directories;
+
+ IndexNode(String index_created, String path,
+ SortedSet<FileNode> files,
+ SortedSet<DirectoryNode> directories) {
+ this.index_created = index_created;
+ this.path = path;
+ this.files = files;
+ this.directories = directories;
+ }
+ }
+
+ static class FileNode implements Comparable<FileNode> {
+ String path;
+ long size;
+ String last_modified;
+
+ FileNode(String path, long size, String last_modified) {
+ this.path = path;
+ this.size = size;
+ this.last_modified = last_modified;
+ }
+
+ public int compareTo(FileNode o) {
+ return this.path.compareTo(o.path);
+ }
+ }
+
+ static DateFormat dateTimeFormat;
+
+ static {
+ dateTimeFormat = new SimpleDateFormat(dateTimePattern,
+ dateTimeLocale);
+ dateTimeFormat.setLenient(false);
+ dateTimeFormat.setTimeZone(dateTimezone);
+ }
+
+ static IndexNode indexDirectories() {
+ SortedSet<DirectoryNode> directoryNodes =
+ new TreeSet<DirectoryNode>();
+ for (File directory : indexedDirectories) {
+ if (directory.exists() && directory.isDirectory()) {
+ directoryNodes.add(indexDirectory(directory));
+ }
+ }
+ return new IndexNode(dateTimeFormat.format(
+ System.currentTimeMillis()), basePath, null, directoryNodes);
+ }
+
+ static DirectoryNode indexDirectory(File directory) {
+ SortedSet<FileNode> fileNodes = new TreeSet<FileNode>();
+ SortedSet<DirectoryNode> directoryNodes =
+ new TreeSet<DirectoryNode>();
+ for (File fileOrDirectory : directory.listFiles()) {
+ if (fileOrDirectory.getName().startsWith(".")) {
+ continue;
+ }
+ if (fileOrDirectory.isFile()) {
+ fileNodes.add(indexFile(fileOrDirectory));
+ } else {
+ directoryNodes.add(indexDirectory(fileOrDirectory));
+ }
+ }
+ DirectoryNode directoryNode = new DirectoryNode(
+ directory.getName(), fileNodes.isEmpty() ? null : fileNodes,
+ directoryNodes.isEmpty() ? null : directoryNodes);
+ return directoryNode;
+ }
+
+ static FileNode indexFile(File file) {
+ FileNode fileNode = new FileNode(file.getName(), file.length(),
+ dateTimeFormat.format(file.lastModified()));
+ return fileNode;
+ }
+
+ static void writeIndex(IndexNode indexNode) throws IOException {
+ Gson gson = new GsonBuilder().create();
+ String indexNodeString = gson.toJson(indexNode);
+ Writer[] writers = new Writer[] {
+ new FileWriter(indexJsonFile),
+ new OutputStreamWriter(new GZIPOutputStream(
+ new FileOutputStream(indexJsonFile + ".gz"))),
+ new OutputStreamWriter(new XZCompressorOutputStream(
+ new FileOutputStream(indexJsonFile + ".xz"))),
+ new OutputStreamWriter(new BZip2CompressorOutputStream(
+ new FileOutputStream(indexJsonFile + ".bz2")))
+ };
+ for (Writer writer : writers) {
+ BufferedWriter bufferedWriter = new BufferedWriter(writer);
+ bufferedWriter.write(indexNodeString);
+ bufferedWriter.close();
+ }
+ }
+}
+
diff --git a/src/main/java/org/torproject/collector/main/Configuration.java b/src/main/java/org/torproject/collector/main/Configuration.java
new file mode 100644
index 0000000..aee1d02
--- /dev/null
+++ b/src/main/java/org/torproject/collector/main/Configuration.java
@@ -0,0 +1,318 @@
+/* Copyright 2010--2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.main;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Initialize configuration with hard-coded defaults, overwrite with
+ * configuration in config file, if exists, and answer Main.java about our
+ * configuration.
+ */
+public class Configuration {
+ private String directoryArchivesOutputDirectory =
+ "out/relay-descriptors/";
+ private boolean importCachedRelayDescriptors = false;
+ private List<String> cachedRelayDescriptorsDirectory =
+ new ArrayList<String>(Arrays.asList(
+ "in/relay-descriptors/cacheddesc/".split(",")));
+ private boolean importDirectoryArchives = false;
+ private String directoryArchivesDirectory =
+ "in/relay-descriptors/archives/";
+ private boolean keepDirectoryArchiveImportHistory = false;
+ private boolean replaceIPAddressesWithHashes = false;
+ private long limitBridgeDescriptorMappings = -1L;
+ private String sanitizedBridgesWriteDirectory =
+ "out/bridge-descriptors/";
+ private String bridgeSnapshotsDirectory = "in/bridge-descriptors/";
+ private boolean downloadRelayDescriptors = false;
+ private List<String> downloadFromDirectoryAuthorities = Arrays.asList((
+ "86.59.21.38,76.73.17.194:9030,171.25.193.9:443,"
+ + "193.23.244.244,208.83.223.34:443,128.31.0.34:9131,"
+ + "194.109.206.212,212.112.245.170,154.35.32.5").split(","));
+ private List<String> downloadVotesByFingerprint = Arrays.asList((
+ "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,"
+ + "27B6B5996C426270A5C95488AA5BCEB6BCC86956,"
+ + "49015F787433103580E3B66A1707A00E60F2D15B,"
+ + "585769C78764D58426B8B52B6651A5A71137189A,"
+ + "80550987E1D626E3EBA5E5E75A458DE0626D088C,"
+ + "D586D18309DED4CD6D57C18FDB97EFA96D330566,"
+ + "E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,"
+ + "ED03BB616EB2F60BEC80151114BB25CEF515B226,"
+ + "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97").split(","));
+ private boolean downloadCurrentConsensus = true;
+ private boolean downloadCurrentMicrodescConsensus = true;
+ private boolean downloadCurrentVotes = true;
+ private boolean downloadMissingServerDescriptors = true;
+ private boolean downloadMissingExtraInfoDescriptors = true;
+ private boolean downloadMissingMicrodescriptors = true;
+ private boolean downloadAllServerDescriptors = false;
+ private boolean downloadAllExtraInfoDescriptors = false;
+ private boolean compressRelayDescriptorDownloads;
+ private String torperfOutputDirectory = "out/torperf/";
+ private SortedMap<String, String> torperfSources = null;
+ private List<String> torperfFiles = null;
+
+ public Configuration() {
+
+ /* Initialize logger. */
+ Logger logger = Logger.getLogger(Configuration.class.getName());
+
+ /* Read config file, if present. */
+ File configFile = new File("config");
+ if (!configFile.exists()) {
+ logger.warning("Could not find config file. In the default "
+ + "configuration, we are not configured to read data from any "
+ + "data source or write data to any data sink. You need to "
+ + "create a config file (" + configFile.getAbsolutePath()
+ + ") and provide at least one data source and one data sink. "
+ + "Refer to the manual for more information.");
+ return;
+ }
+ String line = null;
+ boolean containsCachedRelayDescriptorsDirectory = false;
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(configFile));
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("#") || line.length() < 1) {
+ continue;
+ } else if (line.startsWith("DirectoryArchivesOutputDirectory")) {
+ this.directoryArchivesOutputDirectory = line.split(" ")[1];
+ } else if (line.startsWith("ImportCachedRelayDescriptors")) {
+ this.importCachedRelayDescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("CachedRelayDescriptorsDirectory")) {
+ if (!containsCachedRelayDescriptorsDirectory) {
+ this.cachedRelayDescriptorsDirectory.clear();
+ containsCachedRelayDescriptorsDirectory = true;
+ }
+ this.cachedRelayDescriptorsDirectory.add(line.split(" ")[1]);
+ } else if (line.startsWith("ImportDirectoryArchives")) {
+ this.importDirectoryArchives = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DirectoryArchivesDirectory")) {
+ this.directoryArchivesDirectory = line.split(" ")[1];
+ } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
+ this.keepDirectoryArchiveImportHistory = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("ReplaceIPAddressesWithHashes")) {
+ this.replaceIPAddressesWithHashes = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("LimitBridgeDescriptorMappings")) {
+ this.limitBridgeDescriptorMappings = Long.parseLong(
+ line.split(" ")[1]);
+ } else if (line.startsWith("SanitizedBridgesWriteDirectory")) {
+ this.sanitizedBridgesWriteDirectory = line.split(" ")[1];
+ } else if (line.startsWith("BridgeSnapshotsDirectory")) {
+ this.bridgeSnapshotsDirectory = line.split(" ")[1];
+ } else if (line.startsWith("DownloadRelayDescriptors")) {
+ this.downloadRelayDescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadFromDirectoryAuthorities")) {
+ this.downloadFromDirectoryAuthorities = new ArrayList<String>();
+ for (String dir : line.split(" ")[1].split(",")) {
+ // test if IP:port pair has correct format
+ if (dir.length() < 1) {
+ logger.severe("Configuration file contains directory "
+ + "authority IP:port of length 0 in line '" + line
+ + "'! Exiting!");
+ System.exit(1);
+ }
+ new URL("http://" + dir + "/");
+ this.downloadFromDirectoryAuthorities.add(dir);
+ }
+ } else if (line.startsWith("DownloadVotesByFingerprint")) {
+ this.downloadVotesByFingerprint = new ArrayList<String>();
+ for (String fingerprint : line.split(" ")[1].split(",")) {
+ this.downloadVotesByFingerprint.add(fingerprint);
+ }
+ } else if (line.startsWith("DownloadCurrentConsensus")) {
+ this.downloadCurrentConsensus = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadCurrentMicrodescConsensus")) {
+ this.downloadCurrentMicrodescConsensus = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadCurrentVotes")) {
+ this.downloadCurrentVotes = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadMissingServerDescriptors")) {
+ this.downloadMissingServerDescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith(
+ "DownloadMissingExtraInfoDescriptors")) {
+ this.downloadMissingExtraInfoDescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadMissingMicrodescriptors")) {
+ this.downloadMissingMicrodescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadAllServerDescriptors")) {
+ this.downloadAllServerDescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("DownloadAllExtraInfoDescriptors")) {
+ this.downloadAllExtraInfoDescriptors = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("CompressRelayDescriptorDownloads")) {
+ this.compressRelayDescriptorDownloads = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("TorperfOutputDirectory")) {
+ this.torperfOutputDirectory = line.split(" ")[1];
+ } else if (line.startsWith("TorperfSource")) {
+ if (this.torperfSources == null) {
+ this.torperfSources = new TreeMap<String, String>();
+ }
+ String[] parts = line.split(" ");
+ String sourceName = parts[1];
+ String baseUrl = parts[2];
+ this.torperfSources.put(sourceName, baseUrl);
+ } else if (line.startsWith("TorperfFiles")) {
+ if (this.torperfFiles == null) {
+ this.torperfFiles = new ArrayList<String>();
+ }
+ String[] parts = line.split(" ");
+ if (parts.length != 5) {
+ logger.severe("Configuration file contains TorperfFiles "
+ + "option with wrong number of values in line '" + line
+ + "'! Exiting!");
+ System.exit(1);
+ }
+ this.torperfFiles.add(line);
+ } else {
+ logger.severe("Configuration file contains unrecognized "
+ + "configuration key in line '" + line + "'! Exiting!");
+ System.exit(1);
+ }
+ }
+ br.close();
+ } catch (ArrayIndexOutOfBoundsException e) {
+ logger.severe("Configuration file contains configuration key "
+ + "without value in line '" + line + "'. Exiting!");
+ System.exit(1);
+ } catch (MalformedURLException e) {
+ logger.severe("Configuration file contains illegal URL or IP:port "
+ + "pair in line '" + line + "'. Exiting!");
+ System.exit(1);
+ } catch (NumberFormatException e) {
+ logger.severe("Configuration file contains illegal value in line '"
+ + line + "' with legal values being 0 or 1. Exiting!");
+ System.exit(1);
+ } catch (IOException e) {
+ logger.log(Level.SEVERE, "Unknown problem while reading config "
+ + "file! Exiting!", e);
+ System.exit(1);
+ }
+ }
+
+ public String getDirectoryArchivesOutputDirectory() {
+ return this.directoryArchivesOutputDirectory;
+ }
+
+ public boolean getImportCachedRelayDescriptors() {
+ return this.importCachedRelayDescriptors;
+ }
+
+ public List<String> getCachedRelayDescriptorDirectory() {
+ return this.cachedRelayDescriptorsDirectory;
+ }
+
+ public boolean getImportDirectoryArchives() {
+ return this.importDirectoryArchives;
+ }
+
+ public String getDirectoryArchivesDirectory() {
+ return this.directoryArchivesDirectory;
+ }
+
+ public boolean getKeepDirectoryArchiveImportHistory() {
+ return this.keepDirectoryArchiveImportHistory;
+ }
+
+ public boolean getReplaceIPAddressesWithHashes() {
+ return this.replaceIPAddressesWithHashes;
+ }
+
+ public long getLimitBridgeDescriptorMappings() {
+ return this.limitBridgeDescriptorMappings;
+ }
+
+ public String getSanitizedBridgesWriteDirectory() {
+ return this.sanitizedBridgesWriteDirectory;
+ }
+
+ public String getBridgeSnapshotsDirectory() {
+ return this.bridgeSnapshotsDirectory;
+ }
+
+ public boolean getDownloadRelayDescriptors() {
+ return this.downloadRelayDescriptors;
+ }
+
+ public List<String> getDownloadFromDirectoryAuthorities() {
+ return this.downloadFromDirectoryAuthorities;
+ }
+
+ public List<String> getDownloadVotesByFingerprint() {
+ return this.downloadVotesByFingerprint;
+ }
+
+ public boolean getDownloadCurrentConsensus() {
+ return this.downloadCurrentConsensus;
+ }
+
+ public boolean getDownloadCurrentMicrodescConsensus() {
+ return this.downloadCurrentMicrodescConsensus;
+ }
+
+ public boolean getDownloadCurrentVotes() {
+ return this.downloadCurrentVotes;
+ }
+
+ public boolean getDownloadMissingServerDescriptors() {
+ return this.downloadMissingServerDescriptors;
+ }
+
+ public boolean getDownloadMissingExtraInfoDescriptors() {
+ return this.downloadMissingExtraInfoDescriptors;
+ }
+
+ public boolean getDownloadMissingMicrodescriptors() {
+ return this.downloadMissingMicrodescriptors;
+ }
+
+ public boolean getDownloadAllServerDescriptors() {
+ return this.downloadAllServerDescriptors;
+ }
+
+ public boolean getDownloadAllExtraInfoDescriptors() {
+ return this.downloadAllExtraInfoDescriptors;
+ }
+
+ public boolean getCompressRelayDescriptorDownloads() {
+ return this.compressRelayDescriptorDownloads;
+ }
+
+ public String getTorperfOutputDirectory() {
+ return this.torperfOutputDirectory;
+ }
+
+ public SortedMap<String, String> getTorperfSources() {
+ return this.torperfSources;
+ }
+
+ public List<String> getTorperfFiles() {
+ return this.torperfFiles;
+ }
+}
+
diff --git a/src/main/java/org/torproject/collector/main/LockFile.java b/src/main/java/org/torproject/collector/main/LockFile.java
new file mode 100644
index 0000000..b07d4b1
--- /dev/null
+++ b/src/main/java/org/torproject/collector/main/LockFile.java
@@ -0,0 +1,56 @@
+/* Copyright 2010--2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.main;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.logging.Logger;
+
+public class LockFile {
+
+ private File lockFile;
+ private Logger logger;
+
+ public LockFile(String moduleName) {
+ this.lockFile = new File("lock/" + moduleName);
+ this.logger = Logger.getLogger(LockFile.class.getName());
+ }
+
+ public boolean acquireLock() {
+ this.logger.fine("Trying to acquire lock...");
+ try {
+ if (this.lockFile.exists()) {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.lockFile));
+ long runStarted = Long.parseLong(br.readLine());
+ br.close();
+ if (System.currentTimeMillis() - runStarted < 55L * 60L * 1000L) {
+ return false;
+ }
+ }
+ this.lockFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.lockFile));
+ bw.append("" + System.currentTimeMillis() + "\n");
+ bw.close();
+ this.logger.fine("Acquired lock.");
+ return true;
+ } catch (IOException e) {
+ this.logger.warning("Caught exception while trying to acquire "
+ + "lock!");
+ return false;
+ }
+ }
+
+ public void releaseLock() {
+ this.logger.fine("Releasing lock...");
+ this.lockFile.delete();
+ this.logger.fine("Released lock.");
+ }
+}
+
diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveReader.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveReader.java
new file mode 100644
index 0000000..72f8231
--- /dev/null
+++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveReader.java
@@ -0,0 +1,286 @@
+/* Copyright 2010--2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.relaydescs;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.UnsupportedEncodingException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.Stack;
+import java.util.TimeZone;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Read in all files in a given directory and pass buffered readers of
+ * them to the relay descriptor parser.
+ */
+public class ArchiveReader {
+
+ private Map<String, Set<String>> microdescriptorValidAfterTimes =
+ new HashMap<String, Set<String>>();
+
+ public ArchiveReader(RelayDescriptorParser rdp, File archivesDirectory,
+ File statsDirectory, boolean keepImportHistory) {
+
+ if (rdp == null || archivesDirectory == null
+ || statsDirectory == null) {
+ throw new IllegalArgumentException();
+ }
+
+ rdp.setArchiveReader(this);
+ int parsedFiles = 0;
+ int ignoredFiles = 0;
+ Logger logger = Logger.getLogger(ArchiveReader.class.getName());
+ SortedSet<String> archivesImportHistory = new TreeSet<String>();
+ File archivesImportHistoryFile = new File(statsDirectory,
+ "archives-import-history");
+ if (keepImportHistory && archivesImportHistoryFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ archivesImportHistoryFile));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ archivesImportHistory.add(line);
+ }
+ br.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not read in archives import "
+ + "history file. Skipping.");
+ }
+ }
+ if (archivesDirectory.exists()) {
+ logger.fine("Importing files in directory " + archivesDirectory
+ + "/...");
+ Stack<File> filesInInputDir = new Stack<File>();
+ filesInInputDir.add(archivesDirectory);
+ List<File> problems = new ArrayList<File>();
+ Set<File> filesToRetry = new HashSet<File>();
+ while (!filesInInputDir.isEmpty()) {
+ File pop = filesInInputDir.pop();
+ if (pop.isDirectory()) {
+ for (File f : pop.listFiles()) {
+ filesInInputDir.add(f);
+ }
+ } else {
+ if (rdp != null) {
+ try {
+ BufferedInputStream bis = null;
+ if (keepImportHistory
+ && archivesImportHistory.contains(pop.getName())) {
+ ignoredFiles++;
+ continue;
+ } else if (pop.getName().endsWith(".tar.bz2")) {
+ logger.warning("Cannot parse compressed tarball "
+ + pop.getAbsolutePath() + ". Skipping.");
+ continue;
+ } else if (pop.getName().endsWith(".bz2")) {
+ FileInputStream fis = new FileInputStream(pop);
+ BZip2CompressorInputStream bcis =
+ new BZip2CompressorInputStream(fis);
+ bis = new BufferedInputStream(bcis);
+ } else {
+ FileInputStream fis = new FileInputStream(pop);
+ bis = new BufferedInputStream(fis);
+ }
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ bis.close();
+ byte[] allData = baos.toByteArray();
+ boolean stored = rdp.parse(allData);
+ if (!stored) {
+ filesToRetry.add(pop);
+ continue;
+ }
+ if (keepImportHistory) {
+ archivesImportHistory.add(pop.getName());
+ }
+ parsedFiles++;
+ } catch (IOException e) {
+ problems.add(pop);
+ if (problems.size() > 3) {
+ break;
+ }
+ }
+ }
+ }
+ }
+ for (File pop : filesToRetry) {
+ /* TODO We need to parse microdescriptors ourselves, rather than
+ * RelayDescriptorParser, because only we know the valid-after
+ * time(s) of microdesc consensus(es) containing this
+ * microdescriptor. However, this breaks functional abstraction
+ * pretty badly. */
+ if (rdp != null) {
+ try {
+ BufferedInputStream bis = null;
+ if (pop.getName().endsWith(".bz2")) {
+ FileInputStream fis = new FileInputStream(pop);
+ BZip2CompressorInputStream bcis =
+ new BZip2CompressorInputStream(fis);
+ bis = new BufferedInputStream(bcis);
+ } else {
+ FileInputStream fis = new FileInputStream(pop);
+ bis = new BufferedInputStream(fis);
+ }
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ bis.close();
+ byte[] allData = baos.toByteArray();
+ BufferedReader br = new BufferedReader(new StringReader(
+ new String(allData, "US-ASCII")));
+ String line;
+ do {
+ line = br.readLine();
+ } while (line != null && line.startsWith("@"));
+ br.close();
+ if (line == null) {
+ logger.fine("We were given an empty descriptor for "
+ + "parsing. Ignoring.");
+ continue;
+ }
+ if (!line.equals("onion-key")) {
+ logger.fine("Skipping non-recognized descriptor.");
+ continue;
+ }
+ SimpleDateFormat parseFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String ascii = null;
+ try {
+ ascii = new String(allData, "US-ASCII");
+ } catch (UnsupportedEncodingException e) {
+ /* No way that US-ASCII is not supported. */
+ }
+ int start = -1;
+ int end = -1;
+ String startToken = "onion-key\n";
+ while (end < ascii.length()) {
+ start = ascii.indexOf(startToken, end);
+ if (start < 0) {
+ break;
+ }
+ end = ascii.indexOf(startToken, start + 1);
+ if (end < 0) {
+ end = ascii.length();
+ if (end <= start) {
+ break;
+ }
+ }
+ byte[] descBytes = new byte[end - start];
+ System.arraycopy(allData, start, descBytes, 0, end - start);
+ String digest256Base64 = Base64.encodeBase64String(
+ DigestUtils.sha256(descBytes)).replaceAll("=", "");
+ String digest256Hex = DigestUtils.sha256Hex(descBytes);
+ if (!this.microdescriptorValidAfterTimes.containsKey(
+ digest256Hex)) {
+ logger.fine("Could not store microdescriptor '"
+ + digest256Hex + "', which was not contained in a "
+ + "microdesc consensus.");
+ continue;
+ }
+ for (String validAfterTime :
+ this.microdescriptorValidAfterTimes.get(digest256Hex)) {
+ try {
+ long validAfter =
+ parseFormat.parse(validAfterTime).getTime();
+ rdp.storeMicrodescriptor(descBytes, digest256Hex,
+ digest256Base64, validAfter);
+ } catch (ParseException e) {
+ logger.log(Level.WARNING, "Could not parse "
+ + "valid-after time '" + validAfterTime + "'. Not "
+ + "storing microdescriptor.", e);
+ }
+ }
+ }
+ if (keepImportHistory) {
+ archivesImportHistory.add(pop.getName());
+ }
+ parsedFiles++;
+ } catch (IOException e) {
+ problems.add(pop);
+ if (problems.size() > 3) {
+ break;
+ }
+ }
+ }
+ }
+ if (problems.isEmpty()) {
+ logger.fine("Finished importing files in directory "
+ + archivesDirectory + "/.");
+ } else {
+ StringBuilder sb = new StringBuilder("Failed importing files in "
+ + "directory " + archivesDirectory + "/:");
+ int printed = 0;
+ for (File f : problems) {
+ sb.append("\n " + f.getAbsolutePath());
+ if (++printed >= 3) {
+ sb.append("\n ... more");
+ break;
+ }
+ }
+ }
+ }
+ if (keepImportHistory) {
+ try {
+ archivesImportHistoryFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ archivesImportHistoryFile));
+ for (String line : archivesImportHistory) {
+ bw.write(line + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not write archives import "
+ + "history file.");
+ }
+ }
+ logger.info("Finished importing relay descriptors from local "
+ + "directory:\nParsed " + parsedFiles + ", ignored "
+ + ignoredFiles + " files.");
+ }
+
+ public void haveParsedMicrodescConsensus(String validAfterTime,
+ SortedSet<String> microdescriptorDigests) {
+ for (String microdescriptor : microdescriptorDigests) {
+ if (!this.microdescriptorValidAfterTimes.containsKey(
+ microdescriptor)) {
+ this.microdescriptorValidAfterTimes.put(microdescriptor,
+ new HashSet<String>());
+ }
+ this.microdescriptorValidAfterTimes.get(microdescriptor).add(
+ validAfterTime);
+ }
+ }
+}
+
diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
new file mode 100644
index 0000000..cf603d1
--- /dev/null
+++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
@@ -0,0 +1,845 @@
+/* Copyright 2010--2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.relaydescs;
+
+import org.torproject.collector.main.Configuration;
+import org.torproject.collector.main.LockFile;
+import org.torproject.descriptor.DescriptorParseException;
+import org.torproject.descriptor.DescriptorParser;
+import org.torproject.descriptor.DescriptorSourceFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.Stack;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class ArchiveWriter extends Thread {
+
+ private Configuration config;
+
+ private long now = System.currentTimeMillis();
+ private Logger logger;
+ private File outputDirectory;
+ private String rsyncCatString;
+ private DescriptorParser descriptorParser;
+ private int storedConsensusesCounter = 0;
+ private int storedMicrodescConsensusesCounter = 0;
+ private int storedVotesCounter = 0;
+ private int storedCertsCounter = 0;
+ private int storedServerDescriptorsCounter = 0;
+ private int storedExtraInfoDescriptorsCounter = 0;
+ private int storedMicrodescriptorsCounter = 0;
+
+ private SortedMap<Long, SortedSet<String>> storedConsensuses =
+ new TreeMap<Long, SortedSet<String>>();
+ private SortedMap<Long, SortedSet<String>> storedMicrodescConsensuses =
+ new TreeMap<Long, SortedSet<String>>();
+ private SortedMap<Long, Integer> expectedVotes =
+ new TreeMap<Long, Integer>();
+ private SortedMap<Long, SortedMap<String, SortedSet<String>>>
+ storedVotes =
+ new TreeMap<Long, SortedMap<String, SortedSet<String>>>();
+ private SortedMap<Long, Map<String, String>> storedServerDescriptors =
+ new TreeMap<Long, Map<String, String>>();
+ private SortedMap<Long, Set<String>> storedExtraInfoDescriptors =
+ new TreeMap<Long, Set<String>>();
+ private SortedMap<Long, Set<String>> storedMicrodescriptors =
+ new TreeMap<Long, Set<String>>();
+
+ private File storedServerDescriptorsFile = new File(
+ "stats/stored-server-descriptors");
+ private File storedExtraInfoDescriptorsFile = new File(
+ "stats/stored-extra-info-descriptors");
+ private File storedMicrodescriptorsFile = new File(
+ "stats/stored-microdescriptors");
+
+ private static final byte[] CONSENSUS_ANNOTATION =
+ "@type network-status-consensus-3 1.0\n".getBytes();
+
+ private static final byte[] MICRODESCCONSENSUS_ANNOTATION =
+ "@type network-status-microdesc-consensus-3 1.0\n".getBytes();
+
+ private static final byte[] VOTE_ANNOTATION =
+ "@type network-status-vote-3 1.0\n".getBytes();
+
+ private static final byte[] CERTIFICATE_ANNOTATION =
+ "@type dir-key-certificate-3 1.0\n".getBytes();
+
+ private static final byte[] SERVER_DESCRIPTOR_ANNOTATION =
+ "@type server-descriptor 1.0\n".getBytes();
+
+ private static final byte[] EXTRA_INFO_ANNOTATION =
+ "@type extra-info 1.0\n".getBytes();
+
+ private static final byte[] MICRODESCRIPTOR_ANNOTATION =
+ "@type microdescriptor 1.0\n".getBytes();
+
+ private StringBuilder intermediateStats = new StringBuilder();
+
+ public static void main(String[] args) {
+
+ Logger logger = Logger.getLogger(ArchiveWriter.class.getName());
+ logger.info("Starting relay-descriptors module of CollecTor.");
+
+ // Initialize configuration
+ Configuration config = new Configuration();
+
+ // Use lock file to avoid overlapping runs
+ LockFile lf = new LockFile("relay-descriptors");
+ if (!lf.acquireLock()) {
+ logger.severe("Warning: CollecTor is already running or has not exited "
+ + "cleanly! Exiting!");
+ System.exit(1);
+ }
+
+ // Import/download relay descriptors from the various sources
+ new ArchiveWriter(config).run();
+
+ new ReferenceChecker(new File("recent/relay-descriptors"),
+ new File("stats/references"),
+ new File("stats/references-history")).check();
+
+ // Remove lock file
+ lf.releaseLock();
+
+ logger.info("Terminating relay-descriptors module of CollecTor.");
+ }
+
+ public ArchiveWriter(Configuration config) {
+ this.config = config;
+ }
+
+ public void run() {
+
+ File outputDirectory =
+ new File(config.getDirectoryArchivesOutputDirectory());
+ File statsDirectory = new File("stats");
+
+ this.logger = Logger.getLogger(ArchiveWriter.class.getName());
+ this.outputDirectory = outputDirectory;
+ SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
+ "yyyy-MM-dd-HH-mm-ss");
+ rsyncCatFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ this.rsyncCatString = rsyncCatFormat.format(
+ System.currentTimeMillis());
+ this.descriptorParser =
+ DescriptorSourceFactory.createDescriptorParser();
+
+ this.loadDescriptorDigests();
+
+ // Prepare relay descriptor parser
+ RelayDescriptorParser rdp = new RelayDescriptorParser(this);
+
+ RelayDescriptorDownloader rdd = null;
+ if (config.getDownloadRelayDescriptors()) {
+ List<String> dirSources =
+ config.getDownloadFromDirectoryAuthorities();
+ rdd = new RelayDescriptorDownloader(rdp, dirSources,
+ config.getDownloadVotesByFingerprint(),
+ config.getDownloadCurrentConsensus(),
+ config.getDownloadCurrentMicrodescConsensus(),
+ config.getDownloadCurrentVotes(),
+ config.getDownloadMissingServerDescriptors(),
+ config.getDownloadMissingExtraInfoDescriptors(),
+ config.getDownloadMissingMicrodescriptors(),
+ config.getDownloadAllServerDescriptors(),
+ config.getDownloadAllExtraInfoDescriptors(),
+ config.getCompressRelayDescriptorDownloads());
+ rdp.setRelayDescriptorDownloader(rdd);
+ }
+ if (config.getImportCachedRelayDescriptors()) {
+ new CachedRelayDescriptorReader(rdp,
+ config.getCachedRelayDescriptorDirectory(), statsDirectory);
+ this.intermediateStats("importing relay descriptors from local "
+ + "Tor data directories");
+ }
+ if (config.getImportDirectoryArchives()) {
+ new ArchiveReader(rdp,
+ new File(config.getDirectoryArchivesDirectory()),
+ statsDirectory,
+ config.getKeepDirectoryArchiveImportHistory());
+ this.intermediateStats("importing relay descriptors from local "
+ + "directory");
+ }
+ if (rdd != null) {
+ rdd.downloadDescriptors();
+ rdd.writeFile();
+ rdd = null;
+ this.intermediateStats("downloading relay descriptors from the "
+ + "directory authorities");
+ }
+
+ this.checkMissingDescriptors();
+
+ this.checkStaledescriptors();
+
+ this.cleanUpRsyncDirectory();
+
+ this.saveDescriptorDigests();
+ }
+
+ private void loadDescriptorDigests() {
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ try {
+ if (this.storedServerDescriptorsFile.exists()) {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.storedServerDescriptorsFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(",");
+ if (parts.length != 3) {
+ this.logger.warning("Could not load server descriptor "
+ + "digests because of illegal line '" + line + "'. We "
+ + "might not be able to correctly check descriptors for "
+ + "completeness.");
+ break;
+ }
+ long published = dateTimeFormat.parse(parts[0]).getTime();
+ if (published < this.now - 48L * 60L * 60L * 1000L) {
+ continue;
+ }
+ if (!this.storedServerDescriptors.containsKey(published)) {
+ this.storedServerDescriptors.put(published,
+ new HashMap<String, String>());
+ }
+ String serverDescriptorDigest = parts[1];
+ String extraInfoDescriptorDigest = parts[2].equals("NA") ? null
+ : parts[2];
+ this.storedServerDescriptors.get(published).put(
+ serverDescriptorDigest, extraInfoDescriptorDigest);
+ }
+ br.close();
+ }
+ if (this.storedExtraInfoDescriptorsFile.exists()) {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.storedExtraInfoDescriptorsFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(",");
+ if (parts.length != 2) {
+ this.logger.warning("Could not load extra-info descriptor "
+ + "digests because of illegal line '" + line + "'. We "
+ + "might not be able to correctly check descriptors for "
+ + "completeness.");
+ break;
+ }
+ long published = dateTimeFormat.parse(parts[0]).getTime();
+ if (published < this.now - 48L * 60L * 60L * 1000L) {
+ continue;
+ }
+ if (!this.storedExtraInfoDescriptors.containsKey(published)) {
+ this.storedExtraInfoDescriptors.put(published,
+ new HashSet<String>());
+ }
+ String extraInfoDescriptorDigest = parts[1];
+ this.storedExtraInfoDescriptors.get(published).add(
+ extraInfoDescriptorDigest);
+ }
+ br.close();
+ }
+ if (this.storedMicrodescriptorsFile.exists()) {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.storedMicrodescriptorsFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(",");
+ if (parts.length != 2) {
+ this.logger.warning("Could not load microdescriptor digests "
+ + "because of illegal line '" + line + "'. We might not "
+ + "be able to correctly check descriptors for "
+ + "completeness.");
+ break;
+ }
+ long validAfter = dateTimeFormat.parse(parts[0]).getTime();
+ if (validAfter < this.now - 40L * 24L * 60L * 60L * 1000L) {
+ continue;
+ }
+ if (!this.storedMicrodescriptors.containsKey(validAfter)) {
+ this.storedMicrodescriptors.put(validAfter,
+ new HashSet<String>());
+ }
+ String microdescriptorDigest = parts[1];
+ this.storedMicrodescriptors.get(validAfter).add(
+ microdescriptorDigest);
+ }
+ br.close();
+ }
+ } catch (ParseException e) {
+ this.logger.log(Level.WARNING, "Could not load descriptor "
+ + "digests. We might not be able to correctly check "
+ + "descriptors for completeness.", e);
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not load descriptor "
+ + "digests. We might not be able to correctly check "
+ + "descriptors for completeness.", e);
+ }
+ }
+
+ public void intermediateStats(String event) {
+ intermediateStats.append("While " + event + ", we stored "
+ + this.storedConsensusesCounter + " consensus(es), "
+ + this.storedMicrodescConsensusesCounter + " microdesc "
+ + "consensus(es), " + this.storedVotesCounter + " vote(s), "
+ + this.storedCertsCounter + " certificate(s), "
+ + this.storedServerDescriptorsCounter + " server descriptor(s), "
+ + this.storedExtraInfoDescriptorsCounter + " extra-info "
+ + "descriptor(s), and " + this.storedMicrodescriptorsCounter
+ + " microdescriptor(s) to disk.\n");
+ this.storedConsensusesCounter = 0;
+ this.storedMicrodescConsensusesCounter = 0;
+ this.storedVotesCounter = 0;
+ this.storedCertsCounter = 0;
+ this.storedServerDescriptorsCounter = 0;
+ this.storedExtraInfoDescriptorsCounter = 0;
+ this.storedMicrodescriptorsCounter = 0;
+ }
+
+ private void checkMissingDescriptors() {
+ StringBuilder sb = new StringBuilder("Finished writing relay "
+ + "descriptors to disk.\n");
+ sb.append(intermediateStats.toString());
+ sb.append("Statistics on the completeness of written relay "
+ + "descriptors:");
+ SimpleDateFormat dateTimeFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ Map<String, String> knownServerDescriptors =
+ new HashMap<String, String>();
+ for (Map<String, String> descriptors :
+ this.storedServerDescriptors.values()) {
+ knownServerDescriptors.putAll(descriptors);
+ }
+ Set<String> knownExtraInfoDescriptors = new HashSet<String>();
+ for (Set<String> descriptors :
+ this.storedExtraInfoDescriptors.values()) {
+ knownExtraInfoDescriptors.addAll(descriptors);
+ }
+ Set<String> knownMicrodescriptors = new HashSet<String>();
+ for (Set<String> descriptors : this.storedMicrodescriptors.values()) {
+ knownMicrodescriptors.addAll(descriptors);
+ }
+ boolean missingDescriptors = false;
+ boolean missingVotes = false;
+ boolean missingMicrodescConsensus = false;
+ for (Map.Entry<Long, SortedSet<String>> c :
+ this.storedConsensuses.entrySet()) {
+ long validAfterMillis = c.getKey();
+ String validAfterTime = dateTimeFormat.format(validAfterMillis);
+ int allVotes = this.expectedVotes.containsKey(validAfterMillis)
+ ? this.expectedVotes.get(validAfterMillis) : 0;
+ int foundVotes = 0;
+ if (this.storedVotes.containsKey(validAfterMillis)) {
+ foundVotes = this.storedVotes.get(validAfterMillis).size();
+ for (Map.Entry<String, SortedSet<String>> v :
+ this.storedVotes.get(validAfterMillis).entrySet()) {
+ int voteFoundServerDescs = 0;
+ int voteAllServerDescs = 0;
+ int voteFoundExtraInfos = 0;
+ int voteAllExtraInfos = 0;
+ for (String serverDescriptorDigest : v.getValue()) {
+ voteAllServerDescs++;
+ if (knownServerDescriptors.containsKey(
+ serverDescriptorDigest)) {
+ voteFoundServerDescs++;
+ if (knownServerDescriptors.get(serverDescriptorDigest)
+ != null) {
+ String extraInfoDescriptorDigest =
+ knownServerDescriptors.get(serverDescriptorDigest);
+ voteAllExtraInfos++;
+ if (knownExtraInfoDescriptors.contains(
+ extraInfoDescriptorDigest)) {
+ voteFoundExtraInfos++;
+ }
+ }
+ }
+ }
+ sb.append("\nV, " + validAfterTime);
+ if (voteAllServerDescs > 0) {
+ sb.append(String.format(", %d/%d S (%.1f%%)",
+ voteFoundServerDescs, voteAllServerDescs,
+ 100.0D * (double) voteFoundServerDescs
+ / (double) voteAllServerDescs));
+ } else {
+ sb.append(", 0/0 S");
+ }
+ if (voteAllExtraInfos > 0) {
+ sb.append(String.format(", %d/%d E (%.1f%%)",
+ voteFoundExtraInfos, voteAllExtraInfos,
+ 100.0D * (double) voteFoundExtraInfos
+ / (double) voteAllExtraInfos));
+ } else {
+ sb.append(", 0/0 E");
+ }
+ String fingerprint = v.getKey();
+ /* Ignore turtles when warning about missing descriptors. */
+ if (!fingerprint.equalsIgnoreCase(
+ "27B6B5996C426270A5C95488AA5BCEB6BCC86956")
+ && (voteFoundServerDescs * 1000 < voteAllServerDescs * 995
+ || voteFoundExtraInfos * 1000 < voteAllExtraInfos * 995)) {
+ missingDescriptors = true;
+ }
+ }
+ }
+ int foundServerDescs = 0;
+ int allServerDescs = 0;
+ int foundExtraInfos = 0;
+ int allExtraInfos = 0;
+ int foundMicrodescriptors = 0;
+ int allMicrodescriptors = 0;
+ for (String serverDescriptorDigest : c.getValue()) {
+ allServerDescs++;
+ if (knownServerDescriptors.containsKey(
+ serverDescriptorDigest)) {
+ foundServerDescs++;
+ if (knownServerDescriptors.get(
+ serverDescriptorDigest) != null) {
+ allExtraInfos++;
+ String extraInfoDescriptorDigest =
+ knownServerDescriptors.get(serverDescriptorDigest);
+ if (knownExtraInfoDescriptors.contains(
+ extraInfoDescriptorDigest)) {
+ foundExtraInfos++;
+ }
+ }
+ }
+ }
+ sb.append("\nC, " + validAfterTime);
+ if (allVotes > 0) {
+ sb.append(String.format(", %d/%d V (%.1f%%)", foundVotes, allVotes,
+ 100.0D * (double) foundVotes / (double) allVotes));
+ } else {
+ sb.append(", 0/0 V");
+ }
+ if (allServerDescs > 0) {
+ sb.append(String.format(", %d/%d S (%.1f%%)", foundServerDescs,
+ allServerDescs, 100.0D * (double) foundServerDescs
+ / (double) allServerDescs));
+ } else {
+ sb.append(", 0/0 S");
+ }
+ if (allExtraInfos > 0) {
+ sb.append(String.format(", %d/%d E (%.1f%%)", foundExtraInfos,
+ allExtraInfos, 100.0D * (double) foundExtraInfos
+ / (double) allExtraInfos));
+ } else {
+ sb.append(", 0/0 E");
+ }
+ if (this.storedMicrodescConsensuses.containsKey(validAfterMillis)) {
+ for (String microdescriptorDigest :
+ this.storedMicrodescConsensuses.get(validAfterMillis)) {
+ allMicrodescriptors++;
+ if (knownMicrodescriptors.contains(microdescriptorDigest)) {
+ foundMicrodescriptors++;
+ }
+ }
+ sb.append("\nM, " + validAfterTime);
+ if (allMicrodescriptors > 0) {
+ sb.append(String.format(", %d/%d M (%.1f%%)",
+ foundMicrodescriptors, allMicrodescriptors,
+ 100.0D * (double) foundMicrodescriptors
+ / (double) allMicrodescriptors));
+ } else {
+ sb.append(", 0/0 M");
+ }
+ } else {
+ missingMicrodescConsensus = true;
+ }
+ if (foundServerDescs * 1000 < allServerDescs * 995
+ || foundExtraInfos * 1000 < allExtraInfos * 995
+ || foundMicrodescriptors * 1000 < allMicrodescriptors * 995) {
+ missingDescriptors = true;
+ }
+ if (foundVotes < allVotes) {
+ missingVotes = true;
+ }
+ }
+ this.logger.info(sb.toString());
+ if (missingDescriptors) {
+ this.logger.fine("We are missing at least 0.5% of server or "
+ + "extra-info descriptors referenced from a consensus or "
+ + "vote or at least 0.5% of microdescriptors referenced from a "
+ + "microdesc consensus.");
+ }
+ if (missingVotes) {
+ /* TODO Shouldn't warn if we're not trying to archive votes at
+ * all. */
+ this.logger.fine("We are missing at least one vote that was "
+ + "referenced from a consensus.");
+ }
+ if (missingMicrodescConsensus) {
+ /* TODO Shouldn't warn if we're not trying to archive microdesc
+ * consensuses at all. */
+ this.logger.fine("We are missing at least one microdesc "
+ + "consensus that was published together with a known "
+ + "consensus.");
+ }
+ }
+
+ private void checkStaledescriptors() {
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ long tooOldMillis = this.now - 330L * 60L * 1000L;
+ if (!this.storedConsensuses.isEmpty()
+ && this.storedConsensuses.lastKey() < tooOldMillis) {
+ this.logger.warning("The last known relay network status "
+ + "consensus was valid after "
+ + dateTimeFormat.format(this.storedConsensuses.lastKey())
+ + ", which is more than 5:30 hours in the past.");
+ }
+ if (!this.storedMicrodescConsensuses.isEmpty()
+ && this.storedMicrodescConsensuses.lastKey() < tooOldMillis) {
+ this.logger.warning("The last known relay network status "
+ + "microdesc consensus was valid after "
+ + dateTimeFormat.format(
+ this.storedMicrodescConsensuses.lastKey())
+ + ", which is more than 5:30 hours in the past.");
+ }
+ if (!this.storedVotes.isEmpty()
+ && this.storedVotes.lastKey() < tooOldMillis) {
+ this.logger.warning("The last known relay network status vote "
+ + "was valid after " + dateTimeFormat.format(
+ this.storedVotes.lastKey()) + ", which is more than 5:30 hours "
+ + "in the past.");
+ }
+ if (!this.storedServerDescriptors.isEmpty()
+ && this.storedServerDescriptors.lastKey() < tooOldMillis) {
+ this.logger.warning("The last known relay server descriptor was "
+ + "published at "
+ + dateTimeFormat.format(this.storedServerDescriptors.lastKey())
+ + ", which is more than 5:30 hours in the past.");
+ }
+ if (!this.storedExtraInfoDescriptors.isEmpty()
+ && this.storedExtraInfoDescriptors.lastKey() < tooOldMillis) {
+ this.logger.warning("The last known relay extra-info descriptor "
+ + "was published at " + dateTimeFormat.format(
+ this.storedExtraInfoDescriptors.lastKey())
+ + ", which is more than 5:30 hours in the past.");
+ }
+ if (!this.storedMicrodescriptors.isEmpty()
+ && this.storedMicrodescriptors.lastKey() < tooOldMillis) {
+ this.logger.warning("The last known relay microdescriptor was "
+ + "contained in a microdesc consensus that was valid after "
+ + dateTimeFormat.format(this.storedMicrodescriptors.lastKey())
+ + ", which is more than 5:30 hours in the past.");
+ }
+ }
+
+ /* Delete all files from the rsync directory that have not been modified
+ * in the last three days (except for microdescriptors which are kept
+ * for up to thirty days), and remove the .tmp extension from newly
+ * written files. */
+ public void cleanUpRsyncDirectory() {
+ long cutOffMillis = System.currentTimeMillis()
+ - 3L * 24L * 60L * 60L * 1000L;
+ long cutOffMicroMillis = cutOffMillis - 27L * 24L * 60L * 60L * 1000L;
+ Stack<File> allFiles = new Stack<File>();
+ allFiles.add(new File("recent/relay-descriptors"));
+ while (!allFiles.isEmpty()) {
+ File file = allFiles.pop();
+ if (file.isDirectory()) {
+ allFiles.addAll(Arrays.asList(file.listFiles()));
+ } else if (file.getName().endsWith("-micro")) {
+ if (file.lastModified() < cutOffMicroMillis) {
+ file.delete();
+ }
+ } else if (file.lastModified() < cutOffMillis) {
+ file.delete();
+ } else if (file.getName().endsWith(".tmp")) {
+ file.renameTo(new File(file.getParentFile(),
+ file.getName().substring(0,
+ file.getName().lastIndexOf(".tmp"))));
+ }
+ }
+ }
+
+ private void saveDescriptorDigests() {
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ try {
+ this.storedServerDescriptorsFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.storedServerDescriptorsFile));
+ for (Map.Entry<Long, Map<String, String>> e :
+ this.storedServerDescriptors.entrySet()) {
+ String published = dateTimeFormat.format(e.getKey());
+ for (Map.Entry<String, String> f : e.getValue().entrySet()) {
+ String serverDescriptorDigest = f.getKey();
+ String extraInfoDescriptorDigest = f.getValue() == null ? "NA"
+ : f.getValue();
+ bw.write(String.format("%s,%s,%s%n", published,
+ serverDescriptorDigest, extraInfoDescriptorDigest));
+ }
+ }
+ bw.close();
+ this.storedExtraInfoDescriptorsFile.getParentFile().mkdirs();
+ bw = new BufferedWriter(new FileWriter(
+ this.storedExtraInfoDescriptorsFile));
+ for (Map.Entry<Long, Set<String>> e :
+ this.storedExtraInfoDescriptors.entrySet()) {
+ String published = dateTimeFormat.format(e.getKey());
+ for (String extraInfoDescriptorDigest : e.getValue()) {
+ bw.write(String.format("%s,%s%n", published,
+ extraInfoDescriptorDigest));
+ }
+ }
+ bw.close();
+ this.storedMicrodescriptorsFile.getParentFile().mkdirs();
+ bw = new BufferedWriter(new FileWriter(
+ this.storedMicrodescriptorsFile));
+ for (Map.Entry<Long, Set<String>> e :
+ this.storedMicrodescriptors.entrySet()) {
+ String validAfter = dateTimeFormat.format(e.getKey());
+ for (String microdescriptorDigest : e.getValue()) {
+ bw.write(String.format("%s,%s%n", validAfter,
+ microdescriptorDigest));
+ }
+ }
+ bw.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not save descriptor "
+ + "digests. We might not be able to correctly check "
+ + "descriptors for completeness in the next run.", e);
+ }
+ }
+
+ public void storeConsensus(byte[] data, long validAfter,
+ SortedSet<String> dirSources,
+ SortedSet<String> serverDescriptorDigests) {
+ SimpleDateFormat printFormat = new SimpleDateFormat(
+ "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ File tarballFile = new File(this.outputDirectory + "/consensus/"
+ + printFormat.format(new Date(validAfter)) + "-consensus");
+ boolean tarballFileExistedBefore = tarballFile.exists();
+ File rsyncFile = new File("recent/relay-descriptors/consensuses/"
+ + tarballFile.getName());
+ File[] outputFiles = new File[] { tarballFile, rsyncFile };
+ if (this.store(CONSENSUS_ANNOTATION, data, outputFiles, null)) {
+ this.storedConsensusesCounter++;
+ }
+ if (!tarballFileExistedBefore
+ && this.now - validAfter < 3L * 60L * 60L * 1000L) {
+ this.storedConsensuses.put(validAfter, serverDescriptorDigests);
+ this.expectedVotes.put(validAfter, dirSources.size());
+ }
+ }
+
+ public void storeMicrodescConsensus(byte[] data, long validAfter,
+ SortedSet<String> microdescriptorDigests) {
+ SimpleDateFormat yearMonthDirectoryFormat = new SimpleDateFormat(
+ "yyyy/MM");
+ yearMonthDirectoryFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ SimpleDateFormat dayDirectoryFileFormat = new SimpleDateFormat(
+ "dd/yyyy-MM-dd-HH-mm-ss");
+ dayDirectoryFileFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ File tarballFile = new File(this.outputDirectory
+ + "/microdesc/" + yearMonthDirectoryFormat.format(validAfter)
+ + "/consensus-microdesc/"
+ + dayDirectoryFileFormat.format(validAfter)
+ + "-consensus-microdesc");
+ boolean tarballFileExistedBefore = tarballFile.exists();
+ File rsyncFile = new File("recent/relay-descriptors/microdescs/"
+ + "consensus-microdesc/" + tarballFile.getName());
+ File[] outputFiles = new File[] { tarballFile, rsyncFile };
+ if (this.store(MICRODESCCONSENSUS_ANNOTATION, data, outputFiles,
+ null)) {
+ this.storedMicrodescConsensusesCounter++;
+ }
+ if (!tarballFileExistedBefore
+ && this.now - validAfter < 3L * 60L * 60L * 1000L) {
+ this.storedMicrodescConsensuses.put(validAfter,
+ microdescriptorDigests);
+ }
+ }
+
+ public void storeVote(byte[] data, long validAfter,
+ String fingerprint, String digest,
+ SortedSet<String> serverDescriptorDigests) {
+ SimpleDateFormat printFormat = new SimpleDateFormat(
+ "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ File tarballFile = new File(this.outputDirectory + "/vote/"
+ + printFormat.format(new Date(validAfter)) + "-vote-"
+ + fingerprint + "-" + digest);
+ boolean tarballFileExistedBefore = tarballFile.exists();
+ File rsyncFile = new File("recent/relay-descriptors/votes/"
+ + tarballFile.getName());
+ File[] outputFiles = new File[] { tarballFile, rsyncFile };
+ if (this.store(VOTE_ANNOTATION, data, outputFiles, null)) {
+ this.storedVotesCounter++;
+ }
+ if (!tarballFileExistedBefore
+ && this.now - validAfter < 3L * 60L * 60L * 1000L) {
+ if (!this.storedVotes.containsKey(validAfter)) {
+ this.storedVotes.put(validAfter,
+ new TreeMap<String, SortedSet<String>>());
+ }
+ this.storedVotes.get(validAfter).put(fingerprint,
+ serverDescriptorDigests);
+ }
+ }
+
+ public void storeCertificate(byte[] data, String fingerprint,
+ long published) {
+ SimpleDateFormat printFormat = new SimpleDateFormat(
+ "yyyy-MM-dd-HH-mm-ss");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ File tarballFile = new File(this.outputDirectory + "/certs/"
+ + fingerprint + "-" + printFormat.format(new Date(published)));
+ File[] outputFiles = new File[] { tarballFile };
+ if (this.store(CERTIFICATE_ANNOTATION, data, outputFiles, null)) {
+ this.storedCertsCounter++;
+ }
+ }
+
+ public void storeServerDescriptor(byte[] data, String digest,
+ long published, String extraInfoDigest) {
+ SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
+ printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ File tarballFile = new File(this.outputDirectory
+ + "/server-descriptor/" + printFormat.format(new Date(published))
+ + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
+ + digest);
+ boolean tarballFileExistedBefore = tarballFile.exists();
+ File rsyncCatFile = new File("recent/relay-descriptors/"
+ + "server-descriptors/" + this.rsyncCatString
+ + "-server-descriptors.tmp");
+ File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
+ boolean[] append = new boolean[] { false, true };
+ if (this.store(SERVER_DESCRIPTOR_ANNOTATION, data, outputFiles,
+ append)) {
+ this.storedServerDescriptorsCounter++;
+ }
+ if (!tarballFileExistedBefore
+ && this.now - published < 48L * 60L * 60L * 1000L) {
+ if (!this.storedServerDescriptors.containsKey(published)) {
+ this.storedServerDescriptors.put(published,
+ new HashMap<String, String>());
+ }
+ this.storedServerDescriptors.get(published).put(digest,
+ extraInfoDigest);
+ }
+ }
+
+ public void storeExtraInfoDescriptor(byte[] data,
+ String extraInfoDigest, long published) {
+ SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
+ descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ File tarballFile = new File(this.outputDirectory + "/extra-info/"
+ + descriptorFormat.format(new Date(published))
+ + extraInfoDigest.substring(0, 1) + "/"
+ + extraInfoDigest.substring(1, 2) + "/"
+ + extraInfoDigest);
+ boolean tarballFileExistedBefore = tarballFile.exists();
+ File rsyncCatFile = new File("recent/relay-descriptors/"
+ + "extra-infos/" + this.rsyncCatString + "-extra-infos.tmp");
+ File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
+ boolean[] append = new boolean[] { false, true };
+ if (this.store(EXTRA_INFO_ANNOTATION, data, outputFiles, append)) {
+ this.storedExtraInfoDescriptorsCounter++;
+ }
+ if (!tarballFileExistedBefore
+ && this.now - published < 48L * 60L * 60L * 1000L) {
+ if (!this.storedExtraInfoDescriptors.containsKey(published)) {
+ this.storedExtraInfoDescriptors.put(published,
+ new HashSet<String>());
+ }
+ this.storedExtraInfoDescriptors.get(published).add(extraInfoDigest);
+ }
+ }
+
+ public void storeMicrodescriptor(byte[] data,
+ String microdescriptorDigest, long validAfter) {
+ /* TODO We could check here whether we already stored the
+ * microdescriptor in the same valid-after month. This can happen,
+ * e.g., when two relays share the same microdescriptor. In that case
+ * this method gets called twice and the second call overwrites the
+ * file written in the first call. However, this method must be
+ * called twice to store the same microdescriptor in two different
+ * valid-after months. */
+ SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
+ descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ File tarballFile = new File(this.outputDirectory + "/microdesc/"
+ + descriptorFormat.format(validAfter) + "micro/"
+ + microdescriptorDigest.substring(0, 1) + "/"
+ + microdescriptorDigest.substring(1, 2) + "/"
+ + microdescriptorDigest);
+ boolean tarballFileExistedBefore = tarballFile.exists();
+ File rsyncCatFile = new File("recent/relay-descriptors/"
+ + "microdescs/micro/" + this.rsyncCatString
+ + "-micro.tmp");
+ File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
+ boolean[] append = new boolean[] { false, true };
+ if (this.store(MICRODESCRIPTOR_ANNOTATION, data, outputFiles,
+ append)) {
+ this.storedMicrodescriptorsCounter++;
+ }
+ if (!tarballFileExistedBefore
+ && this.now - validAfter < 40L * 24L * 60L * 60L * 1000L) {
+ if (!this.storedMicrodescriptors.containsKey(validAfter)) {
+ this.storedMicrodescriptors.put(validAfter,
+ new HashSet<String>());
+ }
+ this.storedMicrodescriptors.get(validAfter).add(
+ microdescriptorDigest);
+ }
+ }
+
+ private boolean store(byte[] typeAnnotation, byte[] data,
+ File[] outputFiles, boolean[] append) {
+ try {
+ this.logger.finer("Storing " + outputFiles[0]);
+ if (this.descriptorParser.parseDescriptors(data,
+ outputFiles[0].getName()).size() != 1) {
+ this.logger.info("Relay descriptor file " + outputFiles[0]
+ + " doesn't contain exactly one descriptor. Not storing.");
+ return false;
+ }
+ for (int i = 0; i < outputFiles.length; i++) {
+ File outputFile = outputFiles[i];
+ boolean appendToFile = append == null ? false : append[i];
+ outputFile.getParentFile().mkdirs();
+ BufferedOutputStream bos = new BufferedOutputStream(
+ new FileOutputStream(outputFile, appendToFile));
+ if (data.length > 0 && data[0] != '@') {
+ bos.write(typeAnnotation, 0, typeAnnotation.length);
+ }
+ bos.write(data, 0, data.length);
+ bos.close();
+ }
+ return true;
+ } catch (DescriptorParseException e) {
+ this.logger.log(Level.WARNING, "Could not parse relay descriptor "
+ + outputFiles[0] + " before storing it to disk. Skipping.", e);
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not store relay descriptor "
+ + outputFiles[0], e);
+ }
+ return false;
+ }
+}
diff --git a/src/main/java/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java b/src/main/java/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java
new file mode 100644
index 0000000..b9001dd
--- /dev/null
+++ b/src/main/java/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java
@@ -0,0 +1,255 @@
+/* Copyright 2010--2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.relaydescs;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.StringReader;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.Stack;
+import java.util.TimeZone;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Parses all descriptors in local directory cacheddesc/ and sorts them
+ * into directory structure in directory-archive/.
+ */
+public class CachedRelayDescriptorReader {
+ public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
+ List<String> inputDirectories, File statsDirectory) {
+
+ if (rdp == null || inputDirectories == null
+ || inputDirectories.isEmpty() || statsDirectory == null) {
+ throw new IllegalArgumentException();
+ }
+
+ StringBuilder dumpStats = new StringBuilder("Finished importing "
+ + "relay descriptors from local Tor data directories:");
+ Logger logger = Logger.getLogger(
+ CachedRelayDescriptorReader.class.getName());
+
+ /* Read import history containing SHA-1 digests of previously parsed
+ * statuses and descriptors, so that we can skip them in this run. */
+ Set<String> lastImportHistory = new HashSet<String>();
+ Set<String> currentImportHistory = new HashSet<String>();
+ File importHistoryFile = new File(statsDirectory,
+ "cacheddesc-import-history");
+ if (importHistoryFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ importHistoryFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ lastImportHistory.add(line);
+ }
+ br.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not read import history from "
+ + importHistoryFile.getAbsolutePath() + ".", e);
+ }
+ }
+
+ /* Read cached descriptors directories. */
+ for (String inputDirectory : inputDirectories) {
+ File cachedDescDir = new File(inputDirectory);
+ if (!cachedDescDir.exists()) {
+ logger.warning("Directory " + cachedDescDir.getAbsolutePath()
+ + " does not exist. Skipping.");
+ continue;
+ }
+ logger.fine("Reading " + cachedDescDir.getAbsolutePath()
+ + " directory.");
+ SortedSet<File> cachedDescFiles = new TreeSet<File>();
+ Stack<File> files = new Stack<File>();
+ files.add(cachedDescDir);
+ while (!files.isEmpty()) {
+ File file = files.pop();
+ if (file.isDirectory()) {
+ files.addAll(Arrays.asList(file.listFiles()));
+ } else {
+ cachedDescFiles.add(file);
+ }
+ }
+ for (File f : cachedDescFiles) {
+ try {
+ // descriptors may contain non-ASCII chars; read as bytes to
+ // determine digests
+ BufferedInputStream bis =
+ new BufferedInputStream(new FileInputStream(f));
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ bis.close();
+ byte[] allData = baos.toByteArray();
+ if (f.getName().equals("cached-consensus")) {
+ /* Check if directory information is stale. */
+ BufferedReader br = new BufferedReader(new StringReader(
+ new String(allData, "US-ASCII")));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("valid-after ")) {
+ dumpStats.append("\n" + f.getName() + ": " + line.substring(
+ "valid-after ".length()));
+ SimpleDateFormat dateTimeFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ if (dateTimeFormat.parse(line.substring("valid-after "
+ .length())).getTime() < System.currentTimeMillis()
+ - 6L * 60L * 60L * 1000L) {
+ logger.warning("Cached descriptor files in "
+ + cachedDescDir.getAbsolutePath() + " are stale. "
+ + "The valid-after line in cached-consensus is '"
+ + line + "'.");
+ dumpStats.append(" (stale!)");
+ }
+ break;
+ }
+ }
+ br.close();
+
+ /* Parse the cached consensus if we haven't parsed it before
+ * (but regardless of whether it's stale or not). */
+ if (rdp != null) {
+ String digest = Hex.encodeHexString(DigestUtils.sha(
+ allData));
+ if (!lastImportHistory.contains(digest)
+ && !currentImportHistory.contains(digest)) {
+ rdp.parse(allData);
+ } else {
+ dumpStats.append(" (skipped)");
+ }
+ currentImportHistory.add(digest);
+ }
+ } else if (f.getName().equals("v3-status-votes")) {
+ int parsedNum = 0;
+ int skippedNum = 0;
+ String ascii = new String(allData, "US-ASCII");
+ String startToken = "network-status-version ";
+ int end = ascii.length();
+ int start = ascii.indexOf(startToken);
+ while (start >= 0 && start < end) {
+ int next = ascii.indexOf(startToken, start + 1);
+ if (next < 0) {
+ next = end;
+ }
+ if (start < next) {
+ byte[] rawNetworkStatusBytes = new byte[next - start];
+ System.arraycopy(allData, start, rawNetworkStatusBytes, 0,
+ next - start);
+ if (rdp != null) {
+ String digest = Hex.encodeHexString(DigestUtils.sha(
+ rawNetworkStatusBytes));
+ if (!lastImportHistory.contains(digest)
+ && !currentImportHistory.contains(digest)) {
+ rdp.parse(rawNetworkStatusBytes);
+ parsedNum++;
+ } else {
+ skippedNum++;
+ }
+ currentImportHistory.add(digest);
+ }
+ }
+ start = next;
+ }
+ dumpStats.append("\n" + f.getName() + ": parsed " + parsedNum
+ + ", skipped " + skippedNum + " votes");
+ } else if (f.getName().startsWith("cached-descriptors")
+ || f.getName().startsWith("cached-extrainfo")) {
+ String ascii = new String(allData, "US-ASCII");
+ int start = -1;
+ int sig = -1;
+ int end = -1;
+ String startToken =
+ f.getName().startsWith("cached-descriptors")
+ ? "router " : "extra-info ";
+ String sigToken = "\nrouter-signature\n";
+ String endToken = "\n-----END SIGNATURE-----\n";
+ int parsedNum = 0;
+ int skippedNum = 0;
+ while (end < ascii.length()) {
+ start = ascii.indexOf(startToken, end);
+ if (start < 0) {
+ break;
+ }
+ sig = ascii.indexOf(sigToken, start);
+ if (sig < 0) {
+ break;
+ }
+ sig += sigToken.length();
+ end = ascii.indexOf(endToken, sig);
+ if (end < 0) {
+ break;
+ }
+ end += endToken.length();
+ byte[] descBytes = new byte[end - start];
+ System.arraycopy(allData, start, descBytes, 0, end - start);
+ if (rdp != null) {
+ String digest = Hex.encodeHexString(DigestUtils.sha(
+ descBytes));
+ if (!lastImportHistory.contains(digest)
+ && !currentImportHistory.contains(digest)) {
+ rdp.parse(descBytes);
+ parsedNum++;
+ } else {
+ skippedNum++;
+ }
+ currentImportHistory.add(digest);
+ }
+ }
+ dumpStats.append("\n" + f.getName() + ": parsed " + parsedNum
+ + ", skipped " + skippedNum + " "
+ + (f.getName().startsWith("cached-descriptors")
+ ? "server" : "extra-info") + " descriptors");
+ }
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed reading "
+ + cachedDescDir.getAbsolutePath() + " directory.", e);
+ } catch (ParseException e) {
+ logger.log(Level.WARNING, "Failed reading "
+ + cachedDescDir.getAbsolutePath() + " directory.", e);
+ }
+ }
+ logger.fine("Finished reading "
+ + cachedDescDir.getAbsolutePath() + " directory.");
+ }
+
+ /* Write import history containing SHA-1 digests to disk. */
+ try {
+ importHistoryFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ importHistoryFile));
+ for (String digest : currentImportHistory) {
+ bw.write(digest + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not write import history to "
+ + importHistoryFile.getAbsolutePath() + ".", e);
+ }
+
+ logger.info(dumpStats.toString());
+ }
+}
+
diff --git a/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java b/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java
new file mode 100644
index 0000000..9f0f183
--- /dev/null
+++ b/src/main/java/org/torproject/collector/relaydescs/ReferenceChecker.java
@@ -0,0 +1,319 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.relaydescs;
+
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorFile;
+import org.torproject.descriptor.DescriptorReader;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.DirSourceEntry;
+import org.torproject.descriptor.ExtraInfoDescriptor;
+import org.torproject.descriptor.Microdescriptor;
+import org.torproject.descriptor.NetworkStatusEntry;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import org.torproject.descriptor.RelayNetworkStatusVote;
+import org.torproject.descriptor.ServerDescriptor;
+
+import com.google.gson.Gson;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Locale;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class ReferenceChecker {
+
+ private Logger log = Logger.getLogger(ReferenceChecker.class.getName());
+
+ private File descriptorsDir;
+
+ private File referencesFile;
+
+ private File historyFile;
+
+ private long currentTimeMillis;
+
+ private SortedSet<Reference> references = new TreeSet<Reference>();
+
+ private static DateFormat dateTimeFormat;
+
+ static {
+ dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'",
+ Locale.US);
+ dateTimeFormat.setLenient(false);
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ }
+
+ private static final long ONE_HOUR = 60L * 60L * 1000L;
+
+ private static final long THREE_HOURS = 3L * ONE_HOUR;
+
+ private static final long SIX_HOURS = 6L * ONE_HOUR;
+
+ private static final long ONE_DAY = 24L * ONE_HOUR;
+
+ private static final long THIRTY_DAYS = 30L * ONE_DAY;
+
+ public ReferenceChecker(File descriptorsDir, File referencesFile,
+ File historyFile) {
+ this.descriptorsDir = descriptorsDir;
+ this.referencesFile = referencesFile;
+ this.historyFile = historyFile;
+ }
+
+ public void check() {
+ this.getCurrentTimeMillis();
+ this.readReferencesFile();
+ this.readNewDescriptors();
+ this.dropStaleReferences();
+ this.checkReferences();
+ this.writeReferencesFile();
+ }
+
+ private void getCurrentTimeMillis() {
+ this.currentTimeMillis = System.currentTimeMillis();
+ }
+
+ private static class Reference implements Comparable<Reference> {
+
+ private String referencing;
+
+ private String referenced;
+
+ private double weight;
+
+ private long expiresAfterMillis;
+
+ public Reference(String referencing, String referenced, double weight,
+ long expiresAfterMillis) {
+ this.referencing = referencing;
+ this.referenced = referenced;
+ this.weight = weight;
+ this.expiresAfterMillis = expiresAfterMillis;
+ }
+
+ @Override
+ public boolean equals(Object otherObject) {
+ if (!(otherObject instanceof Reference)) {
+ return false;
+ }
+ Reference other = (Reference) otherObject;
+ return this.referencing.equals(other.referencing)
+ && this.referenced.equals(other.referenced);
+ }
+
+ @Override
+ public int hashCode() {
+ return this.referencing.hashCode() + this.referenced.hashCode();
+ }
+
+ @Override
+ public int compareTo(Reference other) {
+ int result = this.referencing.compareTo(other.referencing);
+ if (result == 0) {
+ result = this.referenced.compareTo(other.referenced);
+ }
+ return result;
+ }
+ }
+
+ private void readReferencesFile() {
+ if (!this.referencesFile.exists()) {
+ return;
+ }
+ Gson gson = new Gson();
+ try {
+ FileReader fr = new FileReader(this.referencesFile);
+ this.references.addAll(Arrays.asList(gson.fromJson(fr,
+ Reference[].class)));
+ fr.close();
+ } catch (IOException e) {
+ this.log.log(Level.WARNING, "Cannot read existing references file "
+ + "from previous run.", e);
+ }
+ }
+
+ private void readNewDescriptors() {
+ DescriptorReader descriptorReader =
+ DescriptorSourceFactory.createDescriptorReader();
+ descriptorReader.addDirectory(this.descriptorsDir);
+ descriptorReader.setExcludeFiles(this.historyFile);
+ Iterator<DescriptorFile> descriptorFiles =
+ descriptorReader.readDescriptors();
+ while (descriptorFiles.hasNext()) {
+ DescriptorFile descriptorFile = descriptorFiles.next();
+ for (Descriptor descriptor : descriptorFile.getDescriptors()) {
+ if (descriptor instanceof RelayNetworkStatusConsensus) {
+ RelayNetworkStatusConsensus consensus =
+ (RelayNetworkStatusConsensus) descriptor;
+ String consensusFlavor = consensus.getConsensusFlavor();
+ if (consensusFlavor == null) {
+ this.readRelayNetworkStatusConsensusUnflavored(consensus);
+ } else if (consensusFlavor.equals("microdesc")) {
+ this.readRelayNetworkStatusConsensusMicrodesc(consensus);
+ } else {
+ /* Ignore unknown consensus flavors. */
+ }
+ } else if (descriptor instanceof RelayNetworkStatusVote) {
+ this.readRelayNetworkStatusVote(
+ (RelayNetworkStatusVote) descriptor);
+ } else if (descriptor instanceof ServerDescriptor) {
+ this.readServerDescriptor((ServerDescriptor) descriptor);
+ } else if (descriptor instanceof ExtraInfoDescriptor) {
+ this.readExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
+ } else if (descriptor instanceof Microdescriptor) {
+ readMicrodescriptor((Microdescriptor) descriptor);
+ } else {
+ /* Ignore unknown descriptors. */
+ }
+ }
+ }
+ }
+
+ private void readRelayNetworkStatusConsensusUnflavored(
+ RelayNetworkStatusConsensus consensus) {
+ String validAfter = dateTimeFormat.format(
+ consensus.getValidAfterMillis());
+ String referencing = String.format("C-%s", validAfter);
+ this.addReference(referencing, String.format("M-%s", validAfter), 1.0,
+ consensus.getValidAfterMillis() + THREE_HOURS);
+ for (DirSourceEntry dirSourceEntry :
+ consensus.getDirSourceEntries().values()) {
+ if (!dirSourceEntry.isLegacy()) {
+ this.addReference(referencing, String.format("V-%s-%s",
+ validAfter, dirSourceEntry.getIdentity()), 1.0,
+ consensus.getValidAfterMillis() + THREE_HOURS);
+ }
+ }
+ double entryWeight = 200.0
+ / ((double) consensus.getStatusEntries().size());
+ for (NetworkStatusEntry entry :
+ consensus.getStatusEntries().values()) {
+ this.addReference(referencing,
+ String.format("S-%s", entry.getDescriptor()), entryWeight,
+ entry.getPublishedMillis() + THREE_HOURS);
+ }
+ }
+
+ private void readRelayNetworkStatusConsensusMicrodesc(
+ RelayNetworkStatusConsensus consensus) {
+ String validAfter = dateTimeFormat.format(
+ consensus.getValidAfterMillis());
+ String referencing = String.format("M-%s", validAfter);
+ this.addReference(referencing, String.format("C-%s", validAfter), 1.0,
+ consensus.getValidAfterMillis() + THREE_HOURS);
+ double entryWeight = 200.0
+ / ((double) consensus.getStatusEntries().size());
+ for (NetworkStatusEntry entry :
+ consensus.getStatusEntries().values()) {
+ for (String digest : entry.getMicrodescriptorDigests()) {
+ this.addReference(referencing, String.format("D-%s", digest),
+ entryWeight, entry.getPublishedMillis() + THREE_HOURS);
+ }
+ }
+ }
+
+ private void readRelayNetworkStatusVote(RelayNetworkStatusVote vote) {
+ String validAfter = dateTimeFormat.format(vote.getValidAfterMillis());
+ String referencing = String.format("V-%s-%s", validAfter,
+ vote.getIdentity());
+ double entryWeight = 200.0
+ / ((double) vote.getStatusEntries().size());
+ for (NetworkStatusEntry entry : vote.getStatusEntries().values()) {
+ this.addReference(referencing,
+ String.format("S-%s", entry.getDescriptor()), entryWeight,
+ entry.getPublishedMillis() + SIX_HOURS);
+ }
+ }
+
+ private void readServerDescriptor(ServerDescriptor serverDescriptor) {
+ String referenced = serverDescriptor.getExtraInfoDigest() == null ? ""
+ : String.format("E-%s", serverDescriptor.getExtraInfoDigest());
+ this.addReference(String.format("S-%s",
+ serverDescriptor.getServerDescriptorDigest()), referenced, 0.01,
+ serverDescriptor.getPublishedMillis() + SIX_HOURS);
+ }
+
+ private void readExtraInfoDescriptor(
+ ExtraInfoDescriptor extraInfoDescriptor) {
+ this.addReference(String.format("E-%s",
+ extraInfoDescriptor.getExtraInfoDigest()), "", 0.005,
+ extraInfoDescriptor.getPublishedMillis() + SIX_HOURS);
+ }
+
+ private void readMicrodescriptor(Microdescriptor microdesc) {
+ this.addReference(
+ String.format("D-%s", microdesc.getMicrodescriptorDigest()), "",
+ 0.0, this.currentTimeMillis + THIRTY_DAYS);
+ }
+
+ private void addReference(String referencing, String referenced,
+ double weight, long expiresAfterMillis) {
+ this.references.add(new Reference(referencing.toUpperCase(),
+ referenced.toUpperCase(), weight, expiresAfterMillis));
+ }
+
+ private void dropStaleReferences() {
+ SortedSet<Reference> recentReferences = new TreeSet<Reference>();
+ for (Reference reference : this.references) {
+ if (this.currentTimeMillis <= reference.expiresAfterMillis) {
+ recentReferences.add(reference);
+ }
+ }
+ this.references = recentReferences;
+ }
+
+ private void checkReferences() {
+ Set<String> knownDescriptors = new HashSet<String>();
+ for (Reference reference : this.references) {
+ knownDescriptors.add(reference.referencing);
+ }
+ double totalMissingDescriptorsWeight = 0.0;
+ Set<String> missingDescriptors = new TreeSet<String>();
+ StringBuilder sb = new StringBuilder("Missing referenced "
+ + "descriptors:");
+ for (Reference reference : this.references) {
+ if (reference.referenced.length() > 0
+ && !knownDescriptors.contains(reference.referenced)) {
+ if (!missingDescriptors.contains(reference.referenced)) {
+ totalMissingDescriptorsWeight += reference.weight;
+ }
+ missingDescriptors.add(reference.referenced);
+ sb.append(String.format("%n%s -> %s (%.4f -> %.4f)",
+ reference.referencing, reference.referenced, reference.weight,
+ totalMissingDescriptorsWeight));
+ }
+ }
+ this.log.log(Level.INFO, sb.toString());
+ if (totalMissingDescriptorsWeight > 0.999) {
+ this.log.log(Level.WARNING, "Missing too many referenced "
+ + "descriptors (" + totalMissingDescriptorsWeight + ").");
+ }
+ }
+
+ private void writeReferencesFile() {
+ Gson gson = new Gson();
+ try {
+ FileWriter fw = new FileWriter(this.referencesFile);
+ gson.toJson(this.references, fw);
+ fw.close();
+ } catch (IOException e) {
+ this.log.log(Level.WARNING, "Cannot write references file for next "
+ + "run.", e);
+ }
+ }
+}
+
diff --git a/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java
new file mode 100644
index 0000000..458332a
--- /dev/null
+++ b/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java
@@ -0,0 +1,1134 @@
+/* Copyright 2010--2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.relaydescs;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.digest.DigestUtils;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import java.util.zip.InflaterInputStream;
+
+/**
+ * Downloads relay descriptors from the directory authorities via HTTP.
+ * Keeps a list of missing descriptors that gets updated by parse results
+ * from <code>RelayDescriptorParser</code> and downloads all missing
+ * descriptors that have been published in the last 24 hours. Also
+ * downloads all server and extra-info descriptors known to a directory
+ * authority at most once a day.
+ */
+public class RelayDescriptorDownloader {
+
+ /**
+ * Text file containing the descriptors that we are missing and that we
+ * want to download. Lines are formatted as:
+ *
+ * - "consensus,<validafter>,<parsed>",
+ * - "consensus-microdesc,<validafter>,<parsed>",
+ * - "vote,<validafter>,<fingerprint>,<parsed>",
+ * - "server,<published>,<relayid>,<descid>,<parsed>",
+ * - "extra,<published>,<relayid>,<descid>,<parsed>", or
+ * - "micro,<validafter>,<relayid>,<descid>,<parsed>".
+ */
+ private File missingDescriptorsFile;
+
+ /**
+ * Relay descriptors that we are missing and that we want to download
+ * either in this execution or write to disk and try next time. Map keys
+ * contain comma-separated values as in the missing descriptors files
+ * without the "parsed" column. Map values contain the "parsed" column.
+ */
+ private SortedMap<String, String> missingDescriptors;
+
+ /**
+ * Map from base64 microdescriptor digests to keys in missingDescriptors
+ * ("micro,<validafter>,<relayid>,<descid>"). We need this map, because
+ * we can't learn <validafter> or <relayid> from parsing
+ * microdescriptors, but we need to know <validafter> to store
+ * microdescriptors to disk and both <validafter> and <relayid> to
+ * remove microdescriptors from the missing list. There are potentially
+ * many matching keys in missingDescriptors for the same microdescriptor
+ * digest. Also, in rare cases relays share the same microdescriptor
+ * (which is only possible if they share the same onion key), and then
+ * we don't have to download their microdescriptor more than once.
+ */
+ private Map<String, Set<String>> microdescriptorKeys;
+
+ /**
+ * Set of microdescriptor digests that are currently missing. Used for
+ * logging statistics instead of "micro,<validafter>,..." keys which may
+ * contain the same microdescriptor digest multiple times.
+ */
+ private Set<String> missingMicrodescriptors;
+
+ /**
+ * Text file containing the IP addresses (and Dir ports if not 80) of
+ * directory authorities and when we last downloaded all server and
+ * extra-info descriptors from them, so that we can avoid downloading
+ * them too often.
+ */
+ private File lastDownloadedAllDescriptorsFile;
+
+ /**
+ * Map of directory authorities and when we last downloaded all server
+ * and extra-info descriptors from them. Map keys are IP addresses (and
+ * Dir ports if not 80), map values are timestamps.
+ */
+ private Map<String, String> lastDownloadedAllDescriptors;
+
+ /**
+ * <code>RelayDescriptorParser</code> that we will hand over the
+ * downloaded descriptors for parsing.
+ */
+ private RelayDescriptorParser rdp;
+
+ /**
+ * Directory authorities that we will try to download missing
+ * descriptors from.
+ */
+ private List<String> authorities;
+
+ /**
+ * Fingerprints of directory authorities that we will use to download
+ * votes without requiring a successfully downloaded consensus.
+ */
+ private List<String> authorityFingerprints;
+
+ /**
+ * Should we try to download the current consensus if we don't have it?
+ */
+ private boolean downloadCurrentConsensus;
+
+ /**
+ * Should we try to download the current microdesc consensus if we don't
+ * have it?
+ */
+ private boolean downloadCurrentMicrodescConsensus;
+
+ /**
+ * Should we try to download current votes if we don't have them?
+ */
+ private boolean downloadCurrentVotes;
+
+ /**
+ * Should we try to download missing server descriptors that have been
+ * published within the past 24 hours?
+ */
+ private boolean downloadMissingServerDescriptors;
+
+ /**
+ * Should we try to download missing extra-info descriptors that have
+ * been published within the past 24 hours?
+ */
+ private boolean downloadMissingExtraInfos;
+
+ /**
+ * Should we try to download missing microdescriptors that have been
+ * published within the past 24 hours?
+ */
+ private boolean downloadMissingMicrodescriptors;
+
+ /**
+ * Should we try to download all server descriptors from the authorities
+ * once every 24 hours?
+ */
+ private boolean downloadAllServerDescriptors;
+
+ /**
+ * Should we try to download all extra-info descriptors from the
+ * authorities once every 24 hours?
+ */
+ private boolean downloadAllExtraInfos;
+
+ /**
+ * Should we download zlib-compressed versions of descriptors by adding
+ * ".z" to URLs?
+ */
+ private boolean downloadCompressed;
+
+ /**
+ * valid-after time that we expect the current consensus,
+ * microdescriptor consensus, and votes to have, formatted
+ * "yyyy-MM-dd HH:mm:ss". We only expect to find documents with this
+ * valid-after time on the directory authorities. This time is
+ * initialized as the beginning of the current hour.
+ */
+ private String currentValidAfter;
+
+ /**
+ * Cut-off time for missing server and extra-info descriptors, formatted
+ * "yyyy-MM-dd HH:mm:ss". This time is initialized as the current system
+ * time minus 24 hours.
+ */
+ private String descriptorCutOff;
+
+ /**
+ * Cut-off time for downloading all server and extra-info descriptors
+ * from the directory authorities, formatted "yyyy-MM-dd HH:mm:ss". This
+ * time is initialized as the current system time minus 23:30 hours.
+ */
+ private String downloadAllDescriptorsCutOff;
+
+ /**
+ * Directory authorities that we plan to download all server and
+ * extra-info descriptors from in this execution.
+ */
+ private Set<String> downloadAllDescriptorsFromAuthorities;
+
+ /**
+ * Current timestamp that is written to the missing list for descriptors
+ * that we parsed in this execution and for authorities that we
+ * downloaded all server and extra-info descriptors from.
+ */
+ private String currentTimestamp;
+
+ /**
+ * Logger for this class.
+ */
+ private Logger logger;
+
+ /**
+ * Number of descriptors requested by directory authority to be included
+ * in logs.
+ */
+ private Map<String, Integer> requestsByAuthority;
+
+ /**
+ * Counters for descriptors that we had on the missing list at the
+ * beginning of the execution, that we added to the missing list,
+ * that we requested, and that we successfully downloaded in this
+ * execution.
+ */
+ private int oldMissingConsensuses = 0;
+
+ private int oldMissingMicrodescConsensuses = 0;
+
+ private int oldMissingVotes = 0;
+
+ private int oldMissingServerDescriptors = 0;
+
+ private int oldMissingExtraInfoDescriptors = 0;
+
+ private int oldMissingMicrodescriptors = 0;
+
+ private int newMissingConsensuses = 0;
+
+ private int newMissingMicrodescConsensuses = 0;
+
+ private int newMissingVotes = 0;
+
+ private int newMissingServerDescriptors = 0;
+
+ private int newMissingExtraInfoDescriptors = 0;
+
+ private int newMissingMicrodescriptors = 0;
+
+ private int requestedConsensuses = 0;
+
+ private int requestedMicrodescConsensuses = 0;
+
+ private int requestedVotes = 0;
+
+ private int requestedMissingServerDescriptors = 0;
+
+ private int requestedAllServerDescriptors = 0;
+
+ private int requestedMissingExtraInfoDescriptors = 0;
+
+ private int requestedAllExtraInfoDescriptors = 0;
+
+ private int requestedMissingMicrodescriptors = 0;
+
+ private int downloadedConsensuses = 0;
+
+ private int downloadedMicrodescConsensuses = 0;
+
+ private int downloadedVotes = 0;
+
+ private int downloadedMissingServerDescriptors = 0;
+
+ private int downloadedAllServerDescriptors = 0;
+
+ private int downloadedMissingExtraInfoDescriptors = 0;
+
+ private int downloadedAllExtraInfoDescriptors = 0;
+
+ private int downloadedMissingMicrodescriptors = 0;
+
+ /**
+ * Initializes this class, including reading in missing descriptors from
+ * <code>stats/missing-relay-descriptors</code> and the times when we
+ * last downloaded all server and extra-info descriptors from
+ * <code>stats/last-downloaded-all-descriptors</code>.
+ */
+ public RelayDescriptorDownloader(RelayDescriptorParser rdp,
+ List<String> authorities, List<String> authorityFingerprints,
+ boolean downloadCurrentConsensus,
+ boolean downloadCurrentMicrodescConsensus,
+ boolean downloadCurrentVotes,
+ boolean downloadMissingServerDescriptors,
+ boolean downloadMissingExtraInfos,
+ boolean downloadMissingMicrodescriptors,
+ boolean downloadAllServerDescriptors, boolean downloadAllExtraInfos,
+ boolean downloadCompressed) {
+
+ /* Memorize argument values. */
+ this.rdp = rdp;
+ this.authorities = new ArrayList<String>(authorities);
+ this.authorityFingerprints = new ArrayList<String>(
+ authorityFingerprints);
+ this.downloadCurrentConsensus = downloadCurrentConsensus;
+ this.downloadCurrentMicrodescConsensus =
+ downloadCurrentMicrodescConsensus;
+ this.downloadCurrentVotes = downloadCurrentVotes;
+ this.downloadMissingServerDescriptors =
+ downloadMissingServerDescriptors;
+ this.downloadMissingExtraInfos = downloadMissingExtraInfos;
+ this.downloadMissingMicrodescriptors =
+ downloadMissingMicrodescriptors;
+ this.downloadAllServerDescriptors = downloadAllServerDescriptors;
+ this.downloadAllExtraInfos = downloadAllExtraInfos;
+ this.downloadCompressed = downloadCompressed;
+
+ /* Shuffle list of authorities for better load balancing over time. */
+ Collections.shuffle(this.authorities);
+
+ /* Initialize logger. */
+ this.logger = Logger.getLogger(
+ RelayDescriptorDownloader.class.getName());
+
+ /* Prepare cut-off times and timestamp for the missing descriptors
+ * list and the list of authorities to download all server and
+ * extra-info descriptors from. */
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ format.setTimeZone(TimeZone.getTimeZone("UTC"));
+ long now = System.currentTimeMillis();
+ this.currentValidAfter = format.format((now / (60L * 60L * 1000L))
+ * (60L * 60L * 1000L));
+ this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L);
+ this.currentTimestamp = format.format(now);
+ this.downloadAllDescriptorsCutOff = format.format(now
+ - 23L * 60L * 60L * 1000L - 30L * 60L * 1000L);
+
+ /* Read list of missing descriptors from disk and memorize those that
+ * we are interested in and that are likely to be found on the
+ * directory authorities. */
+ this.missingDescriptors = new TreeMap<String, String>();
+ this.microdescriptorKeys = new HashMap<String, Set<String>>();
+ this.missingMicrodescriptors = new HashSet<String>();
+ this.missingDescriptorsFile = new File(
+ "stats/missing-relay-descriptors");
+ if (this.missingDescriptorsFile.exists()) {
+ try {
+ this.logger.fine("Reading file "
+ + this.missingDescriptorsFile.getAbsolutePath() + "...");
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.missingDescriptorsFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.split(",").length > 2) {
+ String published = line.split(",")[1];
+ if (((line.startsWith("consensus,")
+ || line.startsWith("consensus-microdesc,")
+ || line.startsWith("vote,"))
+ && this.currentValidAfter.equals(published))
+ || ((line.startsWith("server,")
+ || line.startsWith("extra,")
+ || line.startsWith("micro,"))
+ && this.descriptorCutOff.compareTo(published) < 0)) {
+ if (!line.endsWith("NA")) {
+ /* Not missing. */
+ } else if (line.startsWith("consensus,")) {
+ oldMissingConsensuses++;
+ } else if (line.startsWith("consensus-microdesc,")) {
+ oldMissingMicrodescConsensuses++;
+ } else if (line.startsWith("vote,")) {
+ oldMissingVotes++;
+ } else if (line.startsWith("server,")) {
+ oldMissingServerDescriptors++;
+ } else if (line.startsWith("extra,")) {
+ oldMissingExtraInfoDescriptors++;
+ }
+ int separateAt = line.lastIndexOf(",");
+ this.missingDescriptors.put(line.substring(0,
+ separateAt), line.substring(separateAt + 1));
+ if (line.startsWith("micro,")) {
+ String microdescriptorDigest = line.split(",")[3];
+ String microdescriptorKey = line.substring(0,
+ line.lastIndexOf(","));
+ if (!this.microdescriptorKeys.containsKey(
+ microdescriptorDigest)) {
+ this.microdescriptorKeys.put(
+ microdescriptorDigest, new HashSet<String>());
+ }
+ this.microdescriptorKeys.get(microdescriptorDigest).add(
+ microdescriptorKey);
+ if (line.endsWith("NA") && !this.missingMicrodescriptors
+ .contains(microdescriptorDigest)) {
+ this.missingMicrodescriptors.add(microdescriptorDigest);
+ oldMissingMicrodescriptors++;
+ }
+ }
+ }
+ } else {
+ this.logger.fine("Invalid line '" + line + "' in "
+ + this.missingDescriptorsFile.getAbsolutePath()
+ + ". Ignoring.");
+ }
+ }
+ br.close();
+ this.logger.fine("Finished reading file "
+ + this.missingDescriptorsFile.getAbsolutePath() + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed to read file "
+ + this.missingDescriptorsFile.getAbsolutePath()
+ + "! This means that we might forget to dowload relay "
+ + "descriptors we are missing.", e);
+ }
+ }
+
+ /* Read list of directory authorities and when we last downloaded all
+ * server and extra-info descriptors from them. */
+ this.lastDownloadedAllDescriptors = new HashMap<String, String>();
+ this.lastDownloadedAllDescriptorsFile = new File(
+ "stats/last-downloaded-all-descriptors");
+ if (this.lastDownloadedAllDescriptorsFile.exists()) {
+ try {
+ this.logger.fine("Reading file "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+ + "...");
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.lastDownloadedAllDescriptorsFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.split(",").length != 2) {
+ this.logger.fine("Invalid line '" + line + "' in "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+ + ". Ignoring.");
+ } else {
+ String[] parts = line.split(",");
+ String authority = parts[0];
+ String lastDownloaded = parts[1];
+ this.lastDownloadedAllDescriptors.put(authority,
+ lastDownloaded);
+ }
+ }
+ br.close();
+ this.logger.fine("Finished reading file "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+ + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed to read file "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+ + "! This means that we might download all server and "
+ + "extra-info descriptors more often than we should.", e);
+ }
+ }
+
+ /* Make a list of at most two directory authorities that we want to
+ * download all server and extra-info descriptors from. */
+ this.downloadAllDescriptorsFromAuthorities = new HashSet<String>();
+ for (String authority : this.authorities) {
+ if (!this.lastDownloadedAllDescriptors.containsKey(authority)
+ || this.lastDownloadedAllDescriptors.get(authority).compareTo(
+ this.downloadAllDescriptorsCutOff) < 0) {
+ this.downloadAllDescriptorsFromAuthorities.add(authority);
+ }
+ if (this.downloadAllDescriptorsFromAuthorities.size() >= 2) {
+ break;
+ }
+ }
+
+ /* Prepare statistics on this execution. */
+ this.requestsByAuthority = new HashMap<String, Integer>();
+ for (String authority : this.authorities) {
+ this.requestsByAuthority.put(authority, 0);
+ }
+ }
+
+ /**
+ * We have parsed a consensus. Take this consensus off the missing list
+ * and add the votes created by the given <code>authorities</code> and
+ * the <code>serverDescriptors</code> which are in the format
+ * "<published>,<relayid>,<descid>" to that list.
+ */
+ public void haveParsedConsensus(String validAfter,
+ Set<String> authorities, Set<String> serverDescriptors) {
+
+ /* Mark consensus as parsed. */
+ if (this.currentValidAfter.equals(validAfter)) {
+ String consensusKey = "consensus," + validAfter;
+ this.missingDescriptors.put(consensusKey, this.currentTimestamp);
+
+ /* Add votes to missing list. */
+ for (String authority : authorities) {
+ String voteKey = "vote," + validAfter + "," + authority;
+ if (!this.missingDescriptors.containsKey(voteKey)) {
+ this.missingDescriptors.put(voteKey, "NA");
+ this.newMissingVotes++;
+ }
+ }
+ }
+
+ /* Add server descriptors to missing list. */
+ for (String serverDescriptor : serverDescriptors) {
+ String published = serverDescriptor.split(",")[0];
+ if (this.descriptorCutOff.compareTo(published) < 0) {
+ String serverDescriptorKey = "server," + serverDescriptor;
+ if (!this.missingDescriptors.containsKey(
+ serverDescriptorKey)) {
+ this.missingDescriptors.put(serverDescriptorKey, "NA");
+ this.newMissingServerDescriptors++;
+ }
+ }
+ }
+ }
+
+ /**
+ * We have parsed a microdesc consensus. Take this microdesc consensus
+ * off the missing list and add the <code>microdescriptors</code> which
+ * are in the format "<validafter>,<relayid>,<descid>" to that
+ * list.
+ */
+ public void haveParsedMicrodescConsensus(String validAfter,
+ Set<String> microdescriptors) {
+
+ /* Mark microdesc consensus as parsed. */
+ if (this.currentValidAfter.equals(validAfter)) {
+ String microdescConsensusKey = "consensus-microdesc," + validAfter;
+ this.missingDescriptors.put(microdescConsensusKey,
+ this.currentTimestamp);
+ }
+
+ /* Add microdescriptors to missing list. Exclude those that we already
+ * downloaded this month. (We download each microdescriptor at least
+ * once per month to keep the storage logic sane; otherwise we'd have
+ * to copy microdescriptors from the earlier month to the current
+ * month, and that gets messy.) */
+ if (this.descriptorCutOff.compareTo(validAfter) < 0) {
+ String validAfterYearMonth = validAfter.substring(0,
+ "YYYY-MM".length());
+ for (String microdescriptor : microdescriptors) {
+ String microdescriptorKey = "micro," + microdescriptor;
+ String parsed = "NA";
+ String microdescriptorDigest = microdescriptor.split(",")[2];
+ if (this.microdescriptorKeys.containsKey(microdescriptorDigest)) {
+ for (String otherMicrodescriptorKey :
+ this.microdescriptorKeys.get(microdescriptorDigest)) {
+ String otherValidAfter =
+ otherMicrodescriptorKey.split(",")[1];
+ if (!otherValidAfter.startsWith(validAfterYearMonth)) {
+ continue;
+ }
+ String otherParsed = this.missingDescriptors.get(
+ otherMicrodescriptorKey);
+ if (otherParsed != null && !otherParsed.equals("NA")) {
+ parsed = otherParsed;
+ break;
+ }
+ }
+ } else {
+ this.microdescriptorKeys.put(
+ microdescriptorDigest, new HashSet<String>());
+ }
+ this.microdescriptorKeys.get(microdescriptorDigest).add(
+ microdescriptorKey);
+ this.missingDescriptors.put(microdescriptorKey, parsed);
+ if (parsed.equals("NA")
+ && !this.missingMicrodescriptors.contains(microdescriptorDigest)) {
+ this.missingMicrodescriptors.add(microdescriptorDigest);
+ this.newMissingMicrodescriptors++;
+ }
+ }
+ }
+ }
+
+ /**
+ * We have parsed a vote. Take this vote off the missing list and add
+ * the <code>serverDescriptors</code> which are in the format
+ * "<published>,<relayid>,<descid>" to that list.
+ */
+ public void haveParsedVote(String validAfter, String fingerprint,
+ Set<String> serverDescriptors) {
+
+ /* Mark vote as parsed. */
+ if (this.currentValidAfter.equals(validAfter)) {
+ String voteKey = "vote," + validAfter + "," + fingerprint;
+ this.missingDescriptors.put(voteKey, this.currentTimestamp);
+ }
+
+ /* Add server descriptors to missing list. */
+ for (String serverDescriptor : serverDescriptors) {
+ String published = serverDescriptor.split(",")[0];
+ if (this.descriptorCutOff.compareTo(published) < 0) {
+ String serverDescriptorKey = "server," + serverDescriptor;
+ if (!this.missingDescriptors.containsKey(
+ serverDescriptorKey)) {
+ this.missingDescriptors.put(serverDescriptorKey, "NA");
+ this.newMissingServerDescriptors++;
+ }
+ }
+ }
+ }
+
+ /**
+ * We have parsed a server descriptor. Take this server descriptor off
+ * the missing list and put the extra-info descriptor digest on that
+ * list.
+ */
+ public void haveParsedServerDescriptor(String published,
+ String relayIdentity, String serverDescriptorDigest,
+ String extraInfoDigest) {
+
+ /* Mark server descriptor as parsed. */
+ if (this.descriptorCutOff.compareTo(published) < 0) {
+ String serverDescriptorKey = "server," + published + ","
+ + relayIdentity + "," + serverDescriptorDigest;
+ this.missingDescriptors.put(serverDescriptorKey,
+ this.currentTimestamp);
+
+ /* Add extra-info descriptor to missing list. */
+ if (extraInfoDigest != null) {
+ String extraInfoKey = "extra," + published + ","
+ + relayIdentity + "," + extraInfoDigest;
+ if (!this.missingDescriptors.containsKey(extraInfoKey)) {
+ this.missingDescriptors.put(extraInfoKey, "NA");
+ this.newMissingExtraInfoDescriptors++;
+ }
+ }
+ }
+ }
+
+ /**
+ * We have parsed an extra-info descriptor. Take it off the missing
+ * list.
+ */
+ public void haveParsedExtraInfoDescriptor(String published,
+ String relayIdentity, String extraInfoDigest) {
+ if (this.descriptorCutOff.compareTo(published) < 0) {
+ String extraInfoKey = "extra," + published + ","
+ + relayIdentity + "," + extraInfoDigest;
+ this.missingDescriptors.put(extraInfoKey, this.currentTimestamp);
+ }
+ }
+
+ /**
+ * We have parsed a microdescriptor. Take it off the missing list.
+ */
+ public void haveParsedMicrodescriptor(String descriptorDigest) {
+ if (this.microdescriptorKeys.containsKey(descriptorDigest)) {
+ for (String microdescriptorKey :
+ this.microdescriptorKeys.get(descriptorDigest)) {
+ String validAfter = microdescriptorKey.split(",")[0];
+ if (this.descriptorCutOff.compareTo(validAfter) < 0) {
+ this.missingDescriptors.put(microdescriptorKey,
+ this.currentTimestamp);
+ }
+ }
+ this.missingMicrodescriptors.remove(descriptorDigest);
+ }
+ }
+
+ /**
+ * Downloads missing descriptors that we think might still be available
+ * on the directory authorities as well as all server and extra-info
+ * descriptors once per day.
+ */
+ public void downloadDescriptors() {
+
+ /* Put the current consensus and votes on the missing list, unless we
+ * already have them. */
+ String consensusKey = "consensus," + this.currentValidAfter;
+ if (!this.missingDescriptors.containsKey(consensusKey)) {
+ this.missingDescriptors.put(consensusKey, "NA");
+ this.newMissingConsensuses++;
+ }
+ String microdescConsensusKey = "consensus-microdesc,"
+ + this.currentValidAfter;
+ if (!this.missingDescriptors.containsKey(microdescConsensusKey)) {
+ this.missingDescriptors.put(microdescConsensusKey, "NA");
+ this.newMissingMicrodescConsensuses++;
+ }
+ for (String authority : authorityFingerprints) {
+ String voteKey = "vote," + this.currentValidAfter + "," + authority;
+ if (!this.missingDescriptors.containsKey(voteKey)) {
+ this.missingDescriptors.put(voteKey, "NA");
+ this.newMissingVotes++;
+ }
+ }
+
+ /* Download descriptors from authorities which are in random order, so
+ * that we distribute the load somewhat fairly over time. */
+ for (String authority : authorities) {
+
+ /* Make all requests to an authority in a single try block. If
+ * something goes wrong with this authority, we give up on all
+ * downloads and continue with the next authority. */
+ /* TODO Some authorities provide very little bandwidth and could
+ * slow down the entire download process. Ponder adding a timeout of
+ * 3 or 5 minutes per authority to avoid getting in the way of the
+ * next execution. */
+ try {
+
+ /* Start with downloading the current consensus, unless we already
+ * have it. */
+ if (downloadCurrentConsensus) {
+ if (this.missingDescriptors.containsKey(consensusKey)
+ && this.missingDescriptors.get(consensusKey).equals("NA")) {
+ this.requestedConsensuses++;
+ this.downloadedConsensuses +=
+ this.downloadResourceFromAuthority(authority,
+ "/tor/status-vote/current/consensus");
+ }
+ }
+
+ /* Then try to download the microdesc consensus. */
+ if (downloadCurrentMicrodescConsensus) {
+ if (this.missingDescriptors.containsKey(microdescConsensusKey)
+ && this.missingDescriptors.get(microdescConsensusKey)
+ .equals("NA")) {
+ this.requestedMicrodescConsensuses++;
+ this.downloadedMicrodescConsensuses +=
+ this.downloadResourceFromAuthority(authority,
+ "/tor/status-vote/current/consensus-microdesc");
+ }
+ }
+
+ /* Next, try to download current votes that we're missing. */
+ if (downloadCurrentVotes) {
+ String voteKeyPrefix = "vote," + this.currentValidAfter;
+ SortedSet<String> fingerprints = new TreeSet<String>();
+ for (Map.Entry<String, String> e :
+ this.missingDescriptors.entrySet()) {
+ if (e.getValue().equals("NA")
+ && e.getKey().startsWith(voteKeyPrefix)) {
+ String fingerprint = e.getKey().split(",")[2];
+ fingerprints.add(fingerprint);
+ }
+ }
+ for (String fingerprint : fingerprints) {
+ this.requestedVotes++;
+ this.downloadedVotes +=
+ this.downloadResourceFromAuthority(authority,
+ "/tor/status-vote/current/" + fingerprint);
+ }
+ }
+
+ /* Download either all server and extra-info descriptors or only
+ * those that we're missing. Start with server descriptors, then
+ * request extra-info descriptors. Finally, request missing
+ * microdescriptors. */
+ for (String type : new String[] { "server", "extra", "micro" }) {
+
+ /* Download all server or extra-info descriptors from this
+ * authority if we haven't done so for 24 hours and if we're
+ * configured to do so. */
+ if (this.downloadAllDescriptorsFromAuthorities.contains(
+ authority) && ((type.equals("server")
+ && this.downloadAllServerDescriptors)
+ || (type.equals("extra") && this.downloadAllExtraInfos))) {
+ int downloadedAllDescriptors =
+ this.downloadResourceFromAuthority(authority, "/tor/"
+ + type + "/all");
+ if (type.equals("server")) {
+ this.requestedAllServerDescriptors++;
+ this.downloadedAllServerDescriptors +=
+ downloadedAllDescriptors;
+ } else if (type.equals("extra")) {
+ this.requestedAllExtraInfoDescriptors++;
+ this.downloadedAllExtraInfoDescriptors +=
+ downloadedAllDescriptors;
+ }
+
+ /* Download missing server descriptors, extra-info descriptors,
+ * and microdescriptors if we're configured to do so. */
+ } else if ((type.equals("server")
+ && this.downloadMissingServerDescriptors)
+ || (type.equals("extra") && this.downloadMissingExtraInfos)
+ || (type.equals("micro")
+ && this.downloadMissingMicrodescriptors)) {
+
+ /* Go through the list of missing descriptors of this type
+ * and combine the descriptor identifiers to a URL of up to
+ * 96 server or extra-info descriptors or 92 microdescriptors
+ * that we can download at once. */
+ SortedSet<String> descriptorIdentifiers =
+ new TreeSet<String>();
+ for (Map.Entry<String, String> e :
+ this.missingDescriptors.entrySet()) {
+ if (e.getValue().equals("NA")
+ && e.getKey().startsWith(type + ",")
+ && this.descriptorCutOff.compareTo(
+ e.getKey().split(",")[1]) < 0) {
+ String descriptorIdentifier = e.getKey().split(",")[3];
+ descriptorIdentifiers.add(descriptorIdentifier);
+ }
+ }
+ StringBuilder combinedResource = null;
+ int descriptorsInCombinedResource = 0;
+ int requestedDescriptors = 0;
+ int downloadedDescriptors = 0;
+ int maxDescriptorsInCombinedResource =
+ type.equals("micro") ? 92 : 96;
+ String separator = type.equals("micro") ? "-" : "+";
+ for (String descriptorIdentifier : descriptorIdentifiers) {
+ if (descriptorsInCombinedResource
+ >= maxDescriptorsInCombinedResource) {
+ requestedDescriptors += descriptorsInCombinedResource;
+ downloadedDescriptors +=
+ this.downloadResourceFromAuthority(authority,
+ combinedResource.toString());
+ combinedResource = null;
+ descriptorsInCombinedResource = 0;
+ }
+ if (descriptorsInCombinedResource == 0) {
+ combinedResource = new StringBuilder("/tor/" + type
+ + "/d/" + descriptorIdentifier);
+ } else {
+ combinedResource.append(separator + descriptorIdentifier);
+ }
+ descriptorsInCombinedResource++;
+ }
+ if (descriptorsInCombinedResource > 0) {
+ requestedDescriptors += descriptorsInCombinedResource;
+ downloadedDescriptors +=
+ this.downloadResourceFromAuthority(authority,
+ combinedResource.toString());
+ }
+ if (type.equals("server")) {
+ this.requestedMissingServerDescriptors +=
+ requestedDescriptors;
+ this.downloadedMissingServerDescriptors +=
+ downloadedDescriptors;
+ } else if (type.equals("extra")) {
+ this.requestedMissingExtraInfoDescriptors +=
+ requestedDescriptors;
+ this.downloadedMissingExtraInfoDescriptors +=
+ downloadedDescriptors;
+ } else if (type.equals("micro")) {
+ this.requestedMissingMicrodescriptors +=
+ requestedDescriptors;
+ this.downloadedMissingMicrodescriptors +=
+ downloadedDescriptors;
+ }
+ }
+ }
+
+ /* If a download failed, stop requesting descriptors from this
+ * authority and move on to the next. */
+ } catch (IOException e) {
+ logger.log(Level.FINE, "Failed downloading from " + authority
+ + "!", e);
+ }
+ }
+ }
+
+ /**
+ * Attempts to download one or more descriptors identified by a resource
+ * string from a directory authority and passes the returned
+ * descriptor(s) to the <code>RelayDescriptorParser</code> upon success.
+ * Returns the number of descriptors contained in the reply. Throws an
+ * <code>IOException</code> if something goes wrong while downloading.
+ */
+ private int downloadResourceFromAuthority(String authority,
+ String resource) throws IOException {
+ byte[] allData = null;
+ this.requestsByAuthority.put(authority,
+ this.requestsByAuthority.get(authority) + 1);
+ /* TODO Disable compressed downloads for extra-info descriptors,
+ * because zlib decompression doesn't work correctly. Figure out why
+ * this is and fix it. */
+ String fullUrl = "http://" + authority + resource
+ + (this.downloadCompressed && !resource.startsWith("/tor/extra/")
+ ? ".z" : "");
+ URL u = new URL(fullUrl);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.connect();
+ int response = huc.getResponseCode();
+ if (response == 200) {
+ BufferedInputStream in = this.downloadCompressed
+ && !resource.startsWith("/tor/extra/")
+ ? new BufferedInputStream(new InflaterInputStream(
+ huc.getInputStream()))
+ : new BufferedInputStream(huc.getInputStream());
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = in.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ in.close();
+ allData = baos.toByteArray();
+ }
+ logger.fine("Downloaded " + fullUrl + " -> " + response + " ("
+ + (allData == null ? 0 : allData.length) + " bytes)");
+ int receivedDescriptors = 0;
+ if (allData != null) {
+ if (resource.startsWith("/tor/status-vote/current/")) {
+ this.rdp.parse(allData);
+ receivedDescriptors = 1;
+ } else if (resource.startsWith("/tor/server/")
+ || resource.startsWith("/tor/extra/")) {
+ if (resource.equals("/tor/server/all")
+ || resource.equals("/tor/extra/all")) {
+ this.lastDownloadedAllDescriptors.put(authority,
+ this.currentTimestamp);
+ }
+ String ascii = null;
+ try {
+ ascii = new String(allData, "US-ASCII");
+ } catch (UnsupportedEncodingException e) {
+ /* No way that US-ASCII is not supported. */
+ }
+ int start = -1;
+ int sig = -1;
+ int end = -1;
+ String startToken = resource.startsWith("/tor/server/")
+ ? "router " : "extra-info ";
+ String sigToken = "\nrouter-signature\n";
+ String endToken = "\n-----END SIGNATURE-----\n";
+ while (end < ascii.length()) {
+ start = ascii.indexOf(startToken, end);
+ if (start < 0) {
+ break;
+ }
+ sig = ascii.indexOf(sigToken, start);
+ if (sig < 0) {
+ break;
+ }
+ sig += sigToken.length();
+ end = ascii.indexOf(endToken, sig);
+ if (end < 0) {
+ break;
+ }
+ end += endToken.length();
+ byte[] descBytes = new byte[end - start];
+ System.arraycopy(allData, start, descBytes, 0, end - start);
+ this.rdp.parse(descBytes);
+ receivedDescriptors++;
+ }
+ } else if (resource.startsWith("/tor/micro/")) {
+ /* TODO We need to parse microdescriptors ourselves, rather than
+ * RelayDescriptorParser, because only we know the valid-after
+ * time(s) of microdesc consensus(es) containing this
+ * microdescriptor. However, this breaks functional abstraction
+ * pretty badly. */
+ SimpleDateFormat parseFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ String ascii = null;
+ try {
+ ascii = new String(allData, "US-ASCII");
+ } catch (UnsupportedEncodingException e) {
+ /* No way that US-ASCII is not supported. */
+ }
+ int start = -1;
+ int end = -1;
+ String startToken = "onion-key\n";
+ while (end < ascii.length()) {
+ start = ascii.indexOf(startToken, end);
+ if (start < 0) {
+ break;
+ }
+ end = ascii.indexOf(startToken, start + 1);
+ if (end < 0) {
+ end = ascii.length();
+ if (end <= start) {
+ break;
+ }
+ }
+ byte[] descBytes = new byte[end - start];
+ System.arraycopy(allData, start, descBytes, 0, end - start);
+ String digest256Base64 = Base64.encodeBase64String(
+ DigestUtils.sha256(descBytes)).replaceAll("=", "");
+ if (!this.microdescriptorKeys.containsKey(digest256Base64)) {
+ continue;
+ }
+ String digest256Hex = DigestUtils.sha256Hex(descBytes);
+ for (String microdescriptorKey :
+ this.microdescriptorKeys.get(digest256Base64)) {
+ String validAfterTime = microdescriptorKey.split(",")[1];
+ try {
+ long validAfter =
+ parseFormat.parse(validAfterTime).getTime();
+ this.rdp.storeMicrodescriptor(descBytes, digest256Hex,
+ digest256Base64, validAfter);
+ } catch (ParseException e) {
+ this.logger.log(Level.WARNING, "Could not parse "
+ + "valid-after time '" + validAfterTime + "' in "
+ + "microdescriptor key. Not storing microdescriptor.",
+ e);
+ }
+ }
+ receivedDescriptors++;
+ }
+ }
+ }
+ return receivedDescriptors;
+ }
+
+ /**
+ * Writes status files to disk and logs statistics about downloading
+ * relay descriptors in this execution.
+ */
+ public void writeFile() {
+
+ /* Write missing descriptors file to disk. */
+ int missingConsensuses = 0;
+ int missingMicrodescConsensuses = 0;
+ int missingVotes = 0;
+ int missingServerDescriptors = 0;
+ int missingExtraInfoDescriptors = 0;
+ try {
+ this.logger.fine("Writing file "
+ + this.missingDescriptorsFile.getAbsolutePath() + "...");
+ this.missingDescriptorsFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.missingDescriptorsFile));
+ for (Map.Entry<String, String> e :
+ this.missingDescriptors.entrySet()) {
+ String key = e.getKey();
+ String value = e.getValue();
+ if (!value.equals("NA")) {
+ /* Not missing. */
+ } else if (key.startsWith("consensus,")) {
+ missingConsensuses++;
+ } else if (key.startsWith("consensus-microdesc,")) {
+ missingMicrodescConsensuses++;
+ } else if (key.startsWith("vote,")) {
+ missingVotes++;
+ } else if (key.startsWith("server,")) {
+ missingServerDescriptors++;
+ } else if (key.startsWith("extra,")) {
+ missingExtraInfoDescriptors++;
+ } else if (key.startsWith("micro,")) {
+ /* We're counting missing microdescriptors below. */
+ }
+ bw.write(key + "," + value + "\n");
+ }
+ bw.close();
+ this.logger.fine("Finished writing file "
+ + this.missingDescriptorsFile.getAbsolutePath() + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed writing "
+ + this.missingDescriptorsFile.getAbsolutePath() + "!", e);
+ }
+ int missingMicrodescriptors = this.missingMicrodescriptors.size();
+
+ /* Write text file containing the directory authorities and when we
+ * last downloaded all server and extra-info descriptors from them to
+ * disk. */
+ try {
+ this.logger.fine("Writing file "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+ + "...");
+ this.lastDownloadedAllDescriptorsFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.lastDownloadedAllDescriptorsFile));
+ for (Map.Entry<String, String> e :
+ this.lastDownloadedAllDescriptors.entrySet()) {
+ String authority = e.getKey();
+ String lastDownloaded = e.getValue();
+ bw.write(authority + "," + lastDownloaded + "\n");
+ }
+ bw.close();
+ this.logger.fine("Finished writing file "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
+ + ".");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed writing "
+ + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + "!",
+ e);
+ }
+
+ /* Log statistics about this execution. */
+ this.logger.info("Finished downloading relay descriptors from the "
+ + "directory authorities.");
+ this.logger.info("At the beginning of this execution, we were "
+ + "missing " + oldMissingConsensuses + " consensus(es), "
+ + oldMissingMicrodescConsensuses + " microdesc consensus(es), "
+ + oldMissingVotes + " vote(s), " + oldMissingServerDescriptors
+ + " server descriptor(s), " + oldMissingExtraInfoDescriptors
+ + " extra-info descriptor(s), and " + oldMissingMicrodescriptors
+ + " microdescriptor(s).");
+ this.logger.info("During this execution, we added "
+ + this.newMissingConsensuses + " consensus(es), "
+ + this.newMissingMicrodescConsensuses
+ + " microdesc consensus(es), " + this.newMissingVotes
+ + " vote(s), " + this.newMissingServerDescriptors
+ + " server descriptor(s), " + this.newMissingExtraInfoDescriptors
+ + " extra-info descriptor(s), and "
+ + this.newMissingMicrodescriptors + " microdescriptor(s) to the "
+ + "missing list, some of which we also "
+ + "requested and removed from the list again.");
+ this.logger.info("We requested " + this.requestedConsensuses
+ + " consensus(es), " + this.requestedMicrodescConsensuses
+ + " microdesc consensus(es), " + this.requestedVotes
+ + " vote(s), " + this.requestedMissingServerDescriptors
+ + " missing server descriptor(s), "
+ + this.requestedAllServerDescriptors
+ + " times all server descriptors, "
+ + this.requestedMissingExtraInfoDescriptors + " missing "
+ + "extra-info descriptor(s), "
+ + this.requestedAllExtraInfoDescriptors + " times all extra-info "
+ + "descriptors, and " + this.requestedMissingMicrodescriptors
+ + " missing microdescriptor(s) from the directory authorities.");
+ StringBuilder sb = new StringBuilder();
+ for (String authority : this.authorities) {
+ sb.append(" " + authority + "="
+ + this.requestsByAuthority.get(authority));
+ }
+ this.logger.info("We sent these numbers of requests to the directory "
+ + "authorities:" + sb.toString());
+ this.logger.info("We successfully downloaded "
+ + this.downloadedConsensuses + " consensus(es), "
+ + this.downloadedMicrodescConsensuses
+ + " microdesc consensus(es), " + this.downloadedVotes
+ + " vote(s), " + this.downloadedMissingServerDescriptors
+ + " missing server descriptor(s), "
+ + this.downloadedAllServerDescriptors
+ + " server descriptor(s) when downloading all descriptors, "
+ + this.downloadedMissingExtraInfoDescriptors + " missing "
+ + "extra-info descriptor(s), "
+ + this.downloadedAllExtraInfoDescriptors + " extra-info "
+ + "descriptor(s) when downloading all descriptors, and "
+ + this.downloadedMissingMicrodescriptors
+ + " missing microdescriptor(s).");
+ this.logger.info("At the end of this execution, we are missing "
+ + missingConsensuses + " consensus(es), "
+ + missingMicrodescConsensuses + " microdesc consensus(es), "
+ + missingVotes + " vote(s), " + missingServerDescriptors
+ + " server descriptor(s), " + missingExtraInfoDescriptors
+ + " extra-info descriptor(s), and " + missingMicrodescriptors
+ + " microdescriptor(s), some of which we may try in the next "
+ + "execution.");
+ }
+}
+
diff --git a/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorParser.java b/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorParser.java
new file mode 100644
index 0000000..3f9b912
--- /dev/null
+++ b/src/main/java/org/torproject/collector/relaydescs/RelayDescriptorParser.java
@@ -0,0 +1,337 @@
+/* Copyright 2010--2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.relaydescs;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.StringReader;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Parses relay descriptors including network status consensuses and
+ * votes, server and extra-info descriptors, and passes the results to the
+ * stats handlers, to the archive writer, or to the relay descriptor
+ * downloader.
+ */
+public class RelayDescriptorParser {
+
+ /**
+ * File writer that writes descriptor contents to files in a
+ * directory-archive directory structure.
+ */
+ private ArchiveWriter aw;
+
+ private ArchiveReader ar;
+
+ /**
+ * Missing descriptor downloader that uses the parse results to learn
+ * which descriptors we are missing and want to download.
+ */
+ private RelayDescriptorDownloader rdd;
+
+ /**
+ * Logger for this class.
+ */
+ private Logger logger;
+
+ private SimpleDateFormat dateTimeFormat;
+
+ /**
+ * Initializes this class.
+ */
+ public RelayDescriptorParser(ArchiveWriter aw) {
+ this.aw = aw;
+
+ /* Initialize logger. */
+ this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
+
+ this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ }
+
+ public void setRelayDescriptorDownloader(
+ RelayDescriptorDownloader rdd) {
+ this.rdd = rdd;
+ }
+
+ public void setArchiveReader(ArchiveReader ar) {
+ this.ar = ar;
+ }
+
+ public boolean parse(byte[] data) {
+ boolean stored = false;
+ try {
+ /* Convert descriptor to ASCII for parsing. This means we'll lose
+ * the non-ASCII chars, but we don't care about them for parsing
+ * anyway. */
+ BufferedReader br = new BufferedReader(new StringReader(new String(
+ data, "US-ASCII")));
+ String line;
+ do {
+ line = br.readLine();
+ } while (line != null && line.startsWith("@"));
+ if (line == null) {
+ this.logger.fine("We were given an empty descriptor for "
+ + "parsing. Ignoring.");
+ return false;
+ }
+ SimpleDateFormat parseFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ if (line.startsWith("network-status-version 3")) {
+ String statusType = "consensus";
+ if (line.equals("network-status-version 3 microdesc")) {
+ statusType = "consensus-microdesc";
+ }
+ String validAfterTime = null;
+ String fingerprint = null;
+ String dirSource = null;
+ long validAfter = -1L;
+ long dirKeyPublished = -1L;
+ SortedSet<String> dirSources = new TreeSet<String>();
+ SortedSet<String> serverDescriptors = new TreeSet<String>();
+ SortedSet<String> serverDescriptorDigests = new TreeSet<String>();
+ SortedSet<String> microdescriptorKeys = new TreeSet<String>();
+ SortedSet<String> microdescriptorDigests = new TreeSet<String>();
+ StringBuilder certificateStringBuilder = null;
+ String certificateString = null;
+ String lastRelayIdentity = null;
+ while ((line = br.readLine()) != null) {
+ if (certificateStringBuilder != null) {
+ if (line.startsWith("r ")) {
+ certificateString = certificateStringBuilder.toString();
+ certificateStringBuilder = null;
+ } else {
+ certificateStringBuilder.append(line + "\n");
+ }
+ }
+ if (line.equals("vote-status vote")) {
+ statusType = "vote";
+ } else if (line.startsWith("valid-after ")) {
+ validAfterTime = line.substring("valid-after ".length());
+ validAfter = parseFormat.parse(validAfterTime).getTime();
+ } else if (line.startsWith("dir-source ")) {
+ dirSource = line.split(" ")[2];
+ } else if (line.startsWith("vote-digest ")) {
+ dirSources.add(dirSource);
+ } else if (line.startsWith("dir-key-certificate-version ")) {
+ certificateStringBuilder = new StringBuilder();
+ certificateStringBuilder.append(line + "\n");
+ } else if (line.startsWith("fingerprint ")) {
+ fingerprint = line.split(" ")[1];
+ } else if (line.startsWith("dir-key-published ")) {
+ String dirKeyPublishedTime = line.substring(
+ "dir-key-published ".length());
+ dirKeyPublished = parseFormat.parse(dirKeyPublishedTime)
+ .getTime();
+ } else if (line.startsWith("r ")) {
+ String[] parts = line.split(" ");
+ if (parts.length == 8) {
+ lastRelayIdentity = Hex.encodeHexString(Base64.decodeBase64(
+ parts[2] + "=")).toLowerCase();
+ } else if (parts.length == 9) {
+ lastRelayIdentity = Hex.encodeHexString(Base64.decodeBase64(
+ parts[2] + "=")).toLowerCase();
+ String serverDesc = Hex.encodeHexString(Base64.decodeBase64(
+ parts[3] + "=")).toLowerCase();
+ String publishedTime = parts[4] + " " + parts[5];
+ serverDescriptors.add(publishedTime + ","
+ + lastRelayIdentity + "," + serverDesc);
+ serverDescriptorDigests.add(serverDesc);
+ } else {
+ this.logger.log(Level.WARNING, "Could not parse r line '"
+ + line + "' in descriptor. Skipping.");
+ break;
+ }
+ } else if (line.startsWith("m ")) {
+ String[] parts = line.split(" ");
+ if (parts.length == 2 && parts[1].length() == 43) {
+ String digest256Base64 = parts[1];
+ microdescriptorKeys.add(validAfterTime + ","
+ + lastRelayIdentity + "," + digest256Base64);
+ String digest256Hex = Hex.encodeHexString(
+ Base64.decodeBase64(digest256Base64 + "="))
+ .toLowerCase();
+ microdescriptorDigests.add(digest256Hex);
+ } else if (parts.length != 3
+ || !parts[2].startsWith("sha256=")
+ || parts[2].length() != 50) {
+ this.logger.log(Level.WARNING, "Could not parse m line '"
+ + line + "' in descriptor. Skipping.");
+ break;
+ }
+ }
+ }
+ if (statusType.equals("consensus")) {
+ if (this.rdd != null) {
+ this.rdd.haveParsedConsensus(validAfterTime, dirSources,
+ serverDescriptors);
+ }
+ if (this.aw != null) {
+ this.aw.storeConsensus(data, validAfter, dirSources,
+ serverDescriptorDigests);
+ stored = true;
+ }
+ } else if (statusType.equals("consensus-microdesc")) {
+ if (this.rdd != null) {
+ this.rdd.haveParsedMicrodescConsensus(validAfterTime,
+ microdescriptorKeys);
+ }
+ if (this.ar != null) {
+ this.ar.haveParsedMicrodescConsensus(validAfterTime,
+ microdescriptorDigests);
+ }
+ if (this.aw != null) {
+ this.aw.storeMicrodescConsensus(data, validAfter,
+ microdescriptorDigests);
+ stored = true;
+ }
+ } else {
+ if (this.aw != null || this.rdd != null) {
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "network-status-version ";
+ String sigToken = "directory-signature ";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken);
+ if (start >= 0 && sig >= 0 && sig > start) {
+ sig += sigToken.length();
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ String digest = DigestUtils.shaHex(forDigest).toUpperCase();
+ if (this.aw != null) {
+ this.aw.storeVote(data, validAfter, dirSource, digest,
+ serverDescriptorDigests);
+ stored = true;
+ }
+ if (this.rdd != null) {
+ this.rdd.haveParsedVote(validAfterTime, fingerprint,
+ serverDescriptors);
+ }
+ }
+ if (certificateString != null) {
+ if (this.aw != null) {
+ this.aw.storeCertificate(certificateString.getBytes(),
+ dirSource, dirKeyPublished);
+ stored = true;
+ }
+ }
+ }
+ }
+ } else if (line.startsWith("router ")) {
+ String publishedTime = null;
+ String extraInfoDigest = null;
+ String relayIdentifier = null;
+ long published = -1L;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("published ")) {
+ publishedTime = line.substring("published ".length());
+ published = parseFormat.parse(publishedTime).getTime();
+ } else if (line.startsWith("opt fingerprint")
+ || line.startsWith("fingerprint")) {
+ relayIdentifier = line.substring(line.startsWith("opt ")
+ ? "opt fingerprint".length() : "fingerprint".length())
+ .replaceAll(" ", "").toLowerCase();
+ } else if (line.startsWith("opt extra-info-digest ")
+ || line.startsWith("extra-info-digest ")) {
+ extraInfoDigest = line.startsWith("opt ")
+ ? line.split(" ")[2].toLowerCase()
+ : line.split(" ")[1].toLowerCase();
+ }
+ }
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "router ";
+ String sigToken = "\nrouter-signature\n";
+ int start = ascii.indexOf(startToken);
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ String digest = null;
+ if (start >= 0 || sig >= 0 || sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ digest = DigestUtils.shaHex(forDigest);
+ }
+ if (this.aw != null && digest != null) {
+ this.aw.storeServerDescriptor(data, digest, published,
+ extraInfoDigest);
+ stored = true;
+ }
+ if (this.rdd != null && digest != null) {
+ this.rdd.haveParsedServerDescriptor(publishedTime,
+ relayIdentifier, digest, extraInfoDigest);
+ }
+ } else if (line.startsWith("extra-info ")) {
+ String publishedTime = null;
+ String relayIdentifier = line.split(" ")[2];
+ long published = -1L;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("published ")) {
+ publishedTime = line.substring("published ".length());
+ published = parseFormat.parse(publishedTime).getTime();
+ }
+ }
+ String ascii = new String(data, "US-ASCII");
+ String startToken = "extra-info ";
+ String sigToken = "\nrouter-signature\n";
+ String digest = null;
+ int start = ascii.indexOf(startToken);
+ if (start > 0) {
+ /* Do not confuse "extra-info " in "@type extra-info 1.0" with
+ * "extra-info 0000...". TODO This is a hack that should be
+ * solved by using metrics-lib some day. */
+ start = ascii.indexOf("\n" + startToken);
+ if (start > 0) {
+ start++;
+ }
+ }
+ int sig = ascii.indexOf(sigToken) + sigToken.length();
+ if (start >= 0 && sig >= 0 && sig > start) {
+ byte[] forDigest = new byte[sig - start];
+ System.arraycopy(data, start, forDigest, 0, sig - start);
+ digest = DigestUtils.shaHex(forDigest);
+ }
+ if (this.aw != null && digest != null) {
+ this.aw.storeExtraInfoDescriptor(data, digest, published);
+ stored = true;
+ }
+ if (this.rdd != null && digest != null) {
+ this.rdd.haveParsedExtraInfoDescriptor(publishedTime,
+ relayIdentifier.toLowerCase(), digest);
+ }
+ } else if (line.equals("onion-key")) {
+ /* Cannot store microdescriptors without knowing valid-after
+ * time(s) of microdesc consensuses containing them, because we
+ * don't know which month directories to put them in. Have to use
+ * storeMicrodescriptor below. */
+ }
+ br.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not parse descriptor. "
+ + "Skipping.", e);
+ } catch (ParseException e) {
+ this.logger.log(Level.WARNING, "Could not parse descriptor. "
+ + "Skipping.", e);
+ }
+ return stored;
+ }
+
+ public void storeMicrodescriptor(byte[] data, String digest256Hex,
+ String digest256Base64, long validAfter) {
+ if (this.aw != null) {
+ this.aw.storeMicrodescriptor(data, digest256Hex, validAfter);
+ }
+ if (this.rdd != null) {
+ this.rdd.haveParsedMicrodescriptor(digest256Base64);
+ }
+ }
+}
+
diff --git a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
new file mode 100644
index 0000000..7bcfbf3
--- /dev/null
+++ b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
@@ -0,0 +1,643 @@
+/* Copyright 2012-2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.torperf;
+
+import org.torproject.collector.main.Configuration;
+import org.torproject.collector.main.LockFile;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.Stack;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/* Download possibly truncated Torperf .data and .extradata files from
+ * configured sources, append them to the files we already have, and merge
+ * the two files into the .tpf format. */
+public class TorperfDownloader extends Thread {
+
+ public static void main(String[] args) {
+
+ Logger logger = Logger.getLogger(TorperfDownloader.class.getName());
+ logger.info("Starting torperf module of CollecTor.");
+
+ // Initialize configuration
+ Configuration config = new Configuration();
+
+ // Use lock file to avoid overlapping runs
+ LockFile lf = new LockFile("torperf");
+ if (!lf.acquireLock()) {
+ logger.severe("Warning: CollecTor is already running or has not exited "
+ + "cleanly! Exiting!");
+ System.exit(1);
+ }
+
+ // Process Torperf files
+ new TorperfDownloader(config).run();
+
+ // Remove lock file
+ lf.releaseLock();
+
+ logger.info("Terminating torperf module of CollecTor.");
+ }
+
+ private Configuration config;
+
+ public TorperfDownloader(Configuration config) {
+ this.config = config;
+ }
+
+ private File torperfOutputDirectory = null;
+ private SortedMap<String, String> torperfSources = null;
+ private List<String> torperfFilesLines = null;
+ private Logger logger = null;
+ private SimpleDateFormat dateFormat;
+
+ public void run() {
+
+ File torperfOutputDirectory =
+ new File(config.getTorperfOutputDirectory());
+ SortedMap<String, String> torperfSources = config.getTorperfSources();
+ List<String> torperfFilesLines = config.getTorperfFiles();
+
+ this.torperfOutputDirectory = torperfOutputDirectory;
+ this.torperfSources = torperfSources;
+ this.torperfFilesLines = torperfFilesLines;
+ if (!this.torperfOutputDirectory.exists()) {
+ this.torperfOutputDirectory.mkdirs();
+ }
+ this.logger = Logger.getLogger(TorperfDownloader.class.getName());
+ this.dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+ this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ this.readLastMergedTimestamps();
+ for (String torperfFilesLine : this.torperfFilesLines) {
+ this.downloadAndMergeFiles(torperfFilesLine);
+ }
+ this.writeLastMergedTimestamps();
+
+ this.cleanUpRsyncDirectory();
+ }
+
+ private File torperfLastMergedFile =
+ new File("stats/torperf-last-merged");
+
+ SortedMap<String, String> lastMergedTimestamps =
+ new TreeMap<String, String>();
+
+ private void readLastMergedTimestamps() {
+ if (!this.torperfLastMergedFile.exists()) {
+ return;
+ }
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.torperfLastMergedFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(" ");
+ String fileName = null;
+ String timestamp = null;
+ if (parts.length == 2) {
+ try {
+ Double.parseDouble(parts[1]);
+ fileName = parts[0];
+ timestamp = parts[1];
+ } catch (NumberFormatException e) {
+ /* Handle below. */
+ }
+ }
+ if (fileName == null || timestamp == null) {
+ this.logger.log(Level.WARNING, "Invalid line '" + line + "' in "
+ + this.torperfLastMergedFile.getAbsolutePath() + ". "
+ + "Ignoring past history of merging .data and .extradata "
+ + "files.");
+ this.lastMergedTimestamps.clear();
+ break;
+ }
+ this.lastMergedTimestamps.put(fileName, timestamp);
+ }
+ br.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Error while reading '"
+ + this.torperfLastMergedFile.getAbsolutePath() + ". Ignoring "
+ + "past history of merging .data and .extradata files.");
+ this.lastMergedTimestamps.clear();
+ }
+ }
+
+ private void writeLastMergedTimestamps() {
+ try {
+ this.torperfLastMergedFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.torperfLastMergedFile));
+ for (Map.Entry<String, String> e :
+ this.lastMergedTimestamps.entrySet()) {
+ String fileName = e.getKey();
+ String timestamp = e.getValue();
+ bw.write(fileName + " " + timestamp + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Error while writing '"
+ + this.torperfLastMergedFile.getAbsolutePath() + ". This may "
+ + "result in ignoring history of merging .data and .extradata "
+ + "files in the next execution.", e);
+ }
+ }
+
+ private void downloadAndMergeFiles(String torperfFilesLine) {
+ String[] parts = torperfFilesLine.split(" ");
+ String sourceName = parts[1];
+ int fileSize = -1;
+ try {
+ fileSize = Integer.parseInt(parts[2]);
+ } catch (NumberFormatException e) {
+ this.logger.log(Level.WARNING, "Could not parse file size in "
+ + "TorperfFiles configuration line '" + torperfFilesLine
+ + "'.");
+ return;
+ }
+
+ /* Download and append the .data file. */
+ String dataFileName = parts[3];
+ String sourceBaseUrl = torperfSources.get(sourceName);
+ String dataUrl = sourceBaseUrl + dataFileName;
+ String dataOutputFileName = sourceName + "-" + dataFileName;
+ File dataOutputFile = new File(torperfOutputDirectory,
+ dataOutputFileName);
+ boolean downloadedDataFile = this.downloadAndAppendFile(dataUrl,
+ dataOutputFile, true);
+
+ /* Download and append the .extradata file. */
+ String extradataFileName = parts[4];
+ String extradataUrl = sourceBaseUrl + extradataFileName;
+ String extradataOutputFileName = sourceName + "-" + extradataFileName;
+ File extradataOutputFile = new File(torperfOutputDirectory,
+ extradataOutputFileName);
+ boolean downloadedExtradataFile = this.downloadAndAppendFile(
+ extradataUrl, extradataOutputFile, false);
+
+ /* Merge both files into .tpf format. */
+ if (!downloadedDataFile && !downloadedExtradataFile) {
+ return;
+ }
+ String skipUntil = null;
+ if (this.lastMergedTimestamps.containsKey(dataOutputFileName)) {
+ skipUntil = this.lastMergedTimestamps.get(dataOutputFileName);
+ }
+ try {
+ skipUntil = this.mergeFiles(dataOutputFile, extradataOutputFile,
+ sourceName, fileSize, skipUntil);
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed merging " + dataOutputFile
+ + " and " + extradataOutputFile + ".", e);
+ }
+ if (skipUntil != null) {
+ this.lastMergedTimestamps.put(dataOutputFileName, skipUntil);
+ }
+ }
+
+ private boolean downloadAndAppendFile(String url, File outputFile,
+ boolean isDataFile) {
+
+ /* Read an existing output file to determine which line will be the
+ * first to append to it. */
+ String lastTimestampLine = null;
+ int linesAfterLastTimestampLine = 0;
+ if (outputFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ outputFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (isDataFile || line.contains(" LAUNCH")) {
+ lastTimestampLine = line;
+ linesAfterLastTimestampLine = 0;
+ } else {
+ linesAfterLastTimestampLine++;
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed reading '"
+ + outputFile.getAbsolutePath() + "' to determine the first "
+ + "line to append to it.", e);
+ return false;
+ }
+ }
+ try {
+ this.logger.fine("Downloading " + (isDataFile ? ".data" :
+ ".extradata") + " file from '" + url + "' and merging it into "
+ + "'" + outputFile.getAbsolutePath() + "'.");
+ URL u = new URL(url);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.connect();
+ BufferedReader br = new BufferedReader(new InputStreamReader(
+ huc.getInputStream()));
+ String line;
+ BufferedWriter bw = new BufferedWriter(new FileWriter(outputFile,
+ true));
+ boolean copyLines = lastTimestampLine == null;
+ while ((line = br.readLine()) != null) {
+ if (copyLines && linesAfterLastTimestampLine == 0) {
+ if (isDataFile || line.contains(" LAUNCH")) {
+ lastTimestampLine = line;
+ }
+ bw.write(line + "\n");
+ } else if (copyLines && linesAfterLastTimestampLine > 0) {
+ linesAfterLastTimestampLine--;
+ } else if (line.equals(lastTimestampLine)) {
+ copyLines = true;
+ }
+ }
+ bw.close();
+ br.close();
+ if (!copyLines) {
+ this.logger.warning("The last timestamp line in '"
+ + outputFile.getAbsolutePath() + "' is not contained in the "
+ + "new file downloaded from '" + url + "'. Cannot append "
+ + "new lines without possibly leaving a gap. Skipping.");
+ return false;
+ }
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Failed downloading and/or merging '"
+ + url + "'.", e);
+ return false;
+ }
+ if (lastTimestampLine == null) {
+ this.logger.warning("'" + outputFile.getAbsolutePath()
+ + "' doesn't contain any timestamp lines. Unable to check "
+ + "whether that file is stale or not.");
+ } else {
+ long lastTimestampMillis = -1L;
+ if (isDataFile) {
+ lastTimestampMillis = Long.parseLong(lastTimestampLine.substring(
+ 0, lastTimestampLine.indexOf(" "))) * 1000L;
+ } else {
+ lastTimestampMillis = Long.parseLong(lastTimestampLine.substring(
+ lastTimestampLine.indexOf(" LAUNCH=") + " LAUNCH=".length(),
+ lastTimestampLine.indexOf(".",
+ lastTimestampLine.indexOf(" LAUNCH=")))) * 1000L;
+ }
+ if (lastTimestampMillis < System.currentTimeMillis()
+ - 330L * 60L * 1000L) {
+ this.logger.warning("The last timestamp in '"
+ + outputFile.getAbsolutePath() + "' is more than 5:30 hours "
+ + "old: " + lastTimestampMillis);
+ }
+ }
+ return true;
+ }
+
+ private String mergeFiles(File dataFile, File extradataFile,
+ String source, int fileSize, String skipUntil) throws IOException {
+ SortedMap<String, String> config = new TreeMap<String, String>();
+ config.put("SOURCE", source);
+ config.put("FILESIZE", String.valueOf(fileSize));
+ if (!dataFile.exists() || !extradataFile.exists()) {
+ this.logger.warning("File " + dataFile.getAbsolutePath() + " or "
+ + extradataFile.getAbsolutePath() + " is missing.");
+ return null;
+ }
+ this.logger.fine("Merging " + dataFile.getAbsolutePath() + " and "
+ + extradataFile.getAbsolutePath() + " into .tpf format.");
+ BufferedReader brD = new BufferedReader(new FileReader(dataFile));
+ BufferedReader brE = new BufferedReader(new FileReader(extradataFile));
+ String lineD = brD.readLine();
+ String lineE = brE.readLine();
+ int d = 1;
+ int e = 1;
+ String maxDataComplete = null;
+ String maxUsedAt = null;
+ while (lineD != null) {
+
+ /* Parse .data line. Every valid .data line will go into the .tpf
+ * format, either with additional information from the .extradata
+ * file or without it. */
+ if (lineD.isEmpty()) {
+ this.logger.finer("Skipping empty line " + dataFile.getName()
+ + ":" + d++ + ".");
+ lineD = brD.readLine();
+ continue;
+ }
+ SortedMap<String, String> data = this.parseDataLine(lineD);
+ if (data == null) {
+ this.logger.finer("Skipping illegal line " + dataFile.getName()
+ + ":" + d++ + " '" + lineD + "'.");
+ lineD = brD.readLine();
+ continue;
+ }
+ String dataComplete = data.get("DATACOMPLETE");
+ double dataCompleteSeconds = Double.parseDouble(dataComplete);
+ if (skipUntil != null && dataComplete.compareTo(skipUntil) < 0) {
+ this.logger.finer("Skipping " + dataFile.getName() + ":"
+ + d++ + " which we already processed before.");
+ lineD = brD.readLine();
+ continue;
+ }
+ maxDataComplete = dataComplete;
+
+ /* Parse .extradata line if available and try to find the one that
+ * matches the .data line. */
+ SortedMap<String, String> extradata = null;
+ while (lineE != null) {
+ if (lineE.isEmpty()) {
+ this.logger.finer("Skipping " + extradataFile.getName() + ":"
+ + e++ + " which is empty.");
+ lineE = brE.readLine();
+ continue;
+ }
+ if (lineE.startsWith("BUILDTIMEOUT_SET ")) {
+ this.logger.finer("Skipping " + extradataFile.getName() + ":"
+ + e++ + " which is a BUILDTIMEOUT_SET line.");
+ lineE = brE.readLine();
+ continue;
+ } else if (lineE.startsWith("ok ")
+ || lineE.startsWith("error ")) {
+ this.logger.finer("Skipping " + extradataFile.getName() + ":"
+ + e++ + " which is in the old format.");
+ lineE = brE.readLine();
+ continue;
+ }
+ extradata = this.parseExtradataLine(lineE);
+ if (extradata == null) {
+ this.logger.finer("Skipping Illegal line "
+ + extradataFile.getName() + ":" + e++ + " '" + lineE
+ + "'.");
+ lineE = brE.readLine();
+ continue;
+ }
+ if (!extradata.containsKey("USED_AT")) {
+ this.logger.finer("Skipping " + extradataFile.getName() + ":"
+ + e++ + " which doesn't contain a USED_AT element.");
+ lineE = brE.readLine();
+ continue;
+ }
+ String usedAt = extradata.get("USED_AT");
+ double usedAtSeconds = Double.parseDouble(usedAt);
+ if (skipUntil != null && usedAt.compareTo(skipUntil) < 0) {
+ this.logger.finer("Skipping " + extradataFile.getName() + ":"
+ + e++ + " which we already processed before.");
+ lineE = brE.readLine();
+ continue;
+ }
+ maxUsedAt = usedAt;
+ if (Math.abs(usedAtSeconds - dataCompleteSeconds) <= 1.0) {
+ this.logger.fine("Merging " + extradataFile.getName() + ":"
+ + e++ + " into the current .data line.");
+ lineE = brE.readLine();
+ break;
+ } else if (usedAtSeconds > dataCompleteSeconds) {
+ this.logger.finer("Comparing " + extradataFile.getName()
+ + " to the next .data line.");
+ extradata = null;
+ break;
+ } else {
+ this.logger.finer("Skipping " + extradataFile.getName() + ":"
+ + e++ + " which is too old to be merged with "
+ + dataFile.getName() + ":" + d + ".");
+ lineE = brE.readLine();
+ continue;
+ }
+ }
+
+ /* Write output line to .tpf file. */
+ SortedMap<String, String> keysAndValues =
+ new TreeMap<String, String>();
+ if (extradata != null) {
+ keysAndValues.putAll(extradata);
+ }
+ keysAndValues.putAll(data);
+ keysAndValues.putAll(config);
+ this.logger.fine("Writing " + dataFile.getName() + ":" + d++ + ".");
+ lineD = brD.readLine();
+ try {
+ this.writeTpfLine(source, fileSize, keysAndValues);
+ } catch (IOException ex) {
+ this.logger.log(Level.WARNING, "Error writing output line. "
+ + "Aborting to merge " + dataFile.getName() + " and "
+ + extradataFile.getName() + ".", e);
+ break;
+ }
+ }
+ brD.close();
+ brE.close();
+ this.writeCachedTpfLines();
+ if (maxDataComplete == null) {
+ return maxUsedAt;
+ } else if (maxUsedAt == null) {
+ return maxDataComplete;
+ } else if (maxDataComplete.compareTo(maxUsedAt) > 0) {
+ return maxUsedAt;
+ } else {
+ return maxDataComplete;
+ }
+ }
+
+ private SortedMap<Integer, String> dataTimestamps;
+
+ private SortedMap<String, String> parseDataLine(String line) {
+ String[] parts = line.trim().split(" ");
+ if (line.length() == 0 || parts.length < 20) {
+ return null;
+ }
+ if (this.dataTimestamps == null) {
+ this.dataTimestamps = new TreeMap<Integer, String>();
+ this.dataTimestamps.put(0, "START");
+ this.dataTimestamps.put(2, "SOCKET");
+ this.dataTimestamps.put(4, "CONNECT");
+ this.dataTimestamps.put(6, "NEGOTIATE");
+ this.dataTimestamps.put(8, "REQUEST");
+ this.dataTimestamps.put(10, "RESPONSE");
+ this.dataTimestamps.put(12, "DATAREQUEST");
+ this.dataTimestamps.put(14, "DATARESPONSE");
+ this.dataTimestamps.put(16, "DATACOMPLETE");
+ this.dataTimestamps.put(21, "DATAPERC10");
+ this.dataTimestamps.put(23, "DATAPERC20");
+ this.dataTimestamps.put(25, "DATAPERC30");
+ this.dataTimestamps.put(27, "DATAPERC40");
+ this.dataTimestamps.put(29, "DATAPERC50");
+ this.dataTimestamps.put(31, "DATAPERC60");
+ this.dataTimestamps.put(33, "DATAPERC70");
+ this.dataTimestamps.put(35, "DATAPERC80");
+ this.dataTimestamps.put(37, "DATAPERC90");
+ }
+ SortedMap<String, String> data = new TreeMap<String, String>();
+ try {
+ for (Map.Entry<Integer, String> e : this.dataTimestamps.entrySet()) {
+ int i = e.getKey();
+ if (parts.length > i + 1) {
+ String key = e.getValue();
+ String value = String.format("%s.%02d", parts[i],
+ Integer.parseInt(parts[i + 1]) / 10000);
+ data.put(key, value);
+ }
+ }
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ data.put("WRITEBYTES", parts[18]);
+ data.put("READBYTES", parts[19]);
+ if (parts.length >= 21) {
+ data.put("DIDTIMEOUT", parts[20]);
+ }
+ return data;
+ }
+
+ private SortedMap<String, String> parseExtradataLine(String line) {
+ String[] parts = line.split(" ");
+ SortedMap<String, String> extradata = new TreeMap<String, String>();
+ String previousKey = null;
+ for (String part : parts) {
+ String[] keyAndValue = part.split("=", -1);
+ if (keyAndValue.length == 2) {
+ String key = keyAndValue[0];
+ previousKey = key;
+ String value = keyAndValue[1];
+ if (value.contains(".") && value.lastIndexOf(".")
+ == value.length() - 2) {
+ /* Make sure that all floats have two trailing digits. */
+ value += "0";
+ }
+ extradata.put(key, value);
+ } else if (keyAndValue.length == 1 && previousKey != null) {
+ String value = keyAndValue[0];
+ if (previousKey.equals("STREAM_FAIL_REASONS")
+ && (value.equals("MISC") || value.equals("EXITPOLICY")
+ || value.equals("RESOURCELIMIT")
+ || value.equals("RESOLVEFAILED"))) {
+ extradata.put(previousKey, extradata.get(previousKey) + ":"
+ + value);
+ } else {
+ return null;
+ }
+ } else {
+ return null;
+ }
+ }
+ return extradata;
+ }
+
+ private String cachedSource;
+
+ private int cachedFileSize;
+
+ private String cachedStartDate;
+
+ private SortedMap<String, String> cachedTpfLines;
+
+ private void writeTpfLine(String source, int fileSize,
+ SortedMap<String, String> keysAndValues) throws IOException {
+ StringBuilder sb = new StringBuilder();
+ int written = 0;
+ for (Map.Entry<String, String> keyAndValue :
+ keysAndValues.entrySet()) {
+ String key = keyAndValue.getKey();
+ String value = keyAndValue.getValue();
+ sb.append((written++ > 0 ? " " : "") + key + "=" + value);
+ }
+ String line = sb.toString();
+ String startString = keysAndValues.get("START");
+ long startMillis = Long.parseLong(startString.substring(0,
+ startString.indexOf("."))) * 1000L;
+ String startDate = dateFormat.format(startMillis);
+ if (this.cachedTpfLines == null || !source.equals(this.cachedSource)
+ || fileSize != this.cachedFileSize
+ || !startDate.equals(this.cachedStartDate)) {
+ this.writeCachedTpfLines();
+ this.readTpfLinesToCache(source, fileSize, startDate);
+ }
+ if (!this.cachedTpfLines.containsKey(startString)
+ || line.length() > this.cachedTpfLines.get(startString).length()) {
+ this.cachedTpfLines.put(startString, line);
+ }
+ }
+
+ private void readTpfLinesToCache(String source, int fileSize,
+ String startDate) throws IOException {
+ this.cachedTpfLines = new TreeMap<String, String>();
+ this.cachedSource = source;
+ this.cachedFileSize = fileSize;
+ this.cachedStartDate = startDate;
+ File tpfFile = new File(torperfOutputDirectory,
+ startDate.replaceAll("-", "/") + "/"
+ + source + "-" + String.valueOf(fileSize) + "-" + startDate
+ + ".tpf");
+ if (!tpfFile.exists()) {
+ return;
+ }
+ BufferedReader br = new BufferedReader(new FileReader(tpfFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("@type ")) {
+ continue;
+ }
+ if (line.contains("START=")) {
+ String startString = line.substring(line.indexOf("START=")
+ + "START=".length()).split(" ")[0];
+ this.cachedTpfLines.put(startString, line);
+ }
+ }
+ br.close();
+ }
+
+ private void writeCachedTpfLines() throws IOException {
+ if (this.cachedSource == null || this.cachedFileSize == 0
+ || this.cachedStartDate == null || this.cachedTpfLines == null) {
+ return;
+ }
+ File tarballFile = new File(torperfOutputDirectory,
+ this.cachedStartDate.replaceAll("-", "/")
+ + "/" + this.cachedSource + "-"
+ + String.valueOf(this.cachedFileSize) + "-"
+ + this.cachedStartDate + ".tpf");
+ File rsyncFile = new File("recent/torperf/" + tarballFile.getName());
+ File[] outputFiles = new File[] { tarballFile, rsyncFile };
+ for (File outputFile : outputFiles) {
+ outputFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(outputFile));
+ for (String line : this.cachedTpfLines.values()) {
+ bw.write("@type torperf 1.0\n");
+ bw.write(line + "\n");
+ }
+ bw.close();
+ }
+ this.cachedSource = null;
+ this.cachedFileSize = 0;
+ this.cachedStartDate = null;
+ this.cachedTpfLines = null;
+ }
+
+ /* Delete all files from the rsync directory that have not been modified
+ * in the last three days. */
+ public void cleanUpRsyncDirectory() {
+ long cutOffMillis = System.currentTimeMillis()
+ - 3L * 24L * 60L * 60L * 1000L;
+ Stack<File> allFiles = new Stack<File>();
+ allFiles.add(new File("recent/torperf"));
+ while (!allFiles.isEmpty()) {
+ File file = allFiles.pop();
+ if (file.isDirectory()) {
+ allFiles.addAll(Arrays.asList(file.listFiles()));
+ } else if (file.lastModified() < cutOffMillis) {
+ file.delete();
+ }
+ }
+ }
+}
+
diff --git a/src/org/torproject/collector/Main.java b/src/org/torproject/collector/Main.java
deleted file mode 100644
index 9c64696..0000000
--- a/src/org/torproject/collector/Main.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/* Copyright 2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector;
-
-import org.torproject.collector.bridgedescs.SanitizedBridgesWriter;
-import org.torproject.collector.exitlists.ExitListDownloader;
-import org.torproject.collector.index.CreateIndexJson;
-import org.torproject.collector.relaydescs.ArchiveWriter;
-import org.torproject.collector.torperf.TorperfDownloader;
-
-import java.lang.reflect.InvocationTargetException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.logging.Logger;
-
-/**
- * Main class for starting a CollecTor instance.
- * <br>
- * Run without arguments in order to read the usage information, i.e.
- * <br>
- * <code>java -jar collector.jar</code>
- */
-public class Main {
-
- private static Logger log = Logger.getLogger(Main.class.getName());
-
- /** All possible main classes.
- * If a new CollecTorMain class is available, just add it to this map.
- */
- private static final Map<String, Class> collecTorMains = new HashMap<>();
-
- static { // add a new main class here
- collecTorMains.put("bridgedescs", SanitizedBridgesWriter.class);
- collecTorMains.put("exitlists", ExitListDownloader.class);
- collecTorMains.put("updateindex", CreateIndexJson.class);
- collecTorMains.put("relaydescs", ArchiveWriter.class);
- collecTorMains.put("torperf", TorperfDownloader.class);
- }
-
- private static final String modules = collecTorMains.keySet().toString()
- .replace("[", "").replace("]", "").replaceAll(", ", "|");
-
- /**
- * One argument is necessary.
- * See class description {@link Main}.
- */
- public static void main(String[] args) {
- if (null == args || args.length != 1) {
- printUsageAndExit("CollecTor needs exactly one argument.");
- } else {
- invokeGivenMainAndExit(args[0]);
- }
- }
-
- private static void printUsageAndExit(String msg) {
- final String usage = "Usage:\njava -jar collector.jar "
- + "<" + modules + ">";
- System.out.println(msg + "\n" + usage);
- System.exit(0);
- }
-
- private static void invokeGivenMainAndExit(String mainId) {
- Class clazz = collecTorMains.get(mainId);
- if (null == clazz) {
- printUsageAndExit("Unknown argument: " + mainId);
- }
- invokeMainOnClassAndExit(clazz);
- }
-
- private static void invokeMainOnClassAndExit(Class clazz) {
- try {
- clazz.getMethod("main", new Class[] { String[].class })
- .invoke(null, (Object) new String[]{});
- System.exit(0);
- } catch (NoSuchMethodException | IllegalAccessException
- | InvocationTargetException e) {
- log.severe("Cannot invoke 'main' method on "
- + clazz.getName() + ". " + e);
- }
- }
-}
-
diff --git a/src/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java b/src/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java
deleted file mode 100644
index f683ea0..0000000
--- a/src/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/* Copyright 2010--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.bridgedescs;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class BridgeDescriptorParser {
-
- private SanitizedBridgesWriter sbw;
-
- private Logger logger;
-
- public BridgeDescriptorParser(SanitizedBridgesWriter sbw) {
- this.sbw = sbw;
- this.logger =
- Logger.getLogger(BridgeDescriptorParser.class.getName());
- }
-
- public void parse(byte[] allData, String dateTime) {
- try {
- BufferedReader br = new BufferedReader(new StringReader(
- new String(allData, "US-ASCII")));
- String line = br.readLine();
- if (line == null) {
- return;
- } else if (line.startsWith("router ")) {
- if (this.sbw != null) {
- this.sbw.sanitizeAndStoreServerDescriptor(allData);
- }
- } else if (line.startsWith("extra-info ")) {
- if (this.sbw != null) {
- this.sbw.sanitizeAndStoreExtraInfoDescriptor(allData);
- }
- } else {
- if (this.sbw != null) {
- this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime);
- }
- }
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not parse bridge descriptor.",
- e);
- return;
- }
- }
-}
-
diff --git a/src/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java b/src/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java
deleted file mode 100644
index 2d41d18..0000000
--- a/src/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/* Copyright 2010--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.bridgedescs;
-
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.StringReader;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Reads the half-hourly snapshots of bridge descriptors from Tonga.
- */
-public class BridgeSnapshotReader {
- public BridgeSnapshotReader(BridgeDescriptorParser bdp,
- File bridgeDirectoriesDir, File statsDirectory) {
-
- if (bdp == null || bridgeDirectoriesDir == null
- || statsDirectory == null) {
- throw new IllegalArgumentException();
- }
-
- Logger logger =
- Logger.getLogger(BridgeSnapshotReader.class.getName());
- SortedSet<String> parsed = new TreeSet<String>();
- File bdDir = bridgeDirectoriesDir;
- File pbdFile = new File(statsDirectory, "parsed-bridge-directories");
- boolean modified = false;
- if (bdDir.exists()) {
- if (pbdFile.exists()) {
- logger.fine("Reading file " + pbdFile.getAbsolutePath() + "...");
- try {
- BufferedReader br = new BufferedReader(new FileReader(pbdFile));
- String line = null;
- while ((line = br.readLine()) != null) {
- parsed.add(line);
- }
- br.close();
- logger.fine("Finished reading file "
- + pbdFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed reading file "
- + pbdFile.getAbsolutePath() + "!", e);
- return;
- }
- }
- logger.fine("Importing files in directory " + bridgeDirectoriesDir
- + "/...");
- Set<String> descriptorImportHistory = new HashSet<String>();
- int parsedFiles = 0;
- int skippedFiles = 0;
- int parsedStatuses = 0;
- int parsedServerDescriptors = 0;
- int skippedServerDescriptors = 0;
- int parsedExtraInfoDescriptors = 0;
- int skippedExtraInfoDescriptors = 0;
- Stack<File> filesInInputDir = new Stack<File>();
- filesInInputDir.add(bdDir);
- while (!filesInInputDir.isEmpty()) {
- File pop = filesInInputDir.pop();
- if (pop.isDirectory()) {
- for (File f : pop.listFiles()) {
- filesInInputDir.add(f);
- }
- } else if (!parsed.contains(pop.getName())) {
- try {
- FileInputStream in = new FileInputStream(pop);
- if (in.available() > 0) {
- TarArchiveInputStream tais = null;
- if (pop.getName().endsWith(".tar.gz")) {
- GzipCompressorInputStream gcis =
- new GzipCompressorInputStream(in);
- tais = new TarArchiveInputStream(gcis);
- } else if (pop.getName().endsWith(".tar")) {
- tais = new TarArchiveInputStream(in);
- } else {
- continue;
- }
- BufferedInputStream bis = new BufferedInputStream(tais);
- String fn = pop.getName();
- String dateTime = fn.substring(11, 21) + " "
- + fn.substring(22, 24) + ":" + fn.substring(24, 26)
- + ":" + fn.substring(26, 28);
- while ((tais.getNextTarEntry()) != null) {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- byte[] allData = baos.toByteArray();
- if (allData.length == 0) {
- continue;
- }
- String fileDigest = Hex.encodeHexString(DigestUtils.sha(
- allData));
- String ascii = new String(allData, "US-ASCII");
- BufferedReader br3 = new BufferedReader(new StringReader(
- ascii));
- String firstLine = null;
- while ((firstLine = br3.readLine()) != null) {
- if (firstLine.startsWith("@")) {
- continue;
- } else {
- break;
- }
- }
- if (firstLine.startsWith("published ")
- || firstLine.startsWith("flag-thresholds ")
- || firstLine.startsWith("r ")) {
- bdp.parse(allData, dateTime);
- parsedStatuses++;
- } else if (descriptorImportHistory.contains(fileDigest)) {
- /* Skip server descriptors or extra-info descriptors if
- * we parsed them before. */
- skippedFiles++;
- continue;
- } else {
- int start = -1;
- int sig = -1;
- int end = -1;
- String startToken = firstLine.startsWith("router ")
- ? "router " : "extra-info ";
- String sigToken = "\nrouter-signature\n";
- String endToken = "\n-----END SIGNATURE-----\n";
- while (end < ascii.length()) {
- start = ascii.indexOf(startToken, end);
- if (start < 0) {
- break;
- }
- sig = ascii.indexOf(sigToken, start);
- if (sig < 0) {
- break;
- }
- sig += sigToken.length();
- end = ascii.indexOf(endToken, sig);
- if (end < 0) {
- break;
- }
- end += endToken.length();
- byte[] descBytes = new byte[end - start];
- System.arraycopy(allData, start, descBytes, 0,
- end - start);
- String descriptorDigest = Hex.encodeHexString(
- DigestUtils.sha(descBytes));
- if (!descriptorImportHistory.contains(
- descriptorDigest)) {
- bdp.parse(descBytes, dateTime);
- descriptorImportHistory.add(descriptorDigest);
- if (firstLine.startsWith("router ")) {
- parsedServerDescriptors++;
- } else {
- parsedExtraInfoDescriptors++;
- }
- } else {
- if (firstLine.startsWith("router ")) {
- skippedServerDescriptors++;
- } else {
- skippedExtraInfoDescriptors++;
- }
- }
- }
- }
- descriptorImportHistory.add(fileDigest);
- parsedFiles++;
- }
- bis.close();
- }
- in.close();
-
- /* Let's give some memory back, or we'll run out of it. */
- System.gc();
-
- parsed.add(pop.getName());
- modified = true;
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not parse bridge snapshot "
- + pop.getName() + "!", e);
- continue;
- }
- }
- }
- logger.fine("Finished importing files in directory "
- + bridgeDirectoriesDir + "/. In total, we parsed "
- + parsedFiles + " files (skipped " + skippedFiles
- + ") containing " + parsedStatuses + " statuses, "
- + parsedServerDescriptors + " server descriptors (skipped "
- + skippedServerDescriptors + "), and "
- + parsedExtraInfoDescriptors + " extra-info descriptors "
- + "(skipped " + skippedExtraInfoDescriptors + ").");
- if (!parsed.isEmpty() && modified) {
- logger.fine("Writing file " + pbdFile.getAbsolutePath() + "...");
- try {
- pbdFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(pbdFile));
- for (String f : parsed) {
- bw.append(f + "\n");
- }
- bw.close();
- logger.fine("Finished writing file " + pbdFile.getAbsolutePath()
- + ".");
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed writing file "
- + pbdFile.getAbsolutePath() + "!", e);
- }
- }
- }
- }
-}
-
diff --git a/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
deleted file mode 100644
index 3214715..0000000
--- a/src/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
+++ /dev/null
@@ -1,1330 +0,0 @@
-/* Copyright 2010--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.bridgedescs;
-
-import org.torproject.collector.main.Configuration;
-import org.torproject.collector.main.LockFile;
-
-import org.apache.commons.codec.DecoderException;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.StringReader;
-import java.io.UnsupportedEncodingException;
-import java.security.GeneralSecurityException;
-import java.security.SecureRandom;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.Stack;
-import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Sanitizes bridge descriptors, i.e., removes all possibly sensitive
- * information from them, and writes them to a local directory structure.
- * During the sanitizing process, all information about the bridge
- * identity or IP address are removed or replaced. The goal is to keep the
- * sanitized bridge descriptors useful for statistical analysis while not
- * making it easier for an adversary to enumerate bridges.
- *
- * There are three types of bridge descriptors: bridge network statuses
- * (lists of all bridges at a given time), server descriptors (published
- * by the bridge to advertise their capabilities), and extra-info
- * descriptors (published by the bridge, mainly for statistical analysis).
- */
-public class SanitizedBridgesWriter extends Thread {
-
- public static void main(String[] args) {
-
- Logger logger = Logger.getLogger(
- SanitizedBridgesWriter.class.getName());
- logger.info("Starting bridge-descriptors module of CollecTor.");
-
- // Initialize configuration
- Configuration config = new Configuration();
-
- // Use lock file to avoid overlapping runs
- LockFile lf = new LockFile("bridge-descriptors");
- if (!lf.acquireLock()) {
- logger.severe("Warning: CollecTor is already running or has not exited "
- + "cleanly! Exiting!");
- System.exit(1);
- }
-
- // Sanitize bridge descriptors
- new SanitizedBridgesWriter(config).run();
-
- // Remove lock file
- lf.releaseLock();
-
- logger.info("Terminating bridge-descriptors module of CollecTor.");
- }
-
- private Configuration config;
-
- /**
- * Initializes this class.
- */
- public SanitizedBridgesWriter(Configuration config) {
- this.config = config;
- }
-
- /**
- * Logger for this class.
- */
- private Logger logger;
-
- private String rsyncCatString;
-
- private File bridgeDirectoriesDirectory;
-
- /**
- * Output directory for writing sanitized bridge descriptors.
- */
- private File sanitizedBridgesDirectory;
-
- private boolean replaceIPAddressesWithHashes;
-
- private boolean persistenceProblemWithSecrets;
-
- private SortedMap<String, byte[]> secretsForHashingIPAddresses;
-
- private String bridgeSanitizingCutOffTimestamp;
-
- private boolean haveWarnedAboutInterval;
-
- private File bridgeIpSecretsFile;
-
- private SecureRandom secureRandom;
-
- public void run() {
-
- File bridgeDirectoriesDirectory =
- new File(config.getBridgeSnapshotsDirectory());
- File sanitizedBridgesDirectory =
- new File(config.getSanitizedBridgesWriteDirectory());
- boolean replaceIPAddressesWithHashes =
- config.getReplaceIPAddressesWithHashes();
- long limitBridgeSanitizingInterval =
- config.getLimitBridgeDescriptorMappings();
- File statsDirectory = new File("stats");
-
- if (bridgeDirectoriesDirectory == null
- || sanitizedBridgesDirectory == null || statsDirectory == null) {
- throw new IllegalArgumentException();
- }
-
- /* Memorize argument values. */
- this.bridgeDirectoriesDirectory = bridgeDirectoriesDirectory;
- this.sanitizedBridgesDirectory = sanitizedBridgesDirectory;
- this.replaceIPAddressesWithHashes = replaceIPAddressesWithHashes;
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(
- SanitizedBridgesWriter.class.getName());
-
- SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
- "yyyy-MM-dd-HH-mm-ss");
- rsyncCatFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- this.rsyncCatString = rsyncCatFormat.format(
- System.currentTimeMillis());
-
- /* Initialize secure random number generator if we need it. */
- if (this.replaceIPAddressesWithHashes) {
- try {
- this.secureRandom = SecureRandom.getInstance("SHA1PRNG", "SUN");
- } catch (GeneralSecurityException e) {
- this.logger.log(Level.WARNING, "Could not initialize secure "
- + "random number generator! Not calculating any IP address "
- + "hashes in this execution!", e);
- this.persistenceProblemWithSecrets = true;
- }
- }
-
- /* Read hex-encoded secrets for replacing IP addresses with hashes
- * from disk. */
- this.secretsForHashingIPAddresses = new TreeMap<String, byte[]>();
- this.bridgeIpSecretsFile = new File(statsDirectory,
- "bridge-ip-secrets");
- if (this.bridgeIpSecretsFile.exists()) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- this.bridgeIpSecretsFile));
- String line;
- while ((line = br.readLine()) != null) {
- String[] parts = line.split(",");
- if ((line.length() != ("yyyy-MM,".length() + 31 * 2)
- && line.length() != ("yyyy-MM,".length() + 50 * 2))
- || parts.length != 2) {
- this.logger.warning("Invalid line in bridge-ip-secrets file "
- + "starting with '" + line.substring(0, 7) + "'! "
- + "Not calculating any IP address hashes in this "
- + "execution!");
- this.persistenceProblemWithSecrets = true;
- break;
- }
- String month = parts[0];
- byte[] secret = Hex.decodeHex(parts[1].toCharArray());
- this.secretsForHashingIPAddresses.put(month, secret);
- }
- br.close();
- if (!this.persistenceProblemWithSecrets) {
- this.logger.fine("Read "
- + this.secretsForHashingIPAddresses.size() + " secrets for "
- + "hashing bridge IP addresses.");
- }
- } catch (DecoderException e) {
- this.logger.log(Level.WARNING, "Failed to decode hex string in "
- + this.bridgeIpSecretsFile + "! Not calculating any IP "
- + "address hashes in this execution!", e);
- this.persistenceProblemWithSecrets = true;
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to read "
- + this.bridgeIpSecretsFile + "! Not calculating any IP "
- + "address hashes in this execution!", e);
- this.persistenceProblemWithSecrets = true;
- }
- }
-
- /* If we're configured to keep secrets only for a limited time, define
- * the cut-off day and time. */
- if (limitBridgeSanitizingInterval >= 0L) {
- SimpleDateFormat formatter = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
- this.bridgeSanitizingCutOffTimestamp = formatter.format(
- System.currentTimeMillis() - 24L * 60L * 60L * 1000L
- * limitBridgeSanitizingInterval);
- } else {
- this.bridgeSanitizingCutOffTimestamp = "1999-12-31 23:59:59";
- }
-
- // Prepare bridge descriptor parser
- BridgeDescriptorParser bdp = new BridgeDescriptorParser(this);
-
- // Import bridge descriptors
- new BridgeSnapshotReader(bdp, this.bridgeDirectoriesDirectory,
- statsDirectory);
-
- // Finish writing sanitized bridge descriptors to disk
- this.finishWriting();
-
- this.checkStaleDescriptors();
-
- this.cleanUpRsyncDirectory();
- }
-
- private String scrubOrAddress(String orAddress, byte[] fingerprintBytes,
- String published) throws IOException {
- if (!orAddress.contains(":")) {
- /* Malformed or-address or a line. */
- return null;
- }
- String addressPart = orAddress.substring(0,
- orAddress.lastIndexOf(":"));
- String portPart = orAddress.substring(orAddress.lastIndexOf(":") + 1);
- String scrubbedAddressPart = null;
- if (addressPart.startsWith("[")) {
- scrubbedAddressPart = this.scrubIpv6Address(addressPart,
- fingerprintBytes, published);
- } else {
- scrubbedAddressPart = this.scrubIpv4Address(addressPart,
- fingerprintBytes, published);
- }
- return (scrubbedAddressPart == null ? null :
- scrubbedAddressPart + ":" + portPart);
- }
-
- private String scrubIpv4Address(String address, byte[] fingerprintBytes,
- String published) throws IOException {
- if (this.replaceIPAddressesWithHashes) {
- if (this.persistenceProblemWithSecrets) {
- /* There's a persistence problem, so we shouldn't scrub more IP
- * addresses in this execution. */
- return null;
- }
- byte[] hashInput = new byte[4 + 20 + 31];
- String[] ipParts = address.split("\\.");
- for (int i = 0; i < 4; i++) {
- hashInput[i] = (byte) Integer.parseInt(ipParts[i]);
- }
- System.arraycopy(fingerprintBytes, 0, hashInput, 4, 20);
- String month = published.substring(0, "yyyy-MM".length());
- byte[] secret = this.getSecretForMonth(month);
- System.arraycopy(secret, 0, hashInput, 24, 31);
- byte[] hashOutput = DigestUtils.sha256(hashInput);
- String hashedAddress = "10."
- + (((int) hashOutput[0] + 256) % 256) + "."
- + (((int) hashOutput[1] + 256) % 256) + "."
- + (((int) hashOutput[2] + 256) % 256);
- return hashedAddress;
- } else {
- return "127.0.0.1";
- }
- }
-
- private String scrubIpv6Address(String address, byte[] fingerprintBytes,
- String published) throws IOException {
- StringBuilder sb = new StringBuilder("[fd9f:2e19:3bcf::");
- if (this.replaceIPAddressesWithHashes) {
- if (this.persistenceProblemWithSecrets) {
- /* There's a persistence problem, so we shouldn't scrub more IP
- * addresses in this execution. */
- return null;
- }
- byte[] hashInput = new byte[16 + 20 + 19];
- String[] doubleColonSeparatedParts = address.substring(1,
- address.length() - 1).split("::", -1);
- if (doubleColonSeparatedParts.length > 2) {
- /* Invalid IPv6 address. */
- return null;
- }
- List<String> hexParts = new ArrayList<String>();
- for (String doubleColonSeparatedPart : doubleColonSeparatedParts) {
- StringBuilder hexPart = new StringBuilder();
- String[] parts = doubleColonSeparatedPart.split(":", -1);
- if (parts.length < 1 || parts.length > 8) {
- /* Invalid IPv6 address. */
- return null;
- }
- for (int i = 0; i < parts.length; i++) {
- String part = parts[i];
- if (part.contains(".")) {
- String[] ipParts = part.split("\\.");
- byte[] ipv4Bytes = new byte[4];
- if (ipParts.length != 4) {
- /* Invalid IPv4 part in IPv6 address. */
- return null;
- }
- for (int m = 0; m < 4; m++) {
- ipv4Bytes[m] = (byte) Integer.parseInt(ipParts[m]);
- }
- hexPart.append(Hex.encodeHexString(ipv4Bytes));
- } else if (part.length() > 4) {
- /* Invalid IPv6 address. */
- return null;
- } else {
- for (int k = part.length(); k < 4; k++) {
- hexPart.append("0");
- }
- hexPart.append(part);
- }
- }
- hexParts.add(hexPart.toString());
- }
- StringBuilder hex = new StringBuilder();
- hex.append(hexParts.get(0));
- if (hexParts.size() == 2) {
- for (int i = 32 - hexParts.get(0).length()
- - hexParts.get(1).length(); i > 0; i--) {
- hex.append("0");
- }
- hex.append(hexParts.get(1));
- }
- byte[] ipBytes = null;
- try {
- ipBytes = Hex.decodeHex(hex.toString().toCharArray());
- } catch (DecoderException e) {
- /* TODO Invalid IPv6 address. */
- return null;
- }
- if (ipBytes.length != 16) {
- /* TODO Invalid IPv6 address. */
- return null;
- }
- System.arraycopy(ipBytes, 0, hashInput, 0, 16);
- System.arraycopy(fingerprintBytes, 0, hashInput, 16, 20);
- String month = published.substring(0, "yyyy-MM".length());
- byte[] secret = this.getSecretForMonth(month);
- System.arraycopy(secret, 31, hashInput, 36, 19);
- String hashOutput = DigestUtils.sha256Hex(hashInput);
- sb.append(hashOutput.substring(hashOutput.length() - 6,
- hashOutput.length() - 4));
- sb.append(":");
- sb.append(hashOutput.substring(hashOutput.length() - 4));
- }
- sb.append("]");
- return sb.toString();
- }
-
- private byte[] getSecretForMonth(String month) throws IOException {
- if (!this.secretsForHashingIPAddresses.containsKey(month)
- || this.secretsForHashingIPAddresses.get(month).length == 31) {
- byte[] secret = new byte[50];
- this.secureRandom.nextBytes(secret);
- if (this.secretsForHashingIPAddresses.containsKey(month)) {
- System.arraycopy(this.secretsForHashingIPAddresses.get(month), 0,
- secret, 0, 31);
- }
- if (month.compareTo(
- this.bridgeSanitizingCutOffTimestamp) < 0) {
- this.logger.warning("Generated a secret that we won't make "
- + "persistent, because it's outside our bridge descriptor "
- + "sanitizing interval.");
- } else {
- /* Append secret to file on disk immediately before using it, or
- * we might end with inconsistently sanitized bridges. */
- try {
- if (!this.bridgeIpSecretsFile.exists()) {
- this.bridgeIpSecretsFile.getParentFile().mkdirs();
- }
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.bridgeIpSecretsFile,
- this.bridgeIpSecretsFile.exists()));
- bw.write(month + "," + Hex.encodeHexString(secret) + "\n");
- bw.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not store new secret "
- + "to disk! Not calculating any IP address hashes in "
- + "this execution!", e);
- this.persistenceProblemWithSecrets = true;
- throw new IOException(e);
- }
- }
- this.secretsForHashingIPAddresses.put(month, secret);
- }
- return this.secretsForHashingIPAddresses.get(month);
- }
-
- private String maxNetworkStatusPublishedTime = "1970-01-01 00:00:00";
-
- /**
- * Sanitizes a network status and writes it to disk.
- */
- public void sanitizeAndStoreNetworkStatus(byte[] data,
- String publicationTime) {
-
- if (this.persistenceProblemWithSecrets) {
- /* There's a persistence problem, so we shouldn't scrub more IP
- * addresses in this execution. */
- return;
- }
-
- if (publicationTime.compareTo(maxNetworkStatusPublishedTime) > 0) {
- maxNetworkStatusPublishedTime = publicationTime;
- }
-
- if (this.bridgeSanitizingCutOffTimestamp
- .compareTo(publicationTime) > 0) {
- this.logger.log(!this.haveWarnedAboutInterval ? Level.WARNING
- : Level.FINE, "Sanitizing and storing network status with "
- + "publication time outside our descriptor sanitizing "
- + "interval.");
- this.haveWarnedAboutInterval = true;
- }
-
- /* Parse the given network status line by line. */
- StringBuilder header = new StringBuilder();
- SortedMap<String, String> scrubbedLines =
- new TreeMap<String, String>();
- try {
- StringBuilder scrubbed = new StringBuilder();
- BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
- String line = null;
- String mostRecentDescPublished = null;
- byte[] fingerprintBytes = null;
- String descPublicationTime = null;
- String hashedBridgeIdentityHex = null;
- while ((line = br.readLine()) != null) {
-
- /* Use publication time from "published" line instead of the
- * file's last-modified time. Don't copy over the line, because
- * we're going to write a "published" line below. */
- if (line.startsWith("published ")) {
- publicationTime = line.substring("published ".length());
-
- /* Additional header lines don't have to be cleaned up. */
- } else if (line.startsWith("flag-thresholds ")) {
- header.append(line + "\n");
-
- /* r lines contain sensitive information that needs to be removed
- * or replaced. */
- } else if (line.startsWith("r ")) {
-
- /* Clear buffer from previously scrubbed lines. */
- if (scrubbed.length() > 0) {
- String scrubbedLine = scrubbed.toString();
- scrubbedLines.put(hashedBridgeIdentityHex, scrubbedLine);
- scrubbed = new StringBuilder();
- }
-
- /* Parse the relevant parts of this r line. */
- String[] parts = line.split(" ");
- String nickname = parts[1];
- fingerprintBytes = Base64.decodeBase64(parts[2] + "==");
- String descriptorIdentifier = parts[3];
- descPublicationTime = parts[4] + " " + parts[5];
- String address = parts[6];
- String orPort = parts[7];
- String dirPort = parts[8];
-
- /* Determine most recent descriptor publication time. */
- if (descPublicationTime.compareTo(publicationTime) <= 0
- && (mostRecentDescPublished == null
- || descPublicationTime.compareTo(
- mostRecentDescPublished) > 0)) {
- mostRecentDescPublished = descPublicationTime;
- }
-
- /* Write scrubbed r line to buffer. */
- byte[] hashedBridgeIdentity = DigestUtils.sha(fingerprintBytes);
- String hashedBridgeIdentityBase64 = Base64.encodeBase64String(
- hashedBridgeIdentity).substring(0, 27);
- hashedBridgeIdentityHex = Hex.encodeHexString(
- hashedBridgeIdentity);
- String hashedDescriptorIdentifier = Base64.encodeBase64String(
- DigestUtils.sha(Base64.decodeBase64(descriptorIdentifier
- + "=="))).substring(0, 27);
- String scrubbedAddress = scrubIpv4Address(address,
- fingerprintBytes,
- descPublicationTime);
- scrubbed.append("r " + nickname + " "
- + hashedBridgeIdentityBase64 + " "
- + hashedDescriptorIdentifier + " " + descPublicationTime
- + " " + scrubbedAddress + " " + orPort + " " + dirPort
- + "\n");
-
- /* Sanitize any addresses in a lines using the fingerprint and
- * descriptor publication time from the previous r line. */
- } else if (line.startsWith("a ")) {
- String scrubbedOrAddress = scrubOrAddress(
- line.substring("a ".length()), fingerprintBytes,
- descPublicationTime);
- if (scrubbedOrAddress != null) {
- scrubbed.append("a " + scrubbedOrAddress + "\n");
- } else {
- this.logger.warning("Invalid address in line '" + line
- + "' in bridge network status. Skipping line!");
- }
-
- /* Nothing special about s, w, and p lines; just copy them. */
- } else if (line.startsWith("s ") || line.equals("s")
- || line.startsWith("w ") || line.equals("w")
- || line.startsWith("p ") || line.equals("p")) {
- scrubbed.append(line + "\n");
-
- /* There should be nothing else but r, w, p, and s lines in the
- * network status. If there is, we should probably learn before
- * writing anything to the sanitized descriptors. */
- } else {
- this.logger.fine("Unknown line '" + line + "' in bridge "
- + "network status. Not writing to disk!");
- return;
- }
- }
- br.close();
- if (scrubbed.length() > 0) {
- String scrubbedLine = scrubbed.toString();
- scrubbedLines.put(hashedBridgeIdentityHex, scrubbedLine);
- scrubbed = new StringBuilder();
- }
-
- /* Check if we can tell from the descriptor publication times
- * whether this status is possibly stale. */
- SimpleDateFormat formatter = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (formatter.parse(publicationTime).getTime()
- - formatter.parse(mostRecentDescPublished).getTime()
- > 60L * 60L * 1000L) {
- this.logger.warning("The most recent descriptor in the bridge "
- + "network status published at " + publicationTime + " was "
- + "published at " + mostRecentDescPublished + " which is "
- + "more than 1 hour before the status. This is a sign for "
- + "the status being stale. Please check!");
- }
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Could not parse timestamp in "
- + "bridge network status.", e);
- return;
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not parse bridge network "
- + "status.", e);
- return;
- }
-
- /* Write the sanitized network status to disk. */
- try {
- String syear = publicationTime.substring(0, 4);
- String smonth = publicationTime.substring(5, 7);
- String sday = publicationTime.substring(8, 10);
- String stime = publicationTime.substring(11, 13)
- + publicationTime.substring(14, 16)
- + publicationTime.substring(17, 19);
- File tarballFile = new File(
- this.sanitizedBridgesDirectory.getAbsolutePath() + "/" + syear
- + "/" + smonth + "/statuses/" + sday + "/" + syear + smonth
- + sday + "-" + stime + "-"
- + "4A0CCD2DDC7995083D73F5D667100C8A5831F16D");
- File rsyncFile = new File("recent/bridge-descriptors/statuses/"
- + tarballFile.getName());
- File[] outputFiles = new File[] { tarballFile, rsyncFile };
- for (File outputFile : outputFiles) {
- outputFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- outputFile));
- bw.write("@type bridge-network-status 1.0\n");
- bw.write("published " + publicationTime + "\n");
- bw.write(header.toString());
- for (String scrubbed : scrubbedLines.values()) {
- bw.write(scrubbed);
- }
- bw.close();
- }
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not write sanitized bridge "
- + "network status to disk.", e);
- return;
- }
- }
-
- private String maxServerDescriptorPublishedTime = "1970-01-01 00:00:00";
-
- /**
- * Sanitizes a bridge server descriptor and writes it to disk.
- */
- public void sanitizeAndStoreServerDescriptor(byte[] data) {
-
- if (this.persistenceProblemWithSecrets) {
- /* There's a persistence problem, so we shouldn't scrub more IP
- * addresses in this execution. */
- return;
- }
-
- /* Parse descriptor to generate a sanitized version. */
- String scrubbedDesc = null;
- String published = null;
- String masterKeyEd25519FromIdentityEd25519 = null;
- try {
- BufferedReader br = new BufferedReader(new StringReader(
- new String(data, "US-ASCII")));
- StringBuilder scrubbed = new StringBuilder();
- String line = null;
- String hashedBridgeIdentity = null;
- String address = null;
- String routerLine = null;
- String scrubbedAddress = null;
- String masterKeyEd25519 = null;
- List<String> orAddresses = null;
- List<String> scrubbedOrAddresses = null;
- boolean skipCrypto = false;
- while ((line = br.readLine()) != null) {
-
- /* Skip all crypto parts that might be used to derive the bridge's
- * identity fingerprint. */
- if (skipCrypto && !line.startsWith("-----END ")) {
- continue;
-
- /* Store the router line for later processing, because we may need
- * the bridge identity fingerprint for replacing the IP address in
- * the scrubbed version. */
- } else if (line.startsWith("router ")) {
- address = line.split(" ")[2];
- routerLine = line;
-
- /* Store or-address parts in a list and sanitize them when we have
- * read the fingerprint. */
- } else if (line.startsWith("or-address ")) {
- if (orAddresses == null) {
- orAddresses = new ArrayList<String>();
- }
- orAddresses.add(line.substring("or-address ".length()));
-
- /* Parse the publication time to see if we're still inside the
- * sanitizing interval. */
- } else if (line.startsWith("published ")) {
- published = line.substring("published ".length());
- if (published.compareTo(maxServerDescriptorPublishedTime) > 0) {
- maxServerDescriptorPublishedTime = published;
- }
- if (this.bridgeSanitizingCutOffTimestamp
- .compareTo(published) > 0) {
- this.logger.log(!this.haveWarnedAboutInterval
- ? Level.WARNING : Level.FINE, "Sanitizing and storing "
- + "server descriptor with publication time outside our "
- + "descriptor sanitizing interval.");
- this.haveWarnedAboutInterval = true;
- }
- scrubbed.append(line + "\n");
-
- /* Parse the fingerprint to determine the hashed bridge
- * identity. */
- } else if (line.startsWith("opt fingerprint ")
- || line.startsWith("fingerprint ")) {
- String fingerprint = line.substring(line.startsWith("opt ")
- ? "opt fingerprint".length() : "fingerprint".length())
- .replaceAll(" ", "").toLowerCase();
- byte[] fingerprintBytes = Hex.decodeHex(
- fingerprint.toCharArray());
- hashedBridgeIdentity = DigestUtils.shaHex(fingerprintBytes)
- .toLowerCase();
- try {
- scrubbedAddress = scrubIpv4Address(address, fingerprintBytes,
- published);
- if (orAddresses != null) {
- scrubbedOrAddresses = new ArrayList<String>();
- for (String orAddress : orAddresses) {
- String scrubbedOrAddress = scrubOrAddress(orAddress,
- fingerprintBytes, published);
- if (scrubbedOrAddress != null) {
- scrubbedOrAddresses.add(scrubbedOrAddress);
- } else {
- this.logger.warning("Invalid address in line "
- + "'or-address " + orAddress + "' in bridge server "
- + "descriptor. Skipping line!");
- }
- }
- }
- } catch (IOException e) {
- /* There's a persistence problem, so we shouldn't scrub more
- * IP addresses in this execution. */
- this.persistenceProblemWithSecrets = true;
- return;
- }
- scrubbed.append((line.startsWith("opt ") ? "opt " : "")
- + "fingerprint");
- for (int i = 0; i < hashedBridgeIdentity.length() / 4; i++) {
- scrubbed.append(" " + hashedBridgeIdentity.substring(4 * i,
- 4 * (i + 1)).toUpperCase());
- }
- scrubbed.append("\n");
-
- /* Replace the contact line (if present) with a generic one. */
- } else if (line.startsWith("contact ")) {
- scrubbed.append("contact somebody\n");
-
- /* When we reach the signature, we're done. Write the sanitized
- * descriptor to disk below. */
- } else if (line.startsWith("router-signature")) {
- String[] routerLineParts = routerLine.split(" ");
- scrubbedDesc = "router " + routerLineParts[1] + " "
- + scrubbedAddress + " " + routerLineParts[3] + " "
- + routerLineParts[4] + " " + routerLineParts[5] + "\n";
- if (scrubbedOrAddresses != null) {
- for (String scrubbedOrAddress : scrubbedOrAddresses) {
- scrubbedDesc = scrubbedDesc += "or-address "
- + scrubbedOrAddress + "\n";
- }
- }
- scrubbedDesc += scrubbed.toString();
- break;
-
- /* Replace extra-info digest with the hashed digest of the
- * non-scrubbed descriptor. */
- } else if (line.startsWith("opt extra-info-digest ")
- || line.startsWith("extra-info-digest ")) {
- String[] parts = line.split(" ");
- if (line.startsWith("opt ")) {
- scrubbed.append("opt ");
- parts = line.substring(4).split(" ");
- }
- scrubbed.append("extra-info-digest " + DigestUtils.shaHex(
- Hex.decodeHex(parts[1].toCharArray())).toUpperCase());
- if (parts.length > 2) {
- scrubbed.append(" " + Base64.encodeBase64String(
- DigestUtils.sha256(Base64.decodeBase64(parts[2])))
- .replaceAll("=", ""));
- }
- scrubbed.append("\n");
-
- /* Possibly sanitize reject lines if they contain the bridge's own
- * IP address. */
- } else if (line.startsWith("reject ")) {
- if (address != null && line.startsWith("reject " + address)) {
- scrubbed.append("reject " + scrubbedAddress
- + line.substring("reject ".length() + address.length())
- + "\n");
- } else {
- scrubbed.append(line + "\n");
- }
-
- /* Extract master-key-ed25519 from identity-ed25519. */
- } else if (line.equals("identity-ed25519")) {
- StringBuilder sb = new StringBuilder();
- while ((line = br.readLine()) != null
- && !line.equals("-----END ED25519 CERT-----")) {
- if (line.equals("-----BEGIN ED25519 CERT-----")) {
- continue;
- }
- sb.append(line);
- }
- masterKeyEd25519FromIdentityEd25519 =
- this.parseMasterKeyEd25519FromIdentityEd25519(
- sb.toString());
- String sha256MasterKeyEd25519 = Base64.encodeBase64String(
- DigestUtils.sha256(Base64.decodeBase64(
- masterKeyEd25519FromIdentityEd25519 + "=")))
- .replaceAll("=", "");
- scrubbed.append("master-key-ed25519 " + sha256MasterKeyEd25519
- + "\n");
- if (masterKeyEd25519 != null && !masterKeyEd25519.equals(
- masterKeyEd25519FromIdentityEd25519)) {
- this.logger.warning("Mismatch between identity-ed25519 and "
- + "master-key-ed25519. Skipping.");
- return;
- }
-
- /* Verify that identity-ed25519 and master-key-ed25519 match. */
- } else if (line.startsWith("master-key-ed25519 ")) {
- masterKeyEd25519 = line.substring(line.indexOf(" ") + 1);
- if (masterKeyEd25519FromIdentityEd25519 != null
- && !masterKeyEd25519FromIdentityEd25519.equals(
- masterKeyEd25519)) {
- this.logger.warning("Mismatch between identity-ed25519 and "
- + "master-key-ed25519. Skipping.");
- return;
- }
-
- /* Write the following lines unmodified to the sanitized
- * descriptor. */
- } else if (line.startsWith("accept ")
- || line.startsWith("platform ")
- || line.startsWith("opt protocols ")
- || line.startsWith("protocols ")
- || line.startsWith("uptime ")
- || line.startsWith("bandwidth ")
- || line.startsWith("opt hibernating ")
- || line.startsWith("hibernating ")
- || line.startsWith("ntor-onion-key ")
- || line.equals("opt hidden-service-dir")
- || line.equals("hidden-service-dir")
- || line.equals("opt caches-extra-info")
- || line.equals("caches-extra-info")
- || line.equals("opt allow-single-hop-exits")
- || line.equals("allow-single-hop-exits")
- || line.startsWith("ipv6-policy ")
- || line.equals("tunnelled-dir-server")) {
- scrubbed.append(line + "\n");
-
- /* Replace node fingerprints in the family line with their hashes
- * and leave nicknames unchanged. */
- } else if (line.startsWith("family ")) {
- StringBuilder familyLine = new StringBuilder("family");
- for (String s : line.substring(7).split(" ")) {
- if (s.startsWith("$")) {
- familyLine.append(" $" + DigestUtils.shaHex(Hex.decodeHex(
- s.substring(1).toCharArray())).toUpperCase());
- } else {
- familyLine.append(" " + s);
- }
- }
- scrubbed.append(familyLine.toString() + "\n");
-
- /* Skip the purpose line that the bridge authority adds to its
- * cached-descriptors file. */
- } else if (line.startsWith("@purpose ")) {
- continue;
-
- /* Skip all crypto parts that might leak the bridge's identity
- * fingerprint. */
- } else if (line.startsWith("-----BEGIN ")
- || line.equals("onion-key") || line.equals("signing-key")
- || line.equals("onion-key-crosscert")
- || line.startsWith("ntor-onion-key-crosscert ")) {
- skipCrypto = true;
-
- /* Stop skipping lines when the crypto parts are over. */
- } else if (line.startsWith("-----END ")) {
- skipCrypto = false;
-
- /* Skip the ed25519 signature; we'll include a SHA256 digest of
- * the SHA256 descriptor digest in router-digest-sha256. */
- } else if (line.startsWith("router-sig-ed25519 ")) {
- continue;
-
- /* If we encounter an unrecognized line, stop parsing and print
- * out a warning. We might have overlooked sensitive information
- * that we need to remove or replace for the sanitized descriptor
- * version. */
- } else {
- this.logger.warning("Unrecognized line '" + line
- + "'. Skipping.");
- return;
- }
- }
- br.close();
- } catch (Exception e) {
- this.logger.log(Level.WARNING, "Could not parse server "
- + "descriptor.", e);
- return;
- }
-
- /* Determine filename of sanitized server descriptor. */
- String descriptorDigest = null;
- try {
- String ascii = new String(data, "US-ASCII");
- String startToken = "router ";
- String sigToken = "\nrouter-signature\n";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- if (start >= 0 && sig >= 0 && sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- descriptorDigest = DigestUtils.shaHex(DigestUtils.sha(forDigest));
- }
- } catch (UnsupportedEncodingException e) {
- /* Handle below. */
- }
- if (descriptorDigest == null) {
- this.logger.log(Level.WARNING, "Could not calculate server "
- + "descriptor digest.");
- return;
- }
- String descriptorDigestSha256Base64 = null;
- if (masterKeyEd25519FromIdentityEd25519 != null) {
- try {
- String ascii = new String(data, "US-ASCII");
- String startToken = "router ";
- String sigToken = "\n-----END SIGNATURE-----\n";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- if (start >= 0 && sig >= 0 && sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- descriptorDigestSha256Base64 = Base64.encodeBase64String(
- DigestUtils.sha256(DigestUtils.sha256(forDigest)))
- .replaceAll("=", "");
- }
- } catch (UnsupportedEncodingException e) {
- /* Handle below. */
- }
- if (descriptorDigestSha256Base64 == null) {
- this.logger.log(Level.WARNING, "Could not calculate server "
- + "descriptor SHA256 digest.");
- return;
- }
- }
- String dyear = published.substring(0, 4);
- String dmonth = published.substring(5, 7);
- File tarballFile = new File(
- this.sanitizedBridgesDirectory.getAbsolutePath() + "/"
- + dyear + "/" + dmonth + "/server-descriptors/"
- + "/" + descriptorDigest.charAt(0) + "/"
- + descriptorDigest.charAt(1) + "/"
- + descriptorDigest);
- File rsyncCatFile = new File("recent/bridge-descriptors/"
- + "server-descriptors/" + this.rsyncCatString
- + "-server-descriptors.tmp");
- File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
- boolean[] append = new boolean[] { false, true };
- try {
- for (int i = 0; i < outputFiles.length; i++) {
- File outputFile = outputFiles[i];
- boolean appendToFile = append[i];
- if (outputFile.exists() && !appendToFile) {
- /* We already stored this descriptor to disk before, so let's
- * not store it yet another time. */
- break;
- }
- outputFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- outputFile, appendToFile));
- bw.write("@type bridge-server-descriptor 1.1\n");
- bw.write(scrubbedDesc);
- if (descriptorDigestSha256Base64 != null) {
- bw.write("router-digest-sha256 " + descriptorDigestSha256Base64
- + "\n");
- }
- bw.write("router-digest " + descriptorDigest.toUpperCase()
- + "\n");
- bw.close();
- }
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not write sanitized server "
- + "descriptor to disk.", e);
- return;
- }
- }
-
- private String parseMasterKeyEd25519FromIdentityEd25519(
- String identityEd25519Base64) {
- byte[] identityEd25519 = Base64.decodeBase64(identityEd25519Base64);
- if (identityEd25519.length < 40) {
- this.logger.warning("Invalid length of identity-ed25519 (in "
- + "bytes): " + identityEd25519.length);
- } else if (identityEd25519[0] != 0x01) {
- this.logger.warning("Unknown version in identity-ed25519: "
- + identityEd25519[0]);
- } else if (identityEd25519[1] != 0x04) {
- this.logger.warning("Unknown cert type in identity-ed25519: "
- + identityEd25519[1]);
- } else if (identityEd25519[6] != 0x01) {
- this.logger.warning("Unknown certified key type in "
- + "identity-ed25519: " + identityEd25519[1]);
- } else if (identityEd25519[39] == 0x00) {
- this.logger.warning("No extensions in identity-ed25519 (which "
- + "would contain the encoded master-key-ed25519): "
- + identityEd25519[39]);
- } else {
- int extensionStart = 40;
- for (int i = 0; i < (int) identityEd25519[39]; i++) {
- if (identityEd25519.length < extensionStart + 4) {
- this.logger.warning("Invalid extension with id " + i
- + " in identity-ed25519.");
- break;
- }
- int extensionLength = identityEd25519[extensionStart];
- extensionLength <<= 8;
- extensionLength += identityEd25519[extensionStart + 1];
- int extensionType = identityEd25519[extensionStart + 2];
- if (extensionLength == 32 && extensionType == 4) {
- if (identityEd25519.length < extensionStart + 4 + 32) {
- this.logger.warning("Invalid extension with id " + i
- + " in identity-ed25519.");
- break;
- }
- byte[] masterKeyEd25519 = new byte[32];
- System.arraycopy(identityEd25519, extensionStart + 4,
- masterKeyEd25519, 0, masterKeyEd25519.length);
- String masterKeyEd25519Base64 = Base64.encodeBase64String(
- masterKeyEd25519);
- String masterKeyEd25519Base64NoTrailingEqualSigns =
- masterKeyEd25519Base64.replaceAll("=", "");
- return masterKeyEd25519Base64NoTrailingEqualSigns;
- }
- extensionStart += 4 + extensionLength;
- }
- }
- this.logger.warning("Unable to locate master-key-ed25519 in "
- + "identity-ed25519.");
- return null;
- }
-
- private String maxExtraInfoDescriptorPublishedTime =
- "1970-01-01 00:00:00";
-
- /**
- * Sanitizes an extra-info descriptor and writes it to disk.
- */
- public void sanitizeAndStoreExtraInfoDescriptor(byte[] data) {
-
- /* Parse descriptor to generate a sanitized version. */
- String scrubbedDesc = null;
- String published = null;
- String masterKeyEd25519FromIdentityEd25519 = null;
- try {
- BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
- String line = null;
- StringBuilder scrubbed = null;
- String hashedBridgeIdentity = null;
- String masterKeyEd25519 = null;
- while ((line = br.readLine()) != null) {
-
- /* Parse bridge identity from extra-info line and replace it with
- * its hash in the sanitized descriptor. */
- String[] parts = line.split(" ");
- if (line.startsWith("extra-info ")) {
- hashedBridgeIdentity = DigestUtils.shaHex(Hex.decodeHex(
- parts[2].toCharArray())).toLowerCase();
- scrubbed = new StringBuilder("extra-info " + parts[1] + " "
- + hashedBridgeIdentity.toUpperCase() + "\n");
-
- /* Parse the publication time to determine the file name. */
- } else if (line.startsWith("published ")) {
- scrubbed.append(line + "\n");
- published = line.substring("published ".length());
- if (published.compareTo(maxExtraInfoDescriptorPublishedTime)
- > 0) {
- maxExtraInfoDescriptorPublishedTime = published;
- }
-
- /* Remove everything from transport lines except the transport
- * name. */
- } else if (line.startsWith("transport ")) {
- if (parts.length < 3) {
- this.logger.fine("Illegal line in extra-info descriptor: '"
- + line + "'. Skipping descriptor.");
- return;
- }
- scrubbed.append("transport " + parts[1] + "\n");
-
- /* Skip transport-info lines entirely. */
- } else if (line.startsWith("transport-info ")) {
-
- /* Extract master-key-ed25519 from identity-ed25519. */
- } else if (line.equals("identity-ed25519")) {
- StringBuilder sb = new StringBuilder();
- while ((line = br.readLine()) != null
- && !line.equals("-----END ED25519 CERT-----")) {
- if (line.equals("-----BEGIN ED25519 CERT-----")) {
- continue;
- }
- sb.append(line);
- }
- masterKeyEd25519FromIdentityEd25519 =
- this.parseMasterKeyEd25519FromIdentityEd25519(
- sb.toString());
- String sha256MasterKeyEd25519 = Base64.encodeBase64String(
- DigestUtils.sha256(Base64.decodeBase64(
- masterKeyEd25519FromIdentityEd25519 + "=")))
- .replaceAll("=", "");
- scrubbed.append("master-key-ed25519 " + sha256MasterKeyEd25519
- + "\n");
- if (masterKeyEd25519 != null && !masterKeyEd25519.equals(
- masterKeyEd25519FromIdentityEd25519)) {
- this.logger.warning("Mismatch between identity-ed25519 and "
- + "master-key-ed25519. Skipping.");
- return;
- }
-
- /* Verify that identity-ed25519 and master-key-ed25519 match. */
- } else if (line.startsWith("master-key-ed25519 ")) {
- masterKeyEd25519 = line.substring(line.indexOf(" ") + 1);
- if (masterKeyEd25519FromIdentityEd25519 != null
- && !masterKeyEd25519FromIdentityEd25519.equals(
- masterKeyEd25519)) {
- this.logger.warning("Mismatch between identity-ed25519 and "
- + "master-key-ed25519. Skipping.");
- return;
- }
-
- /* Write the following lines unmodified to the sanitized
- * descriptor. */
- } else if (line.startsWith("write-history ")
- || line.startsWith("read-history ")
- || line.startsWith("geoip-start-time ")
- || line.startsWith("geoip-client-origins ")
- || line.startsWith("geoip-db-digest ")
- || line.startsWith("geoip6-db-digest ")
- || line.startsWith("conn-bi-direct ")
- || line.startsWith("bridge-")
- || line.startsWith("dirreq-")
- || line.startsWith("cell-")
- || line.startsWith("entry-")
- || line.startsWith("exit-")) {
- scrubbed.append(line + "\n");
-
- /* When we reach the signature, we're done. Write the sanitized
- * descriptor to disk below. */
- } else if (line.startsWith("router-signature")) {
- scrubbedDesc = scrubbed.toString();
- break;
-
- /* Skip the ed25519 signature; we'll include a SHA256 digest of
- * the SHA256 descriptor digest in router-digest-sha256. */
- } else if (line.startsWith("router-sig-ed25519 ")) {
- continue;
-
- /* If we encounter an unrecognized line, stop parsing and print
- * out a warning. We might have overlooked sensitive information
- * that we need to remove or replace for the sanitized descriptor
- * version. */
- } else {
- this.logger.warning("Unrecognized line '" + line
- + "'. Skipping.");
- return;
- }
- }
- br.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not parse extra-info "
- + "descriptor.", e);
- return;
- } catch (DecoderException e) {
- this.logger.log(Level.WARNING, "Could not parse extra-info "
- + "descriptor.", e);
- return;
- }
-
- /* Determine filename of sanitized extra-info descriptor. */
- String descriptorDigest = null;
- try {
- String ascii = new String(data, "US-ASCII");
- String startToken = "extra-info ";
- String sigToken = "\nrouter-signature\n";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- if (start >= 0 && sig >= 0 && sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- descriptorDigest = DigestUtils.shaHex(DigestUtils.sha(forDigest));
- }
- } catch (UnsupportedEncodingException e) {
- /* Handle below. */
- }
- if (descriptorDigest == null) {
- this.logger.log(Level.WARNING, "Could not calculate extra-info "
- + "descriptor digest.");
- return;
- }
- String descriptorDigestSha256Base64 = null;
- if (masterKeyEd25519FromIdentityEd25519 != null) {
- try {
- String ascii = new String(data, "US-ASCII");
- String startToken = "extra-info ";
- String sigToken = "\n-----END SIGNATURE-----\n";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- if (start >= 0 && sig >= 0 && sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- descriptorDigestSha256Base64 = Base64.encodeBase64String(
- DigestUtils.sha256(DigestUtils.sha256(forDigest)))
- .replaceAll("=", "");
- }
- } catch (UnsupportedEncodingException e) {
- /* Handle below. */
- }
- if (descriptorDigestSha256Base64 == null) {
- this.logger.log(Level.WARNING, "Could not calculate extra-info "
- + "descriptor SHA256 digest.");
- return;
- }
- }
- String dyear = published.substring(0, 4);
- String dmonth = published.substring(5, 7);
- File tarballFile = new File(
- this.sanitizedBridgesDirectory.getAbsolutePath() + "/"
- + dyear + "/" + dmonth + "/extra-infos/"
- + descriptorDigest.charAt(0) + "/"
- + descriptorDigest.charAt(1) + "/"
- + descriptorDigest);
- File rsyncCatFile = new File("recent/bridge-descriptors/"
- + "extra-infos/" + this.rsyncCatString + "-extra-infos.tmp");
- File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
- boolean[] append = new boolean[] { false, true };
- try {
- for (int i = 0; i < outputFiles.length; i++) {
- File outputFile = outputFiles[i];
- boolean appendToFile = append[i];
- if (outputFile.exists() && !appendToFile) {
- /* We already stored this descriptor to disk before, so let's
- * not store it yet another time. */
- break;
- }
- outputFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- outputFile, appendToFile));
- bw.write("@type bridge-extra-info 1.3\n");
- bw.write(scrubbedDesc);
- if (descriptorDigestSha256Base64 != null) {
- bw.write("router-digest-sha256 " + descriptorDigestSha256Base64
- + "\n");
- }
- bw.write("router-digest " + descriptorDigest.toUpperCase()
- + "\n");
- bw.close();
- }
- } catch (Exception e) {
- this.logger.log(Level.WARNING, "Could not write sanitized "
- + "extra-info descriptor to disk.", e);
- }
- }
-
- /**
- * Rewrite all network statuses that might contain references to server
- * descriptors we added or updated in this execution. This applies to
- * all statuses that have been published up to 24 hours after any added
- * or updated server descriptor.
- */
- public void finishWriting() {
-
- /* Delete secrets that we don't need anymore. */
- if (!this.secretsForHashingIPAddresses.isEmpty()
- && this.secretsForHashingIPAddresses.firstKey().compareTo(
- this.bridgeSanitizingCutOffTimestamp) < 0) {
- try {
- int kept = 0;
- int deleted = 0;
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.bridgeIpSecretsFile));
- for (Map.Entry<String, byte[]> e :
- this.secretsForHashingIPAddresses.entrySet()) {
- if (e.getKey().compareTo(
- this.bridgeSanitizingCutOffTimestamp) < 0) {
- deleted++;
- } else {
- bw.write(e.getKey() + "," + Hex.encodeHexString(e.getValue())
- + "\n");
- kept++;
- }
- }
- bw.close();
- this.logger.info("Deleted " + deleted + " secrets that we don't "
- + "need anymore and kept " + kept + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not store reduced set of "
- + "secrets to disk! This is a bad sign, better check what's "
- + "going on!", e);
- }
- }
- }
-
- private void checkStaleDescriptors() {
- SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- long tooOldMillis = System.currentTimeMillis() - 330L * 60L * 1000L;
- try {
- long maxNetworkStatusPublishedMillis =
- dateTimeFormat.parse(maxNetworkStatusPublishedTime).getTime();
- if (maxNetworkStatusPublishedMillis > 0L
- && maxNetworkStatusPublishedMillis < tooOldMillis) {
- this.logger.warning("The last known bridge network status was "
- + "published " + maxNetworkStatusPublishedTime + ", which is "
- + "more than 5:30 hours in the past.");
- }
- long maxServerDescriptorPublishedMillis =
- dateTimeFormat.parse(maxServerDescriptorPublishedTime)
- .getTime();
- if (maxServerDescriptorPublishedMillis > 0L
- && maxServerDescriptorPublishedMillis < tooOldMillis) {
- this.logger.warning("The last known bridge server descriptor was "
- + "published " + maxServerDescriptorPublishedTime + ", which "
- + "is more than 5:30 hours in the past.");
- }
- long maxExtraInfoDescriptorPublishedMillis =
- dateTimeFormat.parse(maxExtraInfoDescriptorPublishedTime)
- .getTime();
- if (maxExtraInfoDescriptorPublishedMillis > 0L
- && maxExtraInfoDescriptorPublishedMillis < tooOldMillis) {
- this.logger.warning("The last known bridge extra-info descriptor "
- + "was published " + maxExtraInfoDescriptorPublishedTime
- + ", which is more than 5:30 hours in the past.");
- }
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Unable to parse timestamp for "
- + "stale check.", e);
- }
- }
-
- /* Delete all files from the rsync directory that have not been modified
- * in the last three days, and remove the .tmp extension from newly
- * written files. */
- public void cleanUpRsyncDirectory() {
- long cutOffMillis = System.currentTimeMillis()
- - 3L * 24L * 60L * 60L * 1000L;
- Stack<File> allFiles = new Stack<File>();
- allFiles.add(new File("recent/bridge-descriptors"));
- while (!allFiles.isEmpty()) {
- File file = allFiles.pop();
- if (file.isDirectory()) {
- allFiles.addAll(Arrays.asList(file.listFiles()));
- } else if (file.lastModified() < cutOffMillis) {
- file.delete();
- } else if (file.getName().endsWith(".tmp")) {
- file.renameTo(new File(file.getParentFile(),
- file.getName().substring(0,
- file.getName().lastIndexOf(".tmp"))));
- }
- }
- }
-}
-
diff --git a/src/org/torproject/collector/exitlists/ExitListDownloader.java b/src/org/torproject/collector/exitlists/ExitListDownloader.java
deleted file mode 100644
index 54fd50f..0000000
--- a/src/org/torproject/collector/exitlists/ExitListDownloader.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/* Copyright 2010--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.exitlists;
-
-import org.torproject.collector.main.Configuration;
-import org.torproject.collector.main.LockFile;
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorParseException;
-import org.torproject.descriptor.DescriptorParser;
-import org.torproject.descriptor.DescriptorSourceFactory;
-import org.torproject.descriptor.ExitList;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.text.SimpleDateFormat;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TimeZone;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class ExitListDownloader extends Thread {
-
- public static void main(String[] args) {
-
- Logger logger = Logger.getLogger(ExitListDownloader.class.getName());
- logger.info("Starting exit-lists module of CollecTor.");
-
- // Initialize configuration
- Configuration config = new Configuration();
-
- // Use lock file to avoid overlapping runs
- LockFile lf = new LockFile("exit-lists");
- if (!lf.acquireLock()) {
- logger.severe("Warning: CollecTor is already running or has not exited "
- + "cleanly! Exiting!");
- System.exit(1);
- }
-
- // Download exit list and store it to disk
- new ExitListDownloader(config).run();
-
- // Remove lock file
- lf.releaseLock();
-
- logger.info("Terminating exit-lists module of CollecTor.");
- }
-
- public ExitListDownloader(Configuration config) {
- }
-
- public void run() {
-
- Logger logger = Logger.getLogger(ExitListDownloader.class.getName());
-
- SimpleDateFormat dateTimeFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-
- Date downloadedDate = new Date();
- String downloadedExitList = null;
- try {
- logger.fine("Downloading exit list...");
- StringBuilder sb = new StringBuilder();
- sb.append("@type tordnsel 1.0\n");
- sb.append("Downloaded " + dateTimeFormat.format(downloadedDate)
- + "\n");
- String exitAddressesUrl =
- "http://exitlist.torproject.org/exit-addresses";
- URL u = new URL(exitAddressesUrl);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.connect();
- int response = huc.getResponseCode();
- if (response != 200) {
- logger.warning("Could not download exit list. Response code "
- + response);
- return;
- }
- BufferedInputStream in = new BufferedInputStream(
- huc.getInputStream());
- int len;
- byte[] data = new byte[1024];
- while ((len = in.read(data, 0, 1024)) >= 0) {
- sb.append(new String(data, 0, len));
- }
- in.close();
- downloadedExitList = sb.toString();
- logger.fine("Finished downloading exit list.");
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed downloading exit list", e);
- return;
- }
- if (downloadedExitList == null) {
- logger.warning("Failed downloading exit list");
- return;
- }
-
- SimpleDateFormat tarballFormat =
- new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
- tarballFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File("out/exit-lists/" + tarballFormat.format(
- downloadedDate));
-
- long maxScanMillis = 0L;
- try {
- DescriptorParser descriptorParser =
- DescriptorSourceFactory.createDescriptorParser();
- List<Descriptor> parsedDescriptors =
- descriptorParser.parseDescriptors(downloadedExitList.getBytes(),
- tarballFile.getName());
- if (parsedDescriptors.size() != 1
- || !(parsedDescriptors.get(0) instanceof ExitList)) {
- logger.warning("Could not parse downloaded exit list");
- return;
- }
- ExitList parsedExitList = (ExitList) parsedDescriptors.get(0);
- for (ExitList.Entry entry : parsedExitList.getEntries()) {
- for (long scanMillis : entry.getExitAddresses().values()) {
- maxScanMillis = Math.max(maxScanMillis, scanMillis);
- }
- }
- } catch (DescriptorParseException e) {
- logger.log(Level.WARNING, "Could not parse downloaded exit list",
- e);
- }
- if (maxScanMillis > 0L
- && maxScanMillis + 330L * 60L * 1000L < System.currentTimeMillis()) {
- logger.warning("The last reported scan in the downloaded exit list "
- + "took place at " + dateTimeFormat.format(maxScanMillis)
- + ", which is more than 5:30 hours in the past.");
- }
-
- /* Write to disk. */
- File rsyncFile = new File("recent/exit-lists/"
- + tarballFile.getName());
- File[] outputFiles = new File[] { tarballFile, rsyncFile };
- for (File outputFile : outputFiles) {
- try {
- outputFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- outputFile));
- bw.write(downloadedExitList);
- bw.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not write downloaded exit list "
- + "to " + outputFile.getAbsolutePath(), e);
- }
- }
-
- /* Write stats. */
- StringBuilder dumpStats = new StringBuilder("Finished downloading "
- + "exit list.\nLast three exit lists are:");
- Stack<File> filesInInputDir = new Stack<File>();
- filesInInputDir.add(new File("out/exit-lists"));
- SortedSet<File> lastThreeExitLists = new TreeSet<File>();
- while (!filesInInputDir.isEmpty()) {
- File pop = filesInInputDir.pop();
- if (pop.isDirectory()) {
- SortedSet<File> lastThreeElements = new TreeSet<File>();
- for (File f : pop.listFiles()) {
- lastThreeElements.add(f);
- }
- while (lastThreeElements.size() > 3) {
- lastThreeElements.remove(lastThreeElements.first());
- }
- for (File f : lastThreeElements) {
- filesInInputDir.add(f);
- }
- } else {
- lastThreeExitLists.add(pop);
- while (lastThreeExitLists.size() > 3) {
- lastThreeExitLists.remove(lastThreeExitLists.first());
- }
- }
- }
- for (File f : lastThreeExitLists) {
- dumpStats.append("\n" + f.getName());
- }
- logger.info(dumpStats.toString());
-
- this.cleanUpRsyncDirectory();
- }
-
- /* Delete all files from the rsync directory that have not been modified
- * in the last three days. */
- public void cleanUpRsyncDirectory() {
- long cutOffMillis = System.currentTimeMillis()
- - 3L * 24L * 60L * 60L * 1000L;
- Stack<File> allFiles = new Stack<File>();
- allFiles.add(new File("recent/exit-lists"));
- while (!allFiles.isEmpty()) {
- File file = allFiles.pop();
- if (file.isDirectory()) {
- allFiles.addAll(Arrays.asList(file.listFiles()));
- } else if (file.lastModified() < cutOffMillis) {
- file.delete();
- }
- }
- }
-}
-
diff --git a/src/org/torproject/collector/index/CreateIndexJson.java b/src/org/torproject/collector/index/CreateIndexJson.java
deleted file mode 100644
index ac5adf5..0000000
--- a/src/org/torproject/collector/index/CreateIndexJson.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/* Copyright 2015--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.index;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-
-import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
-import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Locale;
-import java.util.SortedSet;
-import java.util.TimeZone;
-import java.util.TreeSet;
-import java.util.zip.GZIPOutputStream;
-
-/* Create a fresh index.json containing all directories and files in the
- * archive/ and recent/ directories.
- *
- * Note that if this ever takes longer than a few seconds, we'll have to
- * cache index parts of directories or files that haven't changed.
- * Example: if we parse include cryptographic hashes or @type information,
- * we'll likely have to do that. */
-public class CreateIndexJson {
-
- static final File indexJsonFile = new File("index.json");
-
- static final String basePath = "https://collector.torproject.org";
-
- static final File[] indexedDirectories = new File[] {
- new File("archive"), new File("recent") };
-
- static final String dateTimePattern = "yyyy-MM-dd HH:mm";
-
- static final Locale dateTimeLocale = Locale.US;
-
- static final TimeZone dateTimezone = TimeZone.getTimeZone("UTC");
-
- public static void main(String[] args) throws IOException {
- writeIndex(indexDirectories());
- }
-
- static class DirectoryNode implements Comparable<DirectoryNode> {
- String path;
- SortedSet<FileNode> files;
- SortedSet<DirectoryNode> directories;
-
- DirectoryNode(String path, SortedSet<FileNode> files,
- SortedSet<DirectoryNode> directories) {
- this.path = path;
- this.files = files;
- this.directories = directories;
- }
-
- public int compareTo(DirectoryNode o) {
- return this.path.compareTo(o.path);
- }
- }
-
- static class IndexNode {
- String index_created;
- String path;
- SortedSet<FileNode> files;
- SortedSet<DirectoryNode> directories;
-
- IndexNode(String index_created, String path,
- SortedSet<FileNode> files,
- SortedSet<DirectoryNode> directories) {
- this.index_created = index_created;
- this.path = path;
- this.files = files;
- this.directories = directories;
- }
- }
-
- static class FileNode implements Comparable<FileNode> {
- String path;
- long size;
- String last_modified;
-
- FileNode(String path, long size, String last_modified) {
- this.path = path;
- this.size = size;
- this.last_modified = last_modified;
- }
-
- public int compareTo(FileNode o) {
- return this.path.compareTo(o.path);
- }
- }
-
- static DateFormat dateTimeFormat;
-
- static {
- dateTimeFormat = new SimpleDateFormat(dateTimePattern,
- dateTimeLocale);
- dateTimeFormat.setLenient(false);
- dateTimeFormat.setTimeZone(dateTimezone);
- }
-
- static IndexNode indexDirectories() {
- SortedSet<DirectoryNode> directoryNodes =
- new TreeSet<DirectoryNode>();
- for (File directory : indexedDirectories) {
- if (directory.exists() && directory.isDirectory()) {
- directoryNodes.add(indexDirectory(directory));
- }
- }
- return new IndexNode(dateTimeFormat.format(
- System.currentTimeMillis()), basePath, null, directoryNodes);
- }
-
- static DirectoryNode indexDirectory(File directory) {
- SortedSet<FileNode> fileNodes = new TreeSet<FileNode>();
- SortedSet<DirectoryNode> directoryNodes =
- new TreeSet<DirectoryNode>();
- for (File fileOrDirectory : directory.listFiles()) {
- if (fileOrDirectory.getName().startsWith(".")) {
- continue;
- }
- if (fileOrDirectory.isFile()) {
- fileNodes.add(indexFile(fileOrDirectory));
- } else {
- directoryNodes.add(indexDirectory(fileOrDirectory));
- }
- }
- DirectoryNode directoryNode = new DirectoryNode(
- directory.getName(), fileNodes.isEmpty() ? null : fileNodes,
- directoryNodes.isEmpty() ? null : directoryNodes);
- return directoryNode;
- }
-
- static FileNode indexFile(File file) {
- FileNode fileNode = new FileNode(file.getName(), file.length(),
- dateTimeFormat.format(file.lastModified()));
- return fileNode;
- }
-
- static void writeIndex(IndexNode indexNode) throws IOException {
- Gson gson = new GsonBuilder().create();
- String indexNodeString = gson.toJson(indexNode);
- Writer[] writers = new Writer[] {
- new FileWriter(indexJsonFile),
- new OutputStreamWriter(new GZIPOutputStream(
- new FileOutputStream(indexJsonFile + ".gz"))),
- new OutputStreamWriter(new XZCompressorOutputStream(
- new FileOutputStream(indexJsonFile + ".xz"))),
- new OutputStreamWriter(new BZip2CompressorOutputStream(
- new FileOutputStream(indexJsonFile + ".bz2")))
- };
- for (Writer writer : writers) {
- BufferedWriter bufferedWriter = new BufferedWriter(writer);
- bufferedWriter.write(indexNodeString);
- bufferedWriter.close();
- }
- }
-}
-
diff --git a/src/org/torproject/collector/main/Configuration.java b/src/org/torproject/collector/main/Configuration.java
deleted file mode 100644
index aee1d02..0000000
--- a/src/org/torproject/collector/main/Configuration.java
+++ /dev/null
@@ -1,318 +0,0 @@
-/* Copyright 2010--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.main;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Initialize configuration with hard-coded defaults, overwrite with
- * configuration in config file, if exists, and answer Main.java about our
- * configuration.
- */
-public class Configuration {
- private String directoryArchivesOutputDirectory =
- "out/relay-descriptors/";
- private boolean importCachedRelayDescriptors = false;
- private List<String> cachedRelayDescriptorsDirectory =
- new ArrayList<String>(Arrays.asList(
- "in/relay-descriptors/cacheddesc/".split(",")));
- private boolean importDirectoryArchives = false;
- private String directoryArchivesDirectory =
- "in/relay-descriptors/archives/";
- private boolean keepDirectoryArchiveImportHistory = false;
- private boolean replaceIPAddressesWithHashes = false;
- private long limitBridgeDescriptorMappings = -1L;
- private String sanitizedBridgesWriteDirectory =
- "out/bridge-descriptors/";
- private String bridgeSnapshotsDirectory = "in/bridge-descriptors/";
- private boolean downloadRelayDescriptors = false;
- private List<String> downloadFromDirectoryAuthorities = Arrays.asList((
- "86.59.21.38,76.73.17.194:9030,171.25.193.9:443,"
- + "193.23.244.244,208.83.223.34:443,128.31.0.34:9131,"
- + "194.109.206.212,212.112.245.170,154.35.32.5").split(","));
- private List<String> downloadVotesByFingerprint = Arrays.asList((
- "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4,"
- + "27B6B5996C426270A5C95488AA5BCEB6BCC86956,"
- + "49015F787433103580E3B66A1707A00E60F2D15B,"
- + "585769C78764D58426B8B52B6651A5A71137189A,"
- + "80550987E1D626E3EBA5E5E75A458DE0626D088C,"
- + "D586D18309DED4CD6D57C18FDB97EFA96D330566,"
- + "E8A9C45EDE6D711294FADF8E7951F4DE6CA56B58,"
- + "ED03BB616EB2F60BEC80151114BB25CEF515B226,"
- + "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97").split(","));
- private boolean downloadCurrentConsensus = true;
- private boolean downloadCurrentMicrodescConsensus = true;
- private boolean downloadCurrentVotes = true;
- private boolean downloadMissingServerDescriptors = true;
- private boolean downloadMissingExtraInfoDescriptors = true;
- private boolean downloadMissingMicrodescriptors = true;
- private boolean downloadAllServerDescriptors = false;
- private boolean downloadAllExtraInfoDescriptors = false;
- private boolean compressRelayDescriptorDownloads;
- private String torperfOutputDirectory = "out/torperf/";
- private SortedMap<String, String> torperfSources = null;
- private List<String> torperfFiles = null;
-
- public Configuration() {
-
- /* Initialize logger. */
- Logger logger = Logger.getLogger(Configuration.class.getName());
-
- /* Read config file, if present. */
- File configFile = new File("config");
- if (!configFile.exists()) {
- logger.warning("Could not find config file. In the default "
- + "configuration, we are not configured to read data from any "
- + "data source or write data to any data sink. You need to "
- + "create a config file (" + configFile.getAbsolutePath()
- + ") and provide at least one data source and one data sink. "
- + "Refer to the manual for more information.");
- return;
- }
- String line = null;
- boolean containsCachedRelayDescriptorsDirectory = false;
- try {
- BufferedReader br = new BufferedReader(new FileReader(configFile));
- while ((line = br.readLine()) != null) {
- if (line.startsWith("#") || line.length() < 1) {
- continue;
- } else if (line.startsWith("DirectoryArchivesOutputDirectory")) {
- this.directoryArchivesOutputDirectory = line.split(" ")[1];
- } else if (line.startsWith("ImportCachedRelayDescriptors")) {
- this.importCachedRelayDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("CachedRelayDescriptorsDirectory")) {
- if (!containsCachedRelayDescriptorsDirectory) {
- this.cachedRelayDescriptorsDirectory.clear();
- containsCachedRelayDescriptorsDirectory = true;
- }
- this.cachedRelayDescriptorsDirectory.add(line.split(" ")[1]);
- } else if (line.startsWith("ImportDirectoryArchives")) {
- this.importDirectoryArchives = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DirectoryArchivesDirectory")) {
- this.directoryArchivesDirectory = line.split(" ")[1];
- } else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
- this.keepDirectoryArchiveImportHistory = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("ReplaceIPAddressesWithHashes")) {
- this.replaceIPAddressesWithHashes = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("LimitBridgeDescriptorMappings")) {
- this.limitBridgeDescriptorMappings = Long.parseLong(
- line.split(" ")[1]);
- } else if (line.startsWith("SanitizedBridgesWriteDirectory")) {
- this.sanitizedBridgesWriteDirectory = line.split(" ")[1];
- } else if (line.startsWith("BridgeSnapshotsDirectory")) {
- this.bridgeSnapshotsDirectory = line.split(" ")[1];
- } else if (line.startsWith("DownloadRelayDescriptors")) {
- this.downloadRelayDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadFromDirectoryAuthorities")) {
- this.downloadFromDirectoryAuthorities = new ArrayList<String>();
- for (String dir : line.split(" ")[1].split(",")) {
- // test if IP:port pair has correct format
- if (dir.length() < 1) {
- logger.severe("Configuration file contains directory "
- + "authority IP:port of length 0 in line '" + line
- + "'! Exiting!");
- System.exit(1);
- }
- new URL("http://" + dir + "/");
- this.downloadFromDirectoryAuthorities.add(dir);
- }
- } else if (line.startsWith("DownloadVotesByFingerprint")) {
- this.downloadVotesByFingerprint = new ArrayList<String>();
- for (String fingerprint : line.split(" ")[1].split(",")) {
- this.downloadVotesByFingerprint.add(fingerprint);
- }
- } else if (line.startsWith("DownloadCurrentConsensus")) {
- this.downloadCurrentConsensus = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadCurrentMicrodescConsensus")) {
- this.downloadCurrentMicrodescConsensus = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadCurrentVotes")) {
- this.downloadCurrentVotes = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadMissingServerDescriptors")) {
- this.downloadMissingServerDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith(
- "DownloadMissingExtraInfoDescriptors")) {
- this.downloadMissingExtraInfoDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadMissingMicrodescriptors")) {
- this.downloadMissingMicrodescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadAllServerDescriptors")) {
- this.downloadAllServerDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("DownloadAllExtraInfoDescriptors")) {
- this.downloadAllExtraInfoDescriptors = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("CompressRelayDescriptorDownloads")) {
- this.compressRelayDescriptorDownloads = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("TorperfOutputDirectory")) {
- this.torperfOutputDirectory = line.split(" ")[1];
- } else if (line.startsWith("TorperfSource")) {
- if (this.torperfSources == null) {
- this.torperfSources = new TreeMap<String, String>();
- }
- String[] parts = line.split(" ");
- String sourceName = parts[1];
- String baseUrl = parts[2];
- this.torperfSources.put(sourceName, baseUrl);
- } else if (line.startsWith("TorperfFiles")) {
- if (this.torperfFiles == null) {
- this.torperfFiles = new ArrayList<String>();
- }
- String[] parts = line.split(" ");
- if (parts.length != 5) {
- logger.severe("Configuration file contains TorperfFiles "
- + "option with wrong number of values in line '" + line
- + "'! Exiting!");
- System.exit(1);
- }
- this.torperfFiles.add(line);
- } else {
- logger.severe("Configuration file contains unrecognized "
- + "configuration key in line '" + line + "'! Exiting!");
- System.exit(1);
- }
- }
- br.close();
- } catch (ArrayIndexOutOfBoundsException e) {
- logger.severe("Configuration file contains configuration key "
- + "without value in line '" + line + "'. Exiting!");
- System.exit(1);
- } catch (MalformedURLException e) {
- logger.severe("Configuration file contains illegal URL or IP:port "
- + "pair in line '" + line + "'. Exiting!");
- System.exit(1);
- } catch (NumberFormatException e) {
- logger.severe("Configuration file contains illegal value in line '"
- + line + "' with legal values being 0 or 1. Exiting!");
- System.exit(1);
- } catch (IOException e) {
- logger.log(Level.SEVERE, "Unknown problem while reading config "
- + "file! Exiting!", e);
- System.exit(1);
- }
- }
-
- public String getDirectoryArchivesOutputDirectory() {
- return this.directoryArchivesOutputDirectory;
- }
-
- public boolean getImportCachedRelayDescriptors() {
- return this.importCachedRelayDescriptors;
- }
-
- public List<String> getCachedRelayDescriptorDirectory() {
- return this.cachedRelayDescriptorsDirectory;
- }
-
- public boolean getImportDirectoryArchives() {
- return this.importDirectoryArchives;
- }
-
- public String getDirectoryArchivesDirectory() {
- return this.directoryArchivesDirectory;
- }
-
- public boolean getKeepDirectoryArchiveImportHistory() {
- return this.keepDirectoryArchiveImportHistory;
- }
-
- public boolean getReplaceIPAddressesWithHashes() {
- return this.replaceIPAddressesWithHashes;
- }
-
- public long getLimitBridgeDescriptorMappings() {
- return this.limitBridgeDescriptorMappings;
- }
-
- public String getSanitizedBridgesWriteDirectory() {
- return this.sanitizedBridgesWriteDirectory;
- }
-
- public String getBridgeSnapshotsDirectory() {
- return this.bridgeSnapshotsDirectory;
- }
-
- public boolean getDownloadRelayDescriptors() {
- return this.downloadRelayDescriptors;
- }
-
- public List<String> getDownloadFromDirectoryAuthorities() {
- return this.downloadFromDirectoryAuthorities;
- }
-
- public List<String> getDownloadVotesByFingerprint() {
- return this.downloadVotesByFingerprint;
- }
-
- public boolean getDownloadCurrentConsensus() {
- return this.downloadCurrentConsensus;
- }
-
- public boolean getDownloadCurrentMicrodescConsensus() {
- return this.downloadCurrentMicrodescConsensus;
- }
-
- public boolean getDownloadCurrentVotes() {
- return this.downloadCurrentVotes;
- }
-
- public boolean getDownloadMissingServerDescriptors() {
- return this.downloadMissingServerDescriptors;
- }
-
- public boolean getDownloadMissingExtraInfoDescriptors() {
- return this.downloadMissingExtraInfoDescriptors;
- }
-
- public boolean getDownloadMissingMicrodescriptors() {
- return this.downloadMissingMicrodescriptors;
- }
-
- public boolean getDownloadAllServerDescriptors() {
- return this.downloadAllServerDescriptors;
- }
-
- public boolean getDownloadAllExtraInfoDescriptors() {
- return this.downloadAllExtraInfoDescriptors;
- }
-
- public boolean getCompressRelayDescriptorDownloads() {
- return this.compressRelayDescriptorDownloads;
- }
-
- public String getTorperfOutputDirectory() {
- return this.torperfOutputDirectory;
- }
-
- public SortedMap<String, String> getTorperfSources() {
- return this.torperfSources;
- }
-
- public List<String> getTorperfFiles() {
- return this.torperfFiles;
- }
-}
-
diff --git a/src/org/torproject/collector/main/LockFile.java b/src/org/torproject/collector/main/LockFile.java
deleted file mode 100644
index b07d4b1..0000000
--- a/src/org/torproject/collector/main/LockFile.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/* Copyright 2010--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.main;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.logging.Logger;
-
-public class LockFile {
-
- private File lockFile;
- private Logger logger;
-
- public LockFile(String moduleName) {
- this.lockFile = new File("lock/" + moduleName);
- this.logger = Logger.getLogger(LockFile.class.getName());
- }
-
- public boolean acquireLock() {
- this.logger.fine("Trying to acquire lock...");
- try {
- if (this.lockFile.exists()) {
- BufferedReader br = new BufferedReader(new FileReader(
- this.lockFile));
- long runStarted = Long.parseLong(br.readLine());
- br.close();
- if (System.currentTimeMillis() - runStarted < 55L * 60L * 1000L) {
- return false;
- }
- }
- this.lockFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.lockFile));
- bw.append("" + System.currentTimeMillis() + "\n");
- bw.close();
- this.logger.fine("Acquired lock.");
- return true;
- } catch (IOException e) {
- this.logger.warning("Caught exception while trying to acquire "
- + "lock!");
- return false;
- }
- }
-
- public void releaseLock() {
- this.logger.fine("Releasing lock...");
- this.lockFile.delete();
- this.logger.fine("Released lock.");
- }
-}
-
diff --git a/src/org/torproject/collector/relaydescs/ArchiveReader.java b/src/org/torproject/collector/relaydescs/ArchiveReader.java
deleted file mode 100644
index 72f8231..0000000
--- a/src/org/torproject/collector/relaydescs/ArchiveReader.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/* Copyright 2010--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.relaydescs;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.StringReader;
-import java.io.UnsupportedEncodingException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TimeZone;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Read in all files in a given directory and pass buffered readers of
- * them to the relay descriptor parser.
- */
-public class ArchiveReader {
-
- private Map<String, Set<String>> microdescriptorValidAfterTimes =
- new HashMap<String, Set<String>>();
-
- public ArchiveReader(RelayDescriptorParser rdp, File archivesDirectory,
- File statsDirectory, boolean keepImportHistory) {
-
- if (rdp == null || archivesDirectory == null
- || statsDirectory == null) {
- throw new IllegalArgumentException();
- }
-
- rdp.setArchiveReader(this);
- int parsedFiles = 0;
- int ignoredFiles = 0;
- Logger logger = Logger.getLogger(ArchiveReader.class.getName());
- SortedSet<String> archivesImportHistory = new TreeSet<String>();
- File archivesImportHistoryFile = new File(statsDirectory,
- "archives-import-history");
- if (keepImportHistory && archivesImportHistoryFile.exists()) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- archivesImportHistoryFile));
- String line = null;
- while ((line = br.readLine()) != null) {
- archivesImportHistory.add(line);
- }
- br.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not read in archives import "
- + "history file. Skipping.");
- }
- }
- if (archivesDirectory.exists()) {
- logger.fine("Importing files in directory " + archivesDirectory
- + "/...");
- Stack<File> filesInInputDir = new Stack<File>();
- filesInInputDir.add(archivesDirectory);
- List<File> problems = new ArrayList<File>();
- Set<File> filesToRetry = new HashSet<File>();
- while (!filesInInputDir.isEmpty()) {
- File pop = filesInInputDir.pop();
- if (pop.isDirectory()) {
- for (File f : pop.listFiles()) {
- filesInInputDir.add(f);
- }
- } else {
- if (rdp != null) {
- try {
- BufferedInputStream bis = null;
- if (keepImportHistory
- && archivesImportHistory.contains(pop.getName())) {
- ignoredFiles++;
- continue;
- } else if (pop.getName().endsWith(".tar.bz2")) {
- logger.warning("Cannot parse compressed tarball "
- + pop.getAbsolutePath() + ". Skipping.");
- continue;
- } else if (pop.getName().endsWith(".bz2")) {
- FileInputStream fis = new FileInputStream(pop);
- BZip2CompressorInputStream bcis =
- new BZip2CompressorInputStream(fis);
- bis = new BufferedInputStream(bcis);
- } else {
- FileInputStream fis = new FileInputStream(pop);
- bis = new BufferedInputStream(fis);
- }
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- bis.close();
- byte[] allData = baos.toByteArray();
- boolean stored = rdp.parse(allData);
- if (!stored) {
- filesToRetry.add(pop);
- continue;
- }
- if (keepImportHistory) {
- archivesImportHistory.add(pop.getName());
- }
- parsedFiles++;
- } catch (IOException e) {
- problems.add(pop);
- if (problems.size() > 3) {
- break;
- }
- }
- }
- }
- }
- for (File pop : filesToRetry) {
- /* TODO We need to parse microdescriptors ourselves, rather than
- * RelayDescriptorParser, because only we know the valid-after
- * time(s) of microdesc consensus(es) containing this
- * microdescriptor. However, this breaks functional abstraction
- * pretty badly. */
- if (rdp != null) {
- try {
- BufferedInputStream bis = null;
- if (pop.getName().endsWith(".bz2")) {
- FileInputStream fis = new FileInputStream(pop);
- BZip2CompressorInputStream bcis =
- new BZip2CompressorInputStream(fis);
- bis = new BufferedInputStream(bcis);
- } else {
- FileInputStream fis = new FileInputStream(pop);
- bis = new BufferedInputStream(fis);
- }
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- bis.close();
- byte[] allData = baos.toByteArray();
- BufferedReader br = new BufferedReader(new StringReader(
- new String(allData, "US-ASCII")));
- String line;
- do {
- line = br.readLine();
- } while (line != null && line.startsWith("@"));
- br.close();
- if (line == null) {
- logger.fine("We were given an empty descriptor for "
- + "parsing. Ignoring.");
- continue;
- }
- if (!line.equals("onion-key")) {
- logger.fine("Skipping non-recognized descriptor.");
- continue;
- }
- SimpleDateFormat parseFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String ascii = null;
- try {
- ascii = new String(allData, "US-ASCII");
- } catch (UnsupportedEncodingException e) {
- /* No way that US-ASCII is not supported. */
- }
- int start = -1;
- int end = -1;
- String startToken = "onion-key\n";
- while (end < ascii.length()) {
- start = ascii.indexOf(startToken, end);
- if (start < 0) {
- break;
- }
- end = ascii.indexOf(startToken, start + 1);
- if (end < 0) {
- end = ascii.length();
- if (end <= start) {
- break;
- }
- }
- byte[] descBytes = new byte[end - start];
- System.arraycopy(allData, start, descBytes, 0, end - start);
- String digest256Base64 = Base64.encodeBase64String(
- DigestUtils.sha256(descBytes)).replaceAll("=", "");
- String digest256Hex = DigestUtils.sha256Hex(descBytes);
- if (!this.microdescriptorValidAfterTimes.containsKey(
- digest256Hex)) {
- logger.fine("Could not store microdescriptor '"
- + digest256Hex + "', which was not contained in a "
- + "microdesc consensus.");
- continue;
- }
- for (String validAfterTime :
- this.microdescriptorValidAfterTimes.get(digest256Hex)) {
- try {
- long validAfter =
- parseFormat.parse(validAfterTime).getTime();
- rdp.storeMicrodescriptor(descBytes, digest256Hex,
- digest256Base64, validAfter);
- } catch (ParseException e) {
- logger.log(Level.WARNING, "Could not parse "
- + "valid-after time '" + validAfterTime + "'. Not "
- + "storing microdescriptor.", e);
- }
- }
- }
- if (keepImportHistory) {
- archivesImportHistory.add(pop.getName());
- }
- parsedFiles++;
- } catch (IOException e) {
- problems.add(pop);
- if (problems.size() > 3) {
- break;
- }
- }
- }
- }
- if (problems.isEmpty()) {
- logger.fine("Finished importing files in directory "
- + archivesDirectory + "/.");
- } else {
- StringBuilder sb = new StringBuilder("Failed importing files in "
- + "directory " + archivesDirectory + "/:");
- int printed = 0;
- for (File f : problems) {
- sb.append("\n " + f.getAbsolutePath());
- if (++printed >= 3) {
- sb.append("\n ... more");
- break;
- }
- }
- }
- }
- if (keepImportHistory) {
- try {
- archivesImportHistoryFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- archivesImportHistoryFile));
- for (String line : archivesImportHistory) {
- bw.write(line + "\n");
- }
- bw.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not write archives import "
- + "history file.");
- }
- }
- logger.info("Finished importing relay descriptors from local "
- + "directory:\nParsed " + parsedFiles + ", ignored "
- + ignoredFiles + " files.");
- }
-
- public void haveParsedMicrodescConsensus(String validAfterTime,
- SortedSet<String> microdescriptorDigests) {
- for (String microdescriptor : microdescriptorDigests) {
- if (!this.microdescriptorValidAfterTimes.containsKey(
- microdescriptor)) {
- this.microdescriptorValidAfterTimes.put(microdescriptor,
- new HashSet<String>());
- }
- this.microdescriptorValidAfterTimes.get(microdescriptor).add(
- validAfterTime);
- }
- }
-}
-
diff --git a/src/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/org/torproject/collector/relaydescs/ArchiveWriter.java
deleted file mode 100644
index cf603d1..0000000
--- a/src/org/torproject/collector/relaydescs/ArchiveWriter.java
+++ /dev/null
@@ -1,845 +0,0 @@
-/* Copyright 2010--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.relaydescs;
-
-import org.torproject.collector.main.Configuration;
-import org.torproject.collector.main.LockFile;
-import org.torproject.descriptor.DescriptorParseException;
-import org.torproject.descriptor.DescriptorParser;
-import org.torproject.descriptor.DescriptorSourceFactory;
-
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class ArchiveWriter extends Thread {
-
- private Configuration config;
-
- private long now = System.currentTimeMillis();
- private Logger logger;
- private File outputDirectory;
- private String rsyncCatString;
- private DescriptorParser descriptorParser;
- private int storedConsensusesCounter = 0;
- private int storedMicrodescConsensusesCounter = 0;
- private int storedVotesCounter = 0;
- private int storedCertsCounter = 0;
- private int storedServerDescriptorsCounter = 0;
- private int storedExtraInfoDescriptorsCounter = 0;
- private int storedMicrodescriptorsCounter = 0;
-
- private SortedMap<Long, SortedSet<String>> storedConsensuses =
- new TreeMap<Long, SortedSet<String>>();
- private SortedMap<Long, SortedSet<String>> storedMicrodescConsensuses =
- new TreeMap<Long, SortedSet<String>>();
- private SortedMap<Long, Integer> expectedVotes =
- new TreeMap<Long, Integer>();
- private SortedMap<Long, SortedMap<String, SortedSet<String>>>
- storedVotes =
- new TreeMap<Long, SortedMap<String, SortedSet<String>>>();
- private SortedMap<Long, Map<String, String>> storedServerDescriptors =
- new TreeMap<Long, Map<String, String>>();
- private SortedMap<Long, Set<String>> storedExtraInfoDescriptors =
- new TreeMap<Long, Set<String>>();
- private SortedMap<Long, Set<String>> storedMicrodescriptors =
- new TreeMap<Long, Set<String>>();
-
- private File storedServerDescriptorsFile = new File(
- "stats/stored-server-descriptors");
- private File storedExtraInfoDescriptorsFile = new File(
- "stats/stored-extra-info-descriptors");
- private File storedMicrodescriptorsFile = new File(
- "stats/stored-microdescriptors");
-
- private static final byte[] CONSENSUS_ANNOTATION =
- "@type network-status-consensus-3 1.0\n".getBytes();
-
- private static final byte[] MICRODESCCONSENSUS_ANNOTATION =
- "@type network-status-microdesc-consensus-3 1.0\n".getBytes();
-
- private static final byte[] VOTE_ANNOTATION =
- "@type network-status-vote-3 1.0\n".getBytes();
-
- private static final byte[] CERTIFICATE_ANNOTATION =
- "@type dir-key-certificate-3 1.0\n".getBytes();
-
- private static final byte[] SERVER_DESCRIPTOR_ANNOTATION =
- "@type server-descriptor 1.0\n".getBytes();
-
- private static final byte[] EXTRA_INFO_ANNOTATION =
- "@type extra-info 1.0\n".getBytes();
-
- private static final byte[] MICRODESCRIPTOR_ANNOTATION =
- "@type microdescriptor 1.0\n".getBytes();
-
- private StringBuilder intermediateStats = new StringBuilder();
-
- public static void main(String[] args) {
-
- Logger logger = Logger.getLogger(ArchiveWriter.class.getName());
- logger.info("Starting relay-descriptors module of CollecTor.");
-
- // Initialize configuration
- Configuration config = new Configuration();
-
- // Use lock file to avoid overlapping runs
- LockFile lf = new LockFile("relay-descriptors");
- if (!lf.acquireLock()) {
- logger.severe("Warning: CollecTor is already running or has not exited "
- + "cleanly! Exiting!");
- System.exit(1);
- }
-
- // Import/download relay descriptors from the various sources
- new ArchiveWriter(config).run();
-
- new ReferenceChecker(new File("recent/relay-descriptors"),
- new File("stats/references"),
- new File("stats/references-history")).check();
-
- // Remove lock file
- lf.releaseLock();
-
- logger.info("Terminating relay-descriptors module of CollecTor.");
- }
-
- public ArchiveWriter(Configuration config) {
- this.config = config;
- }
-
- public void run() {
-
- File outputDirectory =
- new File(config.getDirectoryArchivesOutputDirectory());
- File statsDirectory = new File("stats");
-
- this.logger = Logger.getLogger(ArchiveWriter.class.getName());
- this.outputDirectory = outputDirectory;
- SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
- "yyyy-MM-dd-HH-mm-ss");
- rsyncCatFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- this.rsyncCatString = rsyncCatFormat.format(
- System.currentTimeMillis());
- this.descriptorParser =
- DescriptorSourceFactory.createDescriptorParser();
-
- this.loadDescriptorDigests();
-
- // Prepare relay descriptor parser
- RelayDescriptorParser rdp = new RelayDescriptorParser(this);
-
- RelayDescriptorDownloader rdd = null;
- if (config.getDownloadRelayDescriptors()) {
- List<String> dirSources =
- config.getDownloadFromDirectoryAuthorities();
- rdd = new RelayDescriptorDownloader(rdp, dirSources,
- config.getDownloadVotesByFingerprint(),
- config.getDownloadCurrentConsensus(),
- config.getDownloadCurrentMicrodescConsensus(),
- config.getDownloadCurrentVotes(),
- config.getDownloadMissingServerDescriptors(),
- config.getDownloadMissingExtraInfoDescriptors(),
- config.getDownloadMissingMicrodescriptors(),
- config.getDownloadAllServerDescriptors(),
- config.getDownloadAllExtraInfoDescriptors(),
- config.getCompressRelayDescriptorDownloads());
- rdp.setRelayDescriptorDownloader(rdd);
- }
- if (config.getImportCachedRelayDescriptors()) {
- new CachedRelayDescriptorReader(rdp,
- config.getCachedRelayDescriptorDirectory(), statsDirectory);
- this.intermediateStats("importing relay descriptors from local "
- + "Tor data directories");
- }
- if (config.getImportDirectoryArchives()) {
- new ArchiveReader(rdp,
- new File(config.getDirectoryArchivesDirectory()),
- statsDirectory,
- config.getKeepDirectoryArchiveImportHistory());
- this.intermediateStats("importing relay descriptors from local "
- + "directory");
- }
- if (rdd != null) {
- rdd.downloadDescriptors();
- rdd.writeFile();
- rdd = null;
- this.intermediateStats("downloading relay descriptors from the "
- + "directory authorities");
- }
-
- this.checkMissingDescriptors();
-
- this.checkStaledescriptors();
-
- this.cleanUpRsyncDirectory();
-
- this.saveDescriptorDigests();
- }
-
- private void loadDescriptorDigests() {
- SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- try {
- if (this.storedServerDescriptorsFile.exists()) {
- BufferedReader br = new BufferedReader(new FileReader(
- this.storedServerDescriptorsFile));
- String line;
- while ((line = br.readLine()) != null) {
- String[] parts = line.split(",");
- if (parts.length != 3) {
- this.logger.warning("Could not load server descriptor "
- + "digests because of illegal line '" + line + "'. We "
- + "might not be able to correctly check descriptors for "
- + "completeness.");
- break;
- }
- long published = dateTimeFormat.parse(parts[0]).getTime();
- if (published < this.now - 48L * 60L * 60L * 1000L) {
- continue;
- }
- if (!this.storedServerDescriptors.containsKey(published)) {
- this.storedServerDescriptors.put(published,
- new HashMap<String, String>());
- }
- String serverDescriptorDigest = parts[1];
- String extraInfoDescriptorDigest = parts[2].equals("NA") ? null
- : parts[2];
- this.storedServerDescriptors.get(published).put(
- serverDescriptorDigest, extraInfoDescriptorDigest);
- }
- br.close();
- }
- if (this.storedExtraInfoDescriptorsFile.exists()) {
- BufferedReader br = new BufferedReader(new FileReader(
- this.storedExtraInfoDescriptorsFile));
- String line;
- while ((line = br.readLine()) != null) {
- String[] parts = line.split(",");
- if (parts.length != 2) {
- this.logger.warning("Could not load extra-info descriptor "
- + "digests because of illegal line '" + line + "'. We "
- + "might not be able to correctly check descriptors for "
- + "completeness.");
- break;
- }
- long published = dateTimeFormat.parse(parts[0]).getTime();
- if (published < this.now - 48L * 60L * 60L * 1000L) {
- continue;
- }
- if (!this.storedExtraInfoDescriptors.containsKey(published)) {
- this.storedExtraInfoDescriptors.put(published,
- new HashSet<String>());
- }
- String extraInfoDescriptorDigest = parts[1];
- this.storedExtraInfoDescriptors.get(published).add(
- extraInfoDescriptorDigest);
- }
- br.close();
- }
- if (this.storedMicrodescriptorsFile.exists()) {
- BufferedReader br = new BufferedReader(new FileReader(
- this.storedMicrodescriptorsFile));
- String line;
- while ((line = br.readLine()) != null) {
- String[] parts = line.split(",");
- if (parts.length != 2) {
- this.logger.warning("Could not load microdescriptor digests "
- + "because of illegal line '" + line + "'. We might not "
- + "be able to correctly check descriptors for "
- + "completeness.");
- break;
- }
- long validAfter = dateTimeFormat.parse(parts[0]).getTime();
- if (validAfter < this.now - 40L * 24L * 60L * 60L * 1000L) {
- continue;
- }
- if (!this.storedMicrodescriptors.containsKey(validAfter)) {
- this.storedMicrodescriptors.put(validAfter,
- new HashSet<String>());
- }
- String microdescriptorDigest = parts[1];
- this.storedMicrodescriptors.get(validAfter).add(
- microdescriptorDigest);
- }
- br.close();
- }
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Could not load descriptor "
- + "digests. We might not be able to correctly check "
- + "descriptors for completeness.", e);
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not load descriptor "
- + "digests. We might not be able to correctly check "
- + "descriptors for completeness.", e);
- }
- }
-
- public void intermediateStats(String event) {
- intermediateStats.append("While " + event + ", we stored "
- + this.storedConsensusesCounter + " consensus(es), "
- + this.storedMicrodescConsensusesCounter + " microdesc "
- + "consensus(es), " + this.storedVotesCounter + " vote(s), "
- + this.storedCertsCounter + " certificate(s), "
- + this.storedServerDescriptorsCounter + " server descriptor(s), "
- + this.storedExtraInfoDescriptorsCounter + " extra-info "
- + "descriptor(s), and " + this.storedMicrodescriptorsCounter
- + " microdescriptor(s) to disk.\n");
- this.storedConsensusesCounter = 0;
- this.storedMicrodescConsensusesCounter = 0;
- this.storedVotesCounter = 0;
- this.storedCertsCounter = 0;
- this.storedServerDescriptorsCounter = 0;
- this.storedExtraInfoDescriptorsCounter = 0;
- this.storedMicrodescriptorsCounter = 0;
- }
-
- private void checkMissingDescriptors() {
- StringBuilder sb = new StringBuilder("Finished writing relay "
- + "descriptors to disk.\n");
- sb.append(intermediateStats.toString());
- sb.append("Statistics on the completeness of written relay "
- + "descriptors:");
- SimpleDateFormat dateTimeFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- Map<String, String> knownServerDescriptors =
- new HashMap<String, String>();
- for (Map<String, String> descriptors :
- this.storedServerDescriptors.values()) {
- knownServerDescriptors.putAll(descriptors);
- }
- Set<String> knownExtraInfoDescriptors = new HashSet<String>();
- for (Set<String> descriptors :
- this.storedExtraInfoDescriptors.values()) {
- knownExtraInfoDescriptors.addAll(descriptors);
- }
- Set<String> knownMicrodescriptors = new HashSet<String>();
- for (Set<String> descriptors : this.storedMicrodescriptors.values()) {
- knownMicrodescriptors.addAll(descriptors);
- }
- boolean missingDescriptors = false;
- boolean missingVotes = false;
- boolean missingMicrodescConsensus = false;
- for (Map.Entry<Long, SortedSet<String>> c :
- this.storedConsensuses.entrySet()) {
- long validAfterMillis = c.getKey();
- String validAfterTime = dateTimeFormat.format(validAfterMillis);
- int allVotes = this.expectedVotes.containsKey(validAfterMillis)
- ? this.expectedVotes.get(validAfterMillis) : 0;
- int foundVotes = 0;
- if (this.storedVotes.containsKey(validAfterMillis)) {
- foundVotes = this.storedVotes.get(validAfterMillis).size();
- for (Map.Entry<String, SortedSet<String>> v :
- this.storedVotes.get(validAfterMillis).entrySet()) {
- int voteFoundServerDescs = 0;
- int voteAllServerDescs = 0;
- int voteFoundExtraInfos = 0;
- int voteAllExtraInfos = 0;
- for (String serverDescriptorDigest : v.getValue()) {
- voteAllServerDescs++;
- if (knownServerDescriptors.containsKey(
- serverDescriptorDigest)) {
- voteFoundServerDescs++;
- if (knownServerDescriptors.get(serverDescriptorDigest)
- != null) {
- String extraInfoDescriptorDigest =
- knownServerDescriptors.get(serverDescriptorDigest);
- voteAllExtraInfos++;
- if (knownExtraInfoDescriptors.contains(
- extraInfoDescriptorDigest)) {
- voteFoundExtraInfos++;
- }
- }
- }
- }
- sb.append("\nV, " + validAfterTime);
- if (voteAllServerDescs > 0) {
- sb.append(String.format(", %d/%d S (%.1f%%)",
- voteFoundServerDescs, voteAllServerDescs,
- 100.0D * (double) voteFoundServerDescs
- / (double) voteAllServerDescs));
- } else {
- sb.append(", 0/0 S");
- }
- if (voteAllExtraInfos > 0) {
- sb.append(String.format(", %d/%d E (%.1f%%)",
- voteFoundExtraInfos, voteAllExtraInfos,
- 100.0D * (double) voteFoundExtraInfos
- / (double) voteAllExtraInfos));
- } else {
- sb.append(", 0/0 E");
- }
- String fingerprint = v.getKey();
- /* Ignore turtles when warning about missing descriptors. */
- if (!fingerprint.equalsIgnoreCase(
- "27B6B5996C426270A5C95488AA5BCEB6BCC86956")
- && (voteFoundServerDescs * 1000 < voteAllServerDescs * 995
- || voteFoundExtraInfos * 1000 < voteAllExtraInfos * 995)) {
- missingDescriptors = true;
- }
- }
- }
- int foundServerDescs = 0;
- int allServerDescs = 0;
- int foundExtraInfos = 0;
- int allExtraInfos = 0;
- int foundMicrodescriptors = 0;
- int allMicrodescriptors = 0;
- for (String serverDescriptorDigest : c.getValue()) {
- allServerDescs++;
- if (knownServerDescriptors.containsKey(
- serverDescriptorDigest)) {
- foundServerDescs++;
- if (knownServerDescriptors.get(
- serverDescriptorDigest) != null) {
- allExtraInfos++;
- String extraInfoDescriptorDigest =
- knownServerDescriptors.get(serverDescriptorDigest);
- if (knownExtraInfoDescriptors.contains(
- extraInfoDescriptorDigest)) {
- foundExtraInfos++;
- }
- }
- }
- }
- sb.append("\nC, " + validAfterTime);
- if (allVotes > 0) {
- sb.append(String.format(", %d/%d V (%.1f%%)", foundVotes, allVotes,
- 100.0D * (double) foundVotes / (double) allVotes));
- } else {
- sb.append(", 0/0 V");
- }
- if (allServerDescs > 0) {
- sb.append(String.format(", %d/%d S (%.1f%%)", foundServerDescs,
- allServerDescs, 100.0D * (double) foundServerDescs
- / (double) allServerDescs));
- } else {
- sb.append(", 0/0 S");
- }
- if (allExtraInfos > 0) {
- sb.append(String.format(", %d/%d E (%.1f%%)", foundExtraInfos,
- allExtraInfos, 100.0D * (double) foundExtraInfos
- / (double) allExtraInfos));
- } else {
- sb.append(", 0/0 E");
- }
- if (this.storedMicrodescConsensuses.containsKey(validAfterMillis)) {
- for (String microdescriptorDigest :
- this.storedMicrodescConsensuses.get(validAfterMillis)) {
- allMicrodescriptors++;
- if (knownMicrodescriptors.contains(microdescriptorDigest)) {
- foundMicrodescriptors++;
- }
- }
- sb.append("\nM, " + validAfterTime);
- if (allMicrodescriptors > 0) {
- sb.append(String.format(", %d/%d M (%.1f%%)",
- foundMicrodescriptors, allMicrodescriptors,
- 100.0D * (double) foundMicrodescriptors
- / (double) allMicrodescriptors));
- } else {
- sb.append(", 0/0 M");
- }
- } else {
- missingMicrodescConsensus = true;
- }
- if (foundServerDescs * 1000 < allServerDescs * 995
- || foundExtraInfos * 1000 < allExtraInfos * 995
- || foundMicrodescriptors * 1000 < allMicrodescriptors * 995) {
- missingDescriptors = true;
- }
- if (foundVotes < allVotes) {
- missingVotes = true;
- }
- }
- this.logger.info(sb.toString());
- if (missingDescriptors) {
- this.logger.fine("We are missing at least 0.5% of server or "
- + "extra-info descriptors referenced from a consensus or "
- + "vote or at least 0.5% of microdescriptors referenced from a "
- + "microdesc consensus.");
- }
- if (missingVotes) {
- /* TODO Shouldn't warn if we're not trying to archive votes at
- * all. */
- this.logger.fine("We are missing at least one vote that was "
- + "referenced from a consensus.");
- }
- if (missingMicrodescConsensus) {
- /* TODO Shouldn't warn if we're not trying to archive microdesc
- * consensuses at all. */
- this.logger.fine("We are missing at least one microdesc "
- + "consensus that was published together with a known "
- + "consensus.");
- }
- }
-
- private void checkStaledescriptors() {
- SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- long tooOldMillis = this.now - 330L * 60L * 1000L;
- if (!this.storedConsensuses.isEmpty()
- && this.storedConsensuses.lastKey() < tooOldMillis) {
- this.logger.warning("The last known relay network status "
- + "consensus was valid after "
- + dateTimeFormat.format(this.storedConsensuses.lastKey())
- + ", which is more than 5:30 hours in the past.");
- }
- if (!this.storedMicrodescConsensuses.isEmpty()
- && this.storedMicrodescConsensuses.lastKey() < tooOldMillis) {
- this.logger.warning("The last known relay network status "
- + "microdesc consensus was valid after "
- + dateTimeFormat.format(
- this.storedMicrodescConsensuses.lastKey())
- + ", which is more than 5:30 hours in the past.");
- }
- if (!this.storedVotes.isEmpty()
- && this.storedVotes.lastKey() < tooOldMillis) {
- this.logger.warning("The last known relay network status vote "
- + "was valid after " + dateTimeFormat.format(
- this.storedVotes.lastKey()) + ", which is more than 5:30 hours "
- + "in the past.");
- }
- if (!this.storedServerDescriptors.isEmpty()
- && this.storedServerDescriptors.lastKey() < tooOldMillis) {
- this.logger.warning("The last known relay server descriptor was "
- + "published at "
- + dateTimeFormat.format(this.storedServerDescriptors.lastKey())
- + ", which is more than 5:30 hours in the past.");
- }
- if (!this.storedExtraInfoDescriptors.isEmpty()
- && this.storedExtraInfoDescriptors.lastKey() < tooOldMillis) {
- this.logger.warning("The last known relay extra-info descriptor "
- + "was published at " + dateTimeFormat.format(
- this.storedExtraInfoDescriptors.lastKey())
- + ", which is more than 5:30 hours in the past.");
- }
- if (!this.storedMicrodescriptors.isEmpty()
- && this.storedMicrodescriptors.lastKey() < tooOldMillis) {
- this.logger.warning("The last known relay microdescriptor was "
- + "contained in a microdesc consensus that was valid after "
- + dateTimeFormat.format(this.storedMicrodescriptors.lastKey())
- + ", which is more than 5:30 hours in the past.");
- }
- }
-
- /* Delete all files from the rsync directory that have not been modified
- * in the last three days (except for microdescriptors which are kept
- * for up to thirty days), and remove the .tmp extension from newly
- * written files. */
- public void cleanUpRsyncDirectory() {
- long cutOffMillis = System.currentTimeMillis()
- - 3L * 24L * 60L * 60L * 1000L;
- long cutOffMicroMillis = cutOffMillis - 27L * 24L * 60L * 60L * 1000L;
- Stack<File> allFiles = new Stack<File>();
- allFiles.add(new File("recent/relay-descriptors"));
- while (!allFiles.isEmpty()) {
- File file = allFiles.pop();
- if (file.isDirectory()) {
- allFiles.addAll(Arrays.asList(file.listFiles()));
- } else if (file.getName().endsWith("-micro")) {
- if (file.lastModified() < cutOffMicroMillis) {
- file.delete();
- }
- } else if (file.lastModified() < cutOffMillis) {
- file.delete();
- } else if (file.getName().endsWith(".tmp")) {
- file.renameTo(new File(file.getParentFile(),
- file.getName().substring(0,
- file.getName().lastIndexOf(".tmp"))));
- }
- }
- }
-
- private void saveDescriptorDigests() {
- SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- try {
- this.storedServerDescriptorsFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.storedServerDescriptorsFile));
- for (Map.Entry<Long, Map<String, String>> e :
- this.storedServerDescriptors.entrySet()) {
- String published = dateTimeFormat.format(e.getKey());
- for (Map.Entry<String, String> f : e.getValue().entrySet()) {
- String serverDescriptorDigest = f.getKey();
- String extraInfoDescriptorDigest = f.getValue() == null ? "NA"
- : f.getValue();
- bw.write(String.format("%s,%s,%s%n", published,
- serverDescriptorDigest, extraInfoDescriptorDigest));
- }
- }
- bw.close();
- this.storedExtraInfoDescriptorsFile.getParentFile().mkdirs();
- bw = new BufferedWriter(new FileWriter(
- this.storedExtraInfoDescriptorsFile));
- for (Map.Entry<Long, Set<String>> e :
- this.storedExtraInfoDescriptors.entrySet()) {
- String published = dateTimeFormat.format(e.getKey());
- for (String extraInfoDescriptorDigest : e.getValue()) {
- bw.write(String.format("%s,%s%n", published,
- extraInfoDescriptorDigest));
- }
- }
- bw.close();
- this.storedMicrodescriptorsFile.getParentFile().mkdirs();
- bw = new BufferedWriter(new FileWriter(
- this.storedMicrodescriptorsFile));
- for (Map.Entry<Long, Set<String>> e :
- this.storedMicrodescriptors.entrySet()) {
- String validAfter = dateTimeFormat.format(e.getKey());
- for (String microdescriptorDigest : e.getValue()) {
- bw.write(String.format("%s,%s%n", validAfter,
- microdescriptorDigest));
- }
- }
- bw.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not save descriptor "
- + "digests. We might not be able to correctly check "
- + "descriptors for completeness in the next run.", e);
- }
- }
-
- public void storeConsensus(byte[] data, long validAfter,
- SortedSet<String> dirSources,
- SortedSet<String> serverDescriptorDigests) {
- SimpleDateFormat printFormat = new SimpleDateFormat(
- "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
- printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory + "/consensus/"
- + printFormat.format(new Date(validAfter)) + "-consensus");
- boolean tarballFileExistedBefore = tarballFile.exists();
- File rsyncFile = new File("recent/relay-descriptors/consensuses/"
- + tarballFile.getName());
- File[] outputFiles = new File[] { tarballFile, rsyncFile };
- if (this.store(CONSENSUS_ANNOTATION, data, outputFiles, null)) {
- this.storedConsensusesCounter++;
- }
- if (!tarballFileExistedBefore
- && this.now - validAfter < 3L * 60L * 60L * 1000L) {
- this.storedConsensuses.put(validAfter, serverDescriptorDigests);
- this.expectedVotes.put(validAfter, dirSources.size());
- }
- }
-
- public void storeMicrodescConsensus(byte[] data, long validAfter,
- SortedSet<String> microdescriptorDigests) {
- SimpleDateFormat yearMonthDirectoryFormat = new SimpleDateFormat(
- "yyyy/MM");
- yearMonthDirectoryFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- SimpleDateFormat dayDirectoryFileFormat = new SimpleDateFormat(
- "dd/yyyy-MM-dd-HH-mm-ss");
- dayDirectoryFileFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory
- + "/microdesc/" + yearMonthDirectoryFormat.format(validAfter)
- + "/consensus-microdesc/"
- + dayDirectoryFileFormat.format(validAfter)
- + "-consensus-microdesc");
- boolean tarballFileExistedBefore = tarballFile.exists();
- File rsyncFile = new File("recent/relay-descriptors/microdescs/"
- + "consensus-microdesc/" + tarballFile.getName());
- File[] outputFiles = new File[] { tarballFile, rsyncFile };
- if (this.store(MICRODESCCONSENSUS_ANNOTATION, data, outputFiles,
- null)) {
- this.storedMicrodescConsensusesCounter++;
- }
- if (!tarballFileExistedBefore
- && this.now - validAfter < 3L * 60L * 60L * 1000L) {
- this.storedMicrodescConsensuses.put(validAfter,
- microdescriptorDigests);
- }
- }
-
- public void storeVote(byte[] data, long validAfter,
- String fingerprint, String digest,
- SortedSet<String> serverDescriptorDigests) {
- SimpleDateFormat printFormat = new SimpleDateFormat(
- "yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
- printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory + "/vote/"
- + printFormat.format(new Date(validAfter)) + "-vote-"
- + fingerprint + "-" + digest);
- boolean tarballFileExistedBefore = tarballFile.exists();
- File rsyncFile = new File("recent/relay-descriptors/votes/"
- + tarballFile.getName());
- File[] outputFiles = new File[] { tarballFile, rsyncFile };
- if (this.store(VOTE_ANNOTATION, data, outputFiles, null)) {
- this.storedVotesCounter++;
- }
- if (!tarballFileExistedBefore
- && this.now - validAfter < 3L * 60L * 60L * 1000L) {
- if (!this.storedVotes.containsKey(validAfter)) {
- this.storedVotes.put(validAfter,
- new TreeMap<String, SortedSet<String>>());
- }
- this.storedVotes.get(validAfter).put(fingerprint,
- serverDescriptorDigests);
- }
- }
-
- public void storeCertificate(byte[] data, String fingerprint,
- long published) {
- SimpleDateFormat printFormat = new SimpleDateFormat(
- "yyyy-MM-dd-HH-mm-ss");
- printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory + "/certs/"
- + fingerprint + "-" + printFormat.format(new Date(published)));
- File[] outputFiles = new File[] { tarballFile };
- if (this.store(CERTIFICATE_ANNOTATION, data, outputFiles, null)) {
- this.storedCertsCounter++;
- }
- }
-
- public void storeServerDescriptor(byte[] data, String digest,
- long published, String extraInfoDigest) {
- SimpleDateFormat printFormat = new SimpleDateFormat("yyyy/MM/");
- printFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory
- + "/server-descriptor/" + printFormat.format(new Date(published))
- + digest.substring(0, 1) + "/" + digest.substring(1, 2) + "/"
- + digest);
- boolean tarballFileExistedBefore = tarballFile.exists();
- File rsyncCatFile = new File("recent/relay-descriptors/"
- + "server-descriptors/" + this.rsyncCatString
- + "-server-descriptors.tmp");
- File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
- boolean[] append = new boolean[] { false, true };
- if (this.store(SERVER_DESCRIPTOR_ANNOTATION, data, outputFiles,
- append)) {
- this.storedServerDescriptorsCounter++;
- }
- if (!tarballFileExistedBefore
- && this.now - published < 48L * 60L * 60L * 1000L) {
- if (!this.storedServerDescriptors.containsKey(published)) {
- this.storedServerDescriptors.put(published,
- new HashMap<String, String>());
- }
- this.storedServerDescriptors.get(published).put(digest,
- extraInfoDigest);
- }
- }
-
- public void storeExtraInfoDescriptor(byte[] data,
- String extraInfoDigest, long published) {
- SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
- descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory + "/extra-info/"
- + descriptorFormat.format(new Date(published))
- + extraInfoDigest.substring(0, 1) + "/"
- + extraInfoDigest.substring(1, 2) + "/"
- + extraInfoDigest);
- boolean tarballFileExistedBefore = tarballFile.exists();
- File rsyncCatFile = new File("recent/relay-descriptors/"
- + "extra-infos/" + this.rsyncCatString + "-extra-infos.tmp");
- File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
- boolean[] append = new boolean[] { false, true };
- if (this.store(EXTRA_INFO_ANNOTATION, data, outputFiles, append)) {
- this.storedExtraInfoDescriptorsCounter++;
- }
- if (!tarballFileExistedBefore
- && this.now - published < 48L * 60L * 60L * 1000L) {
- if (!this.storedExtraInfoDescriptors.containsKey(published)) {
- this.storedExtraInfoDescriptors.put(published,
- new HashSet<String>());
- }
- this.storedExtraInfoDescriptors.get(published).add(extraInfoDigest);
- }
- }
-
- public void storeMicrodescriptor(byte[] data,
- String microdescriptorDigest, long validAfter) {
- /* TODO We could check here whether we already stored the
- * microdescriptor in the same valid-after month. This can happen,
- * e.g., when two relays share the same microdescriptor. In that case
- * this method gets called twice and the second call overwrites the
- * file written in the first call. However, this method must be
- * called twice to store the same microdescriptor in two different
- * valid-after months. */
- SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
- descriptorFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- File tarballFile = new File(this.outputDirectory + "/microdesc/"
- + descriptorFormat.format(validAfter) + "micro/"
- + microdescriptorDigest.substring(0, 1) + "/"
- + microdescriptorDigest.substring(1, 2) + "/"
- + microdescriptorDigest);
- boolean tarballFileExistedBefore = tarballFile.exists();
- File rsyncCatFile = new File("recent/relay-descriptors/"
- + "microdescs/micro/" + this.rsyncCatString
- + "-micro.tmp");
- File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
- boolean[] append = new boolean[] { false, true };
- if (this.store(MICRODESCRIPTOR_ANNOTATION, data, outputFiles,
- append)) {
- this.storedMicrodescriptorsCounter++;
- }
- if (!tarballFileExistedBefore
- && this.now - validAfter < 40L * 24L * 60L * 60L * 1000L) {
- if (!this.storedMicrodescriptors.containsKey(validAfter)) {
- this.storedMicrodescriptors.put(validAfter,
- new HashSet<String>());
- }
- this.storedMicrodescriptors.get(validAfter).add(
- microdescriptorDigest);
- }
- }
-
- private boolean store(byte[] typeAnnotation, byte[] data,
- File[] outputFiles, boolean[] append) {
- try {
- this.logger.finer("Storing " + outputFiles[0]);
- if (this.descriptorParser.parseDescriptors(data,
- outputFiles[0].getName()).size() != 1) {
- this.logger.info("Relay descriptor file " + outputFiles[0]
- + " doesn't contain exactly one descriptor. Not storing.");
- return false;
- }
- for (int i = 0; i < outputFiles.length; i++) {
- File outputFile = outputFiles[i];
- boolean appendToFile = append == null ? false : append[i];
- outputFile.getParentFile().mkdirs();
- BufferedOutputStream bos = new BufferedOutputStream(
- new FileOutputStream(outputFile, appendToFile));
- if (data.length > 0 && data[0] != '@') {
- bos.write(typeAnnotation, 0, typeAnnotation.length);
- }
- bos.write(data, 0, data.length);
- bos.close();
- }
- return true;
- } catch (DescriptorParseException e) {
- this.logger.log(Level.WARNING, "Could not parse relay descriptor "
- + outputFiles[0] + " before storing it to disk. Skipping.", e);
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not store relay descriptor "
- + outputFiles[0], e);
- }
- return false;
- }
-}
diff --git a/src/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java b/src/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java
deleted file mode 100644
index b9001dd..0000000
--- a/src/org/torproject/collector/relaydescs/CachedRelayDescriptorReader.java
+++ /dev/null
@@ -1,255 +0,0 @@
-/* Copyright 2010--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.relaydescs;
-
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.StringReader;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TimeZone;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Parses all descriptors in local directory cacheddesc/ and sorts them
- * into directory structure in directory-archive/.
- */
-public class CachedRelayDescriptorReader {
- public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
- List<String> inputDirectories, File statsDirectory) {
-
- if (rdp == null || inputDirectories == null
- || inputDirectories.isEmpty() || statsDirectory == null) {
- throw new IllegalArgumentException();
- }
-
- StringBuilder dumpStats = new StringBuilder("Finished importing "
- + "relay descriptors from local Tor data directories:");
- Logger logger = Logger.getLogger(
- CachedRelayDescriptorReader.class.getName());
-
- /* Read import history containing SHA-1 digests of previously parsed
- * statuses and descriptors, so that we can skip them in this run. */
- Set<String> lastImportHistory = new HashSet<String>();
- Set<String> currentImportHistory = new HashSet<String>();
- File importHistoryFile = new File(statsDirectory,
- "cacheddesc-import-history");
- if (importHistoryFile.exists()) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- importHistoryFile));
- String line;
- while ((line = br.readLine()) != null) {
- lastImportHistory.add(line);
- }
- br.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not read import history from "
- + importHistoryFile.getAbsolutePath() + ".", e);
- }
- }
-
- /* Read cached descriptors directories. */
- for (String inputDirectory : inputDirectories) {
- File cachedDescDir = new File(inputDirectory);
- if (!cachedDescDir.exists()) {
- logger.warning("Directory " + cachedDescDir.getAbsolutePath()
- + " does not exist. Skipping.");
- continue;
- }
- logger.fine("Reading " + cachedDescDir.getAbsolutePath()
- + " directory.");
- SortedSet<File> cachedDescFiles = new TreeSet<File>();
- Stack<File> files = new Stack<File>();
- files.add(cachedDescDir);
- while (!files.isEmpty()) {
- File file = files.pop();
- if (file.isDirectory()) {
- files.addAll(Arrays.asList(file.listFiles()));
- } else {
- cachedDescFiles.add(file);
- }
- }
- for (File f : cachedDescFiles) {
- try {
- // descriptors may contain non-ASCII chars; read as bytes to
- // determine digests
- BufferedInputStream bis =
- new BufferedInputStream(new FileInputStream(f));
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- bis.close();
- byte[] allData = baos.toByteArray();
- if (f.getName().equals("cached-consensus")) {
- /* Check if directory information is stale. */
- BufferedReader br = new BufferedReader(new StringReader(
- new String(allData, "US-ASCII")));
- String line = null;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("valid-after ")) {
- dumpStats.append("\n" + f.getName() + ": " + line.substring(
- "valid-after ".length()));
- SimpleDateFormat dateTimeFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (dateTimeFormat.parse(line.substring("valid-after "
- .length())).getTime() < System.currentTimeMillis()
- - 6L * 60L * 60L * 1000L) {
- logger.warning("Cached descriptor files in "
- + cachedDescDir.getAbsolutePath() + " are stale. "
- + "The valid-after line in cached-consensus is '"
- + line + "'.");
- dumpStats.append(" (stale!)");
- }
- break;
- }
- }
- br.close();
-
- /* Parse the cached consensus if we haven't parsed it before
- * (but regardless of whether it's stale or not). */
- if (rdp != null) {
- String digest = Hex.encodeHexString(DigestUtils.sha(
- allData));
- if (!lastImportHistory.contains(digest)
- && !currentImportHistory.contains(digest)) {
- rdp.parse(allData);
- } else {
- dumpStats.append(" (skipped)");
- }
- currentImportHistory.add(digest);
- }
- } else if (f.getName().equals("v3-status-votes")) {
- int parsedNum = 0;
- int skippedNum = 0;
- String ascii = new String(allData, "US-ASCII");
- String startToken = "network-status-version ";
- int end = ascii.length();
- int start = ascii.indexOf(startToken);
- while (start >= 0 && start < end) {
- int next = ascii.indexOf(startToken, start + 1);
- if (next < 0) {
- next = end;
- }
- if (start < next) {
- byte[] rawNetworkStatusBytes = new byte[next - start];
- System.arraycopy(allData, start, rawNetworkStatusBytes, 0,
- next - start);
- if (rdp != null) {
- String digest = Hex.encodeHexString(DigestUtils.sha(
- rawNetworkStatusBytes));
- if (!lastImportHistory.contains(digest)
- && !currentImportHistory.contains(digest)) {
- rdp.parse(rawNetworkStatusBytes);
- parsedNum++;
- } else {
- skippedNum++;
- }
- currentImportHistory.add(digest);
- }
- }
- start = next;
- }
- dumpStats.append("\n" + f.getName() + ": parsed " + parsedNum
- + ", skipped " + skippedNum + " votes");
- } else if (f.getName().startsWith("cached-descriptors")
- || f.getName().startsWith("cached-extrainfo")) {
- String ascii = new String(allData, "US-ASCII");
- int start = -1;
- int sig = -1;
- int end = -1;
- String startToken =
- f.getName().startsWith("cached-descriptors")
- ? "router " : "extra-info ";
- String sigToken = "\nrouter-signature\n";
- String endToken = "\n-----END SIGNATURE-----\n";
- int parsedNum = 0;
- int skippedNum = 0;
- while (end < ascii.length()) {
- start = ascii.indexOf(startToken, end);
- if (start < 0) {
- break;
- }
- sig = ascii.indexOf(sigToken, start);
- if (sig < 0) {
- break;
- }
- sig += sigToken.length();
- end = ascii.indexOf(endToken, sig);
- if (end < 0) {
- break;
- }
- end += endToken.length();
- byte[] descBytes = new byte[end - start];
- System.arraycopy(allData, start, descBytes, 0, end - start);
- if (rdp != null) {
- String digest = Hex.encodeHexString(DigestUtils.sha(
- descBytes));
- if (!lastImportHistory.contains(digest)
- && !currentImportHistory.contains(digest)) {
- rdp.parse(descBytes);
- parsedNum++;
- } else {
- skippedNum++;
- }
- currentImportHistory.add(digest);
- }
- }
- dumpStats.append("\n" + f.getName() + ": parsed " + parsedNum
- + ", skipped " + skippedNum + " "
- + (f.getName().startsWith("cached-descriptors")
- ? "server" : "extra-info") + " descriptors");
- }
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed reading "
- + cachedDescDir.getAbsolutePath() + " directory.", e);
- } catch (ParseException e) {
- logger.log(Level.WARNING, "Failed reading "
- + cachedDescDir.getAbsolutePath() + " directory.", e);
- }
- }
- logger.fine("Finished reading "
- + cachedDescDir.getAbsolutePath() + " directory.");
- }
-
- /* Write import history containing SHA-1 digests to disk. */
- try {
- importHistoryFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- importHistoryFile));
- for (String digest : currentImportHistory) {
- bw.write(digest + "\n");
- }
- bw.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Could not write import history to "
- + importHistoryFile.getAbsolutePath() + ".", e);
- }
-
- logger.info(dumpStats.toString());
- }
-}
-
diff --git a/src/org/torproject/collector/relaydescs/ReferenceChecker.java b/src/org/torproject/collector/relaydescs/ReferenceChecker.java
deleted file mode 100644
index 9f0f183..0000000
--- a/src/org/torproject/collector/relaydescs/ReferenceChecker.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/* Copyright 2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.relaydescs;
-
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
-import org.torproject.descriptor.DescriptorReader;
-import org.torproject.descriptor.DescriptorSourceFactory;
-import org.torproject.descriptor.DirSourceEntry;
-import org.torproject.descriptor.ExtraInfoDescriptor;
-import org.torproject.descriptor.Microdescriptor;
-import org.torproject.descriptor.NetworkStatusEntry;
-import org.torproject.descriptor.RelayNetworkStatusConsensus;
-import org.torproject.descriptor.RelayNetworkStatusVote;
-import org.torproject.descriptor.ServerDescriptor;
-
-import com.google.gson.Gson;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Locale;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TimeZone;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-public class ReferenceChecker {
-
- private Logger log = Logger.getLogger(ReferenceChecker.class.getName());
-
- private File descriptorsDir;
-
- private File referencesFile;
-
- private File historyFile;
-
- private long currentTimeMillis;
-
- private SortedSet<Reference> references = new TreeSet<Reference>();
-
- private static DateFormat dateTimeFormat;
-
- static {
- dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'",
- Locale.US);
- dateTimeFormat.setLenient(false);
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- }
-
- private static final long ONE_HOUR = 60L * 60L * 1000L;
-
- private static final long THREE_HOURS = 3L * ONE_HOUR;
-
- private static final long SIX_HOURS = 6L * ONE_HOUR;
-
- private static final long ONE_DAY = 24L * ONE_HOUR;
-
- private static final long THIRTY_DAYS = 30L * ONE_DAY;
-
- public ReferenceChecker(File descriptorsDir, File referencesFile,
- File historyFile) {
- this.descriptorsDir = descriptorsDir;
- this.referencesFile = referencesFile;
- this.historyFile = historyFile;
- }
-
- public void check() {
- this.getCurrentTimeMillis();
- this.readReferencesFile();
- this.readNewDescriptors();
- this.dropStaleReferences();
- this.checkReferences();
- this.writeReferencesFile();
- }
-
- private void getCurrentTimeMillis() {
- this.currentTimeMillis = System.currentTimeMillis();
- }
-
- private static class Reference implements Comparable<Reference> {
-
- private String referencing;
-
- private String referenced;
-
- private double weight;
-
- private long expiresAfterMillis;
-
- public Reference(String referencing, String referenced, double weight,
- long expiresAfterMillis) {
- this.referencing = referencing;
- this.referenced = referenced;
- this.weight = weight;
- this.expiresAfterMillis = expiresAfterMillis;
- }
-
- @Override
- public boolean equals(Object otherObject) {
- if (!(otherObject instanceof Reference)) {
- return false;
- }
- Reference other = (Reference) otherObject;
- return this.referencing.equals(other.referencing)
- && this.referenced.equals(other.referenced);
- }
-
- @Override
- public int hashCode() {
- return this.referencing.hashCode() + this.referenced.hashCode();
- }
-
- @Override
- public int compareTo(Reference other) {
- int result = this.referencing.compareTo(other.referencing);
- if (result == 0) {
- result = this.referenced.compareTo(other.referenced);
- }
- return result;
- }
- }
-
- private void readReferencesFile() {
- if (!this.referencesFile.exists()) {
- return;
- }
- Gson gson = new Gson();
- try {
- FileReader fr = new FileReader(this.referencesFile);
- this.references.addAll(Arrays.asList(gson.fromJson(fr,
- Reference[].class)));
- fr.close();
- } catch (IOException e) {
- this.log.log(Level.WARNING, "Cannot read existing references file "
- + "from previous run.", e);
- }
- }
-
- private void readNewDescriptors() {
- DescriptorReader descriptorReader =
- DescriptorSourceFactory.createDescriptorReader();
- descriptorReader.addDirectory(this.descriptorsDir);
- descriptorReader.setExcludeFiles(this.historyFile);
- Iterator<DescriptorFile> descriptorFiles =
- descriptorReader.readDescriptors();
- while (descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = descriptorFiles.next();
- for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- if (descriptor instanceof RelayNetworkStatusConsensus) {
- RelayNetworkStatusConsensus consensus =
- (RelayNetworkStatusConsensus) descriptor;
- String consensusFlavor = consensus.getConsensusFlavor();
- if (consensusFlavor == null) {
- this.readRelayNetworkStatusConsensusUnflavored(consensus);
- } else if (consensusFlavor.equals("microdesc")) {
- this.readRelayNetworkStatusConsensusMicrodesc(consensus);
- } else {
- /* Ignore unknown consensus flavors. */
- }
- } else if (descriptor instanceof RelayNetworkStatusVote) {
- this.readRelayNetworkStatusVote(
- (RelayNetworkStatusVote) descriptor);
- } else if (descriptor instanceof ServerDescriptor) {
- this.readServerDescriptor((ServerDescriptor) descriptor);
- } else if (descriptor instanceof ExtraInfoDescriptor) {
- this.readExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
- } else if (descriptor instanceof Microdescriptor) {
- readMicrodescriptor((Microdescriptor) descriptor);
- } else {
- /* Ignore unknown descriptors. */
- }
- }
- }
- }
-
- private void readRelayNetworkStatusConsensusUnflavored(
- RelayNetworkStatusConsensus consensus) {
- String validAfter = dateTimeFormat.format(
- consensus.getValidAfterMillis());
- String referencing = String.format("C-%s", validAfter);
- this.addReference(referencing, String.format("M-%s", validAfter), 1.0,
- consensus.getValidAfterMillis() + THREE_HOURS);
- for (DirSourceEntry dirSourceEntry :
- consensus.getDirSourceEntries().values()) {
- if (!dirSourceEntry.isLegacy()) {
- this.addReference(referencing, String.format("V-%s-%s",
- validAfter, dirSourceEntry.getIdentity()), 1.0,
- consensus.getValidAfterMillis() + THREE_HOURS);
- }
- }
- double entryWeight = 200.0
- / ((double) consensus.getStatusEntries().size());
- for (NetworkStatusEntry entry :
- consensus.getStatusEntries().values()) {
- this.addReference(referencing,
- String.format("S-%s", entry.getDescriptor()), entryWeight,
- entry.getPublishedMillis() + THREE_HOURS);
- }
- }
-
- private void readRelayNetworkStatusConsensusMicrodesc(
- RelayNetworkStatusConsensus consensus) {
- String validAfter = dateTimeFormat.format(
- consensus.getValidAfterMillis());
- String referencing = String.format("M-%s", validAfter);
- this.addReference(referencing, String.format("C-%s", validAfter), 1.0,
- consensus.getValidAfterMillis() + THREE_HOURS);
- double entryWeight = 200.0
- / ((double) consensus.getStatusEntries().size());
- for (NetworkStatusEntry entry :
- consensus.getStatusEntries().values()) {
- for (String digest : entry.getMicrodescriptorDigests()) {
- this.addReference(referencing, String.format("D-%s", digest),
- entryWeight, entry.getPublishedMillis() + THREE_HOURS);
- }
- }
- }
-
- private void readRelayNetworkStatusVote(RelayNetworkStatusVote vote) {
- String validAfter = dateTimeFormat.format(vote.getValidAfterMillis());
- String referencing = String.format("V-%s-%s", validAfter,
- vote.getIdentity());
- double entryWeight = 200.0
- / ((double) vote.getStatusEntries().size());
- for (NetworkStatusEntry entry : vote.getStatusEntries().values()) {
- this.addReference(referencing,
- String.format("S-%s", entry.getDescriptor()), entryWeight,
- entry.getPublishedMillis() + SIX_HOURS);
- }
- }
-
- private void readServerDescriptor(ServerDescriptor serverDescriptor) {
- String referenced = serverDescriptor.getExtraInfoDigest() == null ? ""
- : String.format("E-%s", serverDescriptor.getExtraInfoDigest());
- this.addReference(String.format("S-%s",
- serverDescriptor.getServerDescriptorDigest()), referenced, 0.01,
- serverDescriptor.getPublishedMillis() + SIX_HOURS);
- }
-
- private void readExtraInfoDescriptor(
- ExtraInfoDescriptor extraInfoDescriptor) {
- this.addReference(String.format("E-%s",
- extraInfoDescriptor.getExtraInfoDigest()), "", 0.005,
- extraInfoDescriptor.getPublishedMillis() + SIX_HOURS);
- }
-
- private void readMicrodescriptor(Microdescriptor microdesc) {
- this.addReference(
- String.format("D-%s", microdesc.getMicrodescriptorDigest()), "",
- 0.0, this.currentTimeMillis + THIRTY_DAYS);
- }
-
- private void addReference(String referencing, String referenced,
- double weight, long expiresAfterMillis) {
- this.references.add(new Reference(referencing.toUpperCase(),
- referenced.toUpperCase(), weight, expiresAfterMillis));
- }
-
- private void dropStaleReferences() {
- SortedSet<Reference> recentReferences = new TreeSet<Reference>();
- for (Reference reference : this.references) {
- if (this.currentTimeMillis <= reference.expiresAfterMillis) {
- recentReferences.add(reference);
- }
- }
- this.references = recentReferences;
- }
-
- private void checkReferences() {
- Set<String> knownDescriptors = new HashSet<String>();
- for (Reference reference : this.references) {
- knownDescriptors.add(reference.referencing);
- }
- double totalMissingDescriptorsWeight = 0.0;
- Set<String> missingDescriptors = new TreeSet<String>();
- StringBuilder sb = new StringBuilder("Missing referenced "
- + "descriptors:");
- for (Reference reference : this.references) {
- if (reference.referenced.length() > 0
- && !knownDescriptors.contains(reference.referenced)) {
- if (!missingDescriptors.contains(reference.referenced)) {
- totalMissingDescriptorsWeight += reference.weight;
- }
- missingDescriptors.add(reference.referenced);
- sb.append(String.format("%n%s -> %s (%.4f -> %.4f)",
- reference.referencing, reference.referenced, reference.weight,
- totalMissingDescriptorsWeight));
- }
- }
- this.log.log(Level.INFO, sb.toString());
- if (totalMissingDescriptorsWeight > 0.999) {
- this.log.log(Level.WARNING, "Missing too many referenced "
- + "descriptors (" + totalMissingDescriptorsWeight + ").");
- }
- }
-
- private void writeReferencesFile() {
- Gson gson = new Gson();
- try {
- FileWriter fw = new FileWriter(this.referencesFile);
- gson.toJson(this.references, fw);
- fw.close();
- } catch (IOException e) {
- this.log.log(Level.WARNING, "Cannot write references file for next "
- + "run.", e);
- }
- }
-}
-
diff --git a/src/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java b/src/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java
deleted file mode 100644
index 458332a..0000000
--- a/src/org/torproject/collector/relaydescs/RelayDescriptorDownloader.java
+++ /dev/null
@@ -1,1134 +0,0 @@
-/* Copyright 2010--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.relaydescs;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.digest.DigestUtils;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.zip.InflaterInputStream;
-
-/**
- * Downloads relay descriptors from the directory authorities via HTTP.
- * Keeps a list of missing descriptors that gets updated by parse results
- * from <code>RelayDescriptorParser</code> and downloads all missing
- * descriptors that have been published in the last 24 hours. Also
- * downloads all server and extra-info descriptors known to a directory
- * authority at most once a day.
- */
-public class RelayDescriptorDownloader {
-
- /**
- * Text file containing the descriptors that we are missing and that we
- * want to download. Lines are formatted as:
- *
- * - "consensus,<validafter>,<parsed>",
- * - "consensus-microdesc,<validafter>,<parsed>",
- * - "vote,<validafter>,<fingerprint>,<parsed>",
- * - "server,<published>,<relayid>,<descid>,<parsed>",
- * - "extra,<published>,<relayid>,<descid>,<parsed>", or
- * - "micro,<validafter>,<relayid>,<descid>,<parsed>".
- */
- private File missingDescriptorsFile;
-
- /**
- * Relay descriptors that we are missing and that we want to download
- * either in this execution or write to disk and try next time. Map keys
- * contain comma-separated values as in the missing descriptors files
- * without the "parsed" column. Map values contain the "parsed" column.
- */
- private SortedMap<String, String> missingDescriptors;
-
- /**
- * Map from base64 microdescriptor digests to keys in missingDescriptors
- * ("micro,<validafter>,<relayid>,<descid>"). We need this map, because
- * we can't learn <validafter> or <relayid> from parsing
- * microdescriptors, but we need to know <validafter> to store
- * microdescriptors to disk and both <validafter> and <relayid> to
- * remove microdescriptors from the missing list. There are potentially
- * many matching keys in missingDescriptors for the same microdescriptor
- * digest. Also, in rare cases relays share the same microdescriptor
- * (which is only possible if they share the same onion key), and then
- * we don't have to download their microdescriptor more than once.
- */
- private Map<String, Set<String>> microdescriptorKeys;
-
- /**
- * Set of microdescriptor digests that are currently missing. Used for
- * logging statistics instead of "micro,<validafter>,..." keys which may
- * contain the same microdescriptor digest multiple times.
- */
- private Set<String> missingMicrodescriptors;
-
- /**
- * Text file containing the IP addresses (and Dir ports if not 80) of
- * directory authorities and when we last downloaded all server and
- * extra-info descriptors from them, so that we can avoid downloading
- * them too often.
- */
- private File lastDownloadedAllDescriptorsFile;
-
- /**
- * Map of directory authorities and when we last downloaded all server
- * and extra-info descriptors from them. Map keys are IP addresses (and
- * Dir ports if not 80), map values are timestamps.
- */
- private Map<String, String> lastDownloadedAllDescriptors;
-
- /**
- * <code>RelayDescriptorParser</code> that we will hand over the
- * downloaded descriptors for parsing.
- */
- private RelayDescriptorParser rdp;
-
- /**
- * Directory authorities that we will try to download missing
- * descriptors from.
- */
- private List<String> authorities;
-
- /**
- * Fingerprints of directory authorities that we will use to download
- * votes without requiring a successfully downloaded consensus.
- */
- private List<String> authorityFingerprints;
-
- /**
- * Should we try to download the current consensus if we don't have it?
- */
- private boolean downloadCurrentConsensus;
-
- /**
- * Should we try to download the current microdesc consensus if we don't
- * have it?
- */
- private boolean downloadCurrentMicrodescConsensus;
-
- /**
- * Should we try to download current votes if we don't have them?
- */
- private boolean downloadCurrentVotes;
-
- /**
- * Should we try to download missing server descriptors that have been
- * published within the past 24 hours?
- */
- private boolean downloadMissingServerDescriptors;
-
- /**
- * Should we try to download missing extra-info descriptors that have
- * been published within the past 24 hours?
- */
- private boolean downloadMissingExtraInfos;
-
- /**
- * Should we try to download missing microdescriptors that have been
- * published within the past 24 hours?
- */
- private boolean downloadMissingMicrodescriptors;
-
- /**
- * Should we try to download all server descriptors from the authorities
- * once every 24 hours?
- */
- private boolean downloadAllServerDescriptors;
-
- /**
- * Should we try to download all extra-info descriptors from the
- * authorities once every 24 hours?
- */
- private boolean downloadAllExtraInfos;
-
- /**
- * Should we download zlib-compressed versions of descriptors by adding
- * ".z" to URLs?
- */
- private boolean downloadCompressed;
-
- /**
- * valid-after time that we expect the current consensus,
- * microdescriptor consensus, and votes to have, formatted
- * "yyyy-MM-dd HH:mm:ss". We only expect to find documents with this
- * valid-after time on the directory authorities. This time is
- * initialized as the beginning of the current hour.
- */
- private String currentValidAfter;
-
- /**
- * Cut-off time for missing server and extra-info descriptors, formatted
- * "yyyy-MM-dd HH:mm:ss". This time is initialized as the current system
- * time minus 24 hours.
- */
- private String descriptorCutOff;
-
- /**
- * Cut-off time for downloading all server and extra-info descriptors
- * from the directory authorities, formatted "yyyy-MM-dd HH:mm:ss". This
- * time is initialized as the current system time minus 23:30 hours.
- */
- private String downloadAllDescriptorsCutOff;
-
- /**
- * Directory authorities that we plan to download all server and
- * extra-info descriptors from in this execution.
- */
- private Set<String> downloadAllDescriptorsFromAuthorities;
-
- /**
- * Current timestamp that is written to the missing list for descriptors
- * that we parsed in this execution and for authorities that we
- * downloaded all server and extra-info descriptors from.
- */
- private String currentTimestamp;
-
- /**
- * Logger for this class.
- */
- private Logger logger;
-
- /**
- * Number of descriptors requested by directory authority to be included
- * in logs.
- */
- private Map<String, Integer> requestsByAuthority;
-
- /**
- * Counters for descriptors that we had on the missing list at the
- * beginning of the execution, that we added to the missing list,
- * that we requested, and that we successfully downloaded in this
- * execution.
- */
- private int oldMissingConsensuses = 0;
-
- private int oldMissingMicrodescConsensuses = 0;
-
- private int oldMissingVotes = 0;
-
- private int oldMissingServerDescriptors = 0;
-
- private int oldMissingExtraInfoDescriptors = 0;
-
- private int oldMissingMicrodescriptors = 0;
-
- private int newMissingConsensuses = 0;
-
- private int newMissingMicrodescConsensuses = 0;
-
- private int newMissingVotes = 0;
-
- private int newMissingServerDescriptors = 0;
-
- private int newMissingExtraInfoDescriptors = 0;
-
- private int newMissingMicrodescriptors = 0;
-
- private int requestedConsensuses = 0;
-
- private int requestedMicrodescConsensuses = 0;
-
- private int requestedVotes = 0;
-
- private int requestedMissingServerDescriptors = 0;
-
- private int requestedAllServerDescriptors = 0;
-
- private int requestedMissingExtraInfoDescriptors = 0;
-
- private int requestedAllExtraInfoDescriptors = 0;
-
- private int requestedMissingMicrodescriptors = 0;
-
- private int downloadedConsensuses = 0;
-
- private int downloadedMicrodescConsensuses = 0;
-
- private int downloadedVotes = 0;
-
- private int downloadedMissingServerDescriptors = 0;
-
- private int downloadedAllServerDescriptors = 0;
-
- private int downloadedMissingExtraInfoDescriptors = 0;
-
- private int downloadedAllExtraInfoDescriptors = 0;
-
- private int downloadedMissingMicrodescriptors = 0;
-
- /**
- * Initializes this class, including reading in missing descriptors from
- * <code>stats/missing-relay-descriptors</code> and the times when we
- * last downloaded all server and extra-info descriptors from
- * <code>stats/last-downloaded-all-descriptors</code>.
- */
- public RelayDescriptorDownloader(RelayDescriptorParser rdp,
- List<String> authorities, List<String> authorityFingerprints,
- boolean downloadCurrentConsensus,
- boolean downloadCurrentMicrodescConsensus,
- boolean downloadCurrentVotes,
- boolean downloadMissingServerDescriptors,
- boolean downloadMissingExtraInfos,
- boolean downloadMissingMicrodescriptors,
- boolean downloadAllServerDescriptors, boolean downloadAllExtraInfos,
- boolean downloadCompressed) {
-
- /* Memorize argument values. */
- this.rdp = rdp;
- this.authorities = new ArrayList<String>(authorities);
- this.authorityFingerprints = new ArrayList<String>(
- authorityFingerprints);
- this.downloadCurrentConsensus = downloadCurrentConsensus;
- this.downloadCurrentMicrodescConsensus =
- downloadCurrentMicrodescConsensus;
- this.downloadCurrentVotes = downloadCurrentVotes;
- this.downloadMissingServerDescriptors =
- downloadMissingServerDescriptors;
- this.downloadMissingExtraInfos = downloadMissingExtraInfos;
- this.downloadMissingMicrodescriptors =
- downloadMissingMicrodescriptors;
- this.downloadAllServerDescriptors = downloadAllServerDescriptors;
- this.downloadAllExtraInfos = downloadAllExtraInfos;
- this.downloadCompressed = downloadCompressed;
-
- /* Shuffle list of authorities for better load balancing over time. */
- Collections.shuffle(this.authorities);
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(
- RelayDescriptorDownloader.class.getName());
-
- /* Prepare cut-off times and timestamp for the missing descriptors
- * list and the list of authorities to download all server and
- * extra-info descriptors from. */
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- format.setTimeZone(TimeZone.getTimeZone("UTC"));
- long now = System.currentTimeMillis();
- this.currentValidAfter = format.format((now / (60L * 60L * 1000L))
- * (60L * 60L * 1000L));
- this.descriptorCutOff = format.format(now - 24L * 60L * 60L * 1000L);
- this.currentTimestamp = format.format(now);
- this.downloadAllDescriptorsCutOff = format.format(now
- - 23L * 60L * 60L * 1000L - 30L * 60L * 1000L);
-
- /* Read list of missing descriptors from disk and memorize those that
- * we are interested in and that are likely to be found on the
- * directory authorities. */
- this.missingDescriptors = new TreeMap<String, String>();
- this.microdescriptorKeys = new HashMap<String, Set<String>>();
- this.missingMicrodescriptors = new HashSet<String>();
- this.missingDescriptorsFile = new File(
- "stats/missing-relay-descriptors");
- if (this.missingDescriptorsFile.exists()) {
- try {
- this.logger.fine("Reading file "
- + this.missingDescriptorsFile.getAbsolutePath() + "...");
- BufferedReader br = new BufferedReader(new FileReader(
- this.missingDescriptorsFile));
- String line;
- while ((line = br.readLine()) != null) {
- if (line.split(",").length > 2) {
- String published = line.split(",")[1];
- if (((line.startsWith("consensus,")
- || line.startsWith("consensus-microdesc,")
- || line.startsWith("vote,"))
- && this.currentValidAfter.equals(published))
- || ((line.startsWith("server,")
- || line.startsWith("extra,")
- || line.startsWith("micro,"))
- && this.descriptorCutOff.compareTo(published) < 0)) {
- if (!line.endsWith("NA")) {
- /* Not missing. */
- } else if (line.startsWith("consensus,")) {
- oldMissingConsensuses++;
- } else if (line.startsWith("consensus-microdesc,")) {
- oldMissingMicrodescConsensuses++;
- } else if (line.startsWith("vote,")) {
- oldMissingVotes++;
- } else if (line.startsWith("server,")) {
- oldMissingServerDescriptors++;
- } else if (line.startsWith("extra,")) {
- oldMissingExtraInfoDescriptors++;
- }
- int separateAt = line.lastIndexOf(",");
- this.missingDescriptors.put(line.substring(0,
- separateAt), line.substring(separateAt + 1));
- if (line.startsWith("micro,")) {
- String microdescriptorDigest = line.split(",")[3];
- String microdescriptorKey = line.substring(0,
- line.lastIndexOf(","));
- if (!this.microdescriptorKeys.containsKey(
- microdescriptorDigest)) {
- this.microdescriptorKeys.put(
- microdescriptorDigest, new HashSet<String>());
- }
- this.microdescriptorKeys.get(microdescriptorDigest).add(
- microdescriptorKey);
- if (line.endsWith("NA") && !this.missingMicrodescriptors
- .contains(microdescriptorDigest)) {
- this.missingMicrodescriptors.add(microdescriptorDigest);
- oldMissingMicrodescriptors++;
- }
- }
- }
- } else {
- this.logger.fine("Invalid line '" + line + "' in "
- + this.missingDescriptorsFile.getAbsolutePath()
- + ". Ignoring.");
- }
- }
- br.close();
- this.logger.fine("Finished reading file "
- + this.missingDescriptorsFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to read file "
- + this.missingDescriptorsFile.getAbsolutePath()
- + "! This means that we might forget to dowload relay "
- + "descriptors we are missing.", e);
- }
- }
-
- /* Read list of directory authorities and when we last downloaded all
- * server and extra-info descriptors from them. */
- this.lastDownloadedAllDescriptors = new HashMap<String, String>();
- this.lastDownloadedAllDescriptorsFile = new File(
- "stats/last-downloaded-all-descriptors");
- if (this.lastDownloadedAllDescriptorsFile.exists()) {
- try {
- this.logger.fine("Reading file "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
- + "...");
- BufferedReader br = new BufferedReader(new FileReader(
- this.lastDownloadedAllDescriptorsFile));
- String line;
- while ((line = br.readLine()) != null) {
- if (line.split(",").length != 2) {
- this.logger.fine("Invalid line '" + line + "' in "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
- + ". Ignoring.");
- } else {
- String[] parts = line.split(",");
- String authority = parts[0];
- String lastDownloaded = parts[1];
- this.lastDownloadedAllDescriptors.put(authority,
- lastDownloaded);
- }
- }
- br.close();
- this.logger.fine("Finished reading file "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
- + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed to read file "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
- + "! This means that we might download all server and "
- + "extra-info descriptors more often than we should.", e);
- }
- }
-
- /* Make a list of at most two directory authorities that we want to
- * download all server and extra-info descriptors from. */
- this.downloadAllDescriptorsFromAuthorities = new HashSet<String>();
- for (String authority : this.authorities) {
- if (!this.lastDownloadedAllDescriptors.containsKey(authority)
- || this.lastDownloadedAllDescriptors.get(authority).compareTo(
- this.downloadAllDescriptorsCutOff) < 0) {
- this.downloadAllDescriptorsFromAuthorities.add(authority);
- }
- if (this.downloadAllDescriptorsFromAuthorities.size() >= 2) {
- break;
- }
- }
-
- /* Prepare statistics on this execution. */
- this.requestsByAuthority = new HashMap<String, Integer>();
- for (String authority : this.authorities) {
- this.requestsByAuthority.put(authority, 0);
- }
- }
-
- /**
- * We have parsed a consensus. Take this consensus off the missing list
- * and add the votes created by the given <code>authorities</code> and
- * the <code>serverDescriptors</code> which are in the format
- * "<published>,<relayid>,<descid>" to that list.
- */
- public void haveParsedConsensus(String validAfter,
- Set<String> authorities, Set<String> serverDescriptors) {
-
- /* Mark consensus as parsed. */
- if (this.currentValidAfter.equals(validAfter)) {
- String consensusKey = "consensus," + validAfter;
- this.missingDescriptors.put(consensusKey, this.currentTimestamp);
-
- /* Add votes to missing list. */
- for (String authority : authorities) {
- String voteKey = "vote," + validAfter + "," + authority;
- if (!this.missingDescriptors.containsKey(voteKey)) {
- this.missingDescriptors.put(voteKey, "NA");
- this.newMissingVotes++;
- }
- }
- }
-
- /* Add server descriptors to missing list. */
- for (String serverDescriptor : serverDescriptors) {
- String published = serverDescriptor.split(",")[0];
- if (this.descriptorCutOff.compareTo(published) < 0) {
- String serverDescriptorKey = "server," + serverDescriptor;
- if (!this.missingDescriptors.containsKey(
- serverDescriptorKey)) {
- this.missingDescriptors.put(serverDescriptorKey, "NA");
- this.newMissingServerDescriptors++;
- }
- }
- }
- }
-
- /**
- * We have parsed a microdesc consensus. Take this microdesc consensus
- * off the missing list and add the <code>microdescriptors</code> which
- * are in the format "<validafter>,<relayid>,<descid>" to that
- * list.
- */
- public void haveParsedMicrodescConsensus(String validAfter,
- Set<String> microdescriptors) {
-
- /* Mark microdesc consensus as parsed. */
- if (this.currentValidAfter.equals(validAfter)) {
- String microdescConsensusKey = "consensus-microdesc," + validAfter;
- this.missingDescriptors.put(microdescConsensusKey,
- this.currentTimestamp);
- }
-
- /* Add microdescriptors to missing list. Exclude those that we already
- * downloaded this month. (We download each microdescriptor at least
- * once per month to keep the storage logic sane; otherwise we'd have
- * to copy microdescriptors from the earlier month to the current
- * month, and that gets messy.) */
- if (this.descriptorCutOff.compareTo(validAfter) < 0) {
- String validAfterYearMonth = validAfter.substring(0,
- "YYYY-MM".length());
- for (String microdescriptor : microdescriptors) {
- String microdescriptorKey = "micro," + microdescriptor;
- String parsed = "NA";
- String microdescriptorDigest = microdescriptor.split(",")[2];
- if (this.microdescriptorKeys.containsKey(microdescriptorDigest)) {
- for (String otherMicrodescriptorKey :
- this.microdescriptorKeys.get(microdescriptorDigest)) {
- String otherValidAfter =
- otherMicrodescriptorKey.split(",")[1];
- if (!otherValidAfter.startsWith(validAfterYearMonth)) {
- continue;
- }
- String otherParsed = this.missingDescriptors.get(
- otherMicrodescriptorKey);
- if (otherParsed != null && !otherParsed.equals("NA")) {
- parsed = otherParsed;
- break;
- }
- }
- } else {
- this.microdescriptorKeys.put(
- microdescriptorDigest, new HashSet<String>());
- }
- this.microdescriptorKeys.get(microdescriptorDigest).add(
- microdescriptorKey);
- this.missingDescriptors.put(microdescriptorKey, parsed);
- if (parsed.equals("NA")
- && !this.missingMicrodescriptors.contains(microdescriptorDigest)) {
- this.missingMicrodescriptors.add(microdescriptorDigest);
- this.newMissingMicrodescriptors++;
- }
- }
- }
- }
-
- /**
- * We have parsed a vote. Take this vote off the missing list and add
- * the <code>serverDescriptors</code> which are in the format
- * "<published>,<relayid>,<descid>" to that list.
- */
- public void haveParsedVote(String validAfter, String fingerprint,
- Set<String> serverDescriptors) {
-
- /* Mark vote as parsed. */
- if (this.currentValidAfter.equals(validAfter)) {
- String voteKey = "vote," + validAfter + "," + fingerprint;
- this.missingDescriptors.put(voteKey, this.currentTimestamp);
- }
-
- /* Add server descriptors to missing list. */
- for (String serverDescriptor : serverDescriptors) {
- String published = serverDescriptor.split(",")[0];
- if (this.descriptorCutOff.compareTo(published) < 0) {
- String serverDescriptorKey = "server," + serverDescriptor;
- if (!this.missingDescriptors.containsKey(
- serverDescriptorKey)) {
- this.missingDescriptors.put(serverDescriptorKey, "NA");
- this.newMissingServerDescriptors++;
- }
- }
- }
- }
-
- /**
- * We have parsed a server descriptor. Take this server descriptor off
- * the missing list and put the extra-info descriptor digest on that
- * list.
- */
- public void haveParsedServerDescriptor(String published,
- String relayIdentity, String serverDescriptorDigest,
- String extraInfoDigest) {
-
- /* Mark server descriptor as parsed. */
- if (this.descriptorCutOff.compareTo(published) < 0) {
- String serverDescriptorKey = "server," + published + ","
- + relayIdentity + "," + serverDescriptorDigest;
- this.missingDescriptors.put(serverDescriptorKey,
- this.currentTimestamp);
-
- /* Add extra-info descriptor to missing list. */
- if (extraInfoDigest != null) {
- String extraInfoKey = "extra," + published + ","
- + relayIdentity + "," + extraInfoDigest;
- if (!this.missingDescriptors.containsKey(extraInfoKey)) {
- this.missingDescriptors.put(extraInfoKey, "NA");
- this.newMissingExtraInfoDescriptors++;
- }
- }
- }
- }
-
- /**
- * We have parsed an extra-info descriptor. Take it off the missing
- * list.
- */
- public void haveParsedExtraInfoDescriptor(String published,
- String relayIdentity, String extraInfoDigest) {
- if (this.descriptorCutOff.compareTo(published) < 0) {
- String extraInfoKey = "extra," + published + ","
- + relayIdentity + "," + extraInfoDigest;
- this.missingDescriptors.put(extraInfoKey, this.currentTimestamp);
- }
- }
-
- /**
- * We have parsed a microdescriptor. Take it off the missing list.
- */
- public void haveParsedMicrodescriptor(String descriptorDigest) {
- if (this.microdescriptorKeys.containsKey(descriptorDigest)) {
- for (String microdescriptorKey :
- this.microdescriptorKeys.get(descriptorDigest)) {
- String validAfter = microdescriptorKey.split(",")[0];
- if (this.descriptorCutOff.compareTo(validAfter) < 0) {
- this.missingDescriptors.put(microdescriptorKey,
- this.currentTimestamp);
- }
- }
- this.missingMicrodescriptors.remove(descriptorDigest);
- }
- }
-
- /**
- * Downloads missing descriptors that we think might still be available
- * on the directory authorities as well as all server and extra-info
- * descriptors once per day.
- */
- public void downloadDescriptors() {
-
- /* Put the current consensus and votes on the missing list, unless we
- * already have them. */
- String consensusKey = "consensus," + this.currentValidAfter;
- if (!this.missingDescriptors.containsKey(consensusKey)) {
- this.missingDescriptors.put(consensusKey, "NA");
- this.newMissingConsensuses++;
- }
- String microdescConsensusKey = "consensus-microdesc,"
- + this.currentValidAfter;
- if (!this.missingDescriptors.containsKey(microdescConsensusKey)) {
- this.missingDescriptors.put(microdescConsensusKey, "NA");
- this.newMissingMicrodescConsensuses++;
- }
- for (String authority : authorityFingerprints) {
- String voteKey = "vote," + this.currentValidAfter + "," + authority;
- if (!this.missingDescriptors.containsKey(voteKey)) {
- this.missingDescriptors.put(voteKey, "NA");
- this.newMissingVotes++;
- }
- }
-
- /* Download descriptors from authorities which are in random order, so
- * that we distribute the load somewhat fairly over time. */
- for (String authority : authorities) {
-
- /* Make all requests to an authority in a single try block. If
- * something goes wrong with this authority, we give up on all
- * downloads and continue with the next authority. */
- /* TODO Some authorities provide very little bandwidth and could
- * slow down the entire download process. Ponder adding a timeout of
- * 3 or 5 minutes per authority to avoid getting in the way of the
- * next execution. */
- try {
-
- /* Start with downloading the current consensus, unless we already
- * have it. */
- if (downloadCurrentConsensus) {
- if (this.missingDescriptors.containsKey(consensusKey)
- && this.missingDescriptors.get(consensusKey).equals("NA")) {
- this.requestedConsensuses++;
- this.downloadedConsensuses +=
- this.downloadResourceFromAuthority(authority,
- "/tor/status-vote/current/consensus");
- }
- }
-
- /* Then try to download the microdesc consensus. */
- if (downloadCurrentMicrodescConsensus) {
- if (this.missingDescriptors.containsKey(microdescConsensusKey)
- && this.missingDescriptors.get(microdescConsensusKey)
- .equals("NA")) {
- this.requestedMicrodescConsensuses++;
- this.downloadedMicrodescConsensuses +=
- this.downloadResourceFromAuthority(authority,
- "/tor/status-vote/current/consensus-microdesc");
- }
- }
-
- /* Next, try to download current votes that we're missing. */
- if (downloadCurrentVotes) {
- String voteKeyPrefix = "vote," + this.currentValidAfter;
- SortedSet<String> fingerprints = new TreeSet<String>();
- for (Map.Entry<String, String> e :
- this.missingDescriptors.entrySet()) {
- if (e.getValue().equals("NA")
- && e.getKey().startsWith(voteKeyPrefix)) {
- String fingerprint = e.getKey().split(",")[2];
- fingerprints.add(fingerprint);
- }
- }
- for (String fingerprint : fingerprints) {
- this.requestedVotes++;
- this.downloadedVotes +=
- this.downloadResourceFromAuthority(authority,
- "/tor/status-vote/current/" + fingerprint);
- }
- }
-
- /* Download either all server and extra-info descriptors or only
- * those that we're missing. Start with server descriptors, then
- * request extra-info descriptors. Finally, request missing
- * microdescriptors. */
- for (String type : new String[] { "server", "extra", "micro" }) {
-
- /* Download all server or extra-info descriptors from this
- * authority if we haven't done so for 24 hours and if we're
- * configured to do so. */
- if (this.downloadAllDescriptorsFromAuthorities.contains(
- authority) && ((type.equals("server")
- && this.downloadAllServerDescriptors)
- || (type.equals("extra") && this.downloadAllExtraInfos))) {
- int downloadedAllDescriptors =
- this.downloadResourceFromAuthority(authority, "/tor/"
- + type + "/all");
- if (type.equals("server")) {
- this.requestedAllServerDescriptors++;
- this.downloadedAllServerDescriptors +=
- downloadedAllDescriptors;
- } else if (type.equals("extra")) {
- this.requestedAllExtraInfoDescriptors++;
- this.downloadedAllExtraInfoDescriptors +=
- downloadedAllDescriptors;
- }
-
- /* Download missing server descriptors, extra-info descriptors,
- * and microdescriptors if we're configured to do so. */
- } else if ((type.equals("server")
- && this.downloadMissingServerDescriptors)
- || (type.equals("extra") && this.downloadMissingExtraInfos)
- || (type.equals("micro")
- && this.downloadMissingMicrodescriptors)) {
-
- /* Go through the list of missing descriptors of this type
- * and combine the descriptor identifiers to a URL of up to
- * 96 server or extra-info descriptors or 92 microdescriptors
- * that we can download at once. */
- SortedSet<String> descriptorIdentifiers =
- new TreeSet<String>();
- for (Map.Entry<String, String> e :
- this.missingDescriptors.entrySet()) {
- if (e.getValue().equals("NA")
- && e.getKey().startsWith(type + ",")
- && this.descriptorCutOff.compareTo(
- e.getKey().split(",")[1]) < 0) {
- String descriptorIdentifier = e.getKey().split(",")[3];
- descriptorIdentifiers.add(descriptorIdentifier);
- }
- }
- StringBuilder combinedResource = null;
- int descriptorsInCombinedResource = 0;
- int requestedDescriptors = 0;
- int downloadedDescriptors = 0;
- int maxDescriptorsInCombinedResource =
- type.equals("micro") ? 92 : 96;
- String separator = type.equals("micro") ? "-" : "+";
- for (String descriptorIdentifier : descriptorIdentifiers) {
- if (descriptorsInCombinedResource
- >= maxDescriptorsInCombinedResource) {
- requestedDescriptors += descriptorsInCombinedResource;
- downloadedDescriptors +=
- this.downloadResourceFromAuthority(authority,
- combinedResource.toString());
- combinedResource = null;
- descriptorsInCombinedResource = 0;
- }
- if (descriptorsInCombinedResource == 0) {
- combinedResource = new StringBuilder("/tor/" + type
- + "/d/" + descriptorIdentifier);
- } else {
- combinedResource.append(separator + descriptorIdentifier);
- }
- descriptorsInCombinedResource++;
- }
- if (descriptorsInCombinedResource > 0) {
- requestedDescriptors += descriptorsInCombinedResource;
- downloadedDescriptors +=
- this.downloadResourceFromAuthority(authority,
- combinedResource.toString());
- }
- if (type.equals("server")) {
- this.requestedMissingServerDescriptors +=
- requestedDescriptors;
- this.downloadedMissingServerDescriptors +=
- downloadedDescriptors;
- } else if (type.equals("extra")) {
- this.requestedMissingExtraInfoDescriptors +=
- requestedDescriptors;
- this.downloadedMissingExtraInfoDescriptors +=
- downloadedDescriptors;
- } else if (type.equals("micro")) {
- this.requestedMissingMicrodescriptors +=
- requestedDescriptors;
- this.downloadedMissingMicrodescriptors +=
- downloadedDescriptors;
- }
- }
- }
-
- /* If a download failed, stop requesting descriptors from this
- * authority and move on to the next. */
- } catch (IOException e) {
- logger.log(Level.FINE, "Failed downloading from " + authority
- + "!", e);
- }
- }
- }
-
- /**
- * Attempts to download one or more descriptors identified by a resource
- * string from a directory authority and passes the returned
- * descriptor(s) to the <code>RelayDescriptorParser</code> upon success.
- * Returns the number of descriptors contained in the reply. Throws an
- * <code>IOException</code> if something goes wrong while downloading.
- */
- private int downloadResourceFromAuthority(String authority,
- String resource) throws IOException {
- byte[] allData = null;
- this.requestsByAuthority.put(authority,
- this.requestsByAuthority.get(authority) + 1);
- /* TODO Disable compressed downloads for extra-info descriptors,
- * because zlib decompression doesn't work correctly. Figure out why
- * this is and fix it. */
- String fullUrl = "http://" + authority + resource
- + (this.downloadCompressed && !resource.startsWith("/tor/extra/")
- ? ".z" : "");
- URL u = new URL(fullUrl);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.connect();
- int response = huc.getResponseCode();
- if (response == 200) {
- BufferedInputStream in = this.downloadCompressed
- && !resource.startsWith("/tor/extra/")
- ? new BufferedInputStream(new InflaterInputStream(
- huc.getInputStream()))
- : new BufferedInputStream(huc.getInputStream());
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = in.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- in.close();
- allData = baos.toByteArray();
- }
- logger.fine("Downloaded " + fullUrl + " -> " + response + " ("
- + (allData == null ? 0 : allData.length) + " bytes)");
- int receivedDescriptors = 0;
- if (allData != null) {
- if (resource.startsWith("/tor/status-vote/current/")) {
- this.rdp.parse(allData);
- receivedDescriptors = 1;
- } else if (resource.startsWith("/tor/server/")
- || resource.startsWith("/tor/extra/")) {
- if (resource.equals("/tor/server/all")
- || resource.equals("/tor/extra/all")) {
- this.lastDownloadedAllDescriptors.put(authority,
- this.currentTimestamp);
- }
- String ascii = null;
- try {
- ascii = new String(allData, "US-ASCII");
- } catch (UnsupportedEncodingException e) {
- /* No way that US-ASCII is not supported. */
- }
- int start = -1;
- int sig = -1;
- int end = -1;
- String startToken = resource.startsWith("/tor/server/")
- ? "router " : "extra-info ";
- String sigToken = "\nrouter-signature\n";
- String endToken = "\n-----END SIGNATURE-----\n";
- while (end < ascii.length()) {
- start = ascii.indexOf(startToken, end);
- if (start < 0) {
- break;
- }
- sig = ascii.indexOf(sigToken, start);
- if (sig < 0) {
- break;
- }
- sig += sigToken.length();
- end = ascii.indexOf(endToken, sig);
- if (end < 0) {
- break;
- }
- end += endToken.length();
- byte[] descBytes = new byte[end - start];
- System.arraycopy(allData, start, descBytes, 0, end - start);
- this.rdp.parse(descBytes);
- receivedDescriptors++;
- }
- } else if (resource.startsWith("/tor/micro/")) {
- /* TODO We need to parse microdescriptors ourselves, rather than
- * RelayDescriptorParser, because only we know the valid-after
- * time(s) of microdesc consensus(es) containing this
- * microdescriptor. However, this breaks functional abstraction
- * pretty badly. */
- SimpleDateFormat parseFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String ascii = null;
- try {
- ascii = new String(allData, "US-ASCII");
- } catch (UnsupportedEncodingException e) {
- /* No way that US-ASCII is not supported. */
- }
- int start = -1;
- int end = -1;
- String startToken = "onion-key\n";
- while (end < ascii.length()) {
- start = ascii.indexOf(startToken, end);
- if (start < 0) {
- break;
- }
- end = ascii.indexOf(startToken, start + 1);
- if (end < 0) {
- end = ascii.length();
- if (end <= start) {
- break;
- }
- }
- byte[] descBytes = new byte[end - start];
- System.arraycopy(allData, start, descBytes, 0, end - start);
- String digest256Base64 = Base64.encodeBase64String(
- DigestUtils.sha256(descBytes)).replaceAll("=", "");
- if (!this.microdescriptorKeys.containsKey(digest256Base64)) {
- continue;
- }
- String digest256Hex = DigestUtils.sha256Hex(descBytes);
- for (String microdescriptorKey :
- this.microdescriptorKeys.get(digest256Base64)) {
- String validAfterTime = microdescriptorKey.split(",")[1];
- try {
- long validAfter =
- parseFormat.parse(validAfterTime).getTime();
- this.rdp.storeMicrodescriptor(descBytes, digest256Hex,
- digest256Base64, validAfter);
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Could not parse "
- + "valid-after time '" + validAfterTime + "' in "
- + "microdescriptor key. Not storing microdescriptor.",
- e);
- }
- }
- receivedDescriptors++;
- }
- }
- }
- return receivedDescriptors;
- }
-
- /**
- * Writes status files to disk and logs statistics about downloading
- * relay descriptors in this execution.
- */
- public void writeFile() {
-
- /* Write missing descriptors file to disk. */
- int missingConsensuses = 0;
- int missingMicrodescConsensuses = 0;
- int missingVotes = 0;
- int missingServerDescriptors = 0;
- int missingExtraInfoDescriptors = 0;
- try {
- this.logger.fine("Writing file "
- + this.missingDescriptorsFile.getAbsolutePath() + "...");
- this.missingDescriptorsFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.missingDescriptorsFile));
- for (Map.Entry<String, String> e :
- this.missingDescriptors.entrySet()) {
- String key = e.getKey();
- String value = e.getValue();
- if (!value.equals("NA")) {
- /* Not missing. */
- } else if (key.startsWith("consensus,")) {
- missingConsensuses++;
- } else if (key.startsWith("consensus-microdesc,")) {
- missingMicrodescConsensuses++;
- } else if (key.startsWith("vote,")) {
- missingVotes++;
- } else if (key.startsWith("server,")) {
- missingServerDescriptors++;
- } else if (key.startsWith("extra,")) {
- missingExtraInfoDescriptors++;
- } else if (key.startsWith("micro,")) {
- /* We're counting missing microdescriptors below. */
- }
- bw.write(key + "," + value + "\n");
- }
- bw.close();
- this.logger.fine("Finished writing file "
- + this.missingDescriptorsFile.getAbsolutePath() + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed writing "
- + this.missingDescriptorsFile.getAbsolutePath() + "!", e);
- }
- int missingMicrodescriptors = this.missingMicrodescriptors.size();
-
- /* Write text file containing the directory authorities and when we
- * last downloaded all server and extra-info descriptors from them to
- * disk. */
- try {
- this.logger.fine("Writing file "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
- + "...");
- this.lastDownloadedAllDescriptorsFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.lastDownloadedAllDescriptorsFile));
- for (Map.Entry<String, String> e :
- this.lastDownloadedAllDescriptors.entrySet()) {
- String authority = e.getKey();
- String lastDownloaded = e.getValue();
- bw.write(authority + "," + lastDownloaded + "\n");
- }
- bw.close();
- this.logger.fine("Finished writing file "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath()
- + ".");
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed writing "
- + this.lastDownloadedAllDescriptorsFile.getAbsolutePath() + "!",
- e);
- }
-
- /* Log statistics about this execution. */
- this.logger.info("Finished downloading relay descriptors from the "
- + "directory authorities.");
- this.logger.info("At the beginning of this execution, we were "
- + "missing " + oldMissingConsensuses + " consensus(es), "
- + oldMissingMicrodescConsensuses + " microdesc consensus(es), "
- + oldMissingVotes + " vote(s), " + oldMissingServerDescriptors
- + " server descriptor(s), " + oldMissingExtraInfoDescriptors
- + " extra-info descriptor(s), and " + oldMissingMicrodescriptors
- + " microdescriptor(s).");
- this.logger.info("During this execution, we added "
- + this.newMissingConsensuses + " consensus(es), "
- + this.newMissingMicrodescConsensuses
- + " microdesc consensus(es), " + this.newMissingVotes
- + " vote(s), " + this.newMissingServerDescriptors
- + " server descriptor(s), " + this.newMissingExtraInfoDescriptors
- + " extra-info descriptor(s), and "
- + this.newMissingMicrodescriptors + " microdescriptor(s) to the "
- + "missing list, some of which we also "
- + "requested and removed from the list again.");
- this.logger.info("We requested " + this.requestedConsensuses
- + " consensus(es), " + this.requestedMicrodescConsensuses
- + " microdesc consensus(es), " + this.requestedVotes
- + " vote(s), " + this.requestedMissingServerDescriptors
- + " missing server descriptor(s), "
- + this.requestedAllServerDescriptors
- + " times all server descriptors, "
- + this.requestedMissingExtraInfoDescriptors + " missing "
- + "extra-info descriptor(s), "
- + this.requestedAllExtraInfoDescriptors + " times all extra-info "
- + "descriptors, and " + this.requestedMissingMicrodescriptors
- + " missing microdescriptor(s) from the directory authorities.");
- StringBuilder sb = new StringBuilder();
- for (String authority : this.authorities) {
- sb.append(" " + authority + "="
- + this.requestsByAuthority.get(authority));
- }
- this.logger.info("We sent these numbers of requests to the directory "
- + "authorities:" + sb.toString());
- this.logger.info("We successfully downloaded "
- + this.downloadedConsensuses + " consensus(es), "
- + this.downloadedMicrodescConsensuses
- + " microdesc consensus(es), " + this.downloadedVotes
- + " vote(s), " + this.downloadedMissingServerDescriptors
- + " missing server descriptor(s), "
- + this.downloadedAllServerDescriptors
- + " server descriptor(s) when downloading all descriptors, "
- + this.downloadedMissingExtraInfoDescriptors + " missing "
- + "extra-info descriptor(s), "
- + this.downloadedAllExtraInfoDescriptors + " extra-info "
- + "descriptor(s) when downloading all descriptors, and "
- + this.downloadedMissingMicrodescriptors
- + " missing microdescriptor(s).");
- this.logger.info("At the end of this execution, we are missing "
- + missingConsensuses + " consensus(es), "
- + missingMicrodescConsensuses + " microdesc consensus(es), "
- + missingVotes + " vote(s), " + missingServerDescriptors
- + " server descriptor(s), " + missingExtraInfoDescriptors
- + " extra-info descriptor(s), and " + missingMicrodescriptors
- + " microdescriptor(s), some of which we may try in the next "
- + "execution.");
- }
-}
-
diff --git a/src/org/torproject/collector/relaydescs/RelayDescriptorParser.java b/src/org/torproject/collector/relaydescs/RelayDescriptorParser.java
deleted file mode 100644
index 3f9b912..0000000
--- a/src/org/torproject/collector/relaydescs/RelayDescriptorParser.java
+++ /dev/null
@@ -1,337 +0,0 @@
-/* Copyright 2010--2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.relaydescs;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.StringReader;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.SortedSet;
-import java.util.TimeZone;
-import java.util.TreeSet;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * Parses relay descriptors including network status consensuses and
- * votes, server and extra-info descriptors, and passes the results to the
- * stats handlers, to the archive writer, or to the relay descriptor
- * downloader.
- */
-public class RelayDescriptorParser {
-
- /**
- * File writer that writes descriptor contents to files in a
- * directory-archive directory structure.
- */
- private ArchiveWriter aw;
-
- private ArchiveReader ar;
-
- /**
- * Missing descriptor downloader that uses the parse results to learn
- * which descriptors we are missing and want to download.
- */
- private RelayDescriptorDownloader rdd;
-
- /**
- * Logger for this class.
- */
- private Logger logger;
-
- private SimpleDateFormat dateTimeFormat;
-
- /**
- * Initializes this class.
- */
- public RelayDescriptorParser(ArchiveWriter aw) {
- this.aw = aw;
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(RelayDescriptorParser.class.getName());
-
- this.dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- }
-
- public void setRelayDescriptorDownloader(
- RelayDescriptorDownloader rdd) {
- this.rdd = rdd;
- }
-
- public void setArchiveReader(ArchiveReader ar) {
- this.ar = ar;
- }
-
- public boolean parse(byte[] data) {
- boolean stored = false;
- try {
- /* Convert descriptor to ASCII for parsing. This means we'll lose
- * the non-ASCII chars, but we don't care about them for parsing
- * anyway. */
- BufferedReader br = new BufferedReader(new StringReader(new String(
- data, "US-ASCII")));
- String line;
- do {
- line = br.readLine();
- } while (line != null && line.startsWith("@"));
- if (line == null) {
- this.logger.fine("We were given an empty descriptor for "
- + "parsing. Ignoring.");
- return false;
- }
- SimpleDateFormat parseFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (line.startsWith("network-status-version 3")) {
- String statusType = "consensus";
- if (line.equals("network-status-version 3 microdesc")) {
- statusType = "consensus-microdesc";
- }
- String validAfterTime = null;
- String fingerprint = null;
- String dirSource = null;
- long validAfter = -1L;
- long dirKeyPublished = -1L;
- SortedSet<String> dirSources = new TreeSet<String>();
- SortedSet<String> serverDescriptors = new TreeSet<String>();
- SortedSet<String> serverDescriptorDigests = new TreeSet<String>();
- SortedSet<String> microdescriptorKeys = new TreeSet<String>();
- SortedSet<String> microdescriptorDigests = new TreeSet<String>();
- StringBuilder certificateStringBuilder = null;
- String certificateString = null;
- String lastRelayIdentity = null;
- while ((line = br.readLine()) != null) {
- if (certificateStringBuilder != null) {
- if (line.startsWith("r ")) {
- certificateString = certificateStringBuilder.toString();
- certificateStringBuilder = null;
- } else {
- certificateStringBuilder.append(line + "\n");
- }
- }
- if (line.equals("vote-status vote")) {
- statusType = "vote";
- } else if (line.startsWith("valid-after ")) {
- validAfterTime = line.substring("valid-after ".length());
- validAfter = parseFormat.parse(validAfterTime).getTime();
- } else if (line.startsWith("dir-source ")) {
- dirSource = line.split(" ")[2];
- } else if (line.startsWith("vote-digest ")) {
- dirSources.add(dirSource);
- } else if (line.startsWith("dir-key-certificate-version ")) {
- certificateStringBuilder = new StringBuilder();
- certificateStringBuilder.append(line + "\n");
- } else if (line.startsWith("fingerprint ")) {
- fingerprint = line.split(" ")[1];
- } else if (line.startsWith("dir-key-published ")) {
- String dirKeyPublishedTime = line.substring(
- "dir-key-published ".length());
- dirKeyPublished = parseFormat.parse(dirKeyPublishedTime)
- .getTime();
- } else if (line.startsWith("r ")) {
- String[] parts = line.split(" ");
- if (parts.length == 8) {
- lastRelayIdentity = Hex.encodeHexString(Base64.decodeBase64(
- parts[2] + "=")).toLowerCase();
- } else if (parts.length == 9) {
- lastRelayIdentity = Hex.encodeHexString(Base64.decodeBase64(
- parts[2] + "=")).toLowerCase();
- String serverDesc = Hex.encodeHexString(Base64.decodeBase64(
- parts[3] + "=")).toLowerCase();
- String publishedTime = parts[4] + " " + parts[5];
- serverDescriptors.add(publishedTime + ","
- + lastRelayIdentity + "," + serverDesc);
- serverDescriptorDigests.add(serverDesc);
- } else {
- this.logger.log(Level.WARNING, "Could not parse r line '"
- + line + "' in descriptor. Skipping.");
- break;
- }
- } else if (line.startsWith("m ")) {
- String[] parts = line.split(" ");
- if (parts.length == 2 && parts[1].length() == 43) {
- String digest256Base64 = parts[1];
- microdescriptorKeys.add(validAfterTime + ","
- + lastRelayIdentity + "," + digest256Base64);
- String digest256Hex = Hex.encodeHexString(
- Base64.decodeBase64(digest256Base64 + "="))
- .toLowerCase();
- microdescriptorDigests.add(digest256Hex);
- } else if (parts.length != 3
- || !parts[2].startsWith("sha256=")
- || parts[2].length() != 50) {
- this.logger.log(Level.WARNING, "Could not parse m line '"
- + line + "' in descriptor. Skipping.");
- break;
- }
- }
- }
- if (statusType.equals("consensus")) {
- if (this.rdd != null) {
- this.rdd.haveParsedConsensus(validAfterTime, dirSources,
- serverDescriptors);
- }
- if (this.aw != null) {
- this.aw.storeConsensus(data, validAfter, dirSources,
- serverDescriptorDigests);
- stored = true;
- }
- } else if (statusType.equals("consensus-microdesc")) {
- if (this.rdd != null) {
- this.rdd.haveParsedMicrodescConsensus(validAfterTime,
- microdescriptorKeys);
- }
- if (this.ar != null) {
- this.ar.haveParsedMicrodescConsensus(validAfterTime,
- microdescriptorDigests);
- }
- if (this.aw != null) {
- this.aw.storeMicrodescConsensus(data, validAfter,
- microdescriptorDigests);
- stored = true;
- }
- } else {
- if (this.aw != null || this.rdd != null) {
- String ascii = new String(data, "US-ASCII");
- String startToken = "network-status-version ";
- String sigToken = "directory-signature ";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken);
- if (start >= 0 && sig >= 0 && sig > start) {
- sig += sigToken.length();
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- String digest = DigestUtils.shaHex(forDigest).toUpperCase();
- if (this.aw != null) {
- this.aw.storeVote(data, validAfter, dirSource, digest,
- serverDescriptorDigests);
- stored = true;
- }
- if (this.rdd != null) {
- this.rdd.haveParsedVote(validAfterTime, fingerprint,
- serverDescriptors);
- }
- }
- if (certificateString != null) {
- if (this.aw != null) {
- this.aw.storeCertificate(certificateString.getBytes(),
- dirSource, dirKeyPublished);
- stored = true;
- }
- }
- }
- }
- } else if (line.startsWith("router ")) {
- String publishedTime = null;
- String extraInfoDigest = null;
- String relayIdentifier = null;
- long published = -1L;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("published ")) {
- publishedTime = line.substring("published ".length());
- published = parseFormat.parse(publishedTime).getTime();
- } else if (line.startsWith("opt fingerprint")
- || line.startsWith("fingerprint")) {
- relayIdentifier = line.substring(line.startsWith("opt ")
- ? "opt fingerprint".length() : "fingerprint".length())
- .replaceAll(" ", "").toLowerCase();
- } else if (line.startsWith("opt extra-info-digest ")
- || line.startsWith("extra-info-digest ")) {
- extraInfoDigest = line.startsWith("opt ")
- ? line.split(" ")[2].toLowerCase()
- : line.split(" ")[1].toLowerCase();
- }
- }
- String ascii = new String(data, "US-ASCII");
- String startToken = "router ";
- String sigToken = "\nrouter-signature\n";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- String digest = null;
- if (start >= 0 || sig >= 0 || sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- digest = DigestUtils.shaHex(forDigest);
- }
- if (this.aw != null && digest != null) {
- this.aw.storeServerDescriptor(data, digest, published,
- extraInfoDigest);
- stored = true;
- }
- if (this.rdd != null && digest != null) {
- this.rdd.haveParsedServerDescriptor(publishedTime,
- relayIdentifier, digest, extraInfoDigest);
- }
- } else if (line.startsWith("extra-info ")) {
- String publishedTime = null;
- String relayIdentifier = line.split(" ")[2];
- long published = -1L;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("published ")) {
- publishedTime = line.substring("published ".length());
- published = parseFormat.parse(publishedTime).getTime();
- }
- }
- String ascii = new String(data, "US-ASCII");
- String startToken = "extra-info ";
- String sigToken = "\nrouter-signature\n";
- String digest = null;
- int start = ascii.indexOf(startToken);
- if (start > 0) {
- /* Do not confuse "extra-info " in "@type extra-info 1.0" with
- * "extra-info 0000...". TODO This is a hack that should be
- * solved by using metrics-lib some day. */
- start = ascii.indexOf("\n" + startToken);
- if (start > 0) {
- start++;
- }
- }
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- if (start >= 0 && sig >= 0 && sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(data, start, forDigest, 0, sig - start);
- digest = DigestUtils.shaHex(forDigest);
- }
- if (this.aw != null && digest != null) {
- this.aw.storeExtraInfoDescriptor(data, digest, published);
- stored = true;
- }
- if (this.rdd != null && digest != null) {
- this.rdd.haveParsedExtraInfoDescriptor(publishedTime,
- relayIdentifier.toLowerCase(), digest);
- }
- } else if (line.equals("onion-key")) {
- /* Cannot store microdescriptors without knowing valid-after
- * time(s) of microdesc consensuses containing them, because we
- * don't know which month directories to put them in. Have to use
- * storeMicrodescriptor below. */
- }
- br.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not parse descriptor. "
- + "Skipping.", e);
- } catch (ParseException e) {
- this.logger.log(Level.WARNING, "Could not parse descriptor. "
- + "Skipping.", e);
- }
- return stored;
- }
-
- public void storeMicrodescriptor(byte[] data, String digest256Hex,
- String digest256Base64, long validAfter) {
- if (this.aw != null) {
- this.aw.storeMicrodescriptor(data, digest256Hex, validAfter);
- }
- if (this.rdd != null) {
- this.rdd.haveParsedMicrodescriptor(digest256Base64);
- }
- }
-}
-
diff --git a/src/org/torproject/collector/torperf/TorperfDownloader.java b/src/org/torproject/collector/torperf/TorperfDownloader.java
deleted file mode 100644
index 7bcfbf3..0000000
--- a/src/org/torproject/collector/torperf/TorperfDownloader.java
+++ /dev/null
@@ -1,643 +0,0 @@
-/* Copyright 2012-2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.collector.torperf;
-
-import org.torproject.collector.main.Configuration;
-import org.torproject.collector.main.LockFile;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.text.SimpleDateFormat;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.Stack;
-import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/* Download possibly truncated Torperf .data and .extradata files from
- * configured sources, append them to the files we already have, and merge
- * the two files into the .tpf format. */
-public class TorperfDownloader extends Thread {
-
- public static void main(String[] args) {
-
- Logger logger = Logger.getLogger(TorperfDownloader.class.getName());
- logger.info("Starting torperf module of CollecTor.");
-
- // Initialize configuration
- Configuration config = new Configuration();
-
- // Use lock file to avoid overlapping runs
- LockFile lf = new LockFile("torperf");
- if (!lf.acquireLock()) {
- logger.severe("Warning: CollecTor is already running or has not exited "
- + "cleanly! Exiting!");
- System.exit(1);
- }
-
- // Process Torperf files
- new TorperfDownloader(config).run();
-
- // Remove lock file
- lf.releaseLock();
-
- logger.info("Terminating torperf module of CollecTor.");
- }
-
- private Configuration config;
-
- public TorperfDownloader(Configuration config) {
- this.config = config;
- }
-
- private File torperfOutputDirectory = null;
- private SortedMap<String, String> torperfSources = null;
- private List<String> torperfFilesLines = null;
- private Logger logger = null;
- private SimpleDateFormat dateFormat;
-
- public void run() {
-
- File torperfOutputDirectory =
- new File(config.getTorperfOutputDirectory());
- SortedMap<String, String> torperfSources = config.getTorperfSources();
- List<String> torperfFilesLines = config.getTorperfFiles();
-
- this.torperfOutputDirectory = torperfOutputDirectory;
- this.torperfSources = torperfSources;
- this.torperfFilesLines = torperfFilesLines;
- if (!this.torperfOutputDirectory.exists()) {
- this.torperfOutputDirectory.mkdirs();
- }
- this.logger = Logger.getLogger(TorperfDownloader.class.getName());
- this.dateFormat = new SimpleDateFormat("yyyy-MM-dd");
- this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- this.readLastMergedTimestamps();
- for (String torperfFilesLine : this.torperfFilesLines) {
- this.downloadAndMergeFiles(torperfFilesLine);
- }
- this.writeLastMergedTimestamps();
-
- this.cleanUpRsyncDirectory();
- }
-
- private File torperfLastMergedFile =
- new File("stats/torperf-last-merged");
-
- SortedMap<String, String> lastMergedTimestamps =
- new TreeMap<String, String>();
-
- private void readLastMergedTimestamps() {
- if (!this.torperfLastMergedFile.exists()) {
- return;
- }
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- this.torperfLastMergedFile));
- String line;
- while ((line = br.readLine()) != null) {
- String[] parts = line.split(" ");
- String fileName = null;
- String timestamp = null;
- if (parts.length == 2) {
- try {
- Double.parseDouble(parts[1]);
- fileName = parts[0];
- timestamp = parts[1];
- } catch (NumberFormatException e) {
- /* Handle below. */
- }
- }
- if (fileName == null || timestamp == null) {
- this.logger.log(Level.WARNING, "Invalid line '" + line + "' in "
- + this.torperfLastMergedFile.getAbsolutePath() + ". "
- + "Ignoring past history of merging .data and .extradata "
- + "files.");
- this.lastMergedTimestamps.clear();
- break;
- }
- this.lastMergedTimestamps.put(fileName, timestamp);
- }
- br.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Error while reading '"
- + this.torperfLastMergedFile.getAbsolutePath() + ". Ignoring "
- + "past history of merging .data and .extradata files.");
- this.lastMergedTimestamps.clear();
- }
- }
-
- private void writeLastMergedTimestamps() {
- try {
- this.torperfLastMergedFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.torperfLastMergedFile));
- for (Map.Entry<String, String> e :
- this.lastMergedTimestamps.entrySet()) {
- String fileName = e.getKey();
- String timestamp = e.getValue();
- bw.write(fileName + " " + timestamp + "\n");
- }
- bw.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Error while writing '"
- + this.torperfLastMergedFile.getAbsolutePath() + ". This may "
- + "result in ignoring history of merging .data and .extradata "
- + "files in the next execution.", e);
- }
- }
-
- private void downloadAndMergeFiles(String torperfFilesLine) {
- String[] parts = torperfFilesLine.split(" ");
- String sourceName = parts[1];
- int fileSize = -1;
- try {
- fileSize = Integer.parseInt(parts[2]);
- } catch (NumberFormatException e) {
- this.logger.log(Level.WARNING, "Could not parse file size in "
- + "TorperfFiles configuration line '" + torperfFilesLine
- + "'.");
- return;
- }
-
- /* Download and append the .data file. */
- String dataFileName = parts[3];
- String sourceBaseUrl = torperfSources.get(sourceName);
- String dataUrl = sourceBaseUrl + dataFileName;
- String dataOutputFileName = sourceName + "-" + dataFileName;
- File dataOutputFile = new File(torperfOutputDirectory,
- dataOutputFileName);
- boolean downloadedDataFile = this.downloadAndAppendFile(dataUrl,
- dataOutputFile, true);
-
- /* Download and append the .extradata file. */
- String extradataFileName = parts[4];
- String extradataUrl = sourceBaseUrl + extradataFileName;
- String extradataOutputFileName = sourceName + "-" + extradataFileName;
- File extradataOutputFile = new File(torperfOutputDirectory,
- extradataOutputFileName);
- boolean downloadedExtradataFile = this.downloadAndAppendFile(
- extradataUrl, extradataOutputFile, false);
-
- /* Merge both files into .tpf format. */
- if (!downloadedDataFile && !downloadedExtradataFile) {
- return;
- }
- String skipUntil = null;
- if (this.lastMergedTimestamps.containsKey(dataOutputFileName)) {
- skipUntil = this.lastMergedTimestamps.get(dataOutputFileName);
- }
- try {
- skipUntil = this.mergeFiles(dataOutputFile, extradataOutputFile,
- sourceName, fileSize, skipUntil);
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed merging " + dataOutputFile
- + " and " + extradataOutputFile + ".", e);
- }
- if (skipUntil != null) {
- this.lastMergedTimestamps.put(dataOutputFileName, skipUntil);
- }
- }
-
- private boolean downloadAndAppendFile(String url, File outputFile,
- boolean isDataFile) {
-
- /* Read an existing output file to determine which line will be the
- * first to append to it. */
- String lastTimestampLine = null;
- int linesAfterLastTimestampLine = 0;
- if (outputFile.exists()) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- outputFile));
- String line;
- while ((line = br.readLine()) != null) {
- if (isDataFile || line.contains(" LAUNCH")) {
- lastTimestampLine = line;
- linesAfterLastTimestampLine = 0;
- } else {
- linesAfterLastTimestampLine++;
- }
- }
- br.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed reading '"
- + outputFile.getAbsolutePath() + "' to determine the first "
- + "line to append to it.", e);
- return false;
- }
- }
- try {
- this.logger.fine("Downloading " + (isDataFile ? ".data" :
- ".extradata") + " file from '" + url + "' and merging it into "
- + "'" + outputFile.getAbsolutePath() + "'.");
- URL u = new URL(url);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.connect();
- BufferedReader br = new BufferedReader(new InputStreamReader(
- huc.getInputStream()));
- String line;
- BufferedWriter bw = new BufferedWriter(new FileWriter(outputFile,
- true));
- boolean copyLines = lastTimestampLine == null;
- while ((line = br.readLine()) != null) {
- if (copyLines && linesAfterLastTimestampLine == 0) {
- if (isDataFile || line.contains(" LAUNCH")) {
- lastTimestampLine = line;
- }
- bw.write(line + "\n");
- } else if (copyLines && linesAfterLastTimestampLine > 0) {
- linesAfterLastTimestampLine--;
- } else if (line.equals(lastTimestampLine)) {
- copyLines = true;
- }
- }
- bw.close();
- br.close();
- if (!copyLines) {
- this.logger.warning("The last timestamp line in '"
- + outputFile.getAbsolutePath() + "' is not contained in the "
- + "new file downloaded from '" + url + "'. Cannot append "
- + "new lines without possibly leaving a gap. Skipping.");
- return false;
- }
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Failed downloading and/or merging '"
- + url + "'.", e);
- return false;
- }
- if (lastTimestampLine == null) {
- this.logger.warning("'" + outputFile.getAbsolutePath()
- + "' doesn't contain any timestamp lines. Unable to check "
- + "whether that file is stale or not.");
- } else {
- long lastTimestampMillis = -1L;
- if (isDataFile) {
- lastTimestampMillis = Long.parseLong(lastTimestampLine.substring(
- 0, lastTimestampLine.indexOf(" "))) * 1000L;
- } else {
- lastTimestampMillis = Long.parseLong(lastTimestampLine.substring(
- lastTimestampLine.indexOf(" LAUNCH=") + " LAUNCH=".length(),
- lastTimestampLine.indexOf(".",
- lastTimestampLine.indexOf(" LAUNCH=")))) * 1000L;
- }
- if (lastTimestampMillis < System.currentTimeMillis()
- - 330L * 60L * 1000L) {
- this.logger.warning("The last timestamp in '"
- + outputFile.getAbsolutePath() + "' is more than 5:30 hours "
- + "old: " + lastTimestampMillis);
- }
- }
- return true;
- }
-
- private String mergeFiles(File dataFile, File extradataFile,
- String source, int fileSize, String skipUntil) throws IOException {
- SortedMap<String, String> config = new TreeMap<String, String>();
- config.put("SOURCE", source);
- config.put("FILESIZE", String.valueOf(fileSize));
- if (!dataFile.exists() || !extradataFile.exists()) {
- this.logger.warning("File " + dataFile.getAbsolutePath() + " or "
- + extradataFile.getAbsolutePath() + " is missing.");
- return null;
- }
- this.logger.fine("Merging " + dataFile.getAbsolutePath() + " and "
- + extradataFile.getAbsolutePath() + " into .tpf format.");
- BufferedReader brD = new BufferedReader(new FileReader(dataFile));
- BufferedReader brE = new BufferedReader(new FileReader(extradataFile));
- String lineD = brD.readLine();
- String lineE = brE.readLine();
- int d = 1;
- int e = 1;
- String maxDataComplete = null;
- String maxUsedAt = null;
- while (lineD != null) {
-
- /* Parse .data line. Every valid .data line will go into the .tpf
- * format, either with additional information from the .extradata
- * file or without it. */
- if (lineD.isEmpty()) {
- this.logger.finer("Skipping empty line " + dataFile.getName()
- + ":" + d++ + ".");
- lineD = brD.readLine();
- continue;
- }
- SortedMap<String, String> data = this.parseDataLine(lineD);
- if (data == null) {
- this.logger.finer("Skipping illegal line " + dataFile.getName()
- + ":" + d++ + " '" + lineD + "'.");
- lineD = brD.readLine();
- continue;
- }
- String dataComplete = data.get("DATACOMPLETE");
- double dataCompleteSeconds = Double.parseDouble(dataComplete);
- if (skipUntil != null && dataComplete.compareTo(skipUntil) < 0) {
- this.logger.finer("Skipping " + dataFile.getName() + ":"
- + d++ + " which we already processed before.");
- lineD = brD.readLine();
- continue;
- }
- maxDataComplete = dataComplete;
-
- /* Parse .extradata line if available and try to find the one that
- * matches the .data line. */
- SortedMap<String, String> extradata = null;
- while (lineE != null) {
- if (lineE.isEmpty()) {
- this.logger.finer("Skipping " + extradataFile.getName() + ":"
- + e++ + " which is empty.");
- lineE = brE.readLine();
- continue;
- }
- if (lineE.startsWith("BUILDTIMEOUT_SET ")) {
- this.logger.finer("Skipping " + extradataFile.getName() + ":"
- + e++ + " which is a BUILDTIMEOUT_SET line.");
- lineE = brE.readLine();
- continue;
- } else if (lineE.startsWith("ok ")
- || lineE.startsWith("error ")) {
- this.logger.finer("Skipping " + extradataFile.getName() + ":"
- + e++ + " which is in the old format.");
- lineE = brE.readLine();
- continue;
- }
- extradata = this.parseExtradataLine(lineE);
- if (extradata == null) {
- this.logger.finer("Skipping Illegal line "
- + extradataFile.getName() + ":" + e++ + " '" + lineE
- + "'.");
- lineE = brE.readLine();
- continue;
- }
- if (!extradata.containsKey("USED_AT")) {
- this.logger.finer("Skipping " + extradataFile.getName() + ":"
- + e++ + " which doesn't contain a USED_AT element.");
- lineE = brE.readLine();
- continue;
- }
- String usedAt = extradata.get("USED_AT");
- double usedAtSeconds = Double.parseDouble(usedAt);
- if (skipUntil != null && usedAt.compareTo(skipUntil) < 0) {
- this.logger.finer("Skipping " + extradataFile.getName() + ":"
- + e++ + " which we already processed before.");
- lineE = brE.readLine();
- continue;
- }
- maxUsedAt = usedAt;
- if (Math.abs(usedAtSeconds - dataCompleteSeconds) <= 1.0) {
- this.logger.fine("Merging " + extradataFile.getName() + ":"
- + e++ + " into the current .data line.");
- lineE = brE.readLine();
- break;
- } else if (usedAtSeconds > dataCompleteSeconds) {
- this.logger.finer("Comparing " + extradataFile.getName()
- + " to the next .data line.");
- extradata = null;
- break;
- } else {
- this.logger.finer("Skipping " + extradataFile.getName() + ":"
- + e++ + " which is too old to be merged with "
- + dataFile.getName() + ":" + d + ".");
- lineE = brE.readLine();
- continue;
- }
- }
-
- /* Write output line to .tpf file. */
- SortedMap<String, String> keysAndValues =
- new TreeMap<String, String>();
- if (extradata != null) {
- keysAndValues.putAll(extradata);
- }
- keysAndValues.putAll(data);
- keysAndValues.putAll(config);
- this.logger.fine("Writing " + dataFile.getName() + ":" + d++ + ".");
- lineD = brD.readLine();
- try {
- this.writeTpfLine(source, fileSize, keysAndValues);
- } catch (IOException ex) {
- this.logger.log(Level.WARNING, "Error writing output line. "
- + "Aborting to merge " + dataFile.getName() + " and "
- + extradataFile.getName() + ".", e);
- break;
- }
- }
- brD.close();
- brE.close();
- this.writeCachedTpfLines();
- if (maxDataComplete == null) {
- return maxUsedAt;
- } else if (maxUsedAt == null) {
- return maxDataComplete;
- } else if (maxDataComplete.compareTo(maxUsedAt) > 0) {
- return maxUsedAt;
- } else {
- return maxDataComplete;
- }
- }
-
- private SortedMap<Integer, String> dataTimestamps;
-
- private SortedMap<String, String> parseDataLine(String line) {
- String[] parts = line.trim().split(" ");
- if (line.length() == 0 || parts.length < 20) {
- return null;
- }
- if (this.dataTimestamps == null) {
- this.dataTimestamps = new TreeMap<Integer, String>();
- this.dataTimestamps.put(0, "START");
- this.dataTimestamps.put(2, "SOCKET");
- this.dataTimestamps.put(4, "CONNECT");
- this.dataTimestamps.put(6, "NEGOTIATE");
- this.dataTimestamps.put(8, "REQUEST");
- this.dataTimestamps.put(10, "RESPONSE");
- this.dataTimestamps.put(12, "DATAREQUEST");
- this.dataTimestamps.put(14, "DATARESPONSE");
- this.dataTimestamps.put(16, "DATACOMPLETE");
- this.dataTimestamps.put(21, "DATAPERC10");
- this.dataTimestamps.put(23, "DATAPERC20");
- this.dataTimestamps.put(25, "DATAPERC30");
- this.dataTimestamps.put(27, "DATAPERC40");
- this.dataTimestamps.put(29, "DATAPERC50");
- this.dataTimestamps.put(31, "DATAPERC60");
- this.dataTimestamps.put(33, "DATAPERC70");
- this.dataTimestamps.put(35, "DATAPERC80");
- this.dataTimestamps.put(37, "DATAPERC90");
- }
- SortedMap<String, String> data = new TreeMap<String, String>();
- try {
- for (Map.Entry<Integer, String> e : this.dataTimestamps.entrySet()) {
- int i = e.getKey();
- if (parts.length > i + 1) {
- String key = e.getValue();
- String value = String.format("%s.%02d", parts[i],
- Integer.parseInt(parts[i + 1]) / 10000);
- data.put(key, value);
- }
- }
- } catch (NumberFormatException e) {
- return null;
- }
- data.put("WRITEBYTES", parts[18]);
- data.put("READBYTES", parts[19]);
- if (parts.length >= 21) {
- data.put("DIDTIMEOUT", parts[20]);
- }
- return data;
- }
-
- private SortedMap<String, String> parseExtradataLine(String line) {
- String[] parts = line.split(" ");
- SortedMap<String, String> extradata = new TreeMap<String, String>();
- String previousKey = null;
- for (String part : parts) {
- String[] keyAndValue = part.split("=", -1);
- if (keyAndValue.length == 2) {
- String key = keyAndValue[0];
- previousKey = key;
- String value = keyAndValue[1];
- if (value.contains(".") && value.lastIndexOf(".")
- == value.length() - 2) {
- /* Make sure that all floats have two trailing digits. */
- value += "0";
- }
- extradata.put(key, value);
- } else if (keyAndValue.length == 1 && previousKey != null) {
- String value = keyAndValue[0];
- if (previousKey.equals("STREAM_FAIL_REASONS")
- && (value.equals("MISC") || value.equals("EXITPOLICY")
- || value.equals("RESOURCELIMIT")
- || value.equals("RESOLVEFAILED"))) {
- extradata.put(previousKey, extradata.get(previousKey) + ":"
- + value);
- } else {
- return null;
- }
- } else {
- return null;
- }
- }
- return extradata;
- }
-
- private String cachedSource;
-
- private int cachedFileSize;
-
- private String cachedStartDate;
-
- private SortedMap<String, String> cachedTpfLines;
-
- private void writeTpfLine(String source, int fileSize,
- SortedMap<String, String> keysAndValues) throws IOException {
- StringBuilder sb = new StringBuilder();
- int written = 0;
- for (Map.Entry<String, String> keyAndValue :
- keysAndValues.entrySet()) {
- String key = keyAndValue.getKey();
- String value = keyAndValue.getValue();
- sb.append((written++ > 0 ? " " : "") + key + "=" + value);
- }
- String line = sb.toString();
- String startString = keysAndValues.get("START");
- long startMillis = Long.parseLong(startString.substring(0,
- startString.indexOf("."))) * 1000L;
- String startDate = dateFormat.format(startMillis);
- if (this.cachedTpfLines == null || !source.equals(this.cachedSource)
- || fileSize != this.cachedFileSize
- || !startDate.equals(this.cachedStartDate)) {
- this.writeCachedTpfLines();
- this.readTpfLinesToCache(source, fileSize, startDate);
- }
- if (!this.cachedTpfLines.containsKey(startString)
- || line.length() > this.cachedTpfLines.get(startString).length()) {
- this.cachedTpfLines.put(startString, line);
- }
- }
-
- private void readTpfLinesToCache(String source, int fileSize,
- String startDate) throws IOException {
- this.cachedTpfLines = new TreeMap<String, String>();
- this.cachedSource = source;
- this.cachedFileSize = fileSize;
- this.cachedStartDate = startDate;
- File tpfFile = new File(torperfOutputDirectory,
- startDate.replaceAll("-", "/") + "/"
- + source + "-" + String.valueOf(fileSize) + "-" + startDate
- + ".tpf");
- if (!tpfFile.exists()) {
- return;
- }
- BufferedReader br = new BufferedReader(new FileReader(tpfFile));
- String line;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("@type ")) {
- continue;
- }
- if (line.contains("START=")) {
- String startString = line.substring(line.indexOf("START=")
- + "START=".length()).split(" ")[0];
- this.cachedTpfLines.put(startString, line);
- }
- }
- br.close();
- }
-
- private void writeCachedTpfLines() throws IOException {
- if (this.cachedSource == null || this.cachedFileSize == 0
- || this.cachedStartDate == null || this.cachedTpfLines == null) {
- return;
- }
- File tarballFile = new File(torperfOutputDirectory,
- this.cachedStartDate.replaceAll("-", "/")
- + "/" + this.cachedSource + "-"
- + String.valueOf(this.cachedFileSize) + "-"
- + this.cachedStartDate + ".tpf");
- File rsyncFile = new File("recent/torperf/" + tarballFile.getName());
- File[] outputFiles = new File[] { tarballFile, rsyncFile };
- for (File outputFile : outputFiles) {
- outputFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(outputFile));
- for (String line : this.cachedTpfLines.values()) {
- bw.write("@type torperf 1.0\n");
- bw.write(line + "\n");
- }
- bw.close();
- }
- this.cachedSource = null;
- this.cachedFileSize = 0;
- this.cachedStartDate = null;
- this.cachedTpfLines = null;
- }
-
- /* Delete all files from the rsync directory that have not been modified
- * in the last three days. */
- public void cleanUpRsyncDirectory() {
- long cutOffMillis = System.currentTimeMillis()
- - 3L * 24L * 60L * 60L * 1000L;
- Stack<File> allFiles = new Stack<File>();
- allFiles.add(new File("recent/torperf"));
- while (!allFiles.isEmpty()) {
- File file = allFiles.pop();
- if (file.isDirectory()) {
- allFiles.addAll(Arrays.asList(file.listFiles()));
- } else if (file.lastModified() < cutOffMillis) {
- file.delete();
- }
- }
- }
-}
-
More information about the tor-commits
mailing list