[tor-commits] [metrics-lib/master] Make DescriptorFile obsolete.

20 Jun 2017

commit 1062da637f6f55313f52191314f7fb939355af23
Author: Karsten Loesing karsten.loesing@gmx.net
Date:   Wed Jun 7 17:52:02 2017 +0200
Make DescriptorFile obsolete.
This change simplifies the DescriptorReader interface and allows for
    shorter and more concise application code.  The result is that
    DescriptorReader returns Descriptor instances rather than
    DescriptorFile instances containing Descriptors and accepts a maximum
    queue size in Descriptors, DescriptorFile is deprecated, and
    Descriptor contains a File reference to the descriptor file.
    Implements #22141.
Related to this change, this commit introduces a new
    UnparseableDescriptor to be returned by DescriptorParser and
    DescriptorReader if a descriptor cannot be parsed, as opposed to
    throwing a DescriptorParseException or skipping the entire descriptor
    file (fixes #22139), respectively.
Also related to this change, DescriptorParser now returns an Iterable
    instead of a List, which prepares parsing large descriptor files
    descriptor by descriptor (will be tackled in #20395).
---
 CHANGELOG.md                                       |  13 ++
 .../java/org/torproject/descriptor/Descriptor.java |  19 +++
 .../org/torproject/descriptor/DescriptorFile.java  |   7 +
 .../torproject/descriptor/DescriptorParser.java    |  19 +++
 .../torproject/descriptor/DescriptorReader.java    |  56 +++++--
 .../descriptor/UnparseableDescriptor.java          |  44 ++++++
 .../descriptor/impl/BlockingIteratorImpl.java      |  13 +-
 .../impl/BridgeExtraInfoDescriptorImpl.java        |   9 +-
 .../descriptor/impl/BridgeNetworkStatusImpl.java   |   7 +-
 .../descriptor/impl/BridgePoolAssignmentImpl.java  |   9 +-
 .../impl/BridgeServerDescriptorImpl.java           |   9 +-
 .../torproject/descriptor/impl/DescriptorImpl.java |  23 ++-
 .../descriptor/impl/DescriptorParserImpl.java      | 130 ++++++++++------
 .../descriptor/impl/DescriptorReaderImpl.java      | 164 ++++++++++++++++++++-
 .../impl/DirectoryKeyCertificateImpl.java          |   9 +-
 .../torproject/descriptor/impl/ExitListImpl.java   |   7 +-
 .../descriptor/impl/ExtraInfoDescriptorImpl.java   |   9 +-
 .../descriptor/impl/MicrodescriptorImpl.java       |   7 +-
 .../descriptor/impl/NetworkStatusImpl.java         |   7 +-
 .../descriptor/impl/RelayDirectoryImpl.java        |  10 +-
 .../impl/RelayExtraInfoDescriptorImpl.java         |   9 +-
 .../impl/RelayNetworkStatusConsensusImpl.java      |   9 +-
 .../descriptor/impl/RelayNetworkStatusImpl.java    |   7 +-
 .../impl/RelayNetworkStatusVoteImpl.java           |   7 +-
 .../descriptor/impl/RelayServerDescriptorImpl.java |   9 +-
 .../descriptor/impl/ServerDescriptorImpl.java      |   7 +-
 .../descriptor/impl/TorperfResultImpl.java         |  11 +-
 .../descriptor/impl/UnparseableDescriptorImpl.java |  40 +++++
 .../descriptor/impl/BridgeNetworkStatusTest.java   |   2 +-
 .../descriptor/impl/ConsensusBuilder.java          |   2 +-
 .../descriptor/impl/DescriptorImplTest.java        |   1 -
 .../descriptor/impl/DescriptorParserImplTest.java  |   9 +-
 .../descriptor/impl/ExitListImplTest.java          |  10 +-
 .../impl/ExtraInfoDescriptorImplTest.java          |   8 +-
 .../descriptor/impl/MicrodescriptorImplTest.java   |   2 +-
 .../impl/RelayNetworkStatusConsensusImplTest.java  |   4 +-
 .../impl/RelayNetworkStatusImplTest.java           |   4 +-
 .../impl/RelayNetworkStatusVoteImplTest.java       |   3 +-
 .../descriptor/impl/ServerDescriptorImplTest.java  |   6 +-
 .../torproject/descriptor/impl/TestDescriptor.java |   4 +-
 .../descriptor/impl/TestServerDescriptor.java      |   8 +-
 .../descriptor/impl/TorperfResultImplTest.java     |  10 +-
 42 files changed, 586 insertions(+), 157 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e79811a..44f1130 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,18 @@
 # Changes in version 1.9.0 - 2017-06-??
+ * Major changes
+   - Simplify DescriptorReader by returning Descriptor instances
+     rather than DescriptorFile instances containing Descriptors,
+     deprecate DescriptorFile, and add a File reference to Descriptor.
+   - Introduce a new UnparseableDescriptor to be returned by
+     DescriptorParser and DescriptorReader if a descriptor cannot be
+     parsed, as opposed to throwing a DescriptorParseException or
+     skipping the entire descriptor file, respectively.
+
+ * Medium changes
+   - Let DescriptorParser return an Iterable instead of a List, which
+     prepares parsing large descriptor files descriptor by descriptor.
+
  * Minor changes
    - Fix a bug where NetworkStatusEntry's getMicrodescriptorDigests()
      and getMicrodescriptorDigestsSha256Base64() return hex strings
diff --git a/src/main/java/org/torproject/descriptor/Descriptor.java b/src/main/java/org/torproject/descriptor/Descriptor.java
index be6e02f..fb511bf 100644
--- a/src/main/java/org/torproject/descriptor/Descriptor.java
+++ b/src/main/java/org/torproject/descriptor/Descriptor.java
@@ -3,6 +3,7 @@
package org.torproject.descriptor;
+import java.io.File;
 import java.util.List;
/**
@@ -24,6 +25,10 @@ public interface Descriptor {
    * Return the (possibly empty) list of annotations in the format
    * {@code "@key( value)*"}.
    *
+   * <p>Some implementations might not support this operation and will throw an
+   * {@code UnsupportedOperationException}, e.g.,
+   * {@link UnparseableDescriptor}.</p>
+   *
    * @since 1.0.0
    */
   public List<String> getAnnotations();
@@ -32,8 +37,22 @@ public interface Descriptor {
    * Return any unrecognized lines when parsing this descriptor, or an
    * empty list if there were no unrecognized lines.
    *
+   * <p>Some implementations might not support this operation and will throw an
+   * {@code UnsupportedOperationException}, * e.g.,
+   * {@link UnparseableDescriptor}.</p>
+   *
    * @since 1.0.0
    */
   public List<String> getUnrecognizedLines();
+
+  /**
+   * Return the file, tarball or plain file, that contained this descriptor, or
+   * {@code null} if this descriptor was not read from a file.
+   *
+   * @return Descriptor file that contained this descriptor.
+   *
+   * @since 1.9.0
+   */
+  public File getDescriptorFile();
 }
diff --git a/src/main/java/org/torproject/descriptor/DescriptorFile.java b/src/main/java/org/torproject/descriptor/DescriptorFile.java
index b501186..e8c0d21 100644
--- a/src/main/java/org/torproject/descriptor/DescriptorFile.java
+++ b/src/main/java/org/torproject/descriptor/DescriptorFile.java
@@ -15,6 +15,13 @@ import java.util.List;
  * This container also stores potentially useful meta-data about the
  * descriptor file.</p>
  *
+ * @deprecated Irrelevant after introducing
+ *     {@link DescriptorReader#readDescriptors(File...)}, which returns
+ *     {@link Descriptor}s rather than {@link DescriptorFile}s, and including
+ *     the descriptor file reference in {@link Descriptor#getDescriptorFile()}
+ *     and the first thrown {@link DescriptorParseException} in
+ *     {@link UnparseableDescriptor#getDescriptorParseException()}.
+ *
  * @since 1.0.0
  */
 public interface DescriptorFile {
diff --git a/src/main/java/org/torproject/descriptor/DescriptorParser.java b/src/main/java/org/torproject/descriptor/DescriptorParser.java
index 537a259..61f69f4 100644
--- a/src/main/java/org/torproject/descriptor/DescriptorParser.java
+++ b/src/main/java/org/torproject/descriptor/DescriptorParser.java
@@ -3,6 +3,7 @@
package org.torproject.descriptor;
+import java.io.File;
 import java.util.List;
/**
@@ -48,4 +49,22 @@ public interface DescriptorParser {
    */
   public List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes,
       String fileName) throws DescriptorParseException;
+
+  /**
+   * Parse descriptors in the given byte array and return the parsed/unparseable
+   * descriptors.
+   *
+   * @param rawDescriptorBytes Raw descriptor bytes containing one or more
+   *     descriptors
+   * @param descriptorFile Optional descriptor file reference included in
+   *     parsed/unparseable descriptors
+   * @param fileName Descriptor file name used for parsing the descriptor
+   *     publication time of some descriptor types
+   *
+   * @return Parsed/unparseable descriptors
+   *
+   * @since 1.9.0
+   */
+  public Iterable<Descriptor> parseDescriptors(byte[] rawDescriptorBytes,
+      File descriptorFile, String fileName);
 }
diff --git a/src/main/java/org/torproject/descriptor/DescriptorReader.java b/src/main/java/org/torproject/descriptor/DescriptorReader.java
index 85f8cdd..e15e5d9 100644
--- a/src/main/java/org/torproject/descriptor/DescriptorReader.java
+++ b/src/main/java/org/torproject/descriptor/DescriptorReader.java
@@ -23,18 +23,13 @@ import java.util.SortedMap;
  * DescriptorReader descriptorReader =
  *     DescriptorSourceFactory.createDescriptorReader();
  * // Read descriptors from local directory called in/.
- * descriptorReader.addDirectory(new File("in"));
- * Iterator<DescriptorFile> descriptorFiles =
- *     descriptorReader.readDescriptors();
- * while (descriptorFiles.hasNext()) {
- *   DescriptorFile descriptorFile = descriptorFiles.next();
- *   for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- *     if ((descriptor instanceof RelayNetworkStatusConsensus)) {
- *       // Only process network status consensuses, ignore the rest.
- *       RelayNetworkStatusConsensus consensus =
- *           (RelayNetworkStatusConsensus) descriptor;
- *       processConsensus(consensus);
- *     }
+ * for (Descriptor descriptor :
+ *     descriptorReader.readDescriptors(new File("in")) {
+ *   // Only process network status consensuses, ignore the rest.
+ *   if ((descriptor instanceof RelayNetworkStatusConsensus)) {
+ *     RelayNetworkStatusConsensus consensus =
+ *         (RelayNetworkStatusConsensus) descriptor;
+ *     processConsensus(consensus);
  *   }
  * }}</pre>
  *
@@ -46,6 +41,9 @@ public interface DescriptorReader {
    * Add a local directory to read descriptors from, which may contain
    * descriptor files or tarballs containing descriptor files.
    *
+   * @deprecated Replaced with a parameter in {@link #readDescriptors(File...)},
+   *     which ignores any directories added via this deprecated method.
+   *
    * @since 1.0.0
    */
   public void addDirectory(File directory);
@@ -54,6 +52,9 @@ public interface DescriptorReader {
    * Add a tarball to read descriptors from, which may be uncompressed,
    * bz2-compressed, or xz-compressed.
    *
+   * @deprecated Replaced with a parameter in {@link #readDescriptors(File...)},
+   *     which ignores any tarballs added via this deprecated method.
+   *
    * @since 1.0.0
    */
   public void addTarball(File tarball);
@@ -157,11 +158,23 @@ public interface DescriptorReader {
    * <p>The default is 100, but if descriptor files contain hundreds or
    * even thousands of descriptors, that default may be too high.</p>
    *
+   * @deprecated Replaced with {@link #setMaxDescriptorsInQueue(int)}.
+   *
    * @since 1.0.0
    */
   public void setMaxDescriptorFilesInQueue(int max);
/**
+   * Don't keep more than this number of descriptors in the queue (default:
+   * 100).
+   *
+   * @param maxDescriptorsInQueue Maximum number of descriptors in the queue.
+   *
+   * @since 1.9.0
+   */
+  public void setMaxDescriptorsInQueue(int maxDescriptorsInQueue);
+
+  /**
    * Read the previously configured descriptors and make them available
    * via the returned blocking iterator.
    *
@@ -169,8 +182,27 @@ public interface DescriptorReader {
    * more shortly after, it blocks the caller.  This method can only be
    * run once.</p>
    *
+   * @deprecated Replaced with {@link #readDescriptors(File...)}.
+   *
    * @since 1.0.0
    */
   public Iterator<DescriptorFile> readDescriptors();
+
+  /**
+   * Read descriptors from the given descriptor file(s) and return the parsed
+   * descriptors.
+   *
+   * <p>Whenever the reader runs out of descriptors and expects to provide
+   * more shortly after, it blocks the caller.  This method can only be
+   * run once.</p>
+   *
+   * @param descriptorFiles One or more directories, tarballs, or files
+   *     containing descriptors.
+   *
+   * @return Parsed descriptors.
+   *
+   * @since 1.9.0
+   */
+  public Iterable<Descriptor> readDescriptors(File... descriptorFiles);
 }
diff --git a/src/main/java/org/torproject/descriptor/UnparseableDescriptor.java b/src/main/java/org/torproject/descriptor/UnparseableDescriptor.java
new file mode 100644
index 0000000..6282222
--- /dev/null
+++ b/src/main/java/org/torproject/descriptor/UnparseableDescriptor.java
@@ -0,0 +1,44 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor;
+
+import java.util.List;
+
+/**
+ * Contains a descriptor that could not be parsed.
+ * Only {@link UnparseableDescriptor#getRawDescriptorBytes} and
+ * {@link UnparseableDescriptor#getDescriptorFile} are supported.
+ *
+ * @since 1.9.0
+ */
+public interface UnparseableDescriptor extends Descriptor {
+
+  /**
+   * Return the first exception thrown while attempting to parse this
+   * descriptor.
+   *
+   * @return First exception thrown.
+   *
+   * @since 1.9.0
+   */
+  public DescriptorParseException getDescriptorParseException();
+
+  /**
+   * Will always throw an {@code UnsupportedOperationException}.
+   *
+   * @since 1.9.0
+   */
+  @Override
+  public List<String> getAnnotations();
+
+  /**
+   * Will always throw an {@code UnsupportedOperationException}.
+   *
+   * @since 1.9.0
+   */
+  @Override
+  public List<String> getUnrecognizedLines();
+
+}
+
diff --git a/src/main/java/org/torproject/descriptor/impl/BlockingIteratorImpl.java b/src/main/java/org/torproject/descriptor/impl/BlockingIteratorImpl.java
index 3509026..4921ac4 100644
--- a/src/main/java/org/torproject/descriptor/impl/BlockingIteratorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/BlockingIteratorImpl.java
@@ -11,7 +11,7 @@ import java.util.Queue;
 /* Provide an iterator for a queue of objects and block when there are
  * currently no objects in the queue.  Allow the producer to signal that
  * there won't be further objects and unblock any waiting consumers. */
-public class BlockingIteratorImpl<T> implements Iterator<T> {
+public class BlockingIteratorImpl<T> implements Iterator<T>, Iterable<T> {
/* Queue containing produced elemnts waiting for consumers. */
   private Queue<T> queue = new LinkedList<>();
@@ -24,8 +24,10 @@ public class BlockingIteratorImpl<T> implements Iterator<T> {
   }
/* Create instance with maximum queue size. */
-  protected BlockingIteratorImpl(int maxQueueSize) {
-    this.maxQueueSize = maxQueueSize;
+  protected BlockingIteratorImpl(Integer maxQueueSize) {
+    if (null != maxQueueSize) {
+      this.maxQueueSize = maxQueueSize;
+    }
   }
/* Add an object to the queue if there's still room. */
@@ -99,5 +101,10 @@ public class BlockingIteratorImpl<T> implements Iterator<T> {
   public void remove() {
     throw new UnsupportedOperationException();
   }
+
+  @Override
+  public Iterator<T> iterator() {
+    return this;
+  }
 }
diff --git a/src/main/java/org/torproject/descriptor/impl/BridgeExtraInfoDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgeExtraInfoDescriptorImpl.java
index 7d0af52..b2b3e99 100644
--- a/src/main/java/org/torproject/descriptor/impl/BridgeExtraInfoDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/BridgeExtraInfoDescriptorImpl.java
@@ -6,13 +6,16 @@ package org.torproject.descriptor.impl;
 import org.torproject.descriptor.BridgeExtraInfoDescriptor;
 import org.torproject.descriptor.DescriptorParseException;
+import java.io.File;
+
 public class BridgeExtraInfoDescriptorImpl
     extends ExtraInfoDescriptorImpl implements BridgeExtraInfoDescriptor {
protected BridgeExtraInfoDescriptorImpl(byte[] descriptorBytes,
-      int[] offsetAndLimit, boolean failUnrecognizedDescriptorLines)
-      throws DescriptorParseException {
-    super(descriptorBytes, offsetAndLimit, failUnrecognizedDescriptorLines);
+      int[] offsetAndLimit, File descriptorFile,
+      boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+    super(descriptorBytes, offsetAndLimit, descriptorFile,
+        failUnrecognizedDescriptorLines);
   }
 }
diff --git a/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java
index 83a3a8a..a54c91c 100644
--- a/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java
@@ -6,6 +6,7 @@ package org.torproject.descriptor.impl;
 import org.torproject.descriptor.BridgeNetworkStatus;
 import org.torproject.descriptor.DescriptorParseException;
+import java.io.File;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -19,10 +20,10 @@ public class BridgeNetworkStatusImpl extends NetworkStatusImpl
     implements BridgeNetworkStatus {
protected BridgeNetworkStatusImpl(byte[] rawDescriptorBytes,
-      int[] offsetAndLength, String fileName,
+      int[] offsetAndLength, File descriptorFile, String fileName,
       boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
-    super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines,
-        false, false);
+    super(rawDescriptorBytes, offsetAndLength, descriptorFile,
+        failUnrecognizedDescriptorLines, false, false);
     this.setPublishedMillisFromFileName(fileName);
   }
diff --git a/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java
index ca07c29..b2015ff 100644
--- a/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java
@@ -6,6 +6,7 @@ package org.torproject.descriptor.impl;
 import org.torproject.descriptor.BridgePoolAssignment;
 import org.torproject.descriptor.DescriptorParseException;
+import java.io.File;
 import java.util.EnumSet;
 import java.util.Scanner;
 import java.util.SortedMap;
@@ -15,10 +16,10 @@ public class BridgePoolAssignmentImpl extends DescriptorImpl
     implements BridgePoolAssignment {
protected BridgePoolAssignmentImpl(byte[] rawDescriptorBytes,
-      int[] offsetAndlength, boolean failUnrecognizedDescriptorLines)
-      throws DescriptorParseException {
-    super(rawDescriptorBytes, offsetAndlength, failUnrecognizedDescriptorLines,
-        false);
+      int[] offsetAndlength, File descriptorFile,
+      boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+    super(rawDescriptorBytes, offsetAndlength, descriptorFile,
+        failUnrecognizedDescriptorLines, false);
     this.parseDescriptorBytes();
     this.checkExactlyOnceKeys(EnumSet.of(Key.BRIDGE_POOL_ASSIGNMENT));
     this.checkFirstKey(Key.BRIDGE_POOL_ASSIGNMENT);
diff --git a/src/main/java/org/torproject/descriptor/impl/BridgeServerDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgeServerDescriptorImpl.java
index ac45591..30da220 100644
--- a/src/main/java/org/torproject/descriptor/impl/BridgeServerDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/BridgeServerDescriptorImpl.java
@@ -6,13 +6,16 @@ package org.torproject.descriptor.impl;
 import org.torproject.descriptor.BridgeServerDescriptor;
 import org.torproject.descriptor.DescriptorParseException;
+import java.io.File;
+
 public class BridgeServerDescriptorImpl extends ServerDescriptorImpl
     implements BridgeServerDescriptor {
protected BridgeServerDescriptorImpl(byte[] rawDescriptorBytes,
-      int[] offsetAndLength, boolean failUnrecognizedDescriptorLines)
-      throws DescriptorParseException {
-    super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines);
+      int[] offsetAndLength, File descriptorFile,
+      boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+    super(rawDescriptorBytes, offsetAndLength, descriptorFile,
+        failUnrecognizedDescriptorLines);
   }
 }
diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java
index 52ff6d4..d88b542 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.Descriptor;
 import org.torproject.descriptor.DescriptorParseException;
import java.io.ByteArrayInputStream;
+import java.io.File;
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
@@ -41,6 +42,17 @@ public abstract class DescriptorImpl implements Descriptor {
   protected int length;
/**
+   * Optional reference to the descriptor file, if this descriptor was read from
+   * a descriptor file.
+   */
+  private File descriptorFile;
+
+  @Override
+  public File getDescriptorFile() {
+    return this.descriptorFile;
+  }
+
+  /**
    * Returns a <emph>copy</emph> of the full raw descriptor bytes.
    *
    * <p>If possible, subclasses should avoid retrieving raw descriptor bytes and
@@ -178,8 +190,7 @@ public abstract class DescriptorImpl implements Descriptor {
   }
protected DescriptorImpl(byte[] rawDescriptorBytes, int[] offsetAndLength,
-      boolean failUnrecognizedDescriptorLines, boolean blankLinesAllowed)
-      throws DescriptorParseException {
+      File descriptorFile, boolean failUnrecognizedDescriptorLines) {
     int offset = offsetAndLength[0];
     int length = offsetAndLength[1];
     if (offset < 0 || offset + length > rawDescriptorBytes.length
@@ -191,8 +202,16 @@ public abstract class DescriptorImpl implements Descriptor {
     this.rawDescriptorBytes = rawDescriptorBytes;
     this.offset = offset;
     this.length = length;
+    this.descriptorFile = descriptorFile;
     this.failUnrecognizedDescriptorLines =
         failUnrecognizedDescriptorLines;
+  }
+
+  protected DescriptorImpl(byte[] rawDescriptorBytes, int[] offsetAndLength,
+      File descriptorFile, boolean failUnrecognizedDescriptorLines,
+      boolean blankLinesAllowed) throws DescriptorParseException {
+    this(rawDescriptorBytes, offsetAndLength, descriptorFile,
+        failUnrecognizedDescriptorLines);
     this.cutOffAnnotations();
     this.countKeys(rawDescriptorBytes, blankLinesAllowed);
   }
diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
index e416d93..a0be85c 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
@@ -10,6 +10,7 @@ import org.torproject.descriptor.Descriptor;
 import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.DescriptorParser;
+import java.io.File;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.nio.charset.StandardCharsets;
@@ -31,6 +32,29 @@ public class DescriptorParserImpl implements DescriptorParser {
   public List<Descriptor> parseDescriptors(
       byte[] rawDescriptorBytes, String fileName)
       throws DescriptorParseException {
+    return this.parseDescriptors(rawDescriptorBytes, null, fileName, false);
+  }
+
+  @Override
+  public Iterable<Descriptor> parseDescriptors(byte[] rawDescriptorBytes,
+      File descriptorFile, String fileName) {
+    try {
+      return this.parseDescriptors(rawDescriptorBytes, descriptorFile, fileName,
+          true);
+    } catch (DescriptorParseException e) {
+      /* Looks like we attempted to parse the whole raw descriptor bytes at once
+       * below and ran into a parse issue. */
+      List<Descriptor> parsedDescriptors = new ArrayList<>();
+      parsedDescriptors.add(new UnparseableDescriptorImpl(rawDescriptorBytes,
+          new int[] { 0, rawDescriptorBytes.length }, descriptorFile,
+          failUnrecognizedDescriptorLines, e));
+      return parsedDescriptors;
+    }
+  }
+
+  private List<Descriptor> parseDescriptors(
+      byte[] rawDescriptorBytes, File descriptorFile, String fileName,
+      boolean includeUnparseableDescriptors) throws DescriptorParseException {
     byte[] first100Chars = new byte[Math.min(100,
         rawDescriptorBytes.length)];
     System.arraycopy(rawDescriptorBytes, 0, first100Chars, 0,
@@ -45,9 +69,9 @@ public class DescriptorParserImpl implements DescriptorParser {
         NL + Key.NETWORK_STATUS_VERSION.keyword + SP + "3"))
         && firstLines.contains(
         NL + Key.VOTE_STATUS.keyword + SP + "consensus" + NL))) {
-      return parseDescriptors(rawDescriptorBytes, Key.NETWORK_STATUS_VERSION,
-          RelayNetworkStatusConsensusImpl.class,
-          this.failUnrecognizedDescriptorLines);
+      return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+          Key.NETWORK_STATUS_VERSION, RelayNetworkStatusConsensusImpl.class,
+          this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
     } else if (firstLines.startsWith("@type network-status-vote-3 1.")
         || ((firstLines.startsWith(
         Key.NETWORK_STATUS_VERSION.keyword + SP + "3" + NL)
@@ -55,89 +79,95 @@ public class DescriptorParserImpl implements DescriptorParser {
         NL + Key.NETWORK_STATUS_VERSION.keyword + SP + "3" + NL))
         && firstLines.contains(
         NL + Key.VOTE_STATUS.keyword + SP + "vote" + NL))) {
-      return parseDescriptors(rawDescriptorBytes, Key.NETWORK_STATUS_VERSION,
-          RelayNetworkStatusVoteImpl.class,
-          this.failUnrecognizedDescriptorLines);
+      return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+          Key.NETWORK_STATUS_VERSION, RelayNetworkStatusVoteImpl.class,
+          this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
     } else if (firstLines.startsWith("@type bridge-network-status 1.")
         || firstLines.startsWith(Key.R.keyword + SP)) {
       List<Descriptor> parsedDescriptors = new ArrayList<>();
       parsedDescriptors.add(new BridgeNetworkStatusImpl(
           rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length },
-          fileName, this.failUnrecognizedDescriptorLines));
+          descriptorFile, fileName, this.failUnrecognizedDescriptorLines));
       return parsedDescriptors;
     } else if (firstLines.startsWith("@type bridge-server-descriptor 1.")) {
-      return parseDescriptors(rawDescriptorBytes, Key.ROUTER,
-          BridgeServerDescriptorImpl.class,
-          this.failUnrecognizedDescriptorLines);
+      return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+          Key.ROUTER, BridgeServerDescriptorImpl.class,
+          this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
     } else if (firstLines.startsWith("@type server-descriptor 1.")
         || firstLines.startsWith(Key.ROUTER.keyword + SP)
         || firstLines.contains(NL + Key.ROUTER.keyword + SP)) {
-      return parseDescriptors(rawDescriptorBytes, Key.ROUTER,
-          RelayServerDescriptorImpl.class,
-          this.failUnrecognizedDescriptorLines);
+      return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+          Key.ROUTER, RelayServerDescriptorImpl.class,
+          this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
     } else if (firstLines.startsWith("@type bridge-extra-info 1.")) {
-      return parseDescriptors(rawDescriptorBytes, Key.EXTRA_INFO,
-          BridgeExtraInfoDescriptorImpl.class,
-          this.failUnrecognizedDescriptorLines);
+      return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+          Key.EXTRA_INFO, BridgeExtraInfoDescriptorImpl.class,
+          this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
     } else if (firstLines.startsWith("@type extra-info 1.")
         || firstLines.startsWith(Key.EXTRA_INFO.keyword + SP)
         || firstLines.contains(NL + Key.EXTRA_INFO.keyword + SP)) {
-      return parseDescriptors(rawDescriptorBytes, Key.EXTRA_INFO,
-          RelayExtraInfoDescriptorImpl.class,
-          this.failUnrecognizedDescriptorLines);
+      return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+          Key.EXTRA_INFO, RelayExtraInfoDescriptorImpl.class,
+          this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
     } else if (firstLines.startsWith("@type microdescriptor 1.")
         || firstLines.startsWith(Key.ONION_KEY.keyword + NL)
         || firstLines.contains(NL + Key.ONION_KEY.keyword + NL)) {
-      return parseDescriptors(rawDescriptorBytes, Key.ONION_KEY,
-          MicrodescriptorImpl.class, this.failUnrecognizedDescriptorLines);
+      return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+          Key.ONION_KEY, MicrodescriptorImpl.class,
+          this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
     } else if (firstLines.startsWith("@type bridge-pool-assignment 1.")
         || firstLines.startsWith(Key.BRIDGE_POOL_ASSIGNMENT.keyword + SP)
         || firstLines.contains(NL + Key.BRIDGE_POOL_ASSIGNMENT.keyword + SP)) {
-      return parseDescriptors(rawDescriptorBytes, Key.BRIDGE_POOL_ASSIGNMENT,
-          BridgePoolAssignmentImpl.class, this.failUnrecognizedDescriptorLines);
+      return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+          Key.BRIDGE_POOL_ASSIGNMENT, BridgePoolAssignmentImpl.class,
+          this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
     } else if (firstLines.startsWith("@type dir-key-certificate-3 1.")
         || firstLines.startsWith(Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP)
         || firstLines.contains(
         NL + Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP)) {
-      return parseDescriptors(rawDescriptorBytes,
+      return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
           Key.DIR_KEY_CERTIFICATE_VERSION, DirectoryKeyCertificateImpl.class,
-          this.failUnrecognizedDescriptorLines);
+          this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
     } else if (firstLines.startsWith("@type tordnsel 1.")
         || firstLines.startsWith("ExitNode" + SP)
         || firstLines.contains(NL + "ExitNode" + SP)) {
       List<Descriptor> parsedDescriptors = new ArrayList<>();
-      parsedDescriptors.add(new ExitListImpl(rawDescriptorBytes, fileName,
-          this.failUnrecognizedDescriptorLines));
+      parsedDescriptors.add(new ExitListImpl(rawDescriptorBytes, descriptorFile,
+          fileName, this.failUnrecognizedDescriptorLines));
       return parsedDescriptors;
     } else if (firstLines.startsWith("@type network-status-2 1.")
         || firstLines.startsWith(
         Key.NETWORK_STATUS_VERSION.keyword + SP + "2" + NL)
         || firstLines.contains(
         NL + Key.NETWORK_STATUS_VERSION.keyword + SP + "2" + NL)) {
-      return parseDescriptors(rawDescriptorBytes, Key.NETWORK_STATUS_VERSION,
-          RelayNetworkStatusImpl.class, this.failUnrecognizedDescriptorLines);
+      return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+          Key.NETWORK_STATUS_VERSION, RelayNetworkStatusImpl.class,
+          this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
     } else if (firstLines.startsWith("@type directory 1.")
         || firstLines.startsWith(Key.SIGNED_DIRECTORY.keyword + NL)
         || firstLines.contains(NL + Key.SIGNED_DIRECTORY.keyword + NL)) {
-      return parseDescriptors(rawDescriptorBytes, Key.SIGNED_DIRECTORY,
-          RelayDirectoryImpl.class, this.failUnrecognizedDescriptorLines);
+      return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+          Key.SIGNED_DIRECTORY, RelayDirectoryImpl.class,
+          this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
     } else if (firstLines.startsWith("@type torperf 1.")) {
-      return TorperfResultImpl.parseTorperfResults(
-          rawDescriptorBytes, this.failUnrecognizedDescriptorLines);
+      return TorperfResultImpl.parseTorperfResults(rawDescriptorBytes,
+          descriptorFile, this.failUnrecognizedDescriptorLines);
     } else {
       throw new DescriptorParseException("Could not detect descriptor "
           + "type in descriptor starting with '" + firstLines + "'.");
     }
   }
-  private static List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes,
-      Key key, Class<? extends DescriptorImpl> descriptorClass,
-      boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+  private List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes,
+      File descriptorFile, Key key,
+      Class<? extends DescriptorImpl> descriptorClass,
+      boolean failUnrecognizedDescriptorLines,
+      boolean includeUnparseableDescriptors) throws DescriptorParseException {
     List<Descriptor> parsedDescriptors = new ArrayList<>();
     Constructor<? extends DescriptorImpl> constructor;
     try {
       constructor = descriptorClass.getDeclaredConstructor(byte[].class,
-          int[].class, boolean.class);
+          int[].class, File.class, boolean.class);
     } catch (NoSuchMethodException e) {
       throw new RuntimeException(e);
     }
@@ -181,20 +211,31 @@ public class DescriptorParserImpl implements DescriptorParser {
       endDescriptor += 1;
       int[] offsetAndLength = new int[] { startAnnotations,
           endDescriptor - startAnnotations };
-      parsedDescriptors.add(parseDescriptor(rawDescriptorBytes,
-          offsetAndLength, constructor, failUnrecognizedDescriptorLines));
+      try {
+        parsedDescriptors.add(this.parseDescriptor(rawDescriptorBytes,
+            offsetAndLength, descriptorFile, constructor,
+            failUnrecognizedDescriptorLines));
+      } catch (DescriptorParseException e) {
+        if (includeUnparseableDescriptors) {
+          parsedDescriptors.add(new UnparseableDescriptorImpl(
+              rawDescriptorBytes, offsetAndLength, descriptorFile,
+              failUnrecognizedDescriptorLines, e));
+        } else {
+          throw e;
+        }
+      }
       startAnnotations = endDescriptor;
     }
     return parsedDescriptors;
   }
-  static Descriptor parseDescriptor(byte[] rawDescriptorBytes,
-      int[] offsetAndLength, Constructor<? extends DescriptorImpl> constructor,
-      boolean failUnrecognizedDescriptorLines)
-      throws DescriptorParseException {
+  Descriptor parseDescriptor(byte[] rawDescriptorBytes,
+      int[] offsetAndLength, File descriptorFile,
+      Constructor<? extends DescriptorImpl> constructor,
+      boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
     try {
       return constructor.newInstance(rawDescriptorBytes,
-          offsetAndLength, failUnrecognizedDescriptorLines);
+          offsetAndLength, descriptorFile, failUnrecognizedDescriptorLines);
     } catch (InvocationTargetException e) {
       if (null != e.getCause()
           && e.getCause() instanceof DescriptorParseException) {
@@ -206,5 +247,4 @@ public class DescriptorParserImpl implements DescriptorParser {
       throw new RuntimeException(e);
     }
   }
-
 }
diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
index 12d73ad..54e26fc 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
@@ -135,6 +135,17 @@ public class DescriptorReaderImpl implements DescriptorReader {
     this.maxDescriptorFilesInQueue = max;
   }
+  private int maxDescriptorsInQueue = 100;
+
+  @Override
+  public void setMaxDescriptorsInQueue(int maxDescriptorsInQueue) {
+    if (this.hasStartedReading) {
+      throw new IllegalStateException("Reconfiguration is not permitted "
+          + "after starting to read.");
+    }
+    this.maxDescriptorsInQueue = maxDescriptorsInQueue;
+  }
+
   private DescriptorReaderRunnable reader;
@Override
@@ -160,6 +171,25 @@ public class DescriptorReaderImpl implements DescriptorReader {
   }
@Override
+  public Iterable<Descriptor> readDescriptors(File... descriptorFiles) {
+    if (this.hasStartedReading) {
+      throw new IllegalStateException("Initiating reading is only "
+          + "permitted once.");
+    }
+    this.hasStartedReading = true;
+    BlockingIteratorImpl<Descriptor> descriptorQueue =
+        new BlockingIteratorImpl<>(this.maxDescriptorsInQueue);
+    this.reader = new DescriptorReaderRunnable(
+        descriptorFiles, descriptorQueue, this.autoSaveHistoryFile,
+        this.manualSaveHistoryFile, this.excludedFiles,
+        this.failUnrecognizedDescriptorLines);
+    Thread readerThread = new Thread(this.reader);
+    readerThread.setDaemon(true);
+    readerThread.start();
+    return descriptorQueue;
+  }
+
+  @Override
   public void saveHistoryFile(File historyFile) {
     if (!this.reader.hasFinishedReading) {
       throw new IllegalStateException("Saving history is only permitted after "
@@ -170,11 +200,15 @@ public class DescriptorReaderImpl implements DescriptorReader {
private static class DescriptorReaderRunnable implements Runnable {
+    private File[] descriptorFiles;
+
     private List<File> directories;
private List<File> tarballs;
-    private BlockingIteratorImpl<DescriptorFile> descriptorQueue;
+    private BlockingIteratorImpl<Descriptor> descriptorQueue;
+
+    private BlockingIteratorImpl<DescriptorFile> descriptorFileQueue;
private File autoSaveHistoryFile;
@@ -186,24 +220,45 @@ public class DescriptorReaderImpl implements DescriptorReader {
private SortedMap<String, Long> parsedFilesAfter = new TreeMap<>();
+    private boolean failUnrecognizedDescriptorLines;
+
     private DescriptorParser descriptorParser;
private boolean hasFinishedReading = false;
private DescriptorReaderRunnable(List<File> directories,
         List<File> tarballs,
-        BlockingIteratorImpl<DescriptorFile> descriptorQueue,
+        BlockingIteratorImpl<DescriptorFile> descriptorFileQueue,
         File autoSaveHistoryFile, File manualSaveHistoryFile,
         SortedMap<String, Long> excludedFiles,
         boolean failUnrecognizedDescriptorLines) {
       this.directories = directories;
       this.tarballs = tarballs;
+      this.descriptorFileQueue = descriptorFileQueue;
+      this.autoSaveHistoryFile = autoSaveHistoryFile;
+      this.manualSaveHistoryFile = manualSaveHistoryFile;
+      if (excludedFiles != null) {
+        this.excludedFilesBefore = excludedFiles;
+      }
+      this.failUnrecognizedDescriptorLines = failUnrecognizedDescriptorLines;
+      this.descriptorParser = new DescriptorParserImpl();
+      this.descriptorParser.setFailUnrecognizedDescriptorLines(
+          failUnrecognizedDescriptorLines);
+    }
+
+    private DescriptorReaderRunnable(File[] descriptorFiles,
+        BlockingIteratorImpl<Descriptor> descriptorQueue,
+        File autoSaveHistoryFile, File manualSaveHistoryFile,
+        SortedMap<String, Long> excludedFiles,
+        boolean failUnrecognizedDescriptorLines) {
+      this.descriptorFiles = descriptorFiles;
       this.descriptorQueue = descriptorQueue;
       this.autoSaveHistoryFile = autoSaveHistoryFile;
       this.manualSaveHistoryFile = manualSaveHistoryFile;
       if (excludedFiles != null) {
         this.excludedFilesBefore = excludedFiles;
       }
+      this.failUnrecognizedDescriptorLines = failUnrecognizedDescriptorLines;
       this.descriptorParser = new DescriptorParserImpl();
       this.descriptorParser.setFailUnrecognizedDescriptorLines(
           failUnrecognizedDescriptorLines);
@@ -213,6 +268,7 @@ public class DescriptorReaderImpl implements DescriptorReader {
       try {
         this.readOldHistory(this.autoSaveHistoryFile);
         this.readOldHistory(this.manualSaveHistoryFile);
+        this.readDescriptorFiles();
         this.readDescriptors();
         this.readTarballs();
         this.hasFinishedReading = true;
@@ -220,7 +276,12 @@ public class DescriptorReaderImpl implements DescriptorReader {
         log.error("Bug: uncaught exception or error while "
             + "reading descriptors: " + t.getMessage(), t);
       } finally {
-        this.descriptorQueue.setOutOfDescriptors();
+        if (null != this.descriptorFileQueue) {
+          this.descriptorFileQueue.setOutOfDescriptors();
+        }
+        if (null != this.descriptorQueue) {
+          this.descriptorQueue.setOutOfDescriptors();
+        }
       }
       if (this.hasFinishedReading) {
         this.writeNewHistory(this.autoSaveHistoryFile);
@@ -275,7 +336,97 @@ public class DescriptorReaderImpl implements DescriptorReader {
       }
     }
+    private void readDescriptorFiles() {
+      if (null == this.descriptorFiles) {
+        return;
+      }
+      Stack<File> files = new Stack<>();
+      for (File descriptorFile : this.descriptorFiles) {
+        if (!descriptorFile.exists()) {
+          continue;
+        }
+        files.add(descriptorFile);
+        while (!files.isEmpty()) {
+          File file = files.pop();
+          try {
+            String absolutePath = file.getAbsolutePath();
+            long lastModifiedMillis = file.lastModified();
+            if (this.excludedFilesBefore.containsKey(absolutePath)
+                && this.excludedFilesBefore.get(absolutePath)
+                == lastModifiedMillis) {
+              this.excludedFilesAfter.put(absolutePath, lastModifiedMillis);
+              continue;
+            }
+            if (file.isDirectory()) {
+              files.addAll(Arrays.asList(file.listFiles()));
+              continue;
+            } else if (file.getName().endsWith(".tar")
+                || file.getName().endsWith(".tar.bz2")
+                || file.getName().endsWith(".tar.xz")) {
+              this.readTarball(file);
+            } else {
+              this.readDescriptorFile(file);
+            }
+            this.parsedFilesAfter.put(absolutePath, lastModifiedMillis);
+          } catch (IOException e) {
+            log.warn("Unable to read descriptor file {}.", file, e);
+          }
+        }
+      }
+    }
+
+    private void readTarball(File file) throws IOException {
+      FileInputStream in = new FileInputStream(file);
+      if (in.available() <= 0) {
+        return;
+      }
+      TarArchiveInputStream tais;
+      if (file.getName().endsWith(".tar.bz2")) {
+        tais = new TarArchiveInputStream(new BZip2CompressorInputStream(in));
+      } else if (file.getName().endsWith(".tar.xz")) {
+        tais = new TarArchiveInputStream(new XZCompressorInputStream(in));
+      } else if (file.getName().endsWith(".tar")) {
+        tais = new TarArchiveInputStream(in);
+      } else {
+        return;
+      }
+      BufferedInputStream bis = new BufferedInputStream(tais);
+      TarArchiveEntry tae;
+      while ((tae = tais.getNextTarEntry()) != null) {
+        if (tae.isDirectory()) {
+          continue;
+        }
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        int len;
+        byte[] data = new byte[1024];
+        while ((len = bis.read(data, 0, 1024)) >= 0) {
+          baos.write(data, 0, len);
+        }
+        byte[] rawDescriptorBytes = baos.toByteArray();
+        if (rawDescriptorBytes.length < 1) {
+          continue;
+        }
+        String fileName = tae.getName().substring(
+            tae.getName().lastIndexOf("/") + 1);
+        for (Descriptor descriptor : this.descriptorParser.parseDescriptors(
+            rawDescriptorBytes, file, fileName)) {
+          this.descriptorQueue.add(descriptor);
+        }
+      }
+    }
+
+    private void readDescriptorFile(File file) throws IOException {
+      byte[] rawDescriptorBytes = Files.readAllBytes(file.toPath());
+      for (Descriptor descriptor : this.descriptorParser.parseDescriptors(
+          rawDescriptorBytes, file, file.getName())) {
+        this.descriptorQueue.add(descriptor);
+      }
+    }
+
     private void readDescriptors() {
+      if (null == this.directories) {
+        return;
+      }
       for (File directory : this.directories) {
         if (!directory.exists() || !directory.isDirectory()) {
           continue;
@@ -315,13 +466,16 @@ public class DescriptorReaderImpl implements DescriptorReader {
               descriptorFile.setException(e);
               abortReading = true;
             }
-            this.descriptorQueue.add(descriptorFile);
+            this.descriptorFileQueue.add(descriptorFile);
           }
         }
       }
     }
private void readTarballs() {
+      if (null == this.tarballs) {
+        return;
+      }
       List<File> files = new ArrayList<>(this.tarballs);
       boolean abortReading = false;
       while (!abortReading && !files.isEmpty()) {
@@ -385,7 +539,7 @@ public class DescriptorReaderImpl implements DescriptorReader {
               } catch (DescriptorParseException e) {
                 descriptorFile.setException(e);
               }
-              this.descriptorQueue.add(descriptorFile);
+              this.descriptorFileQueue.add(descriptorFile);
             }
           }
         } catch (IOException e) {
diff --git a/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java b/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java
index 9a06a93..6beb9b5 100644
--- a/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java
@@ -6,6 +6,7 @@ package org.torproject.descriptor.impl;
 import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.DirectoryKeyCertificate;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.Scanner;
@@ -15,10 +16,10 @@ public class DirectoryKeyCertificateImpl extends DescriptorImpl
     implements DirectoryKeyCertificate {
protected DirectoryKeyCertificateImpl(byte[] rawDescriptorBytes,
-      int[] offsetAndLength, boolean failUnrecognizedDescriptorLines)
-      throws DescriptorParseException {
-    super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines,
-        false);
+      int[] offsetAndLength, File descriptorFile,
+      boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+    super(rawDescriptorBytes, offsetAndLength, descriptorFile,
+        failUnrecognizedDescriptorLines, false);
     this.parseDescriptorBytes();
     this.calculateDigestSha1Hex(Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP,
         NL + Key.DIR_KEY_CERTIFICATION.keyword + NL);
diff --git a/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java b/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java
index a87a55f..d16856a 100644
--- a/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.ExitList;
 import org.torproject.descriptor.ExitListEntry;
+import java.io.File;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -18,11 +19,11 @@ import java.util.TimeZone;
public class ExitListImpl extends DescriptorImpl implements ExitList {
-  protected ExitListImpl(byte[] rawDescriptorBytes, String fileName,
-      boolean failUnrecognizedDescriptorLines)
+  protected ExitListImpl(byte[] rawDescriptorBytes, File descriptorfile,
+      String fileName, boolean failUnrecognizedDescriptorLines)
       throws DescriptorParseException {
     super(rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length },
-        failUnrecognizedDescriptorLines, false);
+        descriptorfile, failUnrecognizedDescriptorLines, false);
     this.splitAndParseExitListEntries();
     this.setPublishedMillisFromFileName(fileName);
   }
diff --git a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
index ca7d376..32a4dcc 100644
--- a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.BandwidthHistory;
 import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.ExtraInfoDescriptor;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -34,10 +35,10 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl
       Key.PADDING_COUNTS);
protected ExtraInfoDescriptorImpl(byte[] descriptorBytes,
-      int[] offsetAndLimit, boolean failUnrecognizedDescriptorLines)
-      throws DescriptorParseException {
-    super(descriptorBytes, offsetAndLimit, failUnrecognizedDescriptorLines,
-        false);
+      int[] offsetAndLimit, File descriptorFile,
+      boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+    super(descriptorBytes, offsetAndLimit, descriptorFile,
+        failUnrecognizedDescriptorLines, false);
     this.parseDescriptorBytes();
     this.checkExactlyOnceKeys(exactlyOnceKeys);
     Set<Key> dirreqStatsKeys = EnumSet.of(
diff --git a/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java
index c436120..d9c284e 100644
--- a/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java
@@ -6,6 +6,7 @@ package org.torproject.descriptor.impl;
 import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.Microdescriptor;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
@@ -18,10 +19,10 @@ public class MicrodescriptorImpl extends DescriptorImpl
     implements Microdescriptor {
protected MicrodescriptorImpl(byte[] descriptorBytes, int[] offsetAndLength,
-      boolean failUnrecognizedDescriptorLines)
+      File descriptorFile, boolean failUnrecognizedDescriptorLines)
       throws DescriptorParseException {
-    super(descriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines,
-        false);
+    super(descriptorBytes, offsetAndLength, descriptorFile,
+        failUnrecognizedDescriptorLines, false);
     this.parseDescriptorBytes();
     this.calculateDigestSha256Base64(Key.ONION_KEY.keyword + NL);
     this.checkExactlyOnceKeys(EnumSet.of(Key.ONION_KEY));
diff --git a/src/main/java/org/torproject/descriptor/impl/NetworkStatusImpl.java b/src/main/java/org/torproject/descriptor/impl/NetworkStatusImpl.java
index 6765b6f..5695353 100644
--- a/src/main/java/org/torproject/descriptor/impl/NetworkStatusImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/NetworkStatusImpl.java
@@ -8,6 +8,7 @@ import org.torproject.descriptor.DirSourceEntry;
 import org.torproject.descriptor.DirectorySignature;
 import org.torproject.descriptor.NetworkStatusEntry;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.SortedMap;
@@ -19,11 +20,11 @@ import java.util.TreeMap;
 public abstract class NetworkStatusImpl extends DescriptorImpl {
protected NetworkStatusImpl(byte[] rawDescriptorBytes, int[] offsetAndLength,
-      boolean failUnrecognizedDescriptorLines,
+      File descriptorFile, boolean failUnrecognizedDescriptorLines,
       boolean containsDirSourceEntries, boolean blankLinesAllowed)
       throws DescriptorParseException {
-    super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines,
-        blankLinesAllowed);
+    super(rawDescriptorBytes, offsetAndLength, descriptorFile,
+        failUnrecognizedDescriptorLines, blankLinesAllowed);
     this.splitAndParseParts(containsDirSourceEntries);
   }
diff --git a/src/main/java/org/torproject/descriptor/impl/RelayDirectoryImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayDirectoryImpl.java
index d5108ca..79ab862 100644
--- a/src/main/java/org/torproject/descriptor/impl/RelayDirectoryImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/RelayDirectoryImpl.java
@@ -8,6 +8,7 @@ import org.torproject.descriptor.RelayDirectory;
 import org.torproject.descriptor.RouterStatusEntry;
 import org.torproject.descriptor.ServerDescriptor;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.EnumSet;
 import java.util.List;
@@ -18,10 +19,10 @@ public class RelayDirectoryImpl extends DescriptorImpl
     implements RelayDirectory {
protected RelayDirectoryImpl(byte[] directoryBytes, int[] offsetAndLength,
-      boolean failUnrecognizedDescriptorLines)
+      File descriptorFile, boolean failUnrecognizedDescriptorLines)
       throws DescriptorParseException {
-    super(directoryBytes, offsetAndLength, failUnrecognizedDescriptorLines,
-        true);
+    super(directoryBytes, offsetAndLength, descriptorFile,
+        failUnrecognizedDescriptorLines, true);
     this.splitAndParseParts();
     this.calculateDigestSha1Hex(Key.SIGNED_DIRECTORY.keyword + NL,
         NL + Key.DIRECTORY_SIGNATURE.keyword + SP);
@@ -194,7 +195,8 @@ public class RelayDirectoryImpl extends DescriptorImpl
     try {
       ServerDescriptorImpl serverDescriptor =
           new RelayServerDescriptorImpl(this.rawDescriptorBytes,
-          new int[] { offset, length }, this.failUnrecognizedDescriptorLines);
+          new int[] { offset, length }, this.getDescriptorFile(),
+          this.failUnrecognizedDescriptorLines);
       this.serverDescriptors.add(serverDescriptor);
     } catch (DescriptorParseException e) {
       this.serverDescriptorParseExceptions.add(e);
diff --git a/src/main/java/org/torproject/descriptor/impl/RelayExtraInfoDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayExtraInfoDescriptorImpl.java
index 92cff65..9644f90 100644
--- a/src/main/java/org/torproject/descriptor/impl/RelayExtraInfoDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/RelayExtraInfoDescriptorImpl.java
@@ -6,13 +6,16 @@ package org.torproject.descriptor.impl;
 import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.RelayExtraInfoDescriptor;
+import java.io.File;
+
 public class RelayExtraInfoDescriptorImpl
     extends ExtraInfoDescriptorImpl implements RelayExtraInfoDescriptor {
protected RelayExtraInfoDescriptorImpl(byte[] descriptorBytes,
-      int[] offsetAndLimit, boolean failUnrecognizedDescriptorLines)
-      throws DescriptorParseException {
-    super(descriptorBytes, offsetAndLimit, failUnrecognizedDescriptorLines);
+      int[] offsetAndLimit, File descriptorFile,
+      boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+    super(descriptorBytes, offsetAndLimit, descriptorFile,
+        failUnrecognizedDescriptorLines);
     this.calculateDigestSha1Hex(Key.EXTRA_INFO.keyword + SP,
         NL + Key.ROUTER_SIGNATURE.keyword + NL);
     this.calculateDigestSha256Base64(Key.EXTRA_INFO.keyword + SP,
diff --git a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java
index 12a9dc3..0e853bb 100644
--- a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java
@@ -6,6 +6,7 @@ package org.torproject.descriptor.impl;
 import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
@@ -22,10 +23,10 @@ public class RelayNetworkStatusConsensusImpl extends NetworkStatusImpl
     implements RelayNetworkStatusConsensus {
protected RelayNetworkStatusConsensusImpl(byte[] consensusBytes,
-      int[] offsetAndLimit, boolean failUnrecognizedDescriptorLines)
-      throws DescriptorParseException {
-    super(consensusBytes, offsetAndLimit, failUnrecognizedDescriptorLines, true,
-        false);
+      int[] offsetAndLimit, File descriptorFile,
+      boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+    super(consensusBytes, offsetAndLimit, descriptorFile,
+        failUnrecognizedDescriptorLines, true, false);
     Set<Key> exactlyOnceKeys = EnumSet.of(
         Key.VOTE_STATUS, Key.CONSENSUS_METHOD, Key.VALID_AFTER, Key.FRESH_UNTIL,
         Key.VALID_UNTIL, Key.VOTING_DELAY, Key.KNOWN_FLAGS);
diff --git a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java
index edc483a..8ebea2d 100644
--- a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java
@@ -6,6 +6,7 @@ package org.torproject.descriptor.impl;
 import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.RelayNetworkStatus;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
@@ -19,10 +20,10 @@ public class RelayNetworkStatusImpl extends NetworkStatusImpl
     implements RelayNetworkStatus {
protected RelayNetworkStatusImpl(byte[] statusBytes, int[] offsetAndLength,
-      boolean failUnrecognizedDescriptorLines)
+      File descriptorFile, boolean failUnrecognizedDescriptorLines)
       throws DescriptorParseException {
-    super(statusBytes, offsetAndLength, failUnrecognizedDescriptorLines, false,
-        true);
+    super(statusBytes, offsetAndLength, descriptorFile,
+        failUnrecognizedDescriptorLines, false, true);
     Set<Key> exactlyOnceKeys = EnumSet.of(
         Key.NETWORK_STATUS_VERSION, Key.DIR_SOURCE, Key.FINGERPRINT,
         Key.CONTACT, Key.DIR_SIGNING_KEY, Key.PUBLISHED);
diff --git a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java
index cad5c15..93d6f12 100644
--- a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.DirectorySignature;
 import org.torproject.descriptor.RelayNetworkStatusVote;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
@@ -24,10 +25,10 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl
     implements RelayNetworkStatusVote {
protected RelayNetworkStatusVoteImpl(byte[] voteBytes, int[] offsetAndLength,
-      boolean failUnrecognizedDescriptorLines)
+      File descriptorFile, boolean failUnrecognizedDescriptorLines)
       throws DescriptorParseException {
-    super(voteBytes, offsetAndLength, failUnrecognizedDescriptorLines, false,
-        false);
+    super(voteBytes, offsetAndLength, descriptorFile,
+        failUnrecognizedDescriptorLines, false, false);
     Set<Key> exactlyOnceKeys = EnumSet.of(
         Key.VOTE_STATUS, Key.PUBLISHED, Key.VALID_AFTER, Key.FRESH_UNTIL,
         Key.VALID_UNTIL, Key.VOTING_DELAY, Key.KNOWN_FLAGS, Key.DIR_SOURCE,
diff --git a/src/main/java/org/torproject/descriptor/impl/RelayServerDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayServerDescriptorImpl.java
index 4ccb35a..23e9152 100644
--- a/src/main/java/org/torproject/descriptor/impl/RelayServerDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/RelayServerDescriptorImpl.java
@@ -6,13 +6,16 @@ package org.torproject.descriptor.impl;
 import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.RelayServerDescriptor;
+import java.io.File;
+
 public class RelayServerDescriptorImpl extends ServerDescriptorImpl
     implements RelayServerDescriptor {
protected RelayServerDescriptorImpl(byte[] descriptorBytes,
-      int[] offsetAndLength, boolean failUnrecognizedDescriptorLines)
-      throws DescriptorParseException {
-    super(descriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines);
+      int[] offsetAndLength, File descriptorFile,
+      boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+    super(descriptorBytes, offsetAndLength, descriptorFile,
+        failUnrecognizedDescriptorLines);
     this.calculateDigestSha1Hex(Key.ROUTER.keyword + SP,
         NL + Key.ROUTER_SIGNATURE.keyword + NL);
     this.calculateDigestSha256Base64(Key.ROUTER.keyword + SP,
diff --git a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java
index d438702..f434d84 100644
--- a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.BandwidthHistory;
 import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.ServerDescriptor;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumSet;
@@ -35,10 +36,10 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl
       Key.ROUTER, Key.BANDWIDTH, Key.PUBLISHED);
protected ServerDescriptorImpl(byte[] descriptorBytes, int[] offsetAndLength,
-      boolean failUnrecognizedDescriptorLines)
+      File descriptorFile, boolean failUnrecognizedDescriptorLines)
       throws DescriptorParseException {
-    super(descriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines,
-        false);
+    super(descriptorBytes, offsetAndLength, descriptorFile,
+        failUnrecognizedDescriptorLines, false);
     this.parseDescriptorBytes();
     this.checkExactlyOnceKeys(exactlyOnce);
     this.checkAtMostOnceKeys(atMostOnce);
diff --git a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
index fb00a37..ea9eb4b 100644
--- a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.Descriptor;
 import org.torproject.descriptor.DescriptorParseException;
 import org.torproject.descriptor.TorperfResult;
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
@@ -20,8 +21,8 @@ public class TorperfResultImpl extends DescriptorImpl
     implements TorperfResult {
protected static List<Descriptor> parseTorperfResults(
-      byte[] rawDescriptorBytes, boolean failUnrecognizedDescriptorLines)
-      throws DescriptorParseException {
+      byte[] rawDescriptorBytes, File descriptorFile,
+      boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
     if (rawDescriptorBytes.length == 0) {
       throw new DescriptorParseException("Descriptor is empty.");
     }
@@ -47,7 +48,7 @@ public class TorperfResultImpl extends DescriptorImpl
       } else {
         /* XXX21932 */
         parsedDescriptors.add(new TorperfResultImpl(
-            (typeAnnotation + line).getBytes(),
+            (typeAnnotation + line).getBytes(), descriptorFile,
             failUnrecognizedDescriptorLines));
         typeAnnotation = "";
       }
@@ -55,11 +56,11 @@ public class TorperfResultImpl extends DescriptorImpl
     return parsedDescriptors;
   }
-  protected TorperfResultImpl(byte[] rawDescriptorBytes,
+  protected TorperfResultImpl(byte[] rawDescriptorBytes, File descriptorFile,
       boolean failUnrecognizedDescriptorLines)
       throws DescriptorParseException {
     super(rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length },
-        failUnrecognizedDescriptorLines, false);
+        descriptorFile, failUnrecognizedDescriptorLines, false);
     this.parseTorperfResultLine();
   }
diff --git a/src/main/java/org/torproject/descriptor/impl/UnparseableDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/UnparseableDescriptorImpl.java
new file mode 100644
index 0000000..4a0e772
--- /dev/null
+++ b/src/main/java/org/torproject/descriptor/impl/UnparseableDescriptorImpl.java
@@ -0,0 +1,40 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.impl;
+
+import org.torproject.descriptor.DescriptorParseException;
+import org.torproject.descriptor.UnparseableDescriptor;
+
+import java.io.File;
+import java.util.List;
+
+public class UnparseableDescriptorImpl extends DescriptorImpl
+    implements UnparseableDescriptor {
+
+  protected UnparseableDescriptorImpl(byte[] rawDescriptorBytes,
+      int[] offsetAndLength, File descriptorFile,
+      boolean failUnrecognizedDescriptorLines,
+      DescriptorParseException descriptorParseException) {
+    super(rawDescriptorBytes, offsetAndLength, descriptorFile,
+        failUnrecognizedDescriptorLines);
+    this.descriptorParseException = descriptorParseException;
+  }
+
+  private DescriptorParseException descriptorParseException;
+
+  public DescriptorParseException getDescriptorParseException() {
+    return this.descriptorParseException;
+  }
+
+  @Override
+  public List<String> getAnnotations() {
+    throw new UnsupportedOperationException("This operation is not supported.");
+  }
+
+  @Override
+  public List<String> getUnrecognizedLines() {
+    throw new UnsupportedOperationException("This operation is not supported.");
+  }
+}
+
diff --git a/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java b/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java
index 3739e5d..c44de6d 100644
--- a/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java
@@ -100,7 +100,7 @@ public class BridgeNetworkStatusTest {
         throws DescriptorParseException {
       byte[] statusBytes = this.buildStatusBytes();
       return new BridgeNetworkStatusImpl(statusBytes,
-          new int[] { 0, statusBytes.length }, this.fileName,
+          new int[] { 0, statusBytes.length }, null, this.fileName,
           failUnrecognizedDescriptorLines);
     }
diff --git a/src/test/java/org/torproject/descriptor/impl/ConsensusBuilder.java b/src/test/java/org/torproject/descriptor/impl/ConsensusBuilder.java
index e0d0c99..71bcd94 100644
--- a/src/test/java/org/torproject/descriptor/impl/ConsensusBuilder.java
+++ b/src/test/java/org/torproject/descriptor/impl/ConsensusBuilder.java
@@ -360,7 +360,7 @@ public class ConsensusBuilder {
       throws DescriptorParseException {
     byte[] consensusBytes = this.buildConsensusBytes();
     return new RelayNetworkStatusConsensusImpl(consensusBytes,
-        new int[] { 0, consensusBytes.length },
+        new int[] { 0, consensusBytes.length }, null,
         failUnrecognizedDescriptorLines);
   }
diff --git a/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java
index f1a8935..543254d 100644
--- a/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java
@@ -131,7 +131,6 @@ public class DescriptorImplTest {
     digest = des.getDigestSha256Base64();
     assertEquals("1bEECw9nT5KRzPG8dAzEFJgSI4OBQfyWn+wjREb8oa8", digest);
   }
-
 }
diff --git a/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java b/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java
index 5a328ec..558a395 100644
--- a/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java
@@ -11,6 +11,7 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
+import java.io.File;
 import java.lang.reflect.Constructor;
public class DescriptorParserImplTest {
@@ -46,15 +47,17 @@ public class DescriptorParserImplTest {
     Constructor<? extends DescriptorImpl> constructor;
     try {
       constructor = TestServerDescriptor.class
-        .getDeclaredConstructor(byte[].class, int[].class, boolean.class);
+          .getDeclaredConstructor(byte[].class, int[].class, File.class,
+          boolean.class);
     } catch (NoSuchMethodException e) {
       throw new RuntimeException(e);
     }
     this.thrown.expect(DescriptorParseException.class);
     this.thrown.expectMessage("'176x.158.53.63' in line 'router UbuntuCore169 "
         + "176x.158.53.63 44583 0 0' is not a valid IPv4 address.");
-    DescriptorParserImpl.parseDescriptor(DEFECT.getBytes(),
-        new int[]{0, DEFECT.getBytes().length}, constructor, false);
+    DescriptorParserImpl dpi = new DescriptorParserImpl();
+    dpi.parseDescriptor(DEFECT.getBytes(),
+        new int[]{0, DEFECT.getBytes().length}, null, constructor, false);
   }
private static final String DEFECT =
diff --git a/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java b/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java
index 74417d9..5f31341 100644
--- a/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java
@@ -24,7 +24,7 @@ public class ExitListImplTest {
   @Test
   public void testAnnotatedInput() throws Exception {
     ExitListImpl result = new ExitListImpl((tordnselAnnotation + input)
-        .getBytes("US-ASCII"), fileName, false);
+        .getBytes("US-ASCII"), null, fileName, false);
     assertEquals("Expected one annotation.", 1,
         result.getAnnotations().size());
     assertEquals(tordnselAnnotation.substring(0, 18),
@@ -42,7 +42,7 @@ public class ExitListImplTest {
   public void testMultipleOldExitAddresses() throws Exception {
     ExitListImpl result = new ExitListImpl(
         (tordnselAnnotation + multiExitAddressInput)
-        .getBytes("US-ASCII"), fileName, false);
+        .getBytes("US-ASCII"), null, fileName, false);
     assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(),
         result.getUnrecognizedLines().isEmpty());
     assertEquals("Found: " + result.getExitListEntries(),
@@ -68,7 +68,7 @@ public class ExitListImplTest {
   public void testMultipleExitAddresses() throws Exception {
     ExitListImpl result = new ExitListImpl(
         (tordnselAnnotation + multiExitAddressInput)
-        .getBytes("US-ASCII"), fileName, false);
+        .getBytes("US-ASCII"), null, fileName, false);
     assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(),
         result.getUnrecognizedLines().isEmpty());
     Map<String, Long> map = result.getEntries()
@@ -84,7 +84,7 @@ public class ExitListImplTest {
     this.thrown.expect(DescriptorParseException.class);
     this.thrown.expectMessage("Missing 'ExitAddress' line in exit list entry.");
     new ExitListImpl((tordnselAnnotation + insufficientInput[0])
-        .getBytes("US-ASCII"), fileName, false);
+        .getBytes("US-ASCII"), null, fileName, false);
   }
@Test
@@ -92,7 +92,7 @@ public class ExitListImplTest {
     this.thrown.expect(DescriptorParseException.class);
     this.thrown.expectMessage("Missing 'Published' line in exit list entry.");
     new ExitListImpl((tordnselAnnotation + insufficientInput[1])
-        .getBytes("US-ASCII"), fileName, false);
+        .getBytes("US-ASCII"), null, fileName, false);
   }
private static final String tordnselAnnotation = "@type tordnsel 1.0\n";
diff --git a/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java
index 7b62393..82dd365 100644
--- a/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java
@@ -336,7 +336,7 @@ public class ExtraInfoDescriptorImplTest {
         throws DescriptorParseException {
       byte[] descriptorBytes = this.buildDescriptorBytes();
       return new RelayExtraInfoDescriptorImpl(descriptorBytes,
-          new int[] { 0, descriptorBytes.length},
+          new int[] { 0, descriptorBytes.length}, null,
           failUnrecognizedDescriptorLines);
     }
   }
@@ -2332,7 +2332,7 @@ public class ExtraInfoDescriptorImplTest {
         + "").getBytes();
     RelayExtraInfoDescriptor descriptor =
         new RelayExtraInfoDescriptorImpl(descriptorBytes,
-            new int[] { 0, descriptorBytes.length }, true);
+            new int[] { 0, descriptorBytes.length }, null, true);
     assertEquals("Pt1BtzfRwhYqGCDo8jjchS8nJP3ovrDyHGn+dqPbMgw",
         descriptor.getDigestSha256Base64());
   }
@@ -2357,7 +2357,7 @@ public class ExtraInfoDescriptorImplTest {
         .getBytes();
     BridgeExtraInfoDescriptor descriptor =
         new BridgeExtraInfoDescriptorImpl(descriptorBytes,
-            new int[] { 0, descriptorBytes.length }, true);
+            new int[] { 0, descriptorBytes.length }, null, true);
     assertEquals("TvrqpjI7OmCtwGwair/NHUxg5ROVVQYz6/EDyXsDHR4",
         descriptor.getDigestSha256Base64());
   }
@@ -2379,7 +2379,7 @@ public class ExtraInfoDescriptorImplTest {
         .getBytes();
     BridgeExtraInfoDescriptor descriptor =
         new BridgeExtraInfoDescriptorImpl(descriptorBytes,
-            new int[] { 0, descriptorBytes.length }, true);
+            new int[] { 0, descriptorBytes.length }, null, true);
     assertNull(descriptor.getDigestSha1Hex());
     assertNull(descriptor.getDigestSha256Base64());
   }
diff --git a/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java
index b1d6474..f5c9131 100644
--- a/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java
@@ -66,7 +66,7 @@ public class MicrodescriptorImplTest {
         throws DescriptorParseException {
       byte[] descriptorBytes = this.buildDescriptorBytes();
       return new MicrodescriptorImpl(descriptorBytes,
-          new int[] { 0, descriptorBytes.length },
+          new int[] { 0, descriptorBytes.length }, null,
           failUnrecognizedDescriptorLines);
     }
   }
diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java
index 90128c2..4793e76 100644
--- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java
@@ -1599,7 +1599,7 @@ public class RelayNetworkStatusConsensusImplTest {
     byte[] consensusBytes = cb.buildConsensusBytes();
     consensusBytes[20] = (byte) 200;
     new RelayNetworkStatusConsensusImpl(consensusBytes,
-        new int[] { 0, consensusBytes.length }, true);
+        new int[] { 0, consensusBytes.length }, null, true);
   }
@Test
@@ -1613,7 +1613,7 @@ public class RelayNetworkStatusConsensusImplTest {
     byte[] consensusBytes = cb.buildConsensusBytes();
     consensusBytes[0] = (byte) 200;
     new RelayNetworkStatusConsensusImpl(consensusBytes,
-        new int[] { 0, consensusBytes.length }, true);
+        new int[] { 0, consensusBytes.length }, null, true);
   }
@Test
diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java
index e69a26b..760d04c 100644
--- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java
@@ -49,14 +49,14 @@ public class RelayNetworkStatusImplTest {
     byte[] statusBytes = (validAnnotation + invalidHeader + validFooter)
         .getBytes();
     new RelayNetworkStatusImpl(statusBytes, new int[] { 0, statusBytes.length },
-        true);
+        null, true);
   }
@Test
   public void testValidHeader() throws DescriptorParseException {
     byte[] statusBytes = validStatus.getBytes();
     RelayNetworkStatusImpl rnsi = new RelayNetworkStatusImpl(statusBytes,
-        new int[] { 0, statusBytes.length }, true);
+        new int[] { 0, statusBytes.length }, null, true);
     assertEquals(rnsi.getContactLine(),
         "1024R/8D56913D Alex de Joode adejoode@sabotage.org");
   }
diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java
index 8a85145..9727f50 100644
--- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java
@@ -557,7 +557,8 @@ public class RelayNetworkStatusVoteImplTest {
         throws DescriptorParseException {
       byte[] voteBytes = this.buildVoteBytes();
       return new RelayNetworkStatusVoteImpl(voteBytes,
-          new int[] { 0, voteBytes.length }, failUnrecognizedDescriptorLines);
+          new int[] { 0, voteBytes.length }, null,
+          failUnrecognizedDescriptorLines);
     }
private void appendHeader(StringBuilder sb) {
diff --git a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java
index 9533de3..d0c2561 100644
--- a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java
@@ -456,7 +456,7 @@ public class ServerDescriptorImplTest {
         throws DescriptorParseException {
       byte[] descriptorBytes = this.buildDescriptorBytes();
       return new RelayServerDescriptorImpl(descriptorBytes,
-          new int[] { 0, descriptorBytes.length },
+          new int[] { 0, descriptorBytes.length }, null,
           failUnrecognizedDescriptorLines);
     }
   }
@@ -1963,7 +1963,7 @@ public class ServerDescriptorImplTest {
         + "\nrouter-digest " + digestSha1Hex;
     byte[] descriptorBytes = db.buildDescriptorBytes();
     BridgeServerDescriptor descriptor = new BridgeServerDescriptorImpl(
-        descriptorBytes, new int[] { 0, descriptorBytes.length }, true);
+        descriptorBytes, new int[] { 0, descriptorBytes.length }, null, true);
     assertEquals(digestSha1Hex, descriptor.getDigestSha1Hex());
     assertEquals(digestSha256Base64, descriptor.getDigestSha256Base64());
   }
@@ -1974,7 +1974,7 @@ public class ServerDescriptorImplTest {
     DescriptorBuilder db = new DescriptorBuilder();
     byte[] descriptorBytes = db.buildDescriptorBytes();
     BridgeServerDescriptor descriptor = new BridgeServerDescriptorImpl(
-        descriptorBytes, new int[] { 0, descriptorBytes.length }, true);
+        descriptorBytes, new int[] { 0, descriptorBytes.length }, null, true);
     assertNull(descriptor.getDigestSha1Hex());
     assertNull(descriptor.getDigestSha256Base64());
   }
diff --git a/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java b/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java
index 84aedee..e3a0c7c 100644
--- a/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java
+++ b/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java
@@ -3,12 +3,14 @@
package org.torproject.descriptor.impl;
+import org.torproject.descriptor.DescriptorParseException;
+
 public class TestDescriptor extends DescriptorImpl {
protected TestDescriptor(byte[] rawDescriptorBytes, int[] offsetAndLength,
       boolean failUnrecognizedDescriptorLines, boolean blankLinesAllowed)
       throws DescriptorParseException {
-    super(rawDescriptorBytes, offsetAndLength,
+    super(rawDescriptorBytes, offsetAndLength, null,
         failUnrecognizedDescriptorLines, blankLinesAllowed);
   }
 }
diff --git a/src/test/java/org/torproject/descriptor/impl/TestServerDescriptor.java b/src/test/java/org/torproject/descriptor/impl/TestServerDescriptor.java
index 194df82..0afd7b4 100644
--- a/src/test/java/org/torproject/descriptor/impl/TestServerDescriptor.java
+++ b/src/test/java/org/torproject/descriptor/impl/TestServerDescriptor.java
@@ -3,12 +3,16 @@
package org.torproject.descriptor.impl;
+import java.io.File;
+
 public class TestServerDescriptor extends ServerDescriptorImpl {
protected TestServerDescriptor(byte[] rawDescriptorBytes,
-      int[] offsetAndLength, boolean failUnrecognizedDescriptorLines)
+      int[] offsetAndLength, File descriptorFile,
+      boolean failUnrecognizedDescriptorLines)
       throws DescriptorParseException {
-    super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines);
+    super(rawDescriptorBytes, offsetAndLength, descriptorFile,
+        failUnrecognizedDescriptorLines);
   }
}
diff --git a/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java b/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java
index 122e301..7df4269 100644
--- a/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java
@@ -20,7 +20,7 @@ public class TorperfResultImplTest {
   public void testAnnotatedInput() throws Exception {
     TorperfResultImpl result = (TorperfResultImpl)
         (TorperfResultImpl.parseTorperfResults((torperfAnnotation + input)
-        .getBytes("US-ASCII"), false).get(0));
+        .getBytes("US-ASCII"), null, false).get(0));
     assertEquals("Expected one annotation.", 1,
         result.getAnnotations().size());
     assertEquals(torperfAnnotation.substring(0, 17),
@@ -37,7 +37,7 @@ public class TorperfResultImplTest {
     byte[] asciiBytes = (torperfAnnotation
         + input + input + input).getBytes("US-ASCII");
     List<Descriptor> result = TorperfResultImpl.parseTorperfResults(
-        asciiBytes, false);
+        asciiBytes, null, false);
     assertEquals("Expected one annotation.", 1,
         ((TorperfResultImpl)(result.get(0))).getAnnotations().size());
     assertEquals(3, result.size());
@@ -53,7 +53,7 @@ public class TorperfResultImplTest {
         + torperfAnnotation + input
         + torperfAnnotation + input).getBytes("US-ASCII");
     List<Descriptor> result = TorperfResultImpl.parseTorperfResults(
-        asciiBytes, false);
+        asciiBytes, null, false);
     assertEquals("Expected one annotation.", 1,
         ((TorperfResultImpl)(result.get(0))).getAnnotations().size());
     assertEquals(3, result.size());
@@ -90,7 +90,7 @@ public class TorperfResultImplTest {
   @Test
   public void testDatapercNonNumeric() throws Exception {
     List<Descriptor> result = TorperfResultImpl.parseTorperfResults(
-        ("DATAPERMILLE=2.0 " + input).getBytes(), false);
+        ("DATAPERMILLE=2.0 " + input).getBytes(), null, false);
     assertEquals(1, result.size());
     TorperfResultImpl torperfResult = (TorperfResultImpl) result.get(0);
     assertEquals(1, torperfResult.getUnrecognizedKeys().size());
@@ -125,7 +125,7 @@ public class TorperfResultImplTest {
   public void testEndpointsHostnamesSourceAddress()
       throws DescriptorParseException {
     List<Descriptor> result = TorperfResultImpl.parseTorperfResults(
-        input2.getBytes(), true);
+        input2.getBytes(), null, true);
     assertEquals(1, result.size());
     TorperfResultImpl torperfResult = (TorperfResultImpl) result.get(0);
     assertNull(torperfResult.getUnrecognizedKeys());

    

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

[tor-commits] [metrics-lib/master] Make DescriptorFile obsolete.