commit 26b7c97f93ea5e5ae88998085909e86641265a2a Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue Jan 31 17:47:15 2012 +0100

Add a config option to handle unrecognized lines.

Most applications don't care about unrecognized lines in descriptors as long as the lines they care about are contained. But some applications rather want to skip a descriptor with unrecognized lines, e.g., the bridge descriptor sanitizer.

Add a switch to downloaders and readers to control the desired behavior, that is, to either fail parsing or ignore unrecognized lines. In the latter case, unrecognized are written to a list for later inspection by the application.

Corrupt lines with known keywords always lead to failing the parse step. --- .../descriptor/BridgeDescriptorReader.java | 7 + .../descriptor/BridgePoolAssignmentReader.java | 7 + src/org/torproject/descriptor/Descriptor.java | 6 + .../descriptor/RelayDescriptorDownloader.java | 7 + .../descriptor/RelayDescriptorReader.java | 7 + .../descriptor/impl/BridgeNetworkStatusImpl.java | 9 +- .../descriptor/impl/BridgePoolAssignmentImpl.java | 25 ++-- .../torproject/descriptor/impl/DescriptorImpl.java | 33 +++- .../descriptor/impl/DirSourceEntryImpl.java | 37 ++++- .../descriptor/impl/DirectoryDownloader.java | 10 +- .../descriptor/impl/DownloadCoordinatorImpl.java | 9 +- .../descriptor/impl/ExtraInfoDescriptorImpl.java | 39 ++-- .../descriptor/impl/NetworkStatusEntryImpl.java | 25 ++- .../descriptor/impl/NetworkStatusImpl.java | 43 ++++- .../impl/RelayDescriptorDownloaderImpl.java | 12 ++- .../impl/RelayNetworkStatusConsensusImpl.java | 47 +++--- .../impl/RelayNetworkStatusVoteImpl.java | 75 ++++++--- .../impl/RelayOrBridgeDescriptorReaderImpl.java | 16 ++- .../descriptor/impl/ServerDescriptorImpl.java | 26 ++-- .../descriptor/impl/ConsensusBuilder.java | 101 +++++++++-- .../impl/RelayNetworkStatusConsensusImplTest.java | 125 ++++++++++++-- .../impl/RelayNetworkStatusVoteImplTest.java | 182 +++++++++++++++++--- .../descriptor/impl/ServerDescriptorImplTest.java | 77 ++++++--- 23 files changed, 722 insertions(+), 203 deletions(-)

diff --git a/src/org/torproject/descriptor/BridgeDescriptorReader.java b/src/org/torproject/descriptor/BridgeDescriptorReader.java index 3308e82..d827124 100644 --- a/src/org/torproject/descriptor/BridgeDescriptorReader.java +++ b/src/org/torproject/descriptor/BridgeDescriptorReader.java @@ -18,6 +18,13 @@ public interface BridgeDescriptorReader { * last modified timestamp and the absolute path of a file. */ public void setExcludeFiles(File historyFile);

+ /* Fail descriptor parsing when encountering an unrecognized line. This + * is not set by default, because the Tor specifications allow for new + * lines to be added that shall be ignored by older Tor versions. But + * some applications may want to handle unrecognized descriptor lines + * explicitly. */ + public void setFailUnrecognizedDescriptorLines(); + /* Read the previously configured bridge descriptors and make them * available via the returned blocking iterator. Whenever the reader * runs out of descriptors and expects to provide more shortly after, it diff --git a/src/org/torproject/descriptor/BridgePoolAssignmentReader.java b/src/org/torproject/descriptor/BridgePoolAssignmentReader.java index e84f2b4..13c8ae3 100644 --- a/src/org/torproject/descriptor/BridgePoolAssignmentReader.java +++ b/src/org/torproject/descriptor/BridgePoolAssignmentReader.java @@ -18,6 +18,13 @@ public interface BridgePoolAssignmentReader { * last modified timestamp and the absolute path of a file. */ public void setExcludeFiles(File historyFile);

+ /* Fail descriptor parsing when encountering an unrecognized line. This + * is not set by default, because the Tor specifications allow for new + * lines to be added that shall be ignored by older Tor versions. But + * some applications may want to handle unrecognized descriptor lines + * explicitly. */ + public void setFailUnrecognizedDescriptorLines(); + /* Read the previously configured bridge pool assignments and make them * available via the returned blocking iterator. Whenever the reader * runs out of descriptors and expects to provide more shortly after, it diff --git a/src/org/torproject/descriptor/Descriptor.java b/src/org/torproject/descriptor/Descriptor.java index 76b23a0..6e02966 100644 --- a/src/org/torproject/descriptor/Descriptor.java +++ b/src/org/torproject/descriptor/Descriptor.java @@ -2,11 +2,17 @@ * See LICENSE for licensing information */ package org.torproject.descriptor;

+import java.util.List; + /* Store meta-data about how a descriptor was downloaded or read from * disk. */ public interface Descriptor {

/* Return the raw descriptor bytes. */ public byte[] getRawDescriptorBytes(); + + /* Return any unrecognized lines when parsing this descriptor, or an + * empty list if there were no unrecognized lines. */ + public List<String> getUnrecognizedLines(); }

diff --git a/src/org/torproject/descriptor/RelayDescriptorDownloader.java b/src/org/torproject/descriptor/RelayDescriptorDownloader.java index 754c644..29171d3 100644 --- a/src/org/torproject/descriptor/RelayDescriptorDownloader.java +++ b/src/org/torproject/descriptor/RelayDescriptorDownloader.java @@ -92,6 +92,13 @@ public interface RelayDescriptorDownloader { * hour (60 * 60 * 1000). */ public void setGlobalTimeout(long globalTimeoutMillis);

+ /* Fail descriptor parsing when encountering an unrecognized line. This + * is not set by default, because the Tor specifications allow for new + * lines to be added that shall be ignored by older Tor versions. But + * some applications may want to handle unrecognized descriptor lines + * explicitly. */ + public void setFailUnrecognizedDescriptorLines(); + /* Download the previously configured relay descriptors and make them * available via the returned blocking iterator. Whenever the * downloader runs out of descriptors and expects to provide more diff --git a/src/org/torproject/descriptor/RelayDescriptorReader.java b/src/org/torproject/descriptor/RelayDescriptorReader.java index 021794e..142f79c 100644 --- a/src/org/torproject/descriptor/RelayDescriptorReader.java +++ b/src/org/torproject/descriptor/RelayDescriptorReader.java @@ -18,6 +18,13 @@ public interface RelayDescriptorReader { * last modified timestamp and the absolute path of a file. */ public void setExcludeFiles(File historyFile);

+ /* Fail descriptor parsing when encountering an unrecognized line. This + * is not set by default, because the Tor specifications allow for new + * lines to be added that shall be ignored by older Tor versions. But + * some applications may want to handle unrecognized descriptor lines + * explicitly. */ + public void setFailUnrecognizedDescriptorLines(); + /* Read the previously configured relay descriptors and make them * available via the returned blocking iterator. Whenever the reader * runs out of descriptors and expects to provide more shortly after, it diff --git a/src/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java b/src/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java index 79ee8ce..1d9818c 100644 --- a/src/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java +++ b/src/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java @@ -13,14 +13,15 @@ public class BridgeNetworkStatusImpl extends NetworkStatusImpl implements BridgeNetworkStatus {

protected BridgeNetworkStatusImpl(byte[] statusBytes, - String fileName) throws DescriptorParseException { - super(statusBytes); + String fileName, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + super(statusBytes, failUnrecognizedDescriptorLines); this.setPublishedMillisFromFileName(fileName); }

private void setPublishedMillisFromFileName(String fileName) throws DescriptorParseException { - if (fileName.length() == + if (fileName.length() == "20000101-000000-4A0CCD2DDC7995083D73F5D667100C8A5831F16D". length()) { String publishedString = fileName.substring(0, @@ -33,7 +34,7 @@ public class BridgeNetworkStatusImpl extends NetworkStatusImpl this.publishedMillis = fileNameFormat.parse(publishedString). getTime(); } catch (ParseException e) { - } + } } if (this.publishedMillis == 0L) { throw new DescriptorParseException("Unrecognized bridge network " diff --git a/src/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java b/src/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java index 946d897..8a6857e 100644 --- a/src/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java +++ b/src/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java @@ -20,29 +20,26 @@ public class BridgePoolAssignmentImpl extends DescriptorImpl implements BridgePoolAssignment {

protected static List<BridgePoolAssignment> parseDescriptors( - byte[] descriptorsBytes) { + byte[] descriptorsBytes, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { List<BridgePoolAssignment> parsedDescriptors = new ArrayList<BridgePoolAssignment>(); List<byte[]> splitDescriptorsBytes = DescriptorImpl.splitRawDescriptorBytes(descriptorsBytes, "bridge-pool-assignment "); - try { - for (byte[] descriptorBytes : splitDescriptorsBytes) { - BridgePoolAssignment parsedDescriptor = - new BridgePoolAssignmentImpl(descriptorBytes); - parsedDescriptors.add(parsedDescriptor); - } - } catch (DescriptorParseException e) { - /* TODO Handle this error somehow. */ - System.err.println("Failed to parse descriptor. Skipping."); - e.printStackTrace(); + for (byte[] descriptorBytes : splitDescriptorsBytes) { + BridgePoolAssignment parsedDescriptor = + new BridgePoolAssignmentImpl(descriptorBytes, + failUnrecognizedDescriptorLines); + parsedDescriptors.add(parsedDescriptor); } return parsedDescriptors; }

- protected BridgePoolAssignmentImpl(byte[] descriptorBytes) + protected BridgePoolAssignmentImpl(byte[] descriptorBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(descriptorBytes); + super(descriptorBytes, failUnrecognizedDescriptorLines); this.parseDescriptorBytes(); Set<String> exactlyOnceKeywords = new HashSet<String>(Arrays.asList( new String[] { "bridge-pool-assignment" })); @@ -81,7 +78,7 @@ public class BridgePoolAssignmentImpl extends DescriptorImpl parts, 1, 2); }

- private void parseBridgeLine(String line) + private void parseBridgeLine(String line) throws DescriptorParseException { String[] parts = line.split(" "); if (parts.length < 2) { diff --git a/src/org/torproject/descriptor/impl/DescriptorImpl.java b/src/org/torproject/descriptor/impl/DescriptorImpl.java index 7be2fbb..d8705e2 100644 --- a/src/org/torproject/descriptor/impl/DescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/DescriptorImpl.java @@ -16,7 +16,8 @@ import org.torproject.descriptor.Descriptor; public abstract class DescriptorImpl implements Descriptor {

protected static List<Descriptor> parseRelayOrBridgeDescriptors( - byte[] rawDescriptorBytes, String fileName) + byte[] rawDescriptorBytes, String fileName, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { List<Descriptor> parsedDescriptors = new ArrayList<Descriptor>(); if (rawDescriptorBytes == null) { @@ -31,10 +32,12 @@ public abstract class DescriptorImpl implements Descriptor { firstLines.contains("\nnetwork-status-version 3\n")) { if (firstLines.contains("\nvote-status consensus\n")) { parsedDescriptors.addAll(RelayNetworkStatusConsensusImpl. - parseConsensuses(rawDescriptorBytes)); + parseConsensuses(rawDescriptorBytes, + failUnrecognizedDescriptorLines)); } else if (firstLines.contains("\nvote-status vote\n")) { parsedDescriptors.addAll(RelayNetworkStatusVoteImpl. - parseVotes(rawDescriptorBytes)); + parseVotes(rawDescriptorBytes, + failUnrecognizedDescriptorLines)); } else { throw new DescriptorParseException("Could not detect relay " + "network status type in descriptor starting with '" @@ -42,19 +45,22 @@ public abstract class DescriptorImpl implements Descriptor { } } else if (firstLines.startsWith("r ")) { parsedDescriptors.add(new BridgeNetworkStatusImpl( - rawDescriptorBytes, fileName)); + rawDescriptorBytes, fileName, failUnrecognizedDescriptorLines)); } else if (firstLines.startsWith("router ") || firstLines.contains("\nrouter ")) { parsedDescriptors.addAll(ServerDescriptorImpl. - parseDescriptors(rawDescriptorBytes)); + parseDescriptors(rawDescriptorBytes, + failUnrecognizedDescriptorLines)); } else if (firstLines.startsWith("extra-info ") || firstLines.contains("\nextra-info ")) { parsedDescriptors.addAll(ExtraInfoDescriptorImpl. - parseDescriptors(rawDescriptorBytes)); + parseDescriptors(rawDescriptorBytes, + failUnrecognizedDescriptorLines)); } else if (firstLines.startsWith("bridge-pool-assignment ") || firstLines.contains("\nbridge-pool-assignment ")) { parsedDescriptors.addAll(BridgePoolAssignmentImpl. - parseDescriptors(rawDescriptorBytes)); + parseDescriptors(rawDescriptorBytes, + failUnrecognizedDescriptorLines)); } else { throw new DescriptorParseException("Could not detect descriptor " + "type in descriptor starting with '" + firstLines + "'."); @@ -90,9 +96,20 @@ public abstract class DescriptorImpl implements Descriptor { return this.rawDescriptorBytes; }

- protected DescriptorImpl(byte[] rawDescriptorBytes) + protected boolean failUnrecognizedDescriptorLines = false; + + protected List<String> unrecognizedLines; + public List<String> getUnrecognizedLines() { + return this.unrecognizedLines == null ? new ArrayList<String>() : + new ArrayList<String>(this.unrecognizedLines); + } + + protected DescriptorImpl(byte[] rawDescriptorBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { this.rawDescriptorBytes = rawDescriptorBytes; + this.failUnrecognizedDescriptorLines = + failUnrecognizedDescriptorLines; this.countKeywords(rawDescriptorBytes); }

diff --git a/src/org/torproject/descriptor/impl/DirSourceEntryImpl.java b/src/org/torproject/descriptor/impl/DirSourceEntryImpl.java index 2ee4556..c20ca30 100644 --- a/src/org/torproject/descriptor/impl/DirSourceEntryImpl.java +++ b/src/org/torproject/descriptor/impl/DirSourceEntryImpl.java @@ -5,6 +5,8 @@ package org.torproject.descriptor.impl; import java.io.BufferedReader; import java.io.IOException; import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; import java.util.SortedSet; import java.util.TreeSet;

@@ -17,9 +19,20 @@ public class DirSourceEntryImpl implements DirSourceEntry { return this.dirSourceEntryBytes; }

- protected DirSourceEntryImpl(byte[] dirSourceEntryBytes) + private boolean failUnrecognizedDescriptorLines; + private List<String> unrecognizedLines; + protected List<String> getAndClearUnrecognizedLines() { + List<String> lines = this.unrecognizedLines; + this.unrecognizedLines = null; + return lines; + } + + protected DirSourceEntryImpl(byte[] dirSourceEntryBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { this.dirSourceEntryBytes = dirSourceEntryBytes; + this.failUnrecognizedDescriptorLines = + failUnrecognizedDescriptorLines; this.initializeKeywords(); this.parseDirSourceEntryBytes(); this.checkKeywords(); @@ -65,6 +78,7 @@ public class DirSourceEntryImpl implements DirSourceEntry { BufferedReader br = new BufferedReader(new StringReader( new String(this.dirSourceEntryBytes))); String line; + boolean skipCrypto = false; while ((line = br.readLine()) != null) { if (line.startsWith("dir-source")) { this.parseDirSourceLine(line); @@ -72,10 +86,20 @@ public class DirSourceEntryImpl implements DirSourceEntry { this.parseContactLine(line); } else if (line.startsWith("vote-digest")) { this.parseVoteDigestLine(line); - } else { - /* TODO Should we really throw an exception here? */ - throw new DescriptorParseException("Unknown line '" + line - + "' in dir-source entry."); + } else if (line.startsWith("-----BEGIN")) { + skipCrypto = true; + } else if (line.startsWith("-----END")) { + skipCrypto = false; + } else if (!skipCrypto) { + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in dir-source entry."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } } } } catch (IOException e) { @@ -89,6 +113,9 @@ public class DirSourceEntryImpl implements DirSourceEntry { throws DescriptorParseException { this.parsedExactlyOnceKeyword("dir-source"); String[] parts = line.split(" "); + if (parts.length != 7) { + throw new DescriptorParseException("Invalid line '" + line + "'."); + } String nickname = parts[1]; if (nickname.endsWith("-legacy")) { nickname = nickname.substring(0, nickname.length() diff --git a/src/org/torproject/descriptor/impl/DirectoryDownloader.java b/src/org/torproject/descriptor/impl/DirectoryDownloader.java index 7e9d19c..685ee60 100644 --- a/src/org/torproject/descriptor/impl/DirectoryDownloader.java +++ b/src/org/torproject/descriptor/impl/DirectoryDownloader.java @@ -37,6 +37,13 @@ public class DirectoryDownloader implements Runnable { this.readTimeout = readTimeout; }

+ private boolean failUnrecognizedDescriptorLines; + protected void setFailUnrecognizedDescriptorLines( + boolean failUnrecognizedDescriptorLines) { + this.failUnrecognizedDescriptorLines = + failUnrecognizedDescriptorLines; + } + public void run() { boolean keepRunning = true; do { @@ -70,7 +77,8 @@ public class DirectoryDownloader implements Runnable { request.setResponseBytes(responseBytes); request.setRequestEnd(System.currentTimeMillis()); request.setDescriptors(DescriptorImpl. - parseRelayOrBridgeDescriptors(responseBytes, null)); + parseRelayOrBridgeDescriptors(responseBytes, null, + this.failUnrecognizedDescriptorLines)); } } catch (Exception e) { request.setException(e); diff --git a/src/org/torproject/descriptor/impl/DownloadCoordinatorImpl.java b/src/org/torproject/descriptor/impl/DownloadCoordinatorImpl.java index b390b6d..4224b0e 100644 --- a/src/org/torproject/descriptor/impl/DownloadCoordinatorImpl.java +++ b/src/org/torproject/descriptor/impl/DownloadCoordinatorImpl.java @@ -33,6 +33,7 @@ public class DownloadCoordinatorImpl implements DownloadCoordinator { private long connectTimeoutMillis; private long readTimeoutMillis; private long globalTimeoutMillis; + private boolean failUnrecognizedDescriptorLines;

protected DownloadCoordinatorImpl( SortedMap<String, DirectoryDownloader> directoryAuthorities, @@ -41,7 +42,7 @@ public class DownloadCoordinatorImpl implements DownloadCoordinator { boolean downloadConsensusFromAllAuthorities, Set<String> downloadVotes, boolean includeCurrentReferencedVotes, long connectTimeoutMillis, long readTimeoutMillis, - long globalTimeoutMillis) { + long globalTimeoutMillis, boolean failUnrecognizedDescriptorLines) { this.directoryAuthorities = directoryAuthorities; this.directoryMirrors = directoryMirrors; this.runningDirectories = new TreeSet<String>(); @@ -55,6 +56,8 @@ public class DownloadCoordinatorImpl implements DownloadCoordinator { this.connectTimeoutMillis = connectTimeoutMillis; this.readTimeoutMillis = readTimeoutMillis; this.globalTimeoutMillis = globalTimeoutMillis; + this.failUnrecognizedDescriptorLines = + failUnrecognizedDescriptorLines; if (this.directoryMirrors.isEmpty() && this.directoryAuthorities.isEmpty()) { this.descriptorQueue.setOutOfDescriptors(); @@ -70,6 +73,8 @@ public class DownloadCoordinatorImpl implements DownloadCoordinator { directoryMirror.setDownloadCoordinator(this); directoryMirror.setConnectTimeout(this.connectTimeoutMillis); directoryMirror.setReadTimeout(this.readTimeoutMillis); + directoryMirror.setFailUnrecognizedDescriptorLines( + this.failUnrecognizedDescriptorLines); new Thread(directoryMirror).start(); } for (DirectoryDownloader directoryAuthority : @@ -77,6 +82,8 @@ public class DownloadCoordinatorImpl implements DownloadCoordinator { directoryAuthority.setDownloadCoordinator(this); directoryAuthority.setConnectTimeout(this.connectTimeoutMillis); directoryAuthority.setReadTimeout(this.readTimeoutMillis); + directoryAuthority.setFailUnrecognizedDescriptorLines( + this.failUnrecognizedDescriptorLines); new Thread(directoryAuthority).start(); } } diff --git a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java index 60ef498..73ea040 100644 --- a/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java @@ -22,29 +22,26 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl implements ExtraInfoDescriptor {

protected static List<ExtraInfoDescriptor> parseDescriptors( - byte[] descriptorsBytes) { + byte[] descriptorsBytes, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { List<ExtraInfoDescriptor> parsedDescriptors = new ArrayList<ExtraInfoDescriptor>(); List<byte[]> splitDescriptorsBytes = DescriptorImpl.splitRawDescriptorBytes(descriptorsBytes, "extra-info "); - try { - for (byte[] descriptorBytes : splitDescriptorsBytes) { - ExtraInfoDescriptor parsedDescriptor = - new ExtraInfoDescriptorImpl(descriptorBytes); - parsedDescriptors.add(parsedDescriptor); - } - } catch (DescriptorParseException e) { - /* TODO Handle this error somehow. */ - System.err.println("Failed to parse descriptor. Skipping."); - e.printStackTrace(); + for (byte[] descriptorBytes : splitDescriptorsBytes) { + ExtraInfoDescriptor parsedDescriptor = + new ExtraInfoDescriptorImpl(descriptorBytes, + failUnrecognizedDescriptorLines); + parsedDescriptors.add(parsedDescriptor); } return parsedDescriptors; }

- protected ExtraInfoDescriptorImpl(byte[] descriptorBytes) + protected ExtraInfoDescriptorImpl(byte[] descriptorBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(descriptorBytes); + super(descriptorBytes, failUnrecognizedDescriptorLines); this.parseDescriptorBytes(); Set<String> exactlyOnceKeywords = new HashSet<String>(Arrays.asList(( "extra-info,published").split(","))); @@ -159,13 +156,15 @@ public class ExtraInfoDescriptorImpl extends DescriptorImpl } else if (line.startsWith("-----END")) { skipCrypto = false; } else if (!skipCrypto) { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. In theory, - * dir-spec.txt says that unknown lines should be ignored. This - * also applies to the other descriptors. */ - throw new DescriptorParseException("Unrecognized line '" + line - + "'."); + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in extra-info descriptor."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } } } } catch (IOException e) { diff --git a/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java b/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java index ce235ab..2353bee 100644 --- a/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java +++ b/src/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java @@ -20,9 +20,20 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry { return this.statusEntryBytes; }

- protected NetworkStatusEntryImpl(byte[] statusEntryBytes) + private boolean failUnrecognizedDescriptorLines; + private List<String> unrecognizedLines; + protected List<String> getAndClearUnrecognizedLines() { + List<String> lines = this.unrecognizedLines; + this.unrecognizedLines = null; + return lines; + } + + protected NetworkStatusEntryImpl(byte[] statusEntryBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { this.statusEntryBytes = statusEntryBytes; + this.failUnrecognizedDescriptorLines = + failUnrecognizedDescriptorLines; this.initializeKeywords(); this.parseStatusEntryBytes(); } @@ -73,12 +84,14 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry { this.parsePLine(line, parts); } else if (keyword.equals("m")) { this.parseMLine(line, parts); - } else { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. */ - throw new DescriptorParseException("Unknown line '" + line + } else if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + line + "' in status entry."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); } } } catch (IOException e) { diff --git a/src/org/torproject/descriptor/impl/NetworkStatusImpl.java b/src/org/torproject/descriptor/impl/NetworkStatusImpl.java index d5b73e3..fcbd8f6 100644 --- a/src/org/torproject/descriptor/impl/NetworkStatusImpl.java +++ b/src/org/torproject/descriptor/impl/NetworkStatusImpl.java @@ -18,9 +18,10 @@ import org.torproject.descriptor.NetworkStatusEntry; * delegate the specific parts to the subclasses. */ public abstract class NetworkStatusImpl extends DescriptorImpl {

- protected NetworkStatusImpl(byte[] rawDescriptorBytes) + protected NetworkStatusImpl(byte[] rawDescriptorBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(rawDescriptorBytes); + super(rawDescriptorBytes, failUnrecognizedDescriptorLines); this.splitAndParseParts(rawDescriptorBytes); }

@@ -161,17 +162,33 @@ public abstract class NetworkStatusImpl extends DescriptorImpl {

protected void parseDirSource(byte[] dirSourceBytes) throws DescriptorParseException { - DirSourceEntry dirSourceEntry = new DirSourceEntryImpl( - dirSourceBytes); + DirSourceEntryImpl dirSourceEntry = new DirSourceEntryImpl( + dirSourceBytes, this.failUnrecognizedDescriptorLines); this.dirSourceEntries.put(dirSourceEntry.getIdentity(), dirSourceEntry); + List<String> unrecognizedDirSourceLines = dirSourceEntry. + getAndClearUnrecognizedLines(); + if (unrecognizedDirSourceLines != null) { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.addAll(unrecognizedDirSourceLines); + } }

protected void parseStatusEntry(byte[] statusEntryBytes) throws DescriptorParseException { NetworkStatusEntryImpl statusEntry = new NetworkStatusEntryImpl( - statusEntryBytes); + statusEntryBytes, this.failUnrecognizedDescriptorLines); this.statusEntries.put(statusEntry.getFingerprint(), statusEntry); + List<String> unrecognizedDirSourceLines = statusEntry. + getAndClearUnrecognizedLines(); + if (unrecognizedDirSourceLines != null) { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.addAll(unrecognizedDirSourceLines); + } }

protected abstract void parseFooter(byte[] footerBytes) @@ -186,6 +203,7 @@ public abstract class NetworkStatusImpl extends DescriptorImpl { BufferedReader br = new BufferedReader(new StringReader( new String(directorySignatureBytes))); String line; + boolean skipCrypto = false; while ((line = br.readLine()) != null) { if (line.startsWith("directory-signature ")) { String[] parts = line.split(" ", -1); @@ -198,7 +216,20 @@ public abstract class NetworkStatusImpl extends DescriptorImpl { String signingKeyDigest = ParseHelper.parseTwentyByteHexString( line, parts[2]); this.directorySignatures.put(identity, signingKeyDigest); - break; + } else if (line.startsWith("-----BEGIN")) { + skipCrypto = true; + } else if (line.startsWith("-----END")) { + skipCrypto = false; + } else if (!skipCrypto) { + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in dir-source entry."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } } } } catch (IOException e) { diff --git a/src/org/torproject/descriptor/impl/RelayDescriptorDownloaderImpl.java b/src/org/torproject/descriptor/impl/RelayDescriptorDownloaderImpl.java index 0888d64..6993509 100644 --- a/src/org/torproject/descriptor/impl/RelayDescriptorDownloaderImpl.java +++ b/src/org/torproject/descriptor/impl/RelayDescriptorDownloaderImpl.java @@ -235,6 +235,15 @@ public class RelayDescriptorDownloaderImpl this.globalTimeoutMillis = globalTimeoutMillis; }

+ private boolean failUnrecognizedDescriptorLines = false; + public void setFailUnrecognizedDescriptorLines() { + if (this.hasStartedDownloading) { + throw new IllegalStateException("Reconfiguration is not permitted " + + "after starting to download."); + } + this.failUnrecognizedDescriptorLines = true; + } + public Iterator<DescriptorRequest> downloadDescriptors() { if (this.hasStartedDownloading) { throw new IllegalStateException("Initiating downloads is only " @@ -246,7 +255,8 @@ public class RelayDescriptorDownloaderImpl this.directoryMirrors, this.downloadConsensus, this.downloadConsensusFromAllAuthorities, this.downloadVotes, this.includeCurrentReferencedVotes, this.connectTimeoutMillis, - this.readTimeoutMillis, this.globalTimeoutMillis); + this.readTimeoutMillis, this.globalTimeoutMillis, + this.failUnrecognizedDescriptorLines); Iterator<DescriptorRequest> descriptorQueue = downloadCoordinator. getDescriptorQueue(); return descriptorQueue; diff --git a/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java b/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java index bdf6b86..10f97c0 100644 --- a/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java +++ b/src/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java @@ -22,29 +22,26 @@ public class RelayNetworkStatusConsensusImpl extends NetworkStatusImpl implements RelayNetworkStatusConsensus {

protected static List<RelayNetworkStatusConsensus> parseConsensuses( - byte[] consensusesBytes) { + byte[] consensusesBytes, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { List<RelayNetworkStatusConsensus> parsedConsensuses = new ArrayList<RelayNetworkStatusConsensus>(); List<byte[]> splitConsensusBytes = DescriptorImpl.splitRawDescriptorBytes(consensusesBytes, "network-status-version 3"); - try { - for (byte[] consensusBytes : splitConsensusBytes) { - RelayNetworkStatusConsensus parsedConsensus = - new RelayNetworkStatusConsensusImpl(consensusBytes); - parsedConsensuses.add(parsedConsensus); - } - } catch (DescriptorParseException e) { - /* TODO Handle this error somehow. */ - System.err.println("Failed to parse consensus. Skipping."); - e.printStackTrace(); + for (byte[] consensusBytes : splitConsensusBytes) { + RelayNetworkStatusConsensus parsedConsensus = + new RelayNetworkStatusConsensusImpl(consensusBytes, + failUnrecognizedDescriptorLines); + parsedConsensuses.add(parsedConsensus); } return parsedConsensuses; }

- protected RelayNetworkStatusConsensusImpl(byte[] consensusBytes) + protected RelayNetworkStatusConsensusImpl(byte[] consensusBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(consensusBytes); + super(consensusBytes, failUnrecognizedDescriptorLines); Set<String> exactlyOnceKeywords = new HashSet<String>(Arrays.asList(( "vote-status,consensus-method,valid-after,fresh-until," + "valid-until,voting-delay,known-flags," @@ -88,12 +85,14 @@ public class RelayNetworkStatusConsensusImpl extends NetworkStatusImpl this.parseKnownFlagsLine(line, parts); } else if (keyword.equals("params")) { this.parseParamsLine(line, parts); - } else { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. */ + } else if (this.failUnrecognizedDescriptorLines) { throw new DescriptorParseException("Unrecognized line '" + line - + "'."); + + "' in consensus."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); } } } catch (IOException e) { @@ -115,12 +114,14 @@ public class RelayNetworkStatusConsensusImpl extends NetworkStatusImpl if (keyword.equals("directory-footer")) { } else if (keyword.equals("bandwidth-weights")) { this.parseBandwidthWeightsLine(line, parts); - } else { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. */ + } else if (this.failUnrecognizedDescriptorLines) { throw new DescriptorParseException("Unrecognized line '" + line - + "'."); + + "' in consensus."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); } } } catch (IOException e) { diff --git a/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java b/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java index 1e6f300..3729f75 100644 --- a/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java +++ b/src/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java @@ -22,29 +22,26 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl implements RelayNetworkStatusVote {

protected static List<RelayNetworkStatusVote> parseVotes( - byte[] votesBytes) { + byte[] votesBytes, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { List<RelayNetworkStatusVote> parsedVotes = new ArrayList<RelayNetworkStatusVote>(); List<byte[]> splitVotesBytes = DescriptorImpl.splitRawDescriptorBytes(votesBytes, "network-status-version 3"); - try { - for (byte[] voteBytes : splitVotesBytes) { - RelayNetworkStatusVote parsedVote = - new RelayNetworkStatusVoteImpl(voteBytes); - parsedVotes.add(parsedVote); - } - } catch (DescriptorParseException e) { - /* TODO Handle this error somehow. */ - System.err.println("Failed to parse vote. Skipping."); - e.printStackTrace(); + for (byte[] voteBytes : splitVotesBytes) { + RelayNetworkStatusVote parsedVote = + new RelayNetworkStatusVoteImpl(voteBytes, + failUnrecognizedDescriptorLines); + parsedVotes.add(parsedVote); } return parsedVotes; }

- protected RelayNetworkStatusVoteImpl(byte[] voteBytes) + protected RelayNetworkStatusVoteImpl(byte[] voteBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(voteBytes); + super(voteBytes, failUnrecognizedDescriptorLines); Set<String> exactlyOnceKeywords = new HashSet<String>(Arrays.asList(( "vote-status,consensus-methods,published,valid-after,fresh-until," + "valid-until,voting-delay,known-flags,dir-source," @@ -93,12 +90,14 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl this.parseKnownFlagsLine(line, parts); } else if (keyword.equals("params")) { this.parseParamsLine(line, parts); - } else { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. */ + } else if (this.failUnrecognizedDescriptorLines) { throw new DescriptorParseException("Unrecognized line '" + line - + "'."); + + "' in vote."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); } } } catch (IOException e) { @@ -271,11 +270,15 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl } else if (line.startsWith("-----END")) { skipCrypto = false; } else if (!skipCrypto) { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. */ - throw new DescriptorParseException("Unrecognized line '" + line - + "'."); + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in vote."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } } } } catch (IOException e) { @@ -366,8 +369,30 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl parts, 1, 2); }

- protected void parseFooter(byte[] footerBytes) { - /* There is nothing in the footer that we'd want to parse. */ + protected void parseFooter(byte[] footerBytes) + throws DescriptorParseException { + try { + BufferedReader br = new BufferedReader(new StringReader( + new String(footerBytes))); + String line; + while ((line = br.readLine()) != null) { + if (!line.equals("directory-footer")) { + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in vote."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } + } + } + } catch (IOException e) { + throw new RuntimeException("Internal error: Ran into an " + + "IOException while parsing a String in memory. Something's " + + "really wrong.", e); + } }

private String nickname; diff --git a/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java index 1a686f4..d33dba5 100644 --- a/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java +++ b/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java @@ -40,11 +40,17 @@ public class RelayOrBridgeDescriptorReaderImpl this.historyFile = historyFile; }

+ private boolean failUnrecognizedDescriptorLines = false; + public void setFailUnrecognizedDescriptorLines() { + this.failUnrecognizedDescriptorLines = true; + } + public Iterator<DescriptorFile> readDescriptors() { BlockingIteratorImpl<DescriptorFile> descriptorQueue = new BlockingIteratorImpl<DescriptorFile>(); DescriptorReader reader = new DescriptorReader(this.directories, - descriptorQueue, this.historyFile); + descriptorQueue, this.historyFile, + this.failUnrecognizedDescriptorLines); new Thread(reader).start(); return descriptorQueue; } @@ -53,12 +59,15 @@ public class RelayOrBridgeDescriptorReaderImpl private List<File> directories; private BlockingIteratorImpl<DescriptorFile> descriptorQueue; private File historyFile; + private boolean failUnrecognizedDescriptorLines; private DescriptorReader(List<File> directories, BlockingIteratorImpl<DescriptorFile> descriptorQueue, - File historyFile) { + File historyFile, boolean failUnrecognizedDescriptorLines) { this.directories = directories; this.descriptorQueue = descriptorQueue; this.historyFile = historyFile; + this.failUnrecognizedDescriptorLines = + failUnrecognizedDescriptorLines; } public void run() { this.readOldHistory(); @@ -167,7 +176,8 @@ public class RelayOrBridgeDescriptorReaderImpl bis.close(); byte[] rawDescriptorBytes = baos.toByteArray(); return DescriptorImpl.parseRelayOrBridgeDescriptors( - rawDescriptorBytes, file.getName()); + rawDescriptorBytes, file.getName(), + this.failUnrecognizedDescriptorLines); } } } diff --git a/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java b/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java index 09e9123..76873dc 100644 --- a/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/ServerDescriptorImpl.java @@ -19,7 +19,7 @@ public class ServerDescriptorImpl extends DescriptorImpl implements ServerDescriptor {

protected static List<ServerDescriptor> parseDescriptors( - byte[] descriptorsBytes) { + byte[] descriptorsBytes, boolean failUnrecognizedDescriptorLines) { List<ServerDescriptor> parsedDescriptors = new ArrayList<ServerDescriptor>(); List<byte[]> splitDescriptorsBytes = @@ -28,7 +28,8 @@ public class ServerDescriptorImpl extends DescriptorImpl try { for (byte[] descriptorBytes : splitDescriptorsBytes) { ServerDescriptor parsedDescriptor = - new ServerDescriptorImpl(descriptorBytes); + new ServerDescriptorImpl(descriptorBytes, + failUnrecognizedDescriptorLines); parsedDescriptors.add(parsedDescriptor); } } catch (DescriptorParseException e) { @@ -39,9 +40,10 @@ public class ServerDescriptorImpl extends DescriptorImpl return parsedDescriptors; }

- protected ServerDescriptorImpl(byte[] descriptorBytes) + protected ServerDescriptorImpl(byte[] descriptorBytes, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(descriptorBytes); + super(descriptorBytes, failUnrecognizedDescriptorLines); this.parseDescriptorBytes(); Set<String> exactlyOnceKeywords = new HashSet<String>(Arrays.asList( "router,bandwidth,published".split(","))); @@ -127,13 +129,15 @@ public class ServerDescriptorImpl extends DescriptorImpl } else if (line.startsWith("-----END")) { skipCrypto = false; } else if (!skipCrypto) { - /* TODO Is throwing an exception the right thing to do here? - * This is probably fine for development, but once the library - * is in production use, this seems annoying. In theory, - * dir-spec.txt says that unknown lines should be ignored. This - * also applies to the other descriptors. */ - throw new DescriptorParseException("Unrecognized line '" + line - + "'."); + if (this.failUnrecognizedDescriptorLines) { + throw new DescriptorParseException("Unrecognized line '" + + line + "' in server descriptor."); + } else { + if (this.unrecognizedLines == null) { + this.unrecognizedLines = new ArrayList<String>(); + } + this.unrecognizedLines.add(line); + } } } } catch (IOException e) { diff --git a/test/org/torproject/descriptor/impl/ConsensusBuilder.java b/test/org/torproject/descriptor/impl/ConsensusBuilder.java index 51fff2e..2d0291d 100644 --- a/test/org/torproject/descriptor/impl/ConsensusBuilder.java +++ b/test/org/torproject/descriptor/impl/ConsensusBuilder.java @@ -16,7 +16,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.networkStatusVersionLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String voteStatusLine = "vote-status consensus"; protected static RelayNetworkStatusConsensus @@ -24,7 +24,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.voteStatusLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String consensusMethodLine = "consensus-method 11"; protected static RelayNetworkStatusConsensus @@ -32,7 +32,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.consensusMethodLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String validAfterLine = "valid-after 2011-11-30 09:00:00"; protected static RelayNetworkStatusConsensus @@ -40,7 +40,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.validAfterLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String freshUntilLine = "fresh-until 2011-11-30 10:00:00"; protected static RelayNetworkStatusConsensus @@ -48,7 +48,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.freshUntilLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String validUntilLine = "valid-until 2011-11-30 12:00:00"; protected static RelayNetworkStatusConsensus @@ -56,7 +56,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.validUntilLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String votingDelayLine = "voting-delay 300 300"; protected static RelayNetworkStatusConsensus @@ -64,7 +64,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.votingDelayLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } String clientVersionsLine = "client-versions 0.2.1.31," + "0.2.2.34,0.2.3.6-alpha,0.2.3.7-alpha,0.2.3.8-alpha"; @@ -73,7 +73,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.clientVersionsLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } String serverVersionsLine = "server-versions 0.2.1.31," + "0.2.2.34,0.2.3.6-alpha,0.2.3.7-alpha,0.2.3.8-alpha"; @@ -82,7 +82,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.serverVersionsLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String knownFlagsLine = "known-flags Authority BadExit Exit " + "Fast Guard HSDir Named Running Stable Unnamed V2Dir Valid"; @@ -91,7 +91,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.knownFlagsLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String paramsLine = "params " + "CircuitPriorityHalflifeMsec=30000 bwauthbestratio=1 " @@ -103,7 +103,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.paramsLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } List<String> dirSources = new ArrayList<String>(); List<String> statusEntries = new ArrayList<String>(); @@ -113,7 +113,7 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.directoryFooterLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private String bandwidthWeightsLine = "bandwidth-weights Wbd=285 " + "Wbe=0 Wbg=0 Wbm=10000 Wdb=10000 Web=10000 Wed=1021 Wee=10000 " @@ -124,12 +124,62 @@ public class ConsensusBuilder { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.bandwidthWeightsLine = line; - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); } private List<String> directorySignatures = new ArrayList<String>(); protected void addDirectorySignature(String directorySignatureString) { this.directorySignatures.add(directorySignatureString); } + private String unrecognizedHeaderLine = null; + protected static RelayNetworkStatusConsensus + createWithUnrecognizedHeaderLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + ConsensusBuilder cb = new ConsensusBuilder(); + cb.unrecognizedHeaderLine = line; + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedDirSourceLine = null; + protected static RelayNetworkStatusConsensus + createWithUnrecognizedDirSourceLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + ConsensusBuilder cb = new ConsensusBuilder(); + cb.unrecognizedDirSourceLine = line; + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedStatusEntryLine = null; + protected static RelayNetworkStatusConsensus + createWithUnrecognizedStatusEntryLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + ConsensusBuilder cb = new ConsensusBuilder(); + cb.unrecognizedStatusEntryLine = line; + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedFooterLine = null; + protected static RelayNetworkStatusConsensus + createWithUnrecognizedFooterLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + ConsensusBuilder cb = new ConsensusBuilder(); + cb.unrecognizedFooterLine = line; + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedDirectorySignatureLine = null; + protected static RelayNetworkStatusConsensus + createWithUnrecognizedDirectorySignatureLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + ConsensusBuilder cb = new ConsensusBuilder(); + cb.unrecognizedDirectorySignatureLine = line; + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + failUnrecognizedDescriptorLines); + }

protected ConsensusBuilder() { this.dirSources.add("dir-source tor26 " @@ -170,8 +220,10 @@ public class ConsensusBuilder { protected byte[] buildConsensus() { StringBuilder sb = new StringBuilder(); this.appendHeader(sb); - this.appendBody(sb); + this.appendDirSources(sb); + this.appendStatusEntries(sb); this.appendFooter(sb); + this.appendDirectorySignatures(sb); return sb.toString().getBytes(); } private void appendHeader(StringBuilder sb) { @@ -208,14 +260,25 @@ public class ConsensusBuilder { if (this.paramsLine != null) { sb.append(this.paramsLine + "\n"); } + if (this.unrecognizedHeaderLine != null) { + sb.append(this.unrecognizedHeaderLine + "\n"); + } + } + private void appendDirSources(StringBuilder sb) { for (String dirSource : this.dirSources) { sb.append(dirSource + "\n"); } + if (this.unrecognizedDirSourceLine != null) { + sb.append(this.unrecognizedDirSourceLine + "\n"); + } } - private void appendBody(StringBuilder sb) { + private void appendStatusEntries(StringBuilder sb) { for (String statusEntry : this.statusEntries) { sb.append(statusEntry + "\n"); } + if (this.unrecognizedStatusEntryLine != null) { + sb.append(this.unrecognizedStatusEntryLine + "\n"); + } } private void appendFooter(StringBuilder sb) { if (this.directoryFooterLine != null) { @@ -224,9 +287,17 @@ public class ConsensusBuilder { if (this.bandwidthWeightsLine != null) { sb.append(this.bandwidthWeightsLine + "\n"); } + if (this.unrecognizedFooterLine != null) { + sb.append(this.unrecognizedFooterLine + "\n"); + } + } + private void appendDirectorySignatures(StringBuilder sb) { for (String directorySignature : this.directorySignatures) { sb.append(directorySignature + "\n"); } + if (this.unrecognizedDirectorySignatureLine != null) { + sb.append(this.unrecognizedDirectorySignatureLine + "\n"); + } } }

diff --git a/test/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java b/test/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java index 954a3d3..b37b72a 100644 --- a/test/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java +++ b/test/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java @@ -8,6 +8,9 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue;

+import java.util.ArrayList; +import java.util.List; +

import org.junit.Test; import org.torproject.descriptor.RelayNetworkStatusConsensus; @@ -28,7 +31,8 @@ public class RelayNetworkStatusConsensusImplTest { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.dirSources.add(dirSourceString); - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + true); } private String nickname = "gabelmoo"; private static RelayNetworkStatusConsensus @@ -123,7 +127,8 @@ public class RelayNetworkStatusConsensusImplTest { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.statusEntries.add(statusEntryString); - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + true); } private String nickname = "right2privassy3"; private static RelayNetworkStatusConsensus @@ -239,7 +244,8 @@ public class RelayNetworkStatusConsensusImplTest { throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); cb.addDirectorySignature(directorySignatureString); - return new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + return new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), + true); } private String identity = "ED03BB616EB2F60BEC80151114BB25CEF515B226"; private static RelayNetworkStatusConsensus @@ -276,7 +282,7 @@ public class RelayNetworkStatusConsensusImplTest { public void testSampleConsensus() throws DescriptorParseException { ConsensusBuilder cb = new ConsensusBuilder(); RelayNetworkStatusConsensus consensus = - new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); assertEquals(3, consensus.getNetworkStatusVersion()); assertEquals(11, consensus.getConsensusMethod()); assertEquals(1322643600000L, consensus.getValidAfterMillis()); @@ -301,6 +307,7 @@ public class RelayNetworkStatusConsensusImplTest { consensus.getDirectorySignatures().get( "14C131DFC5C6F93646BE72FA1401C02A8DF2E8B4")); assertEquals(285, (int) consensus.getBandwidthWeights().get("Wbd")); + assertTrue(consensus.getUnrecognizedLines().isEmpty()); }

@Test(expected = DescriptorParseException.class) @@ -544,7 +551,7 @@ public class RelayNetworkStatusConsensusImplTest { cb.clientVersionsLine = null; cb.serverVersionsLine = null; RelayNetworkStatusConsensus consensus = - new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); assertNull(consensus.getRecommendedClientVersions()); assertNull(consensus.getRecommendedServerVersions()); } @@ -905,7 +912,7 @@ public class RelayNetworkStatusConsensusImplTest { sb.sLine = sb.sLine + "\n" + sb.sLine; ConsensusBuilder cb = new ConsensusBuilder(); cb.statusEntries.add(sb.buildStatusEntry()); - new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); }

@Test(expected = DescriptorParseException.class) @@ -929,7 +936,7 @@ public class RelayNetworkStatusConsensusImplTest { sb.wLine = sb.wLine + "\n" + sb.wLine; ConsensusBuilder cb = new ConsensusBuilder(); cb.statusEntries.add(sb.buildStatusEntry()); - new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); }

@Test(expected = DescriptorParseException.class) @@ -958,7 +965,7 @@ public class RelayNetworkStatusConsensusImplTest { sb.pLine = sb.pLine + "\n" + sb.pLine; ConsensusBuilder cb = new ConsensusBuilder(); cb.statusEntries.add(sb.buildStatusEntry()); - new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); }

@Test() @@ -966,7 +973,7 @@ public class RelayNetworkStatusConsensusImplTest { ConsensusBuilder cb = new ConsensusBuilder(); cb.statusEntries.clear(); RelayNetworkStatusConsensus consensus = - new RelayNetworkStatusConsensusImpl(cb.buildConsensus()); + new RelayNetworkStatusConsensusImpl(cb.buildConsensus(), true); assertFalse(consensus.containsStatusEntry( "00795A6E8D91C270FC23B30F388A495553E01894")); } @@ -1045,7 +1052,7 @@ public class RelayNetworkStatusConsensusImplTest { ConsensusBuilder cb = new ConsensusBuilder(); byte[] consensusBytes = cb.buildConsensus(); consensusBytes[20] = (byte) 200; - new RelayNetworkStatusConsensusImpl(consensusBytes); + new RelayNetworkStatusConsensusImpl(consensusBytes, true); }

@Test(expected = DescriptorParseException.class) @@ -1055,7 +1062,103 @@ public class RelayNetworkStatusConsensusImplTest { cb.networkStatusVersionLine = "Xnetwork-status-version 3"; byte[] consensusBytes = cb.buildConsensus(); consensusBytes[0] = (byte) 200; - new RelayNetworkStatusConsensusImpl(consensusBytes); + new RelayNetworkStatusConsensusImpl(consensusBytes, true); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedHeaderLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + ConsensusBuilder.createWithUnrecognizedHeaderLine(unrecognizedLine, + true); + } + + @Test() + public void testUnrecognizedHeaderLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusConsensus consensus = ConsensusBuilder. + createWithUnrecognizedHeaderLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, consensus.getUnrecognizedLines()); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedDirSourceLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + ConsensusBuilder.createWithUnrecognizedDirSourceLine(unrecognizedLine, + true); + } + + @Test() + public void testUnrecognizedDirSourceLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusConsensus consensus = ConsensusBuilder. + createWithUnrecognizedDirSourceLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, consensus.getUnrecognizedLines()); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedStatusEntryLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + ConsensusBuilder.createWithUnrecognizedStatusEntryLine( + unrecognizedLine, true); + } + + @Test() + public void testUnrecognizedStatusEntryLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusConsensus consensus = ConsensusBuilder. + createWithUnrecognizedStatusEntryLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, consensus.getUnrecognizedLines()); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedDirectoryFooterLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + ConsensusBuilder.createWithUnrecognizedFooterLine(unrecognizedLine, + true); + } + + @Test() + public void testUnrecognizedDirectoryFooterLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusConsensus consensus = ConsensusBuilder. + createWithUnrecognizedFooterLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, consensus.getUnrecognizedLines()); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedDirectorySignatureLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + ConsensusBuilder.createWithUnrecognizedDirectorySignatureLine( + unrecognizedLine, true); + } + + @Test() + public void testUnrecognizedDirectorySignatureLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusConsensus consensus = ConsensusBuilder. + createWithUnrecognizedDirectorySignatureLine(unrecognizedLine, + false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, consensus.getUnrecognizedLines()); } }

diff --git a/test/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java b/test/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java index 20aa2da..e907086 100644 --- a/test/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java +++ b/test/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java @@ -29,7 +29,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.networkStatusVersionLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String voteStatusLine = "vote-status vote"; private static RelayNetworkStatusVote @@ -37,7 +37,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.voteStatusLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String consensusMethodsLine = "consensus-methods 1 2 3 4 5 6 7 8 9 10 11"; @@ -46,7 +46,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.consensusMethodsLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String publishedLine = "published 2011-11-30 08:50:01"; private static RelayNetworkStatusVote @@ -54,7 +54,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.publishedLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String validAfterLine = "valid-after 2011-11-30 09:00:00"; private static RelayNetworkStatusVote @@ -62,7 +62,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.validAfterLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String freshUntilLine = "fresh-until 2011-11-30 10:00:00"; private static RelayNetworkStatusVote @@ -70,7 +70,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.freshUntilLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String validUntilLine = "valid-until 2011-11-30 12:00:00"; private static RelayNetworkStatusVote @@ -78,7 +78,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.validUntilLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String votingDelayLine = "voting-delay 300 300"; private static RelayNetworkStatusVote @@ -86,7 +86,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.votingDelayLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String clientVersionsLine = "client-versions 0.2.1.31," + "0.2.2.34,0.2.3.6-alpha,0.2.3.7-alpha,0.2.3.8-alpha"; @@ -95,7 +95,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.clientVersionsLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String serverVersionsLine = "server-versions 0.2.1.31," + "0.2.2.34,0.2.3.6-alpha,0.2.3.7-alpha,0.2.3.8-alpha"; @@ -104,7 +104,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.serverVersionsLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String knownFlagsLine = "known-flags Authority BadExit Exit " + "Fast Guard HSDir Named Running Stable Unnamed V2Dir Valid"; @@ -113,7 +113,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.knownFlagsLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String paramsLine = "params " + "CircuitPriorityHalflifeMsec=30000 bwauthbestratio=1 " @@ -125,7 +125,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.paramsLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirSourceLine = "dir-source urras " + "80550987E1D626E3EBA5E5E75A458DE0626D088C 208.83.223.34 " @@ -135,7 +135,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirSourceLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String contactLine = "contact 4096R/E012B42D Jacob Appelbaum " + "jacob@appelbaum.net"; @@ -144,7 +144,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.contactLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirKeyCertificateVersionLine = "dir-key-certificate-version 3"; @@ -153,7 +153,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirKeyCertificateVersionLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String fingerprintLine = "fingerprint " + "80550987E1D626E3EBA5E5E75A458DE0626D088C"; @@ -162,7 +162,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.fingerprintLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirKeyPublishedLine = "dir-key-published 2011-04-27 " + "05:34:37"; @@ -171,7 +171,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirKeyPublishedLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirKeyExpiresLine = "dir-key-expires 2012-04-27 " + "05:34:37"; @@ -180,7 +180,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirKeyExpiresLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirIdentityKeyLines = "dir-identity-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" @@ -199,7 +199,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirIdentityKeyLines = lines; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirSigningKeyLines = "dir-signing-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" @@ -213,7 +213,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirSigningKeyLines = lines; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirKeyCrosscertLines = "dir-key-crosscert\n" + "-----BEGIN ID SIGNATURE-----\n" @@ -226,7 +226,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirKeyCrosscertLines = lines; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String dirKeyCertificationLines = "dir-key-certification\n" + "-----BEGIN SIGNATURE-----\n" @@ -245,7 +245,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.dirKeyCertificationLines = lines; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private List<String> statusEntries = new ArrayList<String>(); private String directoryFooterLine = "directory-footer"; @@ -254,7 +254,7 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.directoryFooterLine = line; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } private String directorySignatureLines = "directory-signature " + "80550987E1D626E3EBA5E5E75A458DE0626D088C " @@ -269,8 +269,59 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); vb.directorySignatureLines = lines; - return new RelayNetworkStatusVoteImpl(vb.buildVote()); + return new RelayNetworkStatusVoteImpl(vb.buildVote(), true); } + private String unrecognizedHeaderLine = null; + protected static RelayNetworkStatusVote + createWithUnrecognizedHeaderLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + VoteBuilder vb = new VoteBuilder(); + vb.unrecognizedHeaderLine = line; + return new RelayNetworkStatusVoteImpl(vb.buildVote(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedDirSourceLine = null; + protected static RelayNetworkStatusVote + createWithUnrecognizedDirSourceLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + VoteBuilder vb = new VoteBuilder(); + vb.unrecognizedDirSourceLine = line; + return new RelayNetworkStatusVoteImpl(vb.buildVote(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedStatusEntryLine = null; + protected static RelayNetworkStatusVote + createWithUnrecognizedStatusEntryLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + VoteBuilder vb = new VoteBuilder(); + vb.unrecognizedStatusEntryLine = line; + return new RelayNetworkStatusVoteImpl(vb.buildVote(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedFooterLine = null; + protected static RelayNetworkStatusVote + createWithUnrecognizedFooterLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + VoteBuilder vb = new VoteBuilder(); + vb.unrecognizedFooterLine = line; + return new RelayNetworkStatusVoteImpl(vb.buildVote(), + failUnrecognizedDescriptorLines); + } + private String unrecognizedDirectorySignatureLine = null; + protected static RelayNetworkStatusVote + createWithUnrecognizedDirectorySignatureLine(String line, + boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + VoteBuilder vb = new VoteBuilder(); + vb.unrecognizedDirectorySignatureLine = line; + return new RelayNetworkStatusVoteImpl(vb.buildVote(), + failUnrecognizedDescriptorLines); + } + private VoteBuilder() { this.statusEntries.add("r right2privassy3 " + "ADQ6gCT3DiFHKPDFr3rODBUI8HM lJY5Vf7kXec+VdkGW2flEsfkFC8 " @@ -289,8 +340,10 @@ public class RelayNetworkStatusVoteImplTest { private byte[] buildVote() { StringBuilder sb = new StringBuilder(); this.appendHeader(sb); - this.appendBody(sb); + this.appendDirSource(sb); + this.appendStatusEntries(sb); this.appendFooter(sb); + this.appendDirectorySignature(sb); return sb.toString().getBytes(); } private void appendHeader(StringBuilder sb) { @@ -330,6 +383,11 @@ public class RelayNetworkStatusVoteImplTest { if (this.paramsLine != null) { sb.append(this.paramsLine + "\n"); } + if (this.unrecognizedHeaderLine != null) { + sb.append(this.unrecognizedHeaderLine + "\n"); + } + } + private void appendDirSource(StringBuilder sb) { if (this.dirSourceLine != null) { sb.append(this.dirSourceLine + "\n"); } @@ -360,19 +418,33 @@ public class RelayNetworkStatusVoteImplTest { if (this.dirKeyCertificationLines != null) { sb.append(this.dirKeyCertificationLines + "\n"); } + if (this.unrecognizedDirSourceLine != null) { + sb.append(this.unrecognizedDirSourceLine + "\n"); + } } - private void appendBody(StringBuilder sb) { + private void appendStatusEntries(StringBuilder sb) { for (String statusEntry : this.statusEntries) { sb.append(statusEntry + "\n"); } + if (this.unrecognizedStatusEntryLine != null) { + sb.append(this.unrecognizedStatusEntryLine + "\n"); + } } private void appendFooter(StringBuilder sb) { if (this.directoryFooterLine != null) { sb.append(this.directoryFooterLine + "\n"); } + if (this.unrecognizedFooterLine != null) { + sb.append(this.unrecognizedFooterLine + "\n"); + } + } + private void appendDirectorySignature(StringBuilder sb) { if (this.directorySignatureLines != null) { sb.append(directorySignatureLines + "\n"); } + if (this.unrecognizedDirectorySignatureLine != null) { + sb.append(this.unrecognizedDirectorySignatureLine + "\n"); + } } }

@@ -380,7 +452,7 @@ public class RelayNetworkStatusVoteImplTest { public void testSampleVote() throws DescriptorParseException { VoteBuilder vb = new VoteBuilder(); RelayNetworkStatusVote vote = - new RelayNetworkStatusVoteImpl(vb.buildVote()); + new RelayNetworkStatusVoteImpl(vb.buildVote(), true); assertEquals(3, vote.getNetworkStatusVersion()); List<Integer> consensusMethods = Arrays.asList( new Integer[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}); @@ -397,6 +469,7 @@ public class RelayNetworkStatusVoteImplTest { assertEquals("Tor 0.2.1.29 (r8e9b25e6c7a2e70c)", vote.getStatusEntry("00343A8024F70E214728F0C5AF7ACE0C1508F073"). getVersion()); + assertTrue(vote.getUnrecognizedLines().isEmpty()); }

@Test(expected = DescriptorParseException.class) @@ -946,5 +1019,60 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { VoteBuilder.createWithDirectorySignatureLines(null); } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedHeaderLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + VoteBuilder.createWithUnrecognizedHeaderLine(unrecognizedLine, true); + } + + @Test() + public void testUnrecognizedHeaderLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusVote vote = VoteBuilder. + createWithUnrecognizedHeaderLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, vote.getUnrecognizedLines()); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedDirSourceLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + VoteBuilder.createWithUnrecognizedDirSourceLine(unrecognizedLine, + true); + } + + @Test() + public void testUnrecognizedDirSourceLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusVote vote = VoteBuilder. + createWithUnrecognizedDirSourceLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, vote.getUnrecognizedLines()); + } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedFooterLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + VoteBuilder.createWithUnrecognizedFooterLine(unrecognizedLine, true); + } + + @Test() + public void testUnrecognizedFooterLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + RelayNetworkStatusVote vote = VoteBuilder. + createWithUnrecognizedFooterLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, vote.getUnrecognizedLines()); + } }

diff --git a/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java b/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java index eec5594..439e5bb 100644 --- a/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java +++ b/test/org/torproject/descriptor/impl/ServerDescriptorImplTest.java @@ -8,7 +8,9 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue;

+import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.SortedMap;

import org.junit.Test; @@ -27,14 +29,14 @@ public class ServerDescriptorImplTest { String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.routerLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String bandwidthLine = "bandwidth 51200 51200 53470"; private static ServerDescriptor createWithBandwidthLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.bandwidthLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String platformLine = "platform Tor 0.2.2.35 " + "(git-b04388f9e7546a9f) on Linux i686"; @@ -42,14 +44,14 @@ public class ServerDescriptorImplTest { String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.platformLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String publishedLine = "published 2012-01-01 04:03:19"; private static ServerDescriptor createWithPublishedLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.publishedLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String fingerprintLine = "opt fingerprint D873 3048 FC8E " + "C910 2466 AD8F 3098 622B F1BF 71FD"; @@ -57,21 +59,21 @@ public class ServerDescriptorImplTest { String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.fingerprintLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String hibernatingLine = null; private static ServerDescriptor createWithHibernatingLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.hibernatingLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String uptimeLine = "uptime 48"; private static ServerDescriptor createWithUptimeLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.uptimeLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String onionKeyLines = "onion-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" @@ -84,7 +86,7 @@ public class ServerDescriptorImplTest { String lines) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.onionKeyLines = lines; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String signingKeyLines = "signing-key\n" + "-----BEGIN RSA PUBLIC KEY-----\n" @@ -97,14 +99,14 @@ public class ServerDescriptorImplTest { String lines) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.signingKeyLines = lines; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String exitPolicyLines = "reject *:*"; private static ServerDescriptor createWithExitPolicyLines( String lines) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.exitPolicyLines = lines; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String contactLine = "contact Random Person <nobody AT " + "example dot com>"; @@ -112,42 +114,42 @@ public class ServerDescriptorImplTest { String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.contactLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String familyLine = null; private static ServerDescriptor createWithFamilyLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.familyLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String readHistoryLine = null; private static ServerDescriptor createWithReadHistoryLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.readHistoryLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String writeHistoryLine = null; private static ServerDescriptor createWithWriteHistoryLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.writeHistoryLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String eventdnsLine = null; private static ServerDescriptor createWithEventdnsLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.eventdnsLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String cachesExtraInfoLine = null; private static ServerDescriptor createWithCachesExtraInfoLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.cachesExtraInfoLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String extraInfoDigestLine = "opt extra-info-digest " + "1469D1550738A25B1E7B47CDDBCD7B2899F51B74"; @@ -155,21 +157,21 @@ public class ServerDescriptorImplTest { String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.extraInfoDigestLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String hiddenServiceDirLine = "opt hidden-service-dir"; private static ServerDescriptor createWithHiddenServiceDirLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.hiddenServiceDirLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String protocolsLine = "opt protocols Link 1 2 Circuit 1"; private static ServerDescriptor createWithProtocolsLine( String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.protocolsLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String allowSingleHopExitsLine = null; private static ServerDescriptor @@ -177,7 +179,7 @@ public class ServerDescriptorImplTest { throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.allowSingleHopExitsLine = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); } private String routerSignatureLines = "router-signature\n" + "-----BEGIN SIGNATURE-----\n" @@ -189,7 +191,16 @@ public class ServerDescriptorImplTest { String line) throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); db.routerSignatureLines = line; - return new ServerDescriptorImpl(db.buildDescriptor()); + return new ServerDescriptorImpl(db.buildDescriptor(), true); + } + private String unrecognizedLine = null; + private static ServerDescriptor createWithUnrecognizedLine( + String line, boolean failUnrecognizedDescriptorLines) + throws DescriptorParseException { + DescriptorBuilder db = new DescriptorBuilder(); + db.unrecognizedLine = line; + return new ServerDescriptorImpl(db.buildDescriptor(), + failUnrecognizedDescriptorLines); } private byte[] buildDescriptor() { StringBuilder sb = new StringBuilder(); @@ -253,6 +264,9 @@ public class ServerDescriptorImplTest { if (this.allowSingleHopExitsLine != null) { sb.append(this.allowSingleHopExitsLine + "\n"); } + if (this.unrecognizedLine != null) { + sb.append(this.unrecognizedLine + "\n"); + } if (this.routerSignatureLines != null) { sb.append(this.routerSignatureLines + "\n"); } @@ -264,7 +278,7 @@ public class ServerDescriptorImplTest { public void testSampleDescriptor() throws DescriptorParseException { DescriptorBuilder db = new DescriptorBuilder(); ServerDescriptor descriptor = - new ServerDescriptorImpl(db.buildDescriptor()); + new ServerDescriptorImpl(db.buildDescriptor(), true); assertEquals("saberrider2008", descriptor.getNickname()); assertEquals("94.134.192.243", descriptor.getAddress()); assertEquals(9001, (int) descriptor.getOrPort()); @@ -298,6 +312,7 @@ public class ServerDescriptorImplTest { assertFalse(descriptor.getUsesEnhancedDnsLogic()); assertFalse(descriptor.getCachesExtraInfo()); assertFalse(descriptor.getAllowSingleHopExits()); + assertTrue(descriptor.getUnrecognizedLines().isEmpty()); }

@Test(expected = DescriptorParseException.class) @@ -1072,5 +1087,23 @@ public class ServerDescriptorImplTest { DescriptorBuilder.createWithAllowSingleHopExitsLine( "allow-single-hop-exits true"); } + + @Test(expected = DescriptorParseException.class) + public void testUnrecognizedLineFail() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + DescriptorBuilder.createWithUnrecognizedLine(unrecognizedLine, true); + } + + @Test() + public void testUnrecognizedLineIgnore() + throws DescriptorParseException { + String unrecognizedLine = "unrecognized-line 1"; + ServerDescriptor descriptor = DescriptorBuilder. + createWithUnrecognizedLine(unrecognizedLine, false); + List<String> unrecognizedLines = new ArrayList<String>(); + unrecognizedLines.add(unrecognizedLine); + assertEquals(unrecognizedLines, descriptor.getUnrecognizedLines()); + } }