commit 5da91cae110b4eba1e468250de8e08562562b06d
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Wed Sep 28 14:56:02 2016 +0200
Add initial set of tests for bridgedescs module.
---
.../collector/bridgedescs/DescriptorBuilder.java | 77 +++
.../bridgedescs/ExtraInfoDescriptorBuilder.java | 55 +++
.../bridgedescs/NetworkStatusBuilder.java | 30 ++
.../bridgedescs/SanitizedBridgesWriterTest.java | 521 +++++++++++++++++++++
.../bridgedescs/ServerDescriptorBuilder.java | 74 +++
.../collector/bridgedescs/TarballBuilder.java | 108 +++++
6 files changed, 865 insertions(+)
diff --git a/src/test/java/org/torproject/collector/bridgedescs/DescriptorBuilder.java b/src/test/java/org/torproject/collector/bridgedescs/DescriptorBuilder.java
new file mode 100644
index 0000000..bab8126
--- /dev/null
+++ b/src/test/java/org/torproject/collector/bridgedescs/DescriptorBuilder.java
@@ -0,0 +1,77 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.List;
+
+/** Builds a descriptor by concatenating the given lines with newlines and
+ * writing the output to the given output stream. */
+abstract class DescriptorBuilder {
+
+ /** Descriptor lines. */
+ List<String> lines;
+
+ /** Removes all lines. */
+ void removeAllLines() {
+ this.lines.clear();
+ }
+
+ /** Removes the given line, or fails if that line cannot be found. */
+ void removeLine(String line) {
+ if (!this.lines.remove(line)) {
+ fail("Line not contained: " + line);
+ }
+ }
+
+ /** Removes all but the given line, or fails if that line cannot be found. */
+ void removeAllExcept(String line) {
+ assertTrue("Line not contained: " + line, this.lines.contains(line));
+ this.lines.retainAll(Arrays.asList(line));
+ }
+
+ /** Finds the first line that starts with the given line start and inserts the
+ * given lines before it, or fails if no line can be found with that line
+ * start. */
+ void insertBeforeLineStartingWith(String lineStart,
+ List<String> linesToInsert) {
+ for (int i = 0; i < this.lines.size(); i++) {
+ if (this.lines.get(i).startsWith(lineStart)) {
+ this.lines.addAll(i, linesToInsert);
+ return;
+ }
+ }
+ fail("Line start not found: " + lineStart);
+ }
+
+ /** Finds the first line that starts with the given line start and replaces
+ * that line and possibly subsequent lines, or fails if no line can be found
+ * with that line start or there are not enough lines left to replace. */
+ void replaceLineStartingWith(String lineStart, List<String> linesToReplace) {
+ for (int i = 0; i < this.lines.size(); i++) {
+ if (this.lines.get(i).startsWith(lineStart)) {
+ for (int j = 0; j < linesToReplace.size(); j++) {
+ assertTrue("Not enough lines left to replace.",
+ this.lines.size() > i + j);
+ this.lines.set(i + j, linesToReplace.get(j));
+ }
+ return;
+ }
+ }
+ fail("Line start not found: " + lineStart);
+ }
+
+ /** Writes all descriptor lines with newlines to the given output stream. */
+ void build(OutputStream outputStream) throws IOException {
+ for (String line : lines) {
+ outputStream.write((line + "\n").getBytes());
+ }
+ }
+}
+
diff --git a/src/test/java/org/torproject/collector/bridgedescs/ExtraInfoDescriptorBuilder.java b/src/test/java/org/torproject/collector/bridgedescs/ExtraInfoDescriptorBuilder.java
new file mode 100644
index 0000000..a6ffa53
--- /dev/null
+++ b/src/test/java/org/torproject/collector/bridgedescs/ExtraInfoDescriptorBuilder.java
@@ -0,0 +1,55 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+
+/** Builds a non-sanitized bridge extra-info descriptor that comes with an
+ * original bridge descriptor (of a bundled and therefore publicly known bridge)
+ * by default. */
+class ExtraInfoDescriptorBuilder extends DescriptorBuilder {
+
+ /** Initializes the descriptor builder. */
+ ExtraInfoDescriptorBuilder() {
+ this.lines = new ArrayList<>(Arrays.asList(
+ "extra-info MeekGoogle "
+ + "46D4A71197B8FA515A826C6B017C522FE264655B",
+ "identity-ed25519",
+ "-----BEGIN ED25519 CERT-----",
+ "AQQABjliAVz1pof1ijauJttRPRlhPc4GKgp7SWOtFsnvSA3ddIsIAQAgBAA6BoYk",
+ "ZEXE7RkiEJ1l5Ij9hc9TJOpM7/9XSPZnF/PbMfE0u3n3JbOO3s82GN6BPuA0v2Cs",
+ "5eSvciL7+38Ok2eCaMa6vDrXYUSKrN+z9Kz3feL/XDWQy9L9Tkm7bticng0=",
+ "-----END ED25519 CERT-----",
+ "published 2016-06-30 21:43:52",
+ "write-history 2016-06-30 18:40:48 (14400 s) "
+ + "415744,497664,359424,410624,420864,933888",
+ "read-history 2016-06-30 18:40:48 (14400 s) "
+ + "4789248,6237184,4473856,5039104,5567488,5440512",
+ "geoip-db-digest 6346E26E2BC96F8511588CE2695E9B0339A75D32",
+ "geoip6-db-digest 43CCB43DBC653D8CC16396A882C5F116A6004F0C",
+ "dirreq-stats-end 2016-06-30 14:40:48 (86400 s)",
+ "dirreq-v3-ips ",
+ "dirreq-v3-reqs ",
+ "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0,"
+ + "not-modified=0,busy=0",
+ "dirreq-v3-direct-dl complete=0,timeout=0,running=0",
+ "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0",
+ "transport meek 198.50.200.131:8000",
+ "transport meek 198.50.200.131:7443",
+ "bridge-stats-end 2016-06-30 14:41:18 (86400 s)",
+ "bridge-ips ",
+ "bridge-ip-versions v4=0,v6=0",
+ "bridge-ip-transports ",
+ "router-sig-ed25519 xNkIgy3gYoENUDMvMvPj1/qPv4suyODE8PcVNLZpY8/WxKvoniT"
+ + "+2UsWvKsZVAZwFnq7kSByBJUGdxC3YdhSCA",
+ "router-signature",
+ "-----BEGIN SIGNATURE-----",
+ "jwfwSxul/olhO4VzJfBTg+KQf4G+nRwFa9XLMSgBTy6P+hqDkw7TE079BZiYb8+v",
+ "ElS08R1Diq50N8fosR5lqP/Ihhm+V0KcEyWG10+Vl7ADMA3m4GdbGa6dSrdiFMPs",
+ "OYE9aueVDIMgKyiOyNmgK3S8lwjX4v6yhaiJWxDGuKs=",
+ "-----END SIGNATURE-----"));
+ }
+}
+
diff --git a/src/test/java/org/torproject/collector/bridgedescs/NetworkStatusBuilder.java b/src/test/java/org/torproject/collector/bridgedescs/NetworkStatusBuilder.java
new file mode 100644
index 0000000..20088a4
--- /dev/null
+++ b/src/test/java/org/torproject/collector/bridgedescs/NetworkStatusBuilder.java
@@ -0,0 +1,30 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+
+/** Builds a non-sanitized bridge network status that comes with an original
+ * bridge network status entry (of a bundled and therefore publicly known
+ * bridge) by default. */
+class NetworkStatusBuilder extends DescriptorBuilder {
+
+ /** Initializes the descriptor builder. */
+ NetworkStatusBuilder() {
+ this.lines = new ArrayList<>(Arrays.asList(
+ "published 2016-06-30 23:40:28",
+ "flag-thresholds stable-uptime=807660 stable-mtbf=1425164 "
+ + "fast-speed=47000 guard-wfu=98.000% guard-tk=691200 "
+ + "guard-bw-inc-exits=400000 guard-bw-exc-exits=402000 "
+ + "enough-mtbf=1 ignoring-advertised-bws=0",
+ "r MeekGoogle RtSnEZe4+lFagmxrAXxSL+JkZVs "
+ + "g+M7Ww+lGKmv6NW9GRmvzLOiR0Y 2016-06-30 21:43:52 "
+ + "198.50.200.131 8008 0",
+ "s Fast Running Stable Valid",
+ "w Bandwidth=56",
+ "p reject 1-65535"));
+ }
+}
+
diff --git a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
new file mode 100644
index 0000000..0049bb6
--- /dev/null
+++ b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
@@ -0,0 +1,521 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.torproject.collector.Main;
+import org.torproject.collector.conf.Configuration;
+import org.torproject.collector.conf.ConfigurationException;
+import org.torproject.collector.conf.Key;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Tests the bridge descriptor sanitizer by preparing a temporary folder
+ * with non-sanitized bridge descriptors, running the sanitizer, and
+ * verifying the sanitized descriptors. */
+public class SanitizedBridgesWriterTest {
+
+ /** Temporary folder containing all files for this test. */
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ /** Directory containing bridge descriptor tarballs to sanitize. */
+ private File bridgeDirectoriesDir;
+
+ /** Directory holding recent descriptor files served by CollecTor. */
+ private File recentDirectory;
+
+ /** Directory storing all intermediate state that needs to be preserved
+ * between processing runs. */
+ private File statsDirectory;
+
+ /** Directory holding sanitized bridge descriptor files. */
+ private File sanitizedBridgesDirectory;
+
+ /** CollecTor configuration for this test. */
+ private Configuration configuration;
+
+ /** Server descriptor builder used to build the first and only server
+ * descriptor for this test, unless removed from the tarball builder.*/
+ private DescriptorBuilder defaultServerDescriptorBuilder;
+
+ /** Extra-info descriptor builder used to build the first and only
+ * extra-info descriptor for this test, unless removed from the tarball
+ * builder.*/
+ private DescriptorBuilder defaultExtraInfoDescriptorBuilder;
+
+ /** Network status builder used to build the first and only network
+ * status for this test, unless removed from the tarball builder.*/
+ private DescriptorBuilder defaultNetworkStatusBuilder;
+
+ /** Tarball builder to build the first and only tarball, unless removed
+ * from the test. */
+ private TarballBuilder defaultTarballBuilder;
+
+ /** Tarball builder(s) for this test. */
+ private List<TarballBuilder> tarballBuilders;
+
+ /** Parsed sanitized bridge descriptors with keys being file names and
+ * values being sanitized descriptor lines. */
+ private Map<String, List<String>> parsedFiles;
+
+ /** Parsed sanitized default server descriptor. */
+ private List<List<String>> parsedServerDescriptors;
+
+ /** Parsed sanitized default extra-info descriptor. */
+ private List<List<String>> parsedExtraInfoDescriptors;
+
+ /** Parsed sanitized default network status. */
+ private List<List<String>> parsedNetworkStatuses;
+
+ /** Prepares the temporary folder and the various builders for this
+ * test. */
+ @Before
+ public void createTemporaryFolderAndBuilders()
+ throws IOException {
+ this.bridgeDirectoriesDir = this.temporaryFolder.newFolder("in");
+ this.recentDirectory = this.temporaryFolder.newFolder("recent");
+ this.statsDirectory = this.temporaryFolder.newFolder("stats");
+ this.sanitizedBridgesDirectory = this.temporaryFolder.newFolder("out");
+ this.initializeTestConfiguration();
+ this.defaultServerDescriptorBuilder = new ServerDescriptorBuilder();
+ this.defaultExtraInfoDescriptorBuilder = new ExtraInfoDescriptorBuilder();
+ this.defaultNetworkStatusBuilder = new NetworkStatusBuilder();
+ this.defaultTarballBuilder = new TarballBuilder(
+ "from-tonga-2016-07-01T000702Z.tar.gz", 1467331624000L);
+ this.defaultTarballBuilder.add("bridge-descriptors", 1467331622000L,
+ Arrays.asList(new DescriptorBuilder[] {
+ this.defaultServerDescriptorBuilder }));
+ this.defaultTarballBuilder.add("cached-extrainfo", 1467327972000L,
+ Arrays.asList(new DescriptorBuilder[] {
+ this.defaultExtraInfoDescriptorBuilder }));
+ this.defaultTarballBuilder.add("cached-extrainfo.new", 1467331623000L,
+ Arrays.asList(new DescriptorBuilder[] { }));
+ this.defaultTarballBuilder.add("networkstatus-bridges",
+ 1467330028000L, Arrays.asList(new DescriptorBuilder[] {
+ this.defaultNetworkStatusBuilder }));
+ this.tarballBuilders = new ArrayList<>(
+ Arrays.asList(this.defaultTarballBuilder));
+ }
+
+ /** Initializes a configuration for the bridge descriptor sanitizer. */
+ private void initializeTestConfiguration() throws IOException {
+ this.configuration = new Configuration();
+ this.configuration.load(getClass().getClassLoader().getResourceAsStream(
+ Main.CONF_FILE));
+ this.configuration.setProperty(Key.BridgedescsActivated.name(), "true");
+ this.configuration.setProperty(Key.RecentPath.name(),
+ recentDirectory.getAbsolutePath());
+ this.configuration.setProperty(Key.StatsPath.name(),
+ statsDirectory.getAbsolutePath());
+ this.configuration.setProperty(Key.BridgeSnapshotsDirectory.name(),
+ bridgeDirectoriesDir.getAbsolutePath());
+ this.configuration.setProperty(Key.SanitizedBridgesWriteDirectory.name(),
+ sanitizedBridgesDirectory.getAbsolutePath());
+ }
+
+ /** Runs this test by executing all builders, performing the sanitizing
+ * process, and parsing sanitized bridge descriptors for inspection. */
+ private void runTest() throws IOException, ConfigurationException {
+ for (TarballBuilder tarballBuilder : this.tarballBuilders) {
+ tarballBuilder.build(this.bridgeDirectoriesDir);
+ }
+ SanitizedBridgesWriter sbw = new SanitizedBridgesWriter(configuration);
+ sbw.startProcessing();
+ List<File> files = new ArrayList<>();
+ files.add(sanitizedBridgesDirectory);
+ String basePath = sanitizedBridgesDirectory.getAbsolutePath() + "/";
+ this.parsedFiles = new LinkedHashMap<>();
+ this.parsedServerDescriptors = new ArrayList<>();
+ this.parsedExtraInfoDescriptors = new ArrayList<>();
+ this.parsedNetworkStatuses = new ArrayList<>();
+ while (!files.isEmpty()) {
+ File file = files.remove(0);
+ if (file.isDirectory()) {
+ files.addAll(Arrays.asList(file.listFiles()));
+ } else {
+ List<String> parsedLines = new ArrayList<>();
+ BufferedReader reader = new BufferedReader(new FileReader(file));
+ String line;
+ while ((line = reader.readLine()) != null) {
+ parsedLines.add(line);
+ }
+ reader.close();
+ String relativePath = file.getAbsolutePath().substring(
+ basePath.length());
+ if (parsedLines.get(0).startsWith("@type bridge-server-descriptor ")) {
+ this.parsedServerDescriptors.add(parsedLines);
+ } else if (parsedLines.get(0).startsWith("@type bridge-extra-info ")) {
+ this.parsedExtraInfoDescriptors.add(parsedLines);
+ } else if (parsedLines.get(0).startsWith(
+ "@type bridge-network-status ")) {
+ this.parsedNetworkStatuses.add(parsedLines);
+ }
+ this.parsedFiles.put(relativePath, parsedLines);
+ }
+ }
+ }
+
+ @Test
+ public void testServerDescriptorDefault() throws Exception {
+ this.runTest();
+ List<String> expectedLines = Arrays.asList(
+ "@type bridge-server-descriptor 1.2",
+ "router MeekGoogle 127.0.0.1 1 0 0",
+ "master-key-ed25519 3HC9xnykNIfNdFDuJWwxaJSM5GCaIJKUtAYgMixbsOM",
+ "platform Tor 0.2.7.6 on Linux",
+ "protocols Link 1 2 Circuit 1",
+ "published 2016-06-30 21:43:52",
+ "fingerprint 88F7 4584 0F47 CE0C 6A4F E61D 8279 50B0 6F9E 4534",
+ "uptime 543754",
+ "bandwidth 3040870 5242880 56583",
+ "extra-info-digest B026CF0F81712D94BBF1362294882688DF247887 "
+ + "/XWPeILeik+uTGaKL3pnUeQfYS87SfjKVkwTiCmbqi0",
+ "hidden-service-dir",
+ "contact somebody",
+ "ntor-onion-key YjZG5eaQ1gmXvlSMGEBwM7OLswv8AtXZr6ccOnDUKQw=",
+ "reject *:*",
+ "router-digest-sha256 "
+ + "KI4PRYH9rDCDLYPNv9NF53gFy8pJjIpeJ7UkzIGOAnw",
+ "router-digest B6922FF5C045814DF4BCB72A0D6C9417CFFBD80A");
+ assertEquals("Sanitized descriptor does not contain expected lines.",
+ expectedLines, this.parsedServerDescriptors.get(0));
+ assertTrue("Sanitized descriptor file name differs.",
+ this.parsedFiles.containsKey("2016/06/server-descriptors/b/6/"
+ + "b6922ff5c045814df4bcb72a0d6c9417cffbd80a"));
+ }
+
+ @Test
+ public void testServerDescriptorEmpty() throws Exception {
+ this.defaultServerDescriptorBuilder.removeAllLines();
+ this.runTest();
+ assertTrue("No server descriptor provided as input.",
+ this.parsedServerDescriptors.isEmpty());
+ }
+
+ @Test
+ public void testServerDescriptorAdditionalAnnotation()
+ throws Exception {
+ this.defaultServerDescriptorBuilder.insertBeforeLineStartingWith(
+ "@purpose bridge", Arrays.asList("@source 198.50.200.131"));
+ this.runTest();
+ assertEquals("Expected 3 sanitized descriptors.", 3,
+ this.parsedFiles.size());
+ }
+
+ @Test
+ public void testServerDescriptorRouterLineTruncated() throws Exception {
+ this.defaultServerDescriptorBuilder.replaceLineStartingWith("router ",
+ Arrays.asList("router MeekGoogle"));
+ this.runTest();
+ assertTrue("Sanitized server descriptor with invalid router line.",
+ this.parsedServerDescriptors.isEmpty());
+ }
+
+ @Test
+ public void testServerDescriptorFingerprintTruncated() throws Exception {
+ this.defaultServerDescriptorBuilder.replaceLineStartingWith(
+ "fingerprint ", Arrays.asList("fingerprint 4"));
+ this.runTest();
+ assertTrue("Sanitized server descriptor with invalid fingerprint "
+ + "line.", this.parsedServerDescriptors.isEmpty());
+ }
+
+ @Test
+ public void testServerDescriptorFingerprintInvalidHex()
+ throws Exception {
+ this.defaultServerDescriptorBuilder.replaceLineStartingWith(
+ "fingerprint ", Arrays.asList("fingerprint FUN!"));
+ this.runTest();
+ assertTrue("Sanitized server descriptor with invalid fingerprint "
+ + "line.", this.parsedServerDescriptors.isEmpty());
+ }
+
+ @Test
+ public void testServerDescriptorFingerprintOpt() throws Exception {
+ this.defaultServerDescriptorBuilder.replaceLineStartingWith("fingerprint ",
+ Arrays.asList("opt fingerprint 46D4 A711 97B8 FA51 5A82 6C6B 017C 522F "
+ + "E264 655B"));
+ this.runTest();
+ this.parsedServerDescriptors.get(0).contains("opt fingerprint 88F7 "
+ + "4584 0F47 CE0C 6A4F E61D 8279 50B0 6F9E 4534");
+ }
+
+ @Test
+ public void testServerDescriptorExtraInfoDigestInvalidHex()
+ throws Exception {
+ this.defaultServerDescriptorBuilder.replaceLineStartingWith(
+ "extra-info-digest ", Arrays.asList("extra-info-digest 6"));
+ this.runTest();
+ assertTrue("Sanitized server descriptor with invalid extra-info "
+ + "line.", this.parsedServerDescriptors.isEmpty());
+ }
+
+ @Test
+ public void testServerDescriptorExtraInfoDigestSha1Only()
+ throws Exception {
+ this.defaultServerDescriptorBuilder.replaceLineStartingWith(
+ "extra-info-digest ", Arrays.asList("extra-info-digest "
+ + "6D03E80568DEFA102968D144CB35FFA6E3355B8A"));
+ this.runTest();
+ assertTrue("Expected different extra-info-digest line.",
+ this.parsedServerDescriptors.get(0).contains(
+ "extra-info-digest B026CF0F81712D94BBF1362294882688DF247887"));
+ }
+
+ @Test
+ public void testServerDescriptorExtraInfoDigestOpt() throws Exception {
+ this.defaultServerDescriptorBuilder.replaceLineStartingWith(
+ "extra-info-digest ", Arrays.asList("opt extra-info-digest "
+ + "6D03E80568DEFA102968D144CB35FFA6E3355B8A "
+ + "cy/LwP7nxukmmcT1+UnDg4qh0yKbjVUYKhGL8VksoJA"));
+ this.runTest();
+ this.parsedServerDescriptors.get(0).contains("opt extra-info-digest "
+ + "B026CF0F81712D94BBF1362294882688DF247887 "
+ + "/XWPeILeik+uTGaKL3pnUeQfYS87SfjKVkwTiCmbqi0");
+ }
+
+ @Test
+ public void testServerDescriptorRejectOwnAddress() throws Exception {
+ this.defaultServerDescriptorBuilder.insertBeforeLineStartingWith(
+ "reject *:*", Arrays.asList("reject 198.50.200.131:*", "accept *:80"));
+ this.runTest();
+ List<String> parsedLines = this.parsedServerDescriptors.get(0);
+ for (int i = 0; i < parsedLines.size(); i++) {
+ if ("reject 127.0.0.1:*".equals(parsedLines.get(i))) {
+ assertEquals("accept line out of order.", "accept *:80",
+ parsedLines.get(i + 1));
+ assertEquals("reject line out of order.", "reject *:*",
+ parsedLines.get(i + 2));
+ return;
+ }
+ }
+ fail("IP address in reject line should have been replaced: "
+ + parsedLines);
+ }
+
+ @Test
+ public void testServerDescriptorEd25519IdentityMasterKeyMismatch()
+ throws Exception {
+ this.defaultServerDescriptorBuilder.replaceLineStartingWith(
+ "master-key-ed25519 ", Arrays.asList("master-key-ed25519 "
+ + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"));
+ this.runTest();
+ assertTrue("Mismatch between Ed25519 identity and master key.",
+ this.parsedServerDescriptors.isEmpty());
+ }
+
+ @Test
+ public void testServerDescriptorFamilyInvalidFingerprint()
+ throws Exception {
+ this.defaultServerDescriptorBuilder.insertBeforeLineStartingWith(
+ "hidden-service-dir", Arrays.asList("family $0"));
+ this.runTest();
+ assertTrue("Sanitized server descriptor with invalid fingerprint in "
+ + "family line.", this.parsedServerDescriptors.isEmpty());
+ }
+
+ @Test
+ public void testExtraInfoDescriptorDefault() throws Exception {
+ this.runTest();
+ List<String> expectedLines = Arrays.asList(
+ "@type bridge-extra-info 1.3",
+ "extra-info MeekGoogle 88F745840F47CE0C6A4FE61D827950B06F9E4534",
+ "master-key-ed25519 3HC9xnykNIfNdFDuJWwxaJSM5GCaIJKUtAYgMixbsOM",
+ "published 2016-06-30 21:43:52",
+ "write-history 2016-06-30 18:40:48 (14400 s) "
+ + "415744,497664,359424,410624,420864,933888",
+ "read-history 2016-06-30 18:40:48 (14400 s) "
+ + "4789248,6237184,4473856,5039104,5567488,5440512",
+ "geoip-db-digest 6346E26E2BC96F8511588CE2695E9B0339A75D32",
+ "geoip6-db-digest 43CCB43DBC653D8CC16396A882C5F116A6004F0C",
+ "dirreq-stats-end 2016-06-30 14:40:48 (86400 s)",
+ "dirreq-v3-ips ",
+ "dirreq-v3-reqs ",
+ "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0,"
+ + "not-modified=0,busy=0",
+ "dirreq-v3-direct-dl complete=0,timeout=0,running=0",
+ "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0",
+ "transport meek",
+ "transport meek",
+ "bridge-stats-end 2016-06-30 14:41:18 (86400 s)",
+ "bridge-ips ",
+ "bridge-ip-versions v4=0,v6=0",
+ "bridge-ip-transports ",
+ "router-digest-sha256 "
+ + "/XWPeILeik+uTGaKL3pnUeQfYS87SfjKVkwTiCmbqi0",
+ "router-digest B026CF0F81712D94BBF1362294882688DF247887");
+ assertEquals("Sanitized descriptor does not contain expected lines.",
+ expectedLines, this.parsedExtraInfoDescriptors.get(0));
+ assertTrue("Sanitized descriptor file name differs.",
+ this.parsedFiles.containsKey("2016/06/extra-infos/b/0/"
+ + "b026cf0f81712d94bbf1362294882688df247887"));
+ }
+
+ @Test
+ public void testExtraInfoDescriptorExtraInfoInvalidHex()
+ throws Exception {
+ this.defaultExtraInfoDescriptorBuilder.replaceLineStartingWith(
+ "extra-info ", Arrays.asList("extra-info MeekGoogle 4"));
+ this.runTest();
+ assertTrue("Sanitized extra-info descriptor with invalid extra-info "
+ + "line.", this.parsedExtraInfoDescriptors.isEmpty());
+ }
+
+ @Test
+ public void testExtraInfoDescriptorRouterSignatureLineSpace()
+ throws Exception {
+ this.defaultExtraInfoDescriptorBuilder.replaceLineStartingWith(
+ "router-signature", Arrays.asList("router-signature "));
+ this.runTest();
+ assertTrue("Sanitized extra-info descriptor with invalid "
+ + "router-signature line.",
+ this.parsedExtraInfoDescriptors.isEmpty());
+ }
+
+ @Test
+ public void testNetworkStatusDefault() throws Exception {
+ this.runTest();
+ List<String> expectedLines = Arrays.asList(
+ "@type bridge-network-status 1.1",
+ "published 2016-06-30 23:40:28",
+ "flag-thresholds stable-uptime=807660 stable-mtbf=1425164 "
+ + "fast-speed=47000 guard-wfu=98.000% guard-tk=691200 "
+ + "guard-bw-inc-exits=400000 guard-bw-exc-exits=402000 "
+ + "enough-mtbf=1 ignoring-advertised-bws=0",
+ "r MeekGoogle iPdFhA9HzgxqT+YdgnlQsG+eRTQ "
+ + "tpIv9cBFgU30vLcqDWyUF8/72Ao 2016-06-30 21:43:52 127.0.0.1 "
+ + "1 0",
+ "s Fast Running Stable Valid",
+ "w Bandwidth=56",
+ "p reject 1-65535");
+ assertEquals("Sanitized descriptor does not contain expected lines.",
+ expectedLines, this.parsedNetworkStatuses.get(0));
+ assertTrue("Sanitized descriptor file name differs.",
+ this.parsedFiles.containsKey("2016/06/statuses/30/"
+ + "20160630-234028-4A0CCD2DDC7995083D73F5D667100C8A5831F16D"));
+ }
+
+ @Test
+ public void testNetworkStatusPublishedLineMissing() throws Exception {
+ this.defaultNetworkStatusBuilder.removeLine(
+ "published 2016-06-30 23:40:28");
+ this.runTest();
+ String sanitizedNetworkStatusFileName = "2016/07/statuses/01/"
+ + "20160701-000702-4A0CCD2DDC7995083D73F5D667100C8A5831F16D";
+ assertTrue("Sanitized descriptor file does contain published line.",
+ this.parsedFiles.get(sanitizedNetworkStatusFileName)
+ .contains("published 2016-07-01 00:07:02"));
+ }
+
+ @Test
+ public void testNetworkStatusPublishedLineMissingTarballFileNameChange()
+ throws Exception {
+ this.defaultNetworkStatusBuilder.removeLine(
+ "published 2016-06-30 23:40:28");
+ this.defaultTarballBuilder.setTarballFileName(
+ "from-tonga-with-love-2016-07-01T000702Z.tar.gz");
+ this.runTest();
+ assertTrue("Sanitized network status without published line and with "
+ + "changed tarball file name.", this.parsedNetworkStatuses.isEmpty());
+ }
+
+ @Test
+ public void testNetworkStatusAlinePortMissing() throws Exception {
+ this.configuration.setProperty(Key.ReplaceIpAddressesWithHashes.name(),
+ "true");
+ this.defaultNetworkStatusBuilder.insertBeforeLineStartingWith("s ",
+ Arrays.asList("a 198.50.200.132"));
+ this.runTest();
+ for (String line : this.parsedNetworkStatuses.get(0)) {
+ if (line.startsWith("a ")) {
+ fail("Sanitized a line without port: " + line);
+ }
+ }
+ }
+
+ @Test
+ public void testTarballContainsSameFileTwice() throws Exception {
+ this.defaultTarballBuilder.add("cached-extrainfo.new", 1467331623000L,
+ Arrays.asList(new DescriptorBuilder[] {
+ this.defaultExtraInfoDescriptorBuilder }));
+ }
+
+ @Test
+ public void testTarballCorrupt() throws Exception {
+ this.tarballBuilders.clear();
+ File tarballFile = new File(bridgeDirectoriesDir,
+ "from-tonga-2016-07-01T000702Z.tar.gz");
+ FileOutputStream stream = new FileOutputStream(tarballFile);
+ stream.write(new byte[] { 0x00 });
+ stream.close();
+ tarballFile.setLastModified(1467331624000L);
+ this.runTest();
+ assertTrue("Sanitized descriptors from corrupt tarball.",
+ this.parsedFiles.isEmpty());
+ }
+
+ @Test
+ public void testTarballUncompressed() throws Exception {
+ String tarballFileName = this.tarballBuilders.get(0).getTarballFileName();
+ this.tarballBuilders.get(0).setTarballFileName(
+ tarballFileName.substring(0, tarballFileName.length() - 3));
+ this.runTest();
+ assertEquals("Expected 3 sanitized descriptors from uncompressed "
+ + "tarball.", 3, this.parsedFiles.size());
+ }
+
+ @Test
+ public void testTarballBzip2Compressed() throws Exception {
+ String tarballFileName = this.tarballBuilders.get(0).getTarballFileName();
+ this.tarballBuilders.get(0).setTarballFileName(
+ tarballFileName.substring(0, tarballFileName.length() - 3) + ".bz2");
+ this.runTest();
+ assertTrue("Didn't expect sanitized descriptors from unsupported "
+ + "bz2-compressed tarball.", this.parsedFiles.isEmpty());
+ }
+
+ @Test
+ public void testParsedBridgeDirectoriesSkipTarball() throws Exception {
+ File parsedBridgeDirectoriesFile = new File(statsDirectory,
+ "parsed-bridge-directories");
+ BufferedWriter writer = new BufferedWriter(new FileWriter(
+ parsedBridgeDirectoriesFile));
+ writer.write(this.tarballBuilders.get(0).getTarballFileName() + "\n");
+ writer.close();
+ this.runTest();
+ assertTrue("Didn't expect sanitized descriptors after skipping "
+ + "tarball.", this.parsedFiles.isEmpty());
+ }
+
+ @Test
+ public void testParsedBridgeDirectoriesIsDirectory() throws Exception {
+ File parsedBridgeDirectoriesFile = new File(statsDirectory,
+ "parsed-bridge-directories");
+ parsedBridgeDirectoriesFile.mkdirs();
+ this.runTest();
+ assertTrue("Continued despite not being able to read "
+ + "parsed-bridge-directories.", this.parsedFiles.isEmpty());
+ }
+}
+
diff --git a/src/test/java/org/torproject/collector/bridgedescs/ServerDescriptorBuilder.java b/src/test/java/org/torproject/collector/bridgedescs/ServerDescriptorBuilder.java
new file mode 100644
index 0000000..0b5e916
--- /dev/null
+++ b/src/test/java/org/torproject/collector/bridgedescs/ServerDescriptorBuilder.java
@@ -0,0 +1,74 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+
+/** Builds a non-sanitized bridge server descriptor that comes with an original
+ * bridge descriptor (of a bundled and therefore publicly known bridge) by
+ * default. */
+class ServerDescriptorBuilder extends DescriptorBuilder {
+
+ /** Initializes the descriptor builder. */
+ ServerDescriptorBuilder() {
+ this.lines = new ArrayList<>(Arrays.asList(
+ "@purpose bridge",
+ "router MeekGoogle 198.50.200.131 8008 0 0",
+ "identity-ed25519",
+ "-----BEGIN ED25519 CERT-----",
+ "AQQABjliAVz1pof1ijauJttRPRlhPc4GKgp7SWOtFsnvSA3ddIsIAQAgBAA6BoYk",
+ "ZEXE7RkiEJ1l5Ij9hc9TJOpM7/9XSPZnF/PbMfE0u3n3JbOO3s82GN6BPuA0v2Cs",
+ "5eSvciL7+38Ok2eCaMa6vDrXYUSKrN+z9Kz3feL/XDWQy9L9Tkm7bticng0=",
+ "-----END ED25519 CERT-----",
+ "master-key-ed25519 "
+ + "OgaGJGRFxO0ZIhCdZeSI/YXPUyTqTO//V0j2Zxfz2zE",
+ "platform Tor 0.2.7.6 on Linux",
+ "protocols Link 1 2 Circuit 1",
+ "published 2016-06-30 21:43:52",
+ "fingerprint 46D4 A711 97B8 FA51 5A82 6C6B 017C 522F E264 655B",
+ "uptime 543754",
+ "bandwidth 3040870 5242880 56583",
+ "extra-info-digest 6D03E80568DEFA102968D144CB35FFA6E3355B8A "
+ + "cy/LwP7nxukmmcT1+UnDg4qh0yKbjVUYKhGL8VksoJA",
+ "onion-key",
+ "-----BEGIN RSA PUBLIC KEY-----",
+ "MIGJAoGBANcIfT+XV4HHSWEQPGkID0C4OgWQ3Gc/RmfQYLMPe5enDNSLBTstw4ep",
+ "aiScHB1xhN8xRhpVB/qaCcYGpmUltIH0NaWQ3tuRV7rw+fp7amfYZfThUk5OPpF0",
+ "soGd3jRrzX7SEm4YCGdLZALL51Wb2pdOmR93WucOZYav/tGs/d9rAgMBAAE=",
+ "-----END RSA PUBLIC KEY-----",
+ "signing-key",
+ "-----BEGIN RSA PUBLIC KEY-----",
+ "MIGJAoGBAMHXuK8J+5028rDovbEejPrsOJKWtsj7fr4EhMmOmIUM4N2gLdEVyFq7",
+ "sVkHZFf3v04PmOhSJymLmVVcXe+Qsb4U300DwADvnpeFjhU4trrFqZljZM5+gPW6",
+ "ZmK0ViD54td0biJZd9Ow65Od9XzbJTa2acO/sVXD0Q8tIfnEywvZAgMBAAE=",
+ "-----END RSA PUBLIC KEY-----",
+ "onion-key-crosscert",
+ "-----BEGIN CROSSCERT-----",
+ "zQvq4eQYXn9Y2St2Qch4AvwqPAJ+Y+MgTFTf4qYaQ04FXo1csf2eSPB5zbWaUgBb",
+ "GbtKaw1ZJJjEtVzk/HnIWQ/V/ONJUSL4BiF2M4RuhozJoK2BGpYfmcsGWQKeLcPi",
+ "YIVtO5OI2XcvxgGGVz4ZPPiiGDFJ2MxHA1747KnGSo8=",
+ "-----END CROSSCERT-----",
+ "ntor-onion-key-crosscert 1",
+ "-----BEGIN ED25519 CERT-----",
+ "AQoABjjOAToGhiRkRcTtGSIQnWXkiP2Fz1Mk6kzv/1dI9mcX89sxAA0280fSYhvB",
+ "Y39F6J5FuCFcE/B1KDZZP8zY3NYAP4y+jVTG82RRsN87hwZlyShoBxm2q3x4LNPl",
+ "67ZGbPdAUAA=",
+ "-----END ED25519 CERT-----",
+ "hidden-service-dir",
+ "contact jvictors at jessevictors com, PGP 0xC20BEC80, BTC "
+ + "1M6tuPXNmhbgSfaJqnxBUAf5tKi4TVhup8",
+ "ntor-onion-key YjZG5eaQ1gmXvlSMGEBwM7OLswv8AtXZr6ccOnDUKQw=",
+ "reject *:*",
+ "router-sig-ed25519 ObNigP8q0HkRDaDPP84txWH+3TbuL8DUNLAF25OZV9uRovF9j02"
+ + "StDVEdLGR6vOx9lRos0/264n42jEHzmUbBg",
+ "router-signature",
+ "-----BEGIN SIGNATURE-----",
+ "u/J/T0w7JlH4yUbXcg5hDIVBzGZtXxoH+800zOJXIxbIEGqgTxOhA13C6s/j/C0G",
+ "+L6bcrNdqKriJERsJicT2UqVRiIl54c76J9ySsknNKvXuEbZ3RJ71FhzLbi5CQXJ",
+ "N5wdZX+AqHSnSe+ayaB3zVlp97gUbFhg3vE2eWPtRxY=",
+ "-----END SIGNATURE-----"));
+ }
+}
+
diff --git a/src/test/java/org/torproject/collector/bridgedescs/TarballBuilder.java b/src/test/java/org/torproject/collector/bridgedescs/TarballBuilder.java
new file mode 100644
index 0000000..eddfb2b
--- /dev/null
+++ b/src/test/java/org/torproject/collector/bridgedescs/TarballBuilder.java
@@ -0,0 +1,108 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.collector.bridgedescs;
+
+import static org.junit.Assert.fail;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/** Builds a tarball containing non-sanitized bridge descriptors built using
+ * descriptor builders and writes the tarball to a new file with the given file
+ * name. */
+class TarballBuilder {
+
+ /** Internal helper class to store details about a file contained in the
+ * tarball. */
+ private class TarballFile {
+
+ /** Last modified time of the file. */
+ private long modifiedMillis;
+
+ /** Descriptor builders used to generate the file content. */
+ private List<DescriptorBuilder> descriptorBuilders;
+ }
+
+ /** File name of the tarball. */
+ private String tarballFileName;
+
+ String getTarballFileName() {
+ return tarballFileName;
+ }
+
+ void setTarballFileName(String tarballFileName) {
+ this.tarballFileName = tarballFileName;
+ }
+
+ /** Last modified time of the tarball file. */
+ private long modifiedMillis;
+
+ /** Files contained in the tarball. */
+ private Map<String, TarballFile> tarballFiles;
+
+ /** Initializes a new tarball builder that is going to write a tarball to the
+ * file with given file name and last-modified time. */
+ TarballBuilder(String tarballFileName, long modifiedMillis) {
+ this.tarballFileName = tarballFileName;
+ this.modifiedMillis = modifiedMillis;
+ this.tarballFiles = new LinkedHashMap<>();
+ }
+
+ /** Adds a new file to the tarball with given name, last-modified time, and
+ * descriptor builders to generate the file content. */
+ TarballBuilder add(String fileName, long modifiedMillis,
+ List<DescriptorBuilder> descriptorBuilders) throws IOException {
+ TarballFile file = new TarballFile();
+ file.modifiedMillis = modifiedMillis;
+ file.descriptorBuilders = descriptorBuilders;
+ this.tarballFiles.put(fileName, file);
+ return this;
+ }
+
+ /** Writes the previously configured tarball with all contained files to the
+ * given file, or fail if the file extension is not known. */
+ void build(File directory) throws IOException {
+ File tarballFile = new File(directory, this.tarballFileName);
+ TarArchiveOutputStream taos = null;
+ if (this.tarballFileName.endsWith(".tar.gz")) {
+ taos = new TarArchiveOutputStream(new GzipCompressorOutputStream(
+ new BufferedOutputStream(new FileOutputStream(tarballFile))));
+ } else if (this.tarballFileName.endsWith(".tar.bz2")) {
+ taos = new TarArchiveOutputStream(new BZip2CompressorOutputStream(
+ new BufferedOutputStream(new FileOutputStream(tarballFile))));
+ } else if (this.tarballFileName.endsWith(".tar")) {
+ taos = new TarArchiveOutputStream(new BufferedOutputStream(
+ new FileOutputStream(tarballFile)));
+ } else {
+ fail("Unknown file extension: " + this.tarballFileName);
+ }
+ for (Map.Entry<String, TarballFile> file : this.tarballFiles.entrySet()) {
+ TarArchiveEntry tae = new TarArchiveEntry(file.getKey());
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ for (DescriptorBuilder descriptorBuilder
+ : file.getValue().descriptorBuilders) {
+ descriptorBuilder.build(baos);
+ }
+ tae.setSize(baos.size());
+ tae.setModTime(file.getValue().modifiedMillis);
+ taos.putArchiveEntry(tae);
+ taos.write(baos.toByteArray());
+ taos.closeArchiveEntry();
+ }
+ taos.close();
+ tarballFile.setLastModified(this.modifiedMillis);
+ }
+}
+
