commit 5da91cae110b4eba1e468250de8e08562562b06d Author: Karsten Loesing karsten.loesing@gmx.net Date: Wed Sep 28 14:56:02 2016 +0200

Add initial set of tests for bridgedescs module. --- .../collector/bridgedescs/DescriptorBuilder.java | 77 +++ .../bridgedescs/ExtraInfoDescriptorBuilder.java | 55 +++ .../bridgedescs/NetworkStatusBuilder.java | 30 ++ .../bridgedescs/SanitizedBridgesWriterTest.java | 521 +++++++++++++++++++++ .../bridgedescs/ServerDescriptorBuilder.java | 74 +++ .../collector/bridgedescs/TarballBuilder.java | 108 +++++ 6 files changed, 865 insertions(+)

diff --git a/src/test/java/org/torproject/collector/bridgedescs/DescriptorBuilder.java b/src/test/java/org/torproject/collector/bridgedescs/DescriptorBuilder.java new file mode 100644 index 0000000..bab8126 --- /dev/null +++ b/src/test/java/org/torproject/collector/bridgedescs/DescriptorBuilder.java @@ -0,0 +1,77 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.bridgedescs; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.List; + +/** Builds a descriptor by concatenating the given lines with newlines and + * writing the output to the given output stream. */ +abstract class DescriptorBuilder { + + /** Descriptor lines. */ + List<String> lines; + + /** Removes all lines. */ + void removeAllLines() { + this.lines.clear(); + } + + /** Removes the given line, or fails if that line cannot be found. */ + void removeLine(String line) { + if (!this.lines.remove(line)) { + fail("Line not contained: " + line); + } + } + + /** Removes all but the given line, or fails if that line cannot be found. */ + void removeAllExcept(String line) { + assertTrue("Line not contained: " + line, this.lines.contains(line)); + this.lines.retainAll(Arrays.asList(line)); + } + + /** Finds the first line that starts with the given line start and inserts the + * given lines before it, or fails if no line can be found with that line + * start. */ + void insertBeforeLineStartingWith(String lineStart, + List<String> linesToInsert) { + for (int i = 0; i < this.lines.size(); i++) { + if (this.lines.get(i).startsWith(lineStart)) { + this.lines.addAll(i, linesToInsert); + return; + } + } + fail("Line start not found: " + lineStart); + } + + /** Finds the first line that starts with the given line start and replaces + * that line and possibly subsequent lines, or fails if no line can be found + * with that line start or there are not enough lines left to replace. */ + void replaceLineStartingWith(String lineStart, List<String> linesToReplace) { + for (int i = 0; i < this.lines.size(); i++) { + if (this.lines.get(i).startsWith(lineStart)) { + for (int j = 0; j < linesToReplace.size(); j++) { + assertTrue("Not enough lines left to replace.", + this.lines.size() > i + j); + this.lines.set(i + j, linesToReplace.get(j)); + } + return; + } + } + fail("Line start not found: " + lineStart); + } + + /** Writes all descriptor lines with newlines to the given output stream. */ + void build(OutputStream outputStream) throws IOException { + for (String line : lines) { + outputStream.write((line + "\n").getBytes()); + } + } +} + diff --git a/src/test/java/org/torproject/collector/bridgedescs/ExtraInfoDescriptorBuilder.java b/src/test/java/org/torproject/collector/bridgedescs/ExtraInfoDescriptorBuilder.java new file mode 100644 index 0000000..a6ffa53 --- /dev/null +++ b/src/test/java/org/torproject/collector/bridgedescs/ExtraInfoDescriptorBuilder.java @@ -0,0 +1,55 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.bridgedescs; + +import java.util.ArrayList; +import java.util.Arrays; + +/** Builds a non-sanitized bridge extra-info descriptor that comes with an + * original bridge descriptor (of a bundled and therefore publicly known bridge) + * by default. */ +class ExtraInfoDescriptorBuilder extends DescriptorBuilder { + + /** Initializes the descriptor builder. */ + ExtraInfoDescriptorBuilder() { + this.lines = new ArrayList<>(Arrays.asList( + "extra-info MeekGoogle " + + "46D4A71197B8FA515A826C6B017C522FE264655B", + "identity-ed25519", + "-----BEGIN ED25519 CERT-----", + "AQQABjliAVz1pof1ijauJttRPRlhPc4GKgp7SWOtFsnvSA3ddIsIAQAgBAA6BoYk", + "ZEXE7RkiEJ1l5Ij9hc9TJOpM7/9XSPZnF/PbMfE0u3n3JbOO3s82GN6BPuA0v2Cs", + "5eSvciL7+38Ok2eCaMa6vDrXYUSKrN+z9Kz3feL/XDWQy9L9Tkm7bticng0=", + "-----END ED25519 CERT-----", + "published 2016-06-30 21:43:52", + "write-history 2016-06-30 18:40:48 (14400 s) " + + "415744,497664,359424,410624,420864,933888", + "read-history 2016-06-30 18:40:48 (14400 s) " + + "4789248,6237184,4473856,5039104,5567488,5440512", + "geoip-db-digest 6346E26E2BC96F8511588CE2695E9B0339A75D32", + "geoip6-db-digest 43CCB43DBC653D8CC16396A882C5F116A6004F0C", + "dirreq-stats-end 2016-06-30 14:40:48 (86400 s)", + "dirreq-v3-ips ", + "dirreq-v3-reqs ", + "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0," + + "not-modified=0,busy=0", + "dirreq-v3-direct-dl complete=0,timeout=0,running=0", + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0", + "transport meek 198.50.200.131:8000", + "transport meek 198.50.200.131:7443", + "bridge-stats-end 2016-06-30 14:41:18 (86400 s)", + "bridge-ips ", + "bridge-ip-versions v4=0,v6=0", + "bridge-ip-transports ", + "router-sig-ed25519 xNkIgy3gYoENUDMvMvPj1/qPv4suyODE8PcVNLZpY8/WxKvoniT" + + "+2UsWvKsZVAZwFnq7kSByBJUGdxC3YdhSCA", + "router-signature", + "-----BEGIN SIGNATURE-----", + "jwfwSxul/olhO4VzJfBTg+KQf4G+nRwFa9XLMSgBTy6P+hqDkw7TE079BZiYb8+v", + "ElS08R1Diq50N8fosR5lqP/Ihhm+V0KcEyWG10+Vl7ADMA3m4GdbGa6dSrdiFMPs", + "OYE9aueVDIMgKyiOyNmgK3S8lwjX4v6yhaiJWxDGuKs=", + "-----END SIGNATURE-----")); + } +} + diff --git a/src/test/java/org/torproject/collector/bridgedescs/NetworkStatusBuilder.java b/src/test/java/org/torproject/collector/bridgedescs/NetworkStatusBuilder.java new file mode 100644 index 0000000..20088a4 --- /dev/null +++ b/src/test/java/org/torproject/collector/bridgedescs/NetworkStatusBuilder.java @@ -0,0 +1,30 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.bridgedescs; + +import java.util.ArrayList; +import java.util.Arrays; + +/** Builds a non-sanitized bridge network status that comes with an original + * bridge network status entry (of a bundled and therefore publicly known + * bridge) by default. */ +class NetworkStatusBuilder extends DescriptorBuilder { + + /** Initializes the descriptor builder. */ + NetworkStatusBuilder() { + this.lines = new ArrayList<>(Arrays.asList( + "published 2016-06-30 23:40:28", + "flag-thresholds stable-uptime=807660 stable-mtbf=1425164 " + + "fast-speed=47000 guard-wfu=98.000% guard-tk=691200 " + + "guard-bw-inc-exits=400000 guard-bw-exc-exits=402000 " + + "enough-mtbf=1 ignoring-advertised-bws=0", + "r MeekGoogle RtSnEZe4+lFagmxrAXxSL+JkZVs " + + "g+M7Ww+lGKmv6NW9GRmvzLOiR0Y 2016-06-30 21:43:52 " + + "198.50.200.131 8008 0", + "s Fast Running Stable Valid", + "w Bandwidth=56", + "p reject 1-65535")); + } +} + diff --git a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java new file mode 100644 index 0000000..0049bb6 --- /dev/null +++ b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java @@ -0,0 +1,521 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.bridgedescs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import org.torproject.collector.Main; +import org.torproject.collector.conf.Configuration; +import org.torproject.collector.conf.ConfigurationException; +import org.torproject.collector.conf.Key; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Tests the bridge descriptor sanitizer by preparing a temporary folder + * with non-sanitized bridge descriptors, running the sanitizer, and + * verifying the sanitized descriptors. */ +public class SanitizedBridgesWriterTest { + + /** Temporary folder containing all files for this test. */ + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + /** Directory containing bridge descriptor tarballs to sanitize. */ + private File bridgeDirectoriesDir; + + /** Directory holding recent descriptor files served by CollecTor. */ + private File recentDirectory; + + /** Directory storing all intermediate state that needs to be preserved + * between processing runs. */ + private File statsDirectory; + + /** Directory holding sanitized bridge descriptor files. */ + private File sanitizedBridgesDirectory; + + /** CollecTor configuration for this test. */ + private Configuration configuration; + + /** Server descriptor builder used to build the first and only server + * descriptor for this test, unless removed from the tarball builder.*/ + private DescriptorBuilder defaultServerDescriptorBuilder; + + /** Extra-info descriptor builder used to build the first and only + * extra-info descriptor for this test, unless removed from the tarball + * builder.*/ + private DescriptorBuilder defaultExtraInfoDescriptorBuilder; + + /** Network status builder used to build the first and only network + * status for this test, unless removed from the tarball builder.*/ + private DescriptorBuilder defaultNetworkStatusBuilder; + + /** Tarball builder to build the first and only tarball, unless removed + * from the test. */ + private TarballBuilder defaultTarballBuilder; + + /** Tarball builder(s) for this test. */ + private List<TarballBuilder> tarballBuilders; + + /** Parsed sanitized bridge descriptors with keys being file names and + * values being sanitized descriptor lines. */ + private Map<String, List<String>> parsedFiles; + + /** Parsed sanitized default server descriptor. */ + private List<List<String>> parsedServerDescriptors; + + /** Parsed sanitized default extra-info descriptor. */ + private List<List<String>> parsedExtraInfoDescriptors; + + /** Parsed sanitized default network status. */ + private List<List<String>> parsedNetworkStatuses; + + /** Prepares the temporary folder and the various builders for this + * test. */ + @Before + public void createTemporaryFolderAndBuilders() + throws IOException { + this.bridgeDirectoriesDir = this.temporaryFolder.newFolder("in"); + this.recentDirectory = this.temporaryFolder.newFolder("recent"); + this.statsDirectory = this.temporaryFolder.newFolder("stats"); + this.sanitizedBridgesDirectory = this.temporaryFolder.newFolder("out"); + this.initializeTestConfiguration(); + this.defaultServerDescriptorBuilder = new ServerDescriptorBuilder(); + this.defaultExtraInfoDescriptorBuilder = new ExtraInfoDescriptorBuilder(); + this.defaultNetworkStatusBuilder = new NetworkStatusBuilder(); + this.defaultTarballBuilder = new TarballBuilder( + "from-tonga-2016-07-01T000702Z.tar.gz", 1467331624000L); + this.defaultTarballBuilder.add("bridge-descriptors", 1467331622000L, + Arrays.asList(new DescriptorBuilder[] { + this.defaultServerDescriptorBuilder })); + this.defaultTarballBuilder.add("cached-extrainfo", 1467327972000L, + Arrays.asList(new DescriptorBuilder[] { + this.defaultExtraInfoDescriptorBuilder })); + this.defaultTarballBuilder.add("cached-extrainfo.new", 1467331623000L, + Arrays.asList(new DescriptorBuilder[] { })); + this.defaultTarballBuilder.add("networkstatus-bridges", + 1467330028000L, Arrays.asList(new DescriptorBuilder[] { + this.defaultNetworkStatusBuilder })); + this.tarballBuilders = new ArrayList<>( + Arrays.asList(this.defaultTarballBuilder)); + } + + /** Initializes a configuration for the bridge descriptor sanitizer. */ + private void initializeTestConfiguration() throws IOException { + this.configuration = new Configuration(); + this.configuration.load(getClass().getClassLoader().getResourceAsStream( + Main.CONF_FILE)); + this.configuration.setProperty(Key.BridgedescsActivated.name(), "true"); + this.configuration.setProperty(Key.RecentPath.name(), + recentDirectory.getAbsolutePath()); + this.configuration.setProperty(Key.StatsPath.name(), + statsDirectory.getAbsolutePath()); + this.configuration.setProperty(Key.BridgeSnapshotsDirectory.name(), + bridgeDirectoriesDir.getAbsolutePath()); + this.configuration.setProperty(Key.SanitizedBridgesWriteDirectory.name(), + sanitizedBridgesDirectory.getAbsolutePath()); + } + + /** Runs this test by executing all builders, performing the sanitizing + * process, and parsing sanitized bridge descriptors for inspection. */ + private void runTest() throws IOException, ConfigurationException { + for (TarballBuilder tarballBuilder : this.tarballBuilders) { + tarballBuilder.build(this.bridgeDirectoriesDir); + } + SanitizedBridgesWriter sbw = new SanitizedBridgesWriter(configuration); + sbw.startProcessing(); + List<File> files = new ArrayList<>(); + files.add(sanitizedBridgesDirectory); + String basePath = sanitizedBridgesDirectory.getAbsolutePath() + "/"; + this.parsedFiles = new LinkedHashMap<>(); + this.parsedServerDescriptors = new ArrayList<>(); + this.parsedExtraInfoDescriptors = new ArrayList<>(); + this.parsedNetworkStatuses = new ArrayList<>(); + while (!files.isEmpty()) { + File file = files.remove(0); + if (file.isDirectory()) { + files.addAll(Arrays.asList(file.listFiles())); + } else { + List<String> parsedLines = new ArrayList<>(); + BufferedReader reader = new BufferedReader(new FileReader(file)); + String line; + while ((line = reader.readLine()) != null) { + parsedLines.add(line); + } + reader.close(); + String relativePath = file.getAbsolutePath().substring( + basePath.length()); + if (parsedLines.get(0).startsWith("@type bridge-server-descriptor ")) { + this.parsedServerDescriptors.add(parsedLines); + } else if (parsedLines.get(0).startsWith("@type bridge-extra-info ")) { + this.parsedExtraInfoDescriptors.add(parsedLines); + } else if (parsedLines.get(0).startsWith( + "@type bridge-network-status ")) { + this.parsedNetworkStatuses.add(parsedLines); + } + this.parsedFiles.put(relativePath, parsedLines); + } + } + } + + @Test + public void testServerDescriptorDefault() throws Exception { + this.runTest(); + List<String> expectedLines = Arrays.asList( + "@type bridge-server-descriptor 1.2", + "router MeekGoogle 127.0.0.1 1 0 0", + "master-key-ed25519 3HC9xnykNIfNdFDuJWwxaJSM5GCaIJKUtAYgMixbsOM", + "platform Tor 0.2.7.6 on Linux", + "protocols Link 1 2 Circuit 1", + "published 2016-06-30 21:43:52", + "fingerprint 88F7 4584 0F47 CE0C 6A4F E61D 8279 50B0 6F9E 4534", + "uptime 543754", + "bandwidth 3040870 5242880 56583", + "extra-info-digest B026CF0F81712D94BBF1362294882688DF247887 " + + "/XWPeILeik+uTGaKL3pnUeQfYS87SfjKVkwTiCmbqi0", + "hidden-service-dir", + "contact somebody", + "ntor-onion-key YjZG5eaQ1gmXvlSMGEBwM7OLswv8AtXZr6ccOnDUKQw=", + "reject *:*", + "router-digest-sha256 " + + "KI4PRYH9rDCDLYPNv9NF53gFy8pJjIpeJ7UkzIGOAnw", + "router-digest B6922FF5C045814DF4BCB72A0D6C9417CFFBD80A"); + assertEquals("Sanitized descriptor does not contain expected lines.", + expectedLines, this.parsedServerDescriptors.get(0)); + assertTrue("Sanitized descriptor file name differs.", + this.parsedFiles.containsKey("2016/06/server-descriptors/b/6/" + + "b6922ff5c045814df4bcb72a0d6c9417cffbd80a")); + } + + @Test + public void testServerDescriptorEmpty() throws Exception { + this.defaultServerDescriptorBuilder.removeAllLines(); + this.runTest(); + assertTrue("No server descriptor provided as input.", + this.parsedServerDescriptors.isEmpty()); + } + + @Test + public void testServerDescriptorAdditionalAnnotation() + throws Exception { + this.defaultServerDescriptorBuilder.insertBeforeLineStartingWith( + "@purpose bridge", Arrays.asList("@source 198.50.200.131")); + this.runTest(); + assertEquals("Expected 3 sanitized descriptors.", 3, + this.parsedFiles.size()); + } + + @Test + public void testServerDescriptorRouterLineTruncated() throws Exception { + this.defaultServerDescriptorBuilder.replaceLineStartingWith("router ", + Arrays.asList("router MeekGoogle")); + this.runTest(); + assertTrue("Sanitized server descriptor with invalid router line.", + this.parsedServerDescriptors.isEmpty()); + } + + @Test + public void testServerDescriptorFingerprintTruncated() throws Exception { + this.defaultServerDescriptorBuilder.replaceLineStartingWith( + "fingerprint ", Arrays.asList("fingerprint 4")); + this.runTest(); + assertTrue("Sanitized server descriptor with invalid fingerprint " + + "line.", this.parsedServerDescriptors.isEmpty()); + } + + @Test + public void testServerDescriptorFingerprintInvalidHex() + throws Exception { + this.defaultServerDescriptorBuilder.replaceLineStartingWith( + "fingerprint ", Arrays.asList("fingerprint FUN!")); + this.runTest(); + assertTrue("Sanitized server descriptor with invalid fingerprint " + + "line.", this.parsedServerDescriptors.isEmpty()); + } + + @Test + public void testServerDescriptorFingerprintOpt() throws Exception { + this.defaultServerDescriptorBuilder.replaceLineStartingWith("fingerprint ", + Arrays.asList("opt fingerprint 46D4 A711 97B8 FA51 5A82 6C6B 017C 522F " + + "E264 655B")); + this.runTest(); + this.parsedServerDescriptors.get(0).contains("opt fingerprint 88F7 " + + "4584 0F47 CE0C 6A4F E61D 8279 50B0 6F9E 4534"); + } + + @Test + public void testServerDescriptorExtraInfoDigestInvalidHex() + throws Exception { + this.defaultServerDescriptorBuilder.replaceLineStartingWith( + "extra-info-digest ", Arrays.asList("extra-info-digest 6")); + this.runTest(); + assertTrue("Sanitized server descriptor with invalid extra-info " + + "line.", this.parsedServerDescriptors.isEmpty()); + } + + @Test + public void testServerDescriptorExtraInfoDigestSha1Only() + throws Exception { + this.defaultServerDescriptorBuilder.replaceLineStartingWith( + "extra-info-digest ", Arrays.asList("extra-info-digest " + + "6D03E80568DEFA102968D144CB35FFA6E3355B8A")); + this.runTest(); + assertTrue("Expected different extra-info-digest line.", + this.parsedServerDescriptors.get(0).contains( + "extra-info-digest B026CF0F81712D94BBF1362294882688DF247887")); + } + + @Test + public void testServerDescriptorExtraInfoDigestOpt() throws Exception { + this.defaultServerDescriptorBuilder.replaceLineStartingWith( + "extra-info-digest ", Arrays.asList("opt extra-info-digest " + + "6D03E80568DEFA102968D144CB35FFA6E3355B8A " + + "cy/LwP7nxukmmcT1+UnDg4qh0yKbjVUYKhGL8VksoJA")); + this.runTest(); + this.parsedServerDescriptors.get(0).contains("opt extra-info-digest " + + "B026CF0F81712D94BBF1362294882688DF247887 " + + "/XWPeILeik+uTGaKL3pnUeQfYS87SfjKVkwTiCmbqi0"); + } + + @Test + public void testServerDescriptorRejectOwnAddress() throws Exception { + this.defaultServerDescriptorBuilder.insertBeforeLineStartingWith( + "reject *:*", Arrays.asList("reject 198.50.200.131:*", "accept *:80")); + this.runTest(); + List<String> parsedLines = this.parsedServerDescriptors.get(0); + for (int i = 0; i < parsedLines.size(); i++) { + if ("reject 127.0.0.1:*".equals(parsedLines.get(i))) { + assertEquals("accept line out of order.", "accept *:80", + parsedLines.get(i + 1)); + assertEquals("reject line out of order.", "reject *:*", + parsedLines.get(i + 2)); + return; + } + } + fail("IP address in reject line should have been replaced: " + + parsedLines); + } + + @Test + public void testServerDescriptorEd25519IdentityMasterKeyMismatch() + throws Exception { + this.defaultServerDescriptorBuilder.replaceLineStartingWith( + "master-key-ed25519 ", Arrays.asList("master-key-ed25519 " + + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")); + this.runTest(); + assertTrue("Mismatch between Ed25519 identity and master key.", + this.parsedServerDescriptors.isEmpty()); + } + + @Test + public void testServerDescriptorFamilyInvalidFingerprint() + throws Exception { + this.defaultServerDescriptorBuilder.insertBeforeLineStartingWith( + "hidden-service-dir", Arrays.asList("family $0")); + this.runTest(); + assertTrue("Sanitized server descriptor with invalid fingerprint in " + + "family line.", this.parsedServerDescriptors.isEmpty()); + } + + @Test + public void testExtraInfoDescriptorDefault() throws Exception { + this.runTest(); + List<String> expectedLines = Arrays.asList( + "@type bridge-extra-info 1.3", + "extra-info MeekGoogle 88F745840F47CE0C6A4FE61D827950B06F9E4534", + "master-key-ed25519 3HC9xnykNIfNdFDuJWwxaJSM5GCaIJKUtAYgMixbsOM", + "published 2016-06-30 21:43:52", + "write-history 2016-06-30 18:40:48 (14400 s) " + + "415744,497664,359424,410624,420864,933888", + "read-history 2016-06-30 18:40:48 (14400 s) " + + "4789248,6237184,4473856,5039104,5567488,5440512", + "geoip-db-digest 6346E26E2BC96F8511588CE2695E9B0339A75D32", + "geoip6-db-digest 43CCB43DBC653D8CC16396A882C5F116A6004F0C", + "dirreq-stats-end 2016-06-30 14:40:48 (86400 s)", + "dirreq-v3-ips ", + "dirreq-v3-reqs ", + "dirreq-v3-resp ok=0,not-enough-sigs=0,unavailable=0,not-found=0," + + "not-modified=0,busy=0", + "dirreq-v3-direct-dl complete=0,timeout=0,running=0", + "dirreq-v3-tunneled-dl complete=0,timeout=0,running=0", + "transport meek", + "transport meek", + "bridge-stats-end 2016-06-30 14:41:18 (86400 s)", + "bridge-ips ", + "bridge-ip-versions v4=0,v6=0", + "bridge-ip-transports ", + "router-digest-sha256 " + + "/XWPeILeik+uTGaKL3pnUeQfYS87SfjKVkwTiCmbqi0", + "router-digest B026CF0F81712D94BBF1362294882688DF247887"); + assertEquals("Sanitized descriptor does not contain expected lines.", + expectedLines, this.parsedExtraInfoDescriptors.get(0)); + assertTrue("Sanitized descriptor file name differs.", + this.parsedFiles.containsKey("2016/06/extra-infos/b/0/" + + "b026cf0f81712d94bbf1362294882688df247887")); + } + + @Test + public void testExtraInfoDescriptorExtraInfoInvalidHex() + throws Exception { + this.defaultExtraInfoDescriptorBuilder.replaceLineStartingWith( + "extra-info ", Arrays.asList("extra-info MeekGoogle 4")); + this.runTest(); + assertTrue("Sanitized extra-info descriptor with invalid extra-info " + + "line.", this.parsedExtraInfoDescriptors.isEmpty()); + } + + @Test + public void testExtraInfoDescriptorRouterSignatureLineSpace() + throws Exception { + this.defaultExtraInfoDescriptorBuilder.replaceLineStartingWith( + "router-signature", Arrays.asList("router-signature ")); + this.runTest(); + assertTrue("Sanitized extra-info descriptor with invalid " + + "router-signature line.", + this.parsedExtraInfoDescriptors.isEmpty()); + } + + @Test + public void testNetworkStatusDefault() throws Exception { + this.runTest(); + List<String> expectedLines = Arrays.asList( + "@type bridge-network-status 1.1", + "published 2016-06-30 23:40:28", + "flag-thresholds stable-uptime=807660 stable-mtbf=1425164 " + + "fast-speed=47000 guard-wfu=98.000% guard-tk=691200 " + + "guard-bw-inc-exits=400000 guard-bw-exc-exits=402000 " + + "enough-mtbf=1 ignoring-advertised-bws=0", + "r MeekGoogle iPdFhA9HzgxqT+YdgnlQsG+eRTQ " + + "tpIv9cBFgU30vLcqDWyUF8/72Ao 2016-06-30 21:43:52 127.0.0.1 " + + "1 0", + "s Fast Running Stable Valid", + "w Bandwidth=56", + "p reject 1-65535"); + assertEquals("Sanitized descriptor does not contain expected lines.", + expectedLines, this.parsedNetworkStatuses.get(0)); + assertTrue("Sanitized descriptor file name differs.", + this.parsedFiles.containsKey("2016/06/statuses/30/" + + "20160630-234028-4A0CCD2DDC7995083D73F5D667100C8A5831F16D")); + } + + @Test + public void testNetworkStatusPublishedLineMissing() throws Exception { + this.defaultNetworkStatusBuilder.removeLine( + "published 2016-06-30 23:40:28"); + this.runTest(); + String sanitizedNetworkStatusFileName = "2016/07/statuses/01/" + + "20160701-000702-4A0CCD2DDC7995083D73F5D667100C8A5831F16D"; + assertTrue("Sanitized descriptor file does contain published line.", + this.parsedFiles.get(sanitizedNetworkStatusFileName) + .contains("published 2016-07-01 00:07:02")); + } + + @Test + public void testNetworkStatusPublishedLineMissingTarballFileNameChange() + throws Exception { + this.defaultNetworkStatusBuilder.removeLine( + "published 2016-06-30 23:40:28"); + this.defaultTarballBuilder.setTarballFileName( + "from-tonga-with-love-2016-07-01T000702Z.tar.gz"); + this.runTest(); + assertTrue("Sanitized network status without published line and with " + + "changed tarball file name.", this.parsedNetworkStatuses.isEmpty()); + } + + @Test + public void testNetworkStatusAlinePortMissing() throws Exception { + this.configuration.setProperty(Key.ReplaceIpAddressesWithHashes.name(), + "true"); + this.defaultNetworkStatusBuilder.insertBeforeLineStartingWith("s ", + Arrays.asList("a 198.50.200.132")); + this.runTest(); + for (String line : this.parsedNetworkStatuses.get(0)) { + if (line.startsWith("a ")) { + fail("Sanitized a line without port: " + line); + } + } + } + + @Test + public void testTarballContainsSameFileTwice() throws Exception { + this.defaultTarballBuilder.add("cached-extrainfo.new", 1467331623000L, + Arrays.asList(new DescriptorBuilder[] { + this.defaultExtraInfoDescriptorBuilder })); + } + + @Test + public void testTarballCorrupt() throws Exception { + this.tarballBuilders.clear(); + File tarballFile = new File(bridgeDirectoriesDir, + "from-tonga-2016-07-01T000702Z.tar.gz"); + FileOutputStream stream = new FileOutputStream(tarballFile); + stream.write(new byte[] { 0x00 }); + stream.close(); + tarballFile.setLastModified(1467331624000L); + this.runTest(); + assertTrue("Sanitized descriptors from corrupt tarball.", + this.parsedFiles.isEmpty()); + } + + @Test + public void testTarballUncompressed() throws Exception { + String tarballFileName = this.tarballBuilders.get(0).getTarballFileName(); + this.tarballBuilders.get(0).setTarballFileName( + tarballFileName.substring(0, tarballFileName.length() - 3)); + this.runTest(); + assertEquals("Expected 3 sanitized descriptors from uncompressed " + + "tarball.", 3, this.parsedFiles.size()); + } + + @Test + public void testTarballBzip2Compressed() throws Exception { + String tarballFileName = this.tarballBuilders.get(0).getTarballFileName(); + this.tarballBuilders.get(0).setTarballFileName( + tarballFileName.substring(0, tarballFileName.length() - 3) + ".bz2"); + this.runTest(); + assertTrue("Didn't expect sanitized descriptors from unsupported " + + "bz2-compressed tarball.", this.parsedFiles.isEmpty()); + } + + @Test + public void testParsedBridgeDirectoriesSkipTarball() throws Exception { + File parsedBridgeDirectoriesFile = new File(statsDirectory, + "parsed-bridge-directories"); + BufferedWriter writer = new BufferedWriter(new FileWriter( + parsedBridgeDirectoriesFile)); + writer.write(this.tarballBuilders.get(0).getTarballFileName() + "\n"); + writer.close(); + this.runTest(); + assertTrue("Didn't expect sanitized descriptors after skipping " + + "tarball.", this.parsedFiles.isEmpty()); + } + + @Test + public void testParsedBridgeDirectoriesIsDirectory() throws Exception { + File parsedBridgeDirectoriesFile = new File(statsDirectory, + "parsed-bridge-directories"); + parsedBridgeDirectoriesFile.mkdirs(); + this.runTest(); + assertTrue("Continued despite not being able to read " + + "parsed-bridge-directories.", this.parsedFiles.isEmpty()); + } +} + diff --git a/src/test/java/org/torproject/collector/bridgedescs/ServerDescriptorBuilder.java b/src/test/java/org/torproject/collector/bridgedescs/ServerDescriptorBuilder.java new file mode 100644 index 0000000..0b5e916 --- /dev/null +++ b/src/test/java/org/torproject/collector/bridgedescs/ServerDescriptorBuilder.java @@ -0,0 +1,74 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.bridgedescs; + +import java.util.ArrayList; +import java.util.Arrays; + +/** Builds a non-sanitized bridge server descriptor that comes with an original + * bridge descriptor (of a bundled and therefore publicly known bridge) by + * default. */ +class ServerDescriptorBuilder extends DescriptorBuilder { + + /** Initializes the descriptor builder. */ + ServerDescriptorBuilder() { + this.lines = new ArrayList<>(Arrays.asList( + "@purpose bridge", + "router MeekGoogle 198.50.200.131 8008 0 0", + "identity-ed25519", + "-----BEGIN ED25519 CERT-----", + "AQQABjliAVz1pof1ijauJttRPRlhPc4GKgp7SWOtFsnvSA3ddIsIAQAgBAA6BoYk", + "ZEXE7RkiEJ1l5Ij9hc9TJOpM7/9XSPZnF/PbMfE0u3n3JbOO3s82GN6BPuA0v2Cs", + "5eSvciL7+38Ok2eCaMa6vDrXYUSKrN+z9Kz3feL/XDWQy9L9Tkm7bticng0=", + "-----END ED25519 CERT-----", + "master-key-ed25519 " + + "OgaGJGRFxO0ZIhCdZeSI/YXPUyTqTO//V0j2Zxfz2zE", + "platform Tor 0.2.7.6 on Linux", + "protocols Link 1 2 Circuit 1", + "published 2016-06-30 21:43:52", + "fingerprint 46D4 A711 97B8 FA51 5A82 6C6B 017C 522F E264 655B", + "uptime 543754", + "bandwidth 3040870 5242880 56583", + "extra-info-digest 6D03E80568DEFA102968D144CB35FFA6E3355B8A " + + "cy/LwP7nxukmmcT1+UnDg4qh0yKbjVUYKhGL8VksoJA", + "onion-key", + "-----BEGIN RSA PUBLIC KEY-----", + "MIGJAoGBANcIfT+XV4HHSWEQPGkID0C4OgWQ3Gc/RmfQYLMPe5enDNSLBTstw4ep", + "aiScHB1xhN8xRhpVB/qaCcYGpmUltIH0NaWQ3tuRV7rw+fp7amfYZfThUk5OPpF0", + "soGd3jRrzX7SEm4YCGdLZALL51Wb2pdOmR93WucOZYav/tGs/d9rAgMBAAE=", + "-----END RSA PUBLIC KEY-----", + "signing-key", + "-----BEGIN RSA PUBLIC KEY-----", + "MIGJAoGBAMHXuK8J+5028rDovbEejPrsOJKWtsj7fr4EhMmOmIUM4N2gLdEVyFq7", + "sVkHZFf3v04PmOhSJymLmVVcXe+Qsb4U300DwADvnpeFjhU4trrFqZljZM5+gPW6", + "ZmK0ViD54td0biJZd9Ow65Od9XzbJTa2acO/sVXD0Q8tIfnEywvZAgMBAAE=", + "-----END RSA PUBLIC KEY-----", + "onion-key-crosscert", + "-----BEGIN CROSSCERT-----", + "zQvq4eQYXn9Y2St2Qch4AvwqPAJ+Y+MgTFTf4qYaQ04FXo1csf2eSPB5zbWaUgBb", + "GbtKaw1ZJJjEtVzk/HnIWQ/V/ONJUSL4BiF2M4RuhozJoK2BGpYfmcsGWQKeLcPi", + "YIVtO5OI2XcvxgGGVz4ZPPiiGDFJ2MxHA1747KnGSo8=", + "-----END CROSSCERT-----", + "ntor-onion-key-crosscert 1", + "-----BEGIN ED25519 CERT-----", + "AQoABjjOAToGhiRkRcTtGSIQnWXkiP2Fz1Mk6kzv/1dI9mcX89sxAA0280fSYhvB", + "Y39F6J5FuCFcE/B1KDZZP8zY3NYAP4y+jVTG82RRsN87hwZlyShoBxm2q3x4LNPl", + "67ZGbPdAUAA=", + "-----END ED25519 CERT-----", + "hidden-service-dir", + "contact jvictors at jessevictors com, PGP 0xC20BEC80, BTC " + + "1M6tuPXNmhbgSfaJqnxBUAf5tKi4TVhup8", + "ntor-onion-key YjZG5eaQ1gmXvlSMGEBwM7OLswv8AtXZr6ccOnDUKQw=", + "reject *:*", + "router-sig-ed25519 ObNigP8q0HkRDaDPP84txWH+3TbuL8DUNLAF25OZV9uRovF9j02" + + "StDVEdLGR6vOx9lRos0/264n42jEHzmUbBg", + "router-signature", + "-----BEGIN SIGNATURE-----", + "u/J/T0w7JlH4yUbXcg5hDIVBzGZtXxoH+800zOJXIxbIEGqgTxOhA13C6s/j/C0G", + "+L6bcrNdqKriJERsJicT2UqVRiIl54c76J9ySsknNKvXuEbZ3RJ71FhzLbi5CQXJ", + "N5wdZX+AqHSnSe+ayaB3zVlp97gUbFhg3vE2eWPtRxY=", + "-----END SIGNATURE-----")); + } +} + diff --git a/src/test/java/org/torproject/collector/bridgedescs/TarballBuilder.java b/src/test/java/org/torproject/collector/bridgedescs/TarballBuilder.java new file mode 100644 index 0000000..eddfb2b --- /dev/null +++ b/src/test/java/org/torproject/collector/bridgedescs/TarballBuilder.java @@ -0,0 +1,108 @@ +/* Copyright 2016 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.collector.bridgedescs; + +import static org.junit.Assert.fail; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Builds a tarball containing non-sanitized bridge descriptors built using + * descriptor builders and writes the tarball to a new file with the given file + * name. */ +class TarballBuilder { + + /** Internal helper class to store details about a file contained in the + * tarball. */ + private class TarballFile { + + /** Last modified time of the file. */ + private long modifiedMillis; + + /** Descriptor builders used to generate the file content. */ + private List<DescriptorBuilder> descriptorBuilders; + } + + /** File name of the tarball. */ + private String tarballFileName; + + String getTarballFileName() { + return tarballFileName; + } + + void setTarballFileName(String tarballFileName) { + this.tarballFileName = tarballFileName; + } + + /** Last modified time of the tarball file. */ + private long modifiedMillis; + + /** Files contained in the tarball. */ + private Map<String, TarballFile> tarballFiles; + + /** Initializes a new tarball builder that is going to write a tarball to the + * file with given file name and last-modified time. */ + TarballBuilder(String tarballFileName, long modifiedMillis) { + this.tarballFileName = tarballFileName; + this.modifiedMillis = modifiedMillis; + this.tarballFiles = new LinkedHashMap<>(); + } + + /** Adds a new file to the tarball with given name, last-modified time, and + * descriptor builders to generate the file content. */ + TarballBuilder add(String fileName, long modifiedMillis, + List<DescriptorBuilder> descriptorBuilders) throws IOException { + TarballFile file = new TarballFile(); + file.modifiedMillis = modifiedMillis; + file.descriptorBuilders = descriptorBuilders; + this.tarballFiles.put(fileName, file); + return this; + } + + /** Writes the previously configured tarball with all contained files to the + * given file, or fail if the file extension is not known. */ + void build(File directory) throws IOException { + File tarballFile = new File(directory, this.tarballFileName); + TarArchiveOutputStream taos = null; + if (this.tarballFileName.endsWith(".tar.gz")) { + taos = new TarArchiveOutputStream(new GzipCompressorOutputStream( + new BufferedOutputStream(new FileOutputStream(tarballFile)))); + } else if (this.tarballFileName.endsWith(".tar.bz2")) { + taos = new TarArchiveOutputStream(new BZip2CompressorOutputStream( + new BufferedOutputStream(new FileOutputStream(tarballFile)))); + } else if (this.tarballFileName.endsWith(".tar")) { + taos = new TarArchiveOutputStream(new BufferedOutputStream( + new FileOutputStream(tarballFile))); + } else { + fail("Unknown file extension: " + this.tarballFileName); + } + for (Map.Entry<String, TarballFile> file : this.tarballFiles.entrySet()) { + TarArchiveEntry tae = new TarArchiveEntry(file.getKey()); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + for (DescriptorBuilder descriptorBuilder + : file.getValue().descriptorBuilders) { + descriptorBuilder.build(baos); + } + tae.setSize(baos.size()); + tae.setModTime(file.getValue().modifiedMillis); + taos.putArchiveEntry(tae); + taos.write(baos.toByteArray()); + taos.closeArchiveEntry(); + } + taos.close(); + tarballFile.setLastModified(this.modifiedMillis); + } +} +