commit 46ab6609854e1e2f0c3d55a52f9ada1c2dd59793 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Fri Aug 26 12:23:22 2016 +0200 Add support for Bifroest's bridge descriptor tarballs. Implements #20037. --- CHANGELOG.md | 3 ++ .../bridgedescs/BridgeDescriptorParser.java | 5 +-- .../bridgedescs/BridgeSnapshotReader.java | 36 +++++++++++++++++++--- .../bridgedescs/SanitizedBridgesWriter.java | 6 ++-- 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8be9e2..cf1ea12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changes in version 1.1.0 - 2016-08-31 + * Medium changes + - Add support for Bifroest's bridge descriptor tarballs. + * Minor changes - Remove quotes around base URL in index.json. diff --git a/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java b/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java index 511b171..d40052e 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java +++ b/src/main/java/org/torproject/collector/bridgedescs/BridgeDescriptorParser.java @@ -27,7 +27,7 @@ public class BridgeDescriptorParser { /** Parses the first line of the given descriptor data to determine the * descriptor type and passes it to the sanitized bridges writer. */ - public void parse(byte[] allData, String dateTime) + public void parse(byte[] allData, String dateTime, String authorityFingerprint) throws ConfigurationException { try { BufferedReader br = new BufferedReader(new StringReader( @@ -45,7 +45,8 @@ public class BridgeDescriptorParser { } } else { if (this.sbw != null) { - this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime); + this.sbw.sanitizeAndStoreNetworkStatus(allData, dateTime, + authorityFingerprint); } } } catch (IOException e) { diff --git a/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java b/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java index 25bb2ea..046db7d 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java +++ b/src/main/java/org/torproject/collector/bridgedescs/BridgeSnapshotReader.java @@ -100,9 +100,35 @@ public class BridgeSnapshotReader { } BufferedInputStream bis = new BufferedInputStream(tais); String fn = pop.getName(); - String dateTime = fn.substring(11, 21) + " " - + fn.substring(22, 24) + ":" + fn.substring(24, 26) - + ":" + fn.substring(26, 28); + String[] fnParts = fn.split("-"); + if (fnParts.length != 5) { + logger.warn("Invalid bridge descriptor tarball file name: " + + fn + ". Skipping."); + continue; + } + String authorityPart = String.format("%s-%s-", fnParts[0], + fnParts[1]); + String datePart = String.format("%s-%s-%s", fnParts[2], + fnParts[3], fnParts[4]); + String authorityFingerprint; + switch (authorityPart) { + case "from-tonga-": + authorityFingerprint = + "4A0CCD2DDC7995083D73F5D667100C8A5831F16D"; + break; + case "from-bifroest-": + authorityFingerprint = + "1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1"; + break; + default: + logger.warn("Did not recognize the bridge authority that " + + "generated " + fn + ". Skipping."); + continue; + } + String dateTime = datePart.substring(0, 10) + " " + + datePart.substring(11, 13) + ":" + + datePart.substring(13, 15) + ":" + + datePart.substring(15, 17); while ((tais.getNextTarEntry()) != null) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); int len; @@ -130,7 +156,7 @@ public class BridgeSnapshotReader { if (firstLine.startsWith("published ") || firstLine.startsWith("flag-thresholds ") || firstLine.startsWith("r ")) { - bdp.parse(allData, dateTime); + bdp.parse(allData, dateTime, authorityFingerprint); parsedStatuses++; } else if (descriptorImportHistory.contains(fileDigest)) { /* Skip server descriptors or extra-info descriptors if @@ -167,7 +193,7 @@ public class BridgeSnapshotReader { DigestUtils.sha(descBytes)); if (!descriptorImportHistory.contains( descriptorDigest)) { - bdp.parse(descBytes, dateTime); + bdp.parse(descBytes, dateTime, authorityFingerprint); descriptorImportHistory.add(descriptorDigest); if (firstLine.startsWith("router ")) { parsedServerDescriptors++; diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java index dcac024..b787f78 100644 --- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java @@ -379,7 +379,8 @@ public class SanitizedBridgesWriter extends CollecTorMain { * Sanitizes a network status and writes it to disk. */ public void sanitizeAndStoreNetworkStatus(byte[] data, - String publicationTime) throws ConfigurationException { + String publicationTime, String authorityFingerprint) + throws ConfigurationException { if (this.persistenceProblemWithSecrets) { /* There's a persistence problem, so we shouldn't scrub more IP @@ -546,8 +547,7 @@ public class SanitizedBridgesWriter extends CollecTorMain { File tarballFile = new File( this.sanitizedBridgesDirectory.getAbsolutePath() + "/" + syear + "/" + smonth + "/statuses/" + sday + "/" + syear + smonth - + sday + "-" + stime + "-" - + "4A0CCD2DDC7995083D73F5D667100C8A5831F16D"); + + sday + "-" + stime + "-" + authorityFingerprint); File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(), "bridge-descriptors/statuses/" + tarballFile.getName()); File[] outputFiles = new File[] { tarballFile, rsyncFile };