[tor-commits] [metrics-db/master] Sanitize IPv6 bridge addresses.
karsten at torproject.org
karsten at torproject.org
Mon Jan 16 17:07:59 UTC 2012
commit 70a3d9989d62550d28967aab17ab078a91097991
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Mon Jan 16 18:06:15 2012 +0100
Sanitize IPv6 bridge addresses.
---
.../ernie/db/SanitizedBridgesWriter.java | 233 ++++++++++++++++----
1 files changed, 189 insertions(+), 44 deletions(-)
diff --git a/src/org/torproject/ernie/db/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
index a3ead19..7e3055c 100644
--- a/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
+++ b/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
@@ -229,8 +229,10 @@ public class SanitizedBridgesWriter {
this.bridgeIpSecretsFile));
String line;
while ((line = br.readLine()) != null) {
- if (line.length() != ("yyyy-MM,".length() + 31 * 2) ||
- line.split(",").length != 2) {
+ String[] parts = line.split(",");
+ if ((line.length() != ("yyyy-MM,".length() + 31 * 2) &&
+ line.length() != ("yyyy-MM,".length() + 50 * 2)) ||
+ parts.length != 2) {
this.logger.warning("Invalid line in bridge-ip-secrets file "
+ "starting with '" + line.substring(0, 7) + "'! "
+ "Not calculating any IP address hashes in this "
@@ -238,7 +240,6 @@ public class SanitizedBridgesWriter {
this.persistenceProblemWithSecrets = true;
break;
}
- String[] parts = line.split(",");
String month = parts[0];
byte[] secret = Hex.decodeHex(parts[1].toCharArray());
this.secretsForHashingIPAddresses.put(month, secret);
@@ -315,7 +316,28 @@ public class SanitizedBridgesWriter {
}
}
- private String scrubAddress(String address, byte[] fingerprintBytes,
+ private String scrubOrAddress(String orAddress, byte[] fingerprintBytes,
+ String published) throws IOException {
+ if (!orAddress.contains(":")) {
+ /* Malformed or-address or a line. */
+ return null;
+ }
+ String addressPart = orAddress.substring(0,
+ orAddress.lastIndexOf(":"));
+ String portPart = orAddress.substring(orAddress.lastIndexOf(":") + 1);
+ String scrubbedAddressPart = null;
+ if (addressPart.startsWith("[")) {
+ scrubbedAddressPart = this.scrubIpv6Address(addressPart,
+ fingerprintBytes, published);
+ } else {
+ scrubbedAddressPart = this.scrubIpv6Address(addressPart,
+ fingerprintBytes, published);
+ }
+ return (scrubbedAddressPart == null ? null :
+ scrubbedAddressPart + ":" + portPart);
+ }
+
+ private String scrubIpv4Address(String address, byte[] fingerprintBytes,
String published) throws IOException {
if (this.replaceIPAddressesWithHashes) {
if (this.persistenceProblemWithSecrets) {
@@ -330,37 +352,7 @@ public class SanitizedBridgesWriter {
}
System.arraycopy(fingerprintBytes, 0, hashInput, 4, 20);
String month = published.substring(0, "yyyy-MM".length());
- if (!this.secretsForHashingIPAddresses.containsKey(month)) {
- byte[] secret = new byte[31];
- this.secureRandom.nextBytes(secret);
- if (month.compareTo(
- this.bridgeDescriptorMappingsCutOffTimestamp) < 0) {
- this.logger.warning("Generated a secret that we won't make "
- + "persistent, because it's outside our bridge descriptors "
- + "mapping interval.");
- } else {
- /* Append secret to file on disk immediately before using it, or
- * we might end with inconsistently sanitized bridges. */
- try {
- if (!this.bridgeIpSecretsFile.exists()) {
- this.bridgeIpSecretsFile.getParentFile().mkdirs();
- }
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.bridgeIpSecretsFile,
- this.bridgeIpSecretsFile.exists()));
- bw.write(month + "," + Hex.encodeHexString(secret) + "\n");
- bw.close();
- } catch (IOException e) {
- this.logger.log(Level.WARNING, "Could not store new secret "
- + "to disk! Not calculating any IP address hashes in "
- + "this execution!", e);
- this.persistenceProblemWithSecrets = true;
- throw new IOException(e);
- }
- }
- this.secretsForHashingIPAddresses.put(month, secret);
- }
- byte[] secret = this.secretsForHashingIPAddresses.get(month);
+ byte[] secret = this.getSecretForMonth(month);
System.arraycopy(secret, 0, hashInput, 24, 31);
byte[] hashOutput = DigestUtils.sha256(hashInput);
String hashedAddress = "10."
@@ -373,6 +365,119 @@ public class SanitizedBridgesWriter {
}
}
+ private String scrubIpv6Address(String address, byte[] fingerprintBytes,
+ String published) throws IOException {
+ StringBuilder sb = new StringBuilder("[fd9f:2e19:3bcf::");
+ if (this.replaceIPAddressesWithHashes) {
+ if (this.persistenceProblemWithSecrets) {
+ /* There's a persistence problem, so we shouldn't scrub more IP
+ * addresses in this execution. */
+ return null;
+ }
+ byte[] hashInput = new byte[16 + 20 + 19];
+ StringBuilder hex = new StringBuilder();
+ String[] parts = address.substring(1, address.length() - 1).
+ split(":", -1);
+ if (parts.length < 1 || parts.length > 8) {
+ /* Invalid IPv6 address. */
+ return null;
+ }
+ for (int i = 0; i < parts.length; i++) {
+ String part = parts[i];
+ if (part.contains(".")) {
+ String[] ipParts = part.split("\\.");
+ byte[] ipv4Bytes = new byte[4];
+ if (ipParts.length != 4) {
+ /* Invalid IPv4 part in IPv6 address. */
+ return null;
+ }
+ for (int m = 0; m < 4; m++) {
+ ipv4Bytes[m] = (byte) Integer.parseInt(ipParts[m]);
+ }
+ hex.append(Hex.encodeHexString(ipv4Bytes));
+ } else if (part.length() > 4) {
+ /* Invalid IPv6 address. */
+ return null;
+ } else if (part.length() < 1) {
+ int j = parts.length - 1;
+ if (address.contains(".")) {
+ j++;
+ }
+ for (; j < 8; j++) {
+ hex.append("0000");
+ }
+ } else {
+ for (int k = part.length(); k < 4; k++) {
+ hex.append("0");
+ }
+ hex.append(part);
+ }
+ }
+ byte[] ipBytes = null;
+ try {
+ ipBytes = Hex.decodeHex(hex.toString().toCharArray());
+ } catch (DecoderException e) {
+ /* TODO Invalid IPv6 address. */
+ return null;
+ }
+ if (ipBytes.length != 16) {
+ /* TODO Invalid IPv6 address. */
+ return null;
+ }
+ System.arraycopy(ipBytes, 0, hashInput, 0, 16);
+ System.arraycopy(fingerprintBytes, 0, hashInput, 16, 20);
+ String month = published.substring(0, "yyyy-MM".length());
+ byte[] secret = this.getSecretForMonth(month);
+ System.arraycopy(secret, 31, hashInput, 36, 19);
+ String hashOutput = DigestUtils.sha256Hex(hashInput);
+ sb.append(hashOutput.substring(hashOutput.length() - 6,
+ hashOutput.length() - 4));
+ sb.append(":");
+ sb.append(hashOutput.substring(hashOutput.length() - 4));
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
+ private byte[] getSecretForMonth(String month) throws IOException {
+ if (!this.secretsForHashingIPAddresses.containsKey(month) ||
+ this.secretsForHashingIPAddresses.get(month).length == 31) {
+ byte[] secret = new byte[50];
+ this.secureRandom.nextBytes(secret);
+ if (this.secretsForHashingIPAddresses.containsKey(month)) {
+ System.arraycopy(this.secretsForHashingIPAddresses.get(month), 0,
+ secret, 0, 31);
+ }
+ if (month.compareTo(
+ this.bridgeDescriptorMappingsCutOffTimestamp) < 0) {
+ this.logger.warning("Generated a secret that we won't make "
+ + "persistent, because it's outside our bridge descriptors "
+ + "mapping interval.");
+ } else {
+ /* Append secret to file on disk immediately before using it, or
+ * we might end with inconsistently sanitized bridges. */
+ try {
+ if (!this.bridgeIpSecretsFile.exists()) {
+ this.bridgeIpSecretsFile.getParentFile().mkdirs();
+ }
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.bridgeIpSecretsFile,
+ this.bridgeIpSecretsFile.exists()));
+ bw.write(month + "," + Hex.encodeHexString(secret) + "\n");
+ bw.close();
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not store new secret "
+ + "to disk! Not calculating any IP address hashes in "
+ + "this execution!", e);
+ this.persistenceProblemWithSecrets = true;
+ throw new IOException(e);
+ }
+ }
+ this.secretsForHashingIPAddresses.put(month, secret);
+ }
+ return this.secretsForHashingIPAddresses.get(month);
+ }
+
/**
* Sanitizes a network status and writes it to disk. Processes every r
* line separately and looks up whether the descriptor mapping contains
@@ -404,6 +509,8 @@ public class SanitizedBridgesWriter {
data, "US-ASCII")));
String line = null;
String mostRecentDescPublished = null;
+ byte[] fingerprintBytes = null;
+ String descPublicationTime = null;
while ((line = br.readLine()) != null) {
/* r lines contain sensitive information that needs to be removed
@@ -412,8 +519,8 @@ public class SanitizedBridgesWriter {
/* Parse the relevant parts of this r line. */
String[] parts = line.split(" ");
- String bridgeIdentity = parts[2];
- String descPublicationTime = parts[4] + " " + parts[5];
+ fingerprintBytes = Base64.decodeBase64(parts[2] + "==");
+ descPublicationTime = parts[4] + " " + parts[5];
String address = parts[6];
String orPort = parts[7];
String dirPort = parts[8];
@@ -421,8 +528,7 @@ public class SanitizedBridgesWriter {
/* Look up the descriptor in the descriptor mapping, or add a
* new mapping entry if there is none. */
String hashedBridgeIdentityHex = Hex.encodeHexString(
- DigestUtils.sha(Base64.decodeBase64(bridgeIdentity
- + "=="))).toLowerCase();
+ DigestUtils.sha(fingerprintBytes)).toLowerCase();
String mappingKey = hashedBridgeIdentityHex + ","
+ descPublicationTime;
DescriptorMapping mapping = null;
@@ -443,15 +549,13 @@ public class SanitizedBridgesWriter {
/* Write scrubbed r line to buffer. */
String hashedBridgeIdentityBase64 = Base64.encodeBase64String(
- DigestUtils.sha(Base64.decodeBase64(bridgeIdentity
- + "=="))).substring(0, 27);
+ DigestUtils.sha(fingerprintBytes)).substring(0, 27);
String sdi = Base64.encodeBase64String(Hex.decodeHex(
mapping.serverDescriptorIdentifier.toCharArray())).
substring(0, 27);
String scrubbedAddress = null;
try {
- scrubbedAddress = scrubAddress(address,
- Base64.decodeBase64(bridgeIdentity + "=="),
+ scrubbedAddress = scrubIpv4Address(address, fingerprintBytes,
descPublicationTime);
} catch (IOException e) {
return;
@@ -466,6 +570,19 @@ public class SanitizedBridgesWriter {
+ descPublicationTime + " " + scrubbedAddress + " "
+ orPort + " " + dirPort + "\n");
+ /* Sanitize any addresses in a lines using the fingerprint and
+ * descriptor publication time from the previous r line. */
+ } else if (line.startsWith("a ")) {
+ String scrubbedOrAddress = scrubOrAddress(
+ line.substring("a ".length()), fingerprintBytes,
+ descPublicationTime);
+ if (scrubbedOrAddress != null) {
+ scrubbed.append("a " + scrubbedOrAddress + "\n");
+ } else {
+ this.logger.warning("Invalid address in line '" + line
+ + "' in bridge network status. Skipping line!");
+ }
+
/* Nothing special about s, w, and p lines; just copy them. */
} else if (line.startsWith("s ") || line.equals("s") ||
line.startsWith("w ") || line.equals("w") ||
@@ -574,6 +691,7 @@ public class SanitizedBridgesWriter {
StringBuilder scrubbed = new StringBuilder();
String line = null, hashedBridgeIdentity = null, address = null,
published = null, routerLine = null, scrubbedAddress = null;
+ List<String> orAddresses = null, scrubbedOrAddresses = null;
boolean skipCrypto = false;
while ((line = br.readLine()) != null) {
@@ -604,6 +722,14 @@ public class SanitizedBridgesWriter {
address = line.split(" ")[2];
routerLine = line;
+ /* Store or-address parts in a list and sanitize them when we have
+ * read the fingerprint. */
+ } else if (line.startsWith("or-address ")) {
+ if (orAddresses == null) {
+ orAddresses = new ArrayList<String>();
+ }
+ orAddresses.add(line.substring("or-address ".length()));
+
/* Parse the publication time and add it to the list of descriptor
* publication times to re-write network statuses at the end of
* the sanitizing procedure. */
@@ -632,8 +758,21 @@ public class SanitizedBridgesWriter {
hashedBridgeIdentity = DigestUtils.shaHex(fingerprintBytes).
toLowerCase();
try {
- scrubbedAddress = scrubAddress(address, fingerprintBytes,
+ scrubbedAddress = scrubIpv4Address(address, fingerprintBytes,
published);
+ if (orAddresses != null) {
+ scrubbedOrAddresses = new ArrayList<String>();
+ for (String orAddress : orAddresses) {
+ String scrubbedOrAddress = scrubOrAddress(orAddress,
+ fingerprintBytes, published);
+ if (scrubbedOrAddress != null) {
+ scrubbedOrAddresses.add(scrubbedOrAddress);
+ } else {
+ this.logger.warning("Invalid address in line '" + line
+ + "' in bridge server descriptor. Skipping line!");
+ }
+ }
+ }
} catch (IOException e) {
/* There's a persistence problem, so we shouldn't scrub more
* IP addresses in this execution. */
@@ -656,6 +795,12 @@ public class SanitizedBridgesWriter {
scrubbedDesc = "router Unnamed " + scrubbedAddress + " "
+ routerLineParts[3] + " " + routerLineParts[4] + " "
+ routerLineParts[5] + "\n";
+ if (scrubbedOrAddresses != null) {
+ for (String scrubbedOrAddress : scrubbedOrAddresses) {
+ scrubbedDesc = scrubbedDesc += "or-address "
+ + scrubbedOrAddress + "\n";
+ }
+ }
scrubbedDesc += scrubbed.toString();
break;
More information about the tor-commits
mailing list