commit 2e8049a8fdc0c72ba8136a9886ed3af75896cadc
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Sat Mar 17 08:10:34 2012 +0100
Add parsing and graphing code for #4147.
---
task-4147/.gitignore | 13 ++
task-4147/AnalyzeDifferentExitAddress.java | 246 ++++++++++++++++++++++++++++
task-4147/README | 37 ++++
task-4147/different-exit-address-1.R | 12 ++
task-4147/different-exit-address-2.R | 39 +++++
5 files changed, 347 insertions(+), 0 deletions(-)
diff --git a/task-4147/.gitignore b/task-4147/.gitignore
new file mode 100644
index 0000000..7e9e868
--- /dev/null
+++ b/task-4147/.gitignore
@@ -0,0 +1,13 @@
+in/
+out/
+status/
+*.jar
+*.class
+.classpath
+.project
+.settings/
+*.csv
+*.pdf
+*.png
+*.swp
+
diff --git a/task-4147/AnalyzeDifferentExitAddress.java b/task-4147/AnalyzeDifferentExitAddress.java
new file mode 100644
index 0000000..44984a8
--- /dev/null
+++ b/task-4147/AnalyzeDifferentExitAddress.java
@@ -0,0 +1,246 @@
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.torproject.descriptor.BandwidthHistory;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorFile;
+import org.torproject.descriptor.DescriptorReader;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.ExitList;
+import org.torproject.descriptor.ExitListEntry;
+import org.torproject.descriptor.ExtraInfoDescriptor;
+import org.torproject.descriptor.NetworkStatusEntry;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+
+/* Answer the question what fraction of bytes written by relays with the
+ * Exit flag could have used a different address for exiting than the
+ * relay used for registering in the Tor network. */
+public class AnalyzeDifferentExitAddress {
+ public static void main(String[] args) throws Exception {
+
+ System.out.println(new Date() + " Starting.");
+
+ /* Iterate over extra-info descriptors to learn about bandwidth
+ * histories. Append 15-minute intervals of written bytes to
+ * status/written-bytes/$fingerprint. */
+ System.out.println(new Date() + " Reading in/extra-infos/* ...");
+ DescriptorReader extraInfoReader = DescriptorSourceFactory
+ .createDescriptorReader();
+ extraInfoReader.addDirectory(new File("in/extra-infos"));
+ extraInfoReader.setExcludeFiles(new File(
+ "status/extra-info-history"));
+ Iterator<DescriptorFile> extraInfoFiles =
+ extraInfoReader.readDescriptors();
+ while (extraInfoFiles.hasNext()) {
+ DescriptorFile extraInfoFile = extraInfoFiles.next();
+ if (extraInfoFile.getDescriptors() != null) {
+ for (Descriptor descriptor : extraInfoFile.getDescriptors()) {
+ ExtraInfoDescriptor extraInfoDescriptor =
+ (ExtraInfoDescriptor) descriptor;
+ BandwidthHistory writeHistory = extraInfoDescriptor.
+ getWriteHistory();
+ if (writeHistory == null) {
+ continue;
+ }
+ String fingerprint = extraInfoDescriptor.getFingerprint();
+ File writtenBytesFile = new File("status/written-bytes/"
+ + fingerprint);
+ writtenBytesFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ writtenBytesFile, true));
+ for (Map.Entry<Long, Long> e :
+ writeHistory.getBandwidthValues().entrySet()) {
+ long intervalEndMillis = e.getKey();
+ long bytesWritten = e.getValue();
+ bw.write(String.valueOf(intervalEndMillis) + " "
+ + String.valueOf(bytesWritten) + "\n");
+ }
+ bw.close();
+ }
+ }
+ }
+
+ /* Iterate over exit lists to learn about exit IP addresses. Append
+ * lines to status/exit-addresses/$fingerprint. */
+ System.out.println(new Date() + " Reading in/exit-lists/* ...");
+ DescriptorReader exitListReader =
+ DescriptorSourceFactory.createDescriptorReader();
+ exitListReader.addDirectory(new File("in/exit-lists"));
+ exitListReader.setExcludeFiles(new File("status/exit-list-history"));
+ Iterator<DescriptorFile> exitListFiles =
+ exitListReader.readDescriptors();
+ while (exitListFiles.hasNext()) {
+ DescriptorFile exitListFile = exitListFiles.next();
+ if (exitListFile.getDescriptors() != null) {
+ for (Descriptor descriptor : exitListFile.getDescriptors()) {
+ ExitList exitList = (ExitList) descriptor;
+ if (exitList.getExitListEntries() == null) {
+ continue;
+ }
+ for (ExitListEntry exitListEntry :
+ exitList.getExitListEntries()) {
+ String fingerprint = exitListEntry.getFingerprint();
+ File exitAddressesFile = new File("status/exit-addresses/"
+ + fingerprint);
+ exitAddressesFile.getParentFile().mkdirs();
+ long scanMillis = exitListEntry.getScanMillis();
+ String address = exitListEntry.getExitAddress();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ exitAddressesFile, true));
+ bw.write(String.valueOf(scanMillis) + " " + address + "\n");
+ bw.close();
+ }
+ }
+ }
+ }
+
+ /* Iterate over consensuses to learn about OR addresses of relays with
+ * the Exit flag. Append lines to
+ * status/or-addresses/$fingerprint. */
+ System.out.println(new Date() + " Reading in/consensuses/* ...");
+ DescriptorReader consensusReader =
+ DescriptorSourceFactory.createDescriptorReader();
+ consensusReader.addDirectory(new File("in/consensuses"));
+ consensusReader.setExcludeFiles(new File("status/consensus-history"));
+ Iterator<DescriptorFile> consensusFiles =
+ consensusReader.readDescriptors();
+ while (consensusFiles.hasNext()) {
+ DescriptorFile consensusFile = consensusFiles.next();
+ if (consensusFile.getDescriptors() != null) {
+ for (Descriptor descriptor : consensusFile.getDescriptors()) {
+ RelayNetworkStatusConsensus consensus =
+ (RelayNetworkStatusConsensus) descriptor;
+ if (consensus.getStatusEntries() == null) {
+ continue;
+ }
+ long validAfterMillis = consensus.getValidAfterMillis();
+ for (NetworkStatusEntry statusEntry :
+ consensus.getStatusEntries().values()) {
+ if (!statusEntry.getFlags().contains("Exit")) {
+ continue;
+ }
+ String fingerprint = statusEntry.getFingerprint();
+ File orAddressesFile = new File("status/or-addresses/"
+ + fingerprint);
+ orAddressesFile.getParentFile().mkdirs();
+ String address = statusEntry.getAddress();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ orAddressesFile, true));
+ bw.write(String.valueOf(validAfterMillis) + " " + address
+ + "\n");
+ bw.close();
+ }
+ }
+ }
+ }
+
+ /* Make sure not to overwrite existing results, and prepare writing
+ * results otherwise. */
+ File differentExitAddressFile = new File(
+ "out/different-exit-address.csv");
+ if (differentExitAddressFile.exists()) {
+ return;
+ } else {
+ differentExitAddressFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ differentExitAddressFile));
+ bw.write("timestamp,differentaddress,writtenbytes\n");
+ bw.close();
+ }
+
+ /* Iterate over OR addresses of relays with the Exit flag. */
+ System.out.println(new Date() + " Writing "
+ + "out/different-exit-address.csv ...");
+ for (File orAddressesFile :
+ new File("status/or-addresses").listFiles()) {
+ String fingerprint = orAddressesFile.getName();
+
+ /* For every relay, read OR addresses, bandwidth histories, and
+ * exit addresses to memory. */
+ SortedMap<Long, String> orAddresses = new TreeMap<Long, String>();
+ SortedMap<Long, Long> writtenBytes = new TreeMap<Long, Long>();
+ SortedMap<Long, String> exitAddresses = new TreeMap<Long, String>();
+ String line;
+ BufferedReader br = new BufferedReader(new FileReader(
+ orAddressesFile));
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(" ");
+ long validAfterMillis = Long.parseLong(parts[0]);
+ String address = parts[1];
+ orAddresses.put(validAfterMillis, address);
+ }
+ br.close();
+ File writtenBytesFile = new File("status/written-bytes/"
+ + fingerprint);
+ if (!writtenBytesFile.exists()) {
+ continue;
+ }
+ br = new BufferedReader(new FileReader(writtenBytesFile));
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(" ");
+ long intervalEndMillis = Long.parseLong(parts[0]);
+ long bytes = Long.parseLong(parts[1]);
+ writtenBytes.put(intervalEndMillis, bytes);
+ }
+ br.close();
+ File exitAddressesFile = new File("status/exit-addresses/"
+ + fingerprint);
+ if (exitAddressesFile.exists()) {
+ br = new BufferedReader(new FileReader(exitAddressesFile));
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(" ");
+ long scanMillis = Long.parseLong(parts[0]);
+ String address = parts[1];
+ exitAddresses.put(scanMillis, address);
+ }
+ br.close();
+ }
+
+ /* Go through consensuses containing this relay as Exit relay in
+ * chronological order, sum up written bytes in the hour after the
+ * consensuses' valid-after time, and look up any exit addresses
+ * found up to 23 hours before up to 1 hour after the valid-after
+ * time. */
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ differentExitAddressFile, true));
+ for (Map.Entry<Long, String> e : orAddresses.entrySet()) {
+ long validAfterMillis = e.getKey();
+ String currentOrAddress = e.getValue();
+ long currentWrittenBytes = 0L;
+ for (long currentBytes : writtenBytes.tailMap(validAfterMillis).
+ headMap(validAfterMillis + 1L * 60L * 60L * 1000L).values()) {
+ currentWrittenBytes += currentBytes;
+ }
+ if (currentWrittenBytes < 1L) {
+ continue;
+ }
+ Set<String> currentExitAddresses = new HashSet<String>();
+ for (String currentExitAddress : exitAddresses.tailMap(
+ validAfterMillis - 23L * 60L * 60L * 1000L).headMap(
+ validAfterMillis + 1L * 60L * 60L * 1000L).values()) {
+ if (!currentExitAddress.equals(currentOrAddress)) {
+ currentExitAddresses.add(currentExitAddress);
+ }
+ }
+ boolean usedOtherAddress = !currentExitAddresses.isEmpty();
+ bw.write(String.valueOf(validAfterMillis / 1000L) + ","
+ + (usedOtherAddress ? "TRUE" : "FALSE") + ","
+ + currentWrittenBytes + "\n");
+ }
+ bw.close();
+ }
+
+ System.out.println(new Date() + " Terminating.");
+ }
+}
+
diff --git a/task-4147/README b/task-4147/README
new file mode 100644
index 0000000..1af3f3a
--- /dev/null
+++ b/task-4147/README
@@ -0,0 +1,37 @@
+Answer the question what fraction of bytes written by relays with the Exit
+flag could have used a different address for exiting than the relay used
+for registering in the Tor network.
+==========================================================================
+
+Clone the metrics-lib repository, create the descriptor.jar file, and put
+it in this directory.
+
+Obtain the Apache Commons Codec 1.4 .jar file commons-codec-1.4.jar and
+put it in this directory.
+
+Download the metrics tarballs containing consensuses, extra-info
+descriptors, and exit lists for a common time period. Note that the first
+and last 2 days of the period won't be usable. Extract the tarballs and
+put them in in/consensuses/, in/extra-infos/, and in/exit-lists/ in this
+directory.
+
+Compile the Java class:
+
+ $ javac -cp descriptor.jar AnalyzeDifferentExitAddress.java
+
+Run the Java class:
+
+ $ java -cp descriptor.jar:commons-codec-1.4.jar:. \
+ AnalyzeDifferentExitAddress
+
+In order to re-run parts of the analysis, delete files in status/ or the
+results file in out/.
+
+Aggregate the results using R and ggplot2:
+
+ $ R --slave -f different-exit-address-1.R
+
+Draw graphs using R and ggplot2:
+
+ $ R --slave -f different-exit-address-2.R
+
diff --git a/task-4147/different-exit-address-1.R b/task-4147/different-exit-address-1.R
new file mode 100644
index 0000000..b5f172a
--- /dev/null
+++ b/task-4147/different-exit-address-1.R
@@ -0,0 +1,12 @@
+library(ggplot2)
+d <- read.csv("out/different-exit-address.csv", stringsAsFactors = FALSE)
+d <- data.frame(
+ date = as.Date(as.POSIXct(d$timestamp, origin = "1970-01-01 00:00:00")),
+ differentaddress = d$differentaddress,
+ writtenbytes = d$writtenbytes)
+d <- aggregate(list(writtenbytes = d$writtenbytes),
+ by = list(date = d$date, differentaddress = d$differentaddress),
+ FUN = sum)
+write.csv(d, file = "different-exit-address-aggregate.csv", quote = FALSE,
+ row.names = FALSE)
+
diff --git a/task-4147/different-exit-address-2.R b/task-4147/different-exit-address-2.R
new file mode 100644
index 0000000..15f097e
--- /dev/null
+++ b/task-4147/different-exit-address-2.R
@@ -0,0 +1,39 @@
+library(ggplot2)
+d <- read.csv("different-exit-address-aggregate.csv",
+ stringsAsFactors = FALSE)
+
+# Cut off dates before 2012-02-14, because exit lists were stale
+# Cut off dates after 2012-02-27, because we only imported February data
+# Leaves us with 2 weeks of data; should be fine
+d <- d[d$date >= "2012-02-14" & d$date <= "2012-02-27", ]
+
+a <- aggregate(x = list(writtenbytes = d$writtenbytes),
+ by = list(date = d$date), FUN = sum)
+ggplot(a, aes(x = as.Date(date), y = writtenbytes / 2^20 / 86400)) +
+geom_line() +
+scale_x_date(name = "") +
+scale_y_continuous(name = "MiB/s\n",
+ limits = c(0, max(a$writtenbytes) / 2^20 / 86400)) +
+opts(title = "Bytes written by all relays with the Exit flag\n")
+
+ggplot(d, aes(x = as.Date(date), y = writtenbytes / 2^20 / 86400,
+ colour = differentaddress)) +
+geom_line() +
+scale_x_date(name = "") +
+scale_y_continuous(name = "MiB/s\n",
+ limits = c(0, max(d$writtenbytes / 2^20 / 86400))) +
+opts(title = "Bytes written by all relays with the Exit flag\n")
+
+s <- cast(d, date ~ differentaddress)
+s <- data.frame(date = s$date, fracdifferent = s[, "TRUE"] / s[, "FALSE"])
+ggplot(s, aes(x = as.Date(date), y = fracdifferent)) +
+geom_line() +
+scale_x_date(name = "", format = "%Y-%m-%d") +
+scale_y_continuous(name = "", formatter = "percent", limits = c(0, 1)) +
+opts(title =
+ paste("Fraction of bytes written by relays with the Exit flag\n",
+ "which could have used a different address for exiting\n",
+ "than the relay used for registering in the Tor network\n",
+ sep = ""))
+ggsave("different-exit-address.png", width = 8, height = 5, dpi = 72)
+
