commit 0dddee5316aebf046db58858af81220bdf3fcb6a
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Fri Apr 27 08:50:31 2012 +0200
Update analysis code for #3261.
---
task-3261/.gitignore | 2 +
task-3261/AggregateStats.java | 122 +++++++++
task-3261/AnalyzeDescriptorParts.java | 315 ++++++++++++++++++++++
task-3261/AnalyzeStatsCoverage.java | 478 ---------------------------------
task-3261/ExtractDescriptorParts.java | 172 ++++++++++++
task-3261/README | 38 +++-
task-3261/plot.R | 65 +++++
task-3261/stats-coverage.R | 25 --
8 files changed, 711 insertions(+), 506 deletions(-)
diff --git a/task-3261/.gitignore b/task-3261/.gitignore
index 2bfd23b..5f2b4dc 100644
--- a/task-3261/.gitignore
+++ b/task-3261/.gitignore
@@ -2,6 +2,8 @@
*.png
*.pdf
*.csv
+bridge-network-statuses
+parse-history
in/
temp/
*.jar
diff --git a/task-3261/AggregateStats.java b/task-3261/AggregateStats.java
new file mode 100755
index 0000000..73f7279
--- /dev/null
+++ b/task-3261/AggregateStats.java
@@ -0,0 +1,122 @@
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.text.SimpleDateFormat;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+/* Aggregate half-hourly per-bridge data to daily statistics. */
+public class AggregateStats {
+ public static void main(String[] args) throws Exception {
+
+ /* Read file containing publication times of bridge statuses and count
+ * statuses per day. */
+ SortedMap<String, Long> publishedStatuses =
+ new TreeMap<String, Long>();
+ File statusFile = new File("bridge-network-statuses");
+ if (!statusFile.exists()) {
+ System.err.println(statusFile.getAbsolutePath() + " does not "
+ + "exist. Exiting.");
+ System.exit(1);
+ } else {
+ BufferedReader br = new BufferedReader(new FileReader(statusFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ String date = line.split(" ")[0];
+ if (publishedStatuses.containsKey(date)) {
+ publishedStatuses.put(date, publishedStatuses.get(date) + 1L);
+ } else {
+ publishedStatuses.put(date, 1L);
+ }
+ }
+ }
+
+ /* Aggregate single observations in memory. */
+ SortedMap<String, Map<String, Long>> aggregatedStats =
+ new TreeMap<String, Map<String, Long>>();
+ SortedSet<String> allKeys = new TreeSet<String>();
+ File evalOutFile = new File("eval-out.csv");
+ if (!evalOutFile.exists()) {
+ System.err.println(evalOutFile.getAbsolutePath() + " does not "
+ + "exist. Exiting.");
+ System.exit(1);
+ } else {
+ BufferedReader ebr = new BufferedReader(new FileReader(evalOutFile));
+ String line;
+ while ((line = ebr.readLine()) != null) {
+ String[] parts = line.split(",");
+ String date = parts[0].split(" ")[0];
+ String key = parts[2] + "," + parts[3] + "," + parts[4];
+ allKeys.add(key);
+ Map<String, Long> stats = null;
+ if (aggregatedStats.containsKey(date)) {
+ stats = aggregatedStats.get(date);
+ } else {
+ stats = new HashMap<String, Long>();
+ aggregatedStats.put(date, stats);
+ }
+ if (stats.containsKey(key)) {
+ stats.put(key, stats.get(key) + 1L);
+ } else {
+ stats.put(key, 1L);
+ }
+ }
+ ebr.close();
+ }
+
+ /* Write aggregated statistics to aggregated.csv. */
+ File aggregatedFile = new File("aggregated.csv");
+ BufferedWriter abw = new BufferedWriter(new FileWriter(
+ aggregatedFile));
+ abw.write("date,reported,discarded,reason,bridges,statuses\n");
+ long previousDateMillis = -1L;
+ final long DAY = 24L * 60L * 60L * 1000L;
+ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ for (Map.Entry<String, Map<String, Long>> e :
+ aggregatedStats.entrySet()) {
+ String date = e.getKey();
+ long currentDateMillis = dateFormat.parse(date).getTime();
+ while (previousDateMillis > -1L &&
+ currentDateMillis - previousDateMillis > DAY) {
+ previousDateMillis += DAY;
+ String tempDate = dateFormat.format(previousDateMillis);
+ for (String key : allKeys) {
+ abw.write(tempDate + "," + key + ",NA,0\n");
+ }
+ }
+ previousDateMillis = currentDateMillis;
+ String nextDate = dateFormat.format(currentDateMillis + DAY);
+ String nextPlusOneDate = dateFormat.format(currentDateMillis
+ + 2 * DAY);
+ long statuses = publishedStatuses.containsKey(date) ?
+ publishedStatuses.get(date) : 0L;
+ Map<String, Long> stats = e.getValue();
+ if (!aggregatedStats.containsKey(nextDate) ||
+ !aggregatedStats.containsKey(nextPlusOneDate) ||
+ statuses < 40) {
+ for (String key : allKeys) {
+ abw.write(date + "," + key + ",NA," + statuses + "\n");
+ }
+ } else {
+ for (String key : allKeys) {
+ if (stats.containsKey(key)) {
+ abw.write(date + "," + key + "," + (stats.get(key) / statuses)
+ + "," + statuses + "\n");
+ } else {
+ abw.write(date + "," + key + ",0," + statuses + "\n");
+ }
+ }
+ }
+ }
+ abw.close();
+ }
+}
+
diff --git a/task-3261/AnalyzeDescriptorParts.java b/task-3261/AnalyzeDescriptorParts.java
new file mode 100755
index 0000000..7f4bbc4
--- /dev/null
+++ b/task-3261/AnalyzeDescriptorParts.java
@@ -0,0 +1,315 @@
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.text.SimpleDateFormat;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+/* Analyze descriptors parts bridge by bridge and determine whether a
+ * bridge reported usage statistics at a given time, and if not, find out
+ * why not. */
+public class AnalyzeDescriptorParts {
+ public static void main(String[] args) throws Exception {
+
+ /* Define paths: we read descriptor part files from temp/ and append
+ * statistics on half hour detail to eval-out.csv. */
+ File tempDirectory = new File("temp");
+ File evalOutFile = new File("eval-out.csv");
+
+ /* Parse descriptor part files bridge by bridge. */
+ SimpleDateFormat dateTimeFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ final long HALF_HOUR = 30L * 60L * 1000L;
+ BufferedWriter ebw = new BufferedWriter(new FileWriter(evalOutFile));
+ for (File tempFile : tempDirectory.listFiles()) {
+ String fingerprint = tempFile.getName();
+ BufferedReader br = new BufferedReader(new FileReader(tempFile));
+ String line;
+
+ /* For each bridge, determine when it was first seen as relay. All
+ * timestamps are in half hours since 1970-01-01 00:00:00 UTC. */
+ long firstRunningRelay = Long.MAX_VALUE;
+
+ /* For each time the bridge was listed in a bridge network status as
+ * Running, remember the status publication time and referenced
+ * descriptor digest. */
+ SortedMap<Long, String> runningBridgeHalfHours =
+ new TreeMap<Long, String>();
+
+ /* For each descriptor published by the bridge, remember seven
+ * timestamps in an array:
+ * 0: when the bridge was started due to the descriptor publication
+ * time and reported uptime,
+ * 1: when the descriptor was published,
+ * 2: when the descriptor was first referenced in a status,
+ * 3: when the descriptor was last referenced in status,
+ * 4: when the first descriptor in the same uptime session was first
+ * referenced in a status,
+ * 5: when the last descriptor in the same uptime session was last
+ * referenced in a status, and
+ * 6: when the last descriptor in the same uptime session was
+ * published. */
+ Map<String, long[]> descriptorSessions =
+ new HashMap<String, long[]>();
+
+ /* For each descriptor, remember the platform string. */
+ Map<String, String> descriptorPlatforms =
+ new HashMap<String, String>();
+
+ /* For each bridge-stats or geoip-stats line, remember a long[] with
+ * two timestamps and a boolean:
+ * 0: when the statistics interval started,
+ * 1: when the statistics interval ended,
+ * 2: whether the bridge reported its geoip file digest (only
+ * 0.2.3.x or higher). */
+ SortedMap<Long, long[]> bridgeStats = new TreeMap<Long, long[]>(),
+ geoipStats = new TreeMap<Long, long[]>();
+
+ /* Parse the file in temp/ line by line. */
+ while ((line = br.readLine()) != null) {
+
+ /* Remember when a descriptor was published and which platform
+ * string it contained. */
+ if (line.startsWith("server-descriptor ")) {
+ String[] parts = line.split(" ");
+ long publishedMillis = dateTimeFormat.parse(parts[1] + " "
+ + parts[2]).getTime();
+ long publishedHalfHour = publishedMillis / HALF_HOUR + 1L;
+ String descriptor = parts[3];
+ long startedHalfHour = (publishedMillis
+ - Long.parseLong(parts[4]) * 1000L) / HALF_HOUR + 1L;
+ long[] descriptorSession;
+ if (descriptorSessions.containsKey(descriptor)) {
+ descriptorSession = descriptorSessions.get(descriptor);
+ } else {
+ descriptorSession = new long[7];
+ descriptorSessions.put(descriptor, descriptorSession);
+ }
+ if (descriptorSession[0] == 0) {
+ descriptorSession[0] = startedHalfHour;
+ descriptorSession[1] = publishedHalfHour;
+ }
+ String platform = line.substring(line.indexOf("Tor "));
+ descriptorPlatforms.put(descriptor, platform);
+
+ /* Remember when a descriptor was first and last referenced from a
+ * bridge network status. */
+ } else if (line.startsWith("running-bridge ")) {
+ String[] parts = line.split(" ");
+ long publishedMillis = dateTimeFormat.parse(parts[1] + " "
+ + parts[2]).getTime();
+ long publishedHalfHour = publishedMillis / HALF_HOUR;
+ String descriptor = parts[3];
+ long[] descriptorSession;
+ if (descriptorSessions.containsKey(descriptor)) {
+ descriptorSession = descriptorSessions.get(descriptor);
+ if (descriptorSession[2] == 0 ||
+ publishedHalfHour < descriptorSession[2]) {
+ descriptorSession[2] = publishedHalfHour;
+ }
+ if (publishedHalfHour > descriptorSession[3]) {
+ descriptorSession[3] = publishedHalfHour;
+ }
+ } else {
+ descriptorSession = new long[7];
+ descriptorSession[2] = publishedHalfHour;
+ descriptorSession[3] = publishedHalfHour;
+ descriptorSessions.put(descriptor, descriptorSession);
+ }
+ runningBridgeHalfHours.put(publishedHalfHour, descriptor);
+
+ /* Remember the start and end of a bridge-stats or geoip-stats
+ * interval, and remember whether the extra-info descriptor
+ * contained a geoip-db-digest line. */
+ } else if (line.startsWith("bridge-stats ") ||
+ line.startsWith("geoip-stats ")) {
+ String parts[] = line.split(" ");
+ long statsEndMillis = dateTimeFormat.parse(parts[1] + " "
+ + parts[2]).getTime();
+ long statsEnd = statsEndMillis / HALF_HOUR;
+ long statsStart = (statsEndMillis - Long.parseLong(parts[3])
+ * 1000L) / HALF_HOUR;
+ boolean hasGeoipFile = !parts[4].equals("NA");
+ long[] stats = new long[3];
+ stats[0] = statsStart;
+ stats[1] = statsEnd;
+ stats[2] = hasGeoipFile ? 1L : 0L;
+ if (line.startsWith("bridge-stats ")) {
+ bridgeStats.put(statsStart, stats);
+ } else {
+ geoipStats.put(statsStart, stats);
+ }
+
+ /* Remember when this bridge was first seen as a relay in the
+ * consensus. */
+ } else if (line.startsWith("running-relay ")) {
+ long runningRelayMillis = dateTimeFormat.parse(line.substring(
+ "running-relay ".length())).getTime() / HALF_HOUR;
+ firstRunningRelay = Math.min(firstRunningRelay,
+ runningRelayMillis);
+ }
+ }
+ br.close();
+
+ /* Sort descriptors by their first reference in a bridge network
+ * status. */
+ SortedMap<Long, String> descriptorsByFirstReferenced =
+ new TreeMap<Long, String>();
+ for (Map.Entry<String, long[]> e : descriptorSessions.entrySet()) {
+ if (e.getValue()[2] == 0) {
+ continue;
+ }
+ descriptorsByFirstReferenced.put(e.getValue()[2], e.getKey());
+ }
+ if (descriptorsByFirstReferenced.isEmpty()) {
+ continue;
+ }
+
+ /* Go through list of descriptors and see if two or more of them
+ * belong to the same bridge uptime session. Two descriptors are
+ * considered as part of the same uptime session if a) they are
+ * referenced from two subsequent statuses and b) the start time in
+ * the second descriptor lies before the publication time of the
+ * first descriptor. First make a list of all descriptors of a
+ * session and then update their long[] values to contain session
+ * information. */
+ long[] previousDescriptorTimestamps = null;
+ long firstStatusInSession = Long.MAX_VALUE,
+ lastStatusInSession = -1L, lastDescriptorPublished = -1L;
+ Set<String> descriptorsInSession = new HashSet<String>();
+ for (String descriptor : descriptorsByFirstReferenced.values()) {
+ long[] currentDescriptorTimestamps =
+ descriptorSessions.get(descriptor);
+ String currentDescriptor = descriptor;
+ if (previousDescriptorTimestamps != null) {
+ boolean sameSession =
+ previousDescriptorTimestamps[3] + 1L ==
+ currentDescriptorTimestamps[2] &&
+ currentDescriptorTimestamps[0] <=
+ previousDescriptorTimestamps[1];
+ if (!sameSession) {
+ for (String descriptorInSession : descriptorsInSession) {
+ long[] descriptorTimestamps = descriptorSessions.get(
+ descriptorInSession);
+ descriptorTimestamps[4] = firstStatusInSession;
+ descriptorTimestamps[5] = lastStatusInSession;
+ descriptorTimestamps[6] = lastDescriptorPublished;
+ }
+ firstStatusInSession = Long.MAX_VALUE;
+ lastStatusInSession = lastDescriptorPublished = -1L;
+ descriptorsInSession.clear();
+ }
+ }
+ firstStatusInSession = Math.min(firstStatusInSession,
+ currentDescriptorTimestamps[2]);
+ lastStatusInSession = Math.max(lastStatusInSession,
+ currentDescriptorTimestamps[3]);
+ lastDescriptorPublished = Math.max(lastDescriptorPublished,
+ currentDescriptorTimestamps[1]);
+ descriptorsInSession.add(currentDescriptor);
+ previousDescriptorTimestamps = currentDescriptorTimestamps;
+ }
+ for (String descriptorInSession : descriptorsInSession) {
+ long[] descriptorTimestamps = descriptorSessions.get(
+ descriptorInSession);
+ descriptorTimestamps[4] = firstStatusInSession;
+ descriptorTimestamps[5] = lastStatusInSession;
+ descriptorTimestamps[6] = lastDescriptorPublished;
+ }
+
+ /* Go through all statuses listing this bridge as Running, determine
+ * if it reported usage statistics and if they were considered for
+ * aggregation, and find out possible reasons for the bridge not
+ * reporting usage statistics. */
+ for (Map.Entry<Long, String> e :
+ runningBridgeHalfHours.entrySet()) {
+ long statusPublished = e.getKey();
+ String descriptor = e.getValue();
+ String platform = descriptorPlatforms.get(descriptor);
+ boolean reported = false, discarded = false;
+ String reason = "none";
+ if (firstRunningRelay <= statusPublished) {
+ /* The bridge was running as a relay before. */
+ discarded = true;
+ reason = "runasrelay";
+ }
+ if (!geoipStats.headMap(statusPublished + 1).isEmpty()) {
+ long[] stats = geoipStats.get(geoipStats.headMap(statusPublished
+ + 1).lastKey());
+ if (stats[0] <= statusPublished && stats[1] > statusPublished) {
+ /* Status publication time falls into stats interval. */
+ reported = true;
+ if (platform != null && platform.compareTo("Tor 0.2.2") > 0) {
+ /* geoip stats published by versions 0.2.2.x or higher are
+ * buggy and therefore discarded. */
+ discarded = true;
+ reason = "geoip022";
+ }
+ }
+ }
+ if (!bridgeStats.headMap(statusPublished + 1).isEmpty()) {
+ long[] stats = bridgeStats.get(bridgeStats.headMap(
+ statusPublished + 1).lastKey());
+ if (stats[0] <= statusPublished && stats[1] > statusPublished) {
+ /* Status publication time falls into stats interval. */
+ reported = true;
+ if (platform != null && platform.compareTo("Tor 0.2.3") > 0 &&
+ stats[2] == 0) {
+ /* The bridge running version 0.2.3.x did not have a geoip
+ * file and therefore published bad bridge statistics. */
+ discarded = true;
+ reason = "nogeoipfile";
+ }
+ }
+ }
+ if (!reported) {
+ /* The bridge didn't report statistics, so it doesn't matter
+ * whether we'd have discarded them. */
+ discarded = false;
+ if (!descriptorSessions.containsKey(descriptor)) {
+ /* The descriptor referenced in the bridge network status is
+ * unavailable, which means we cannot make any statement why the
+ * bridge did not report usage statistics. */
+ reason = "noserverdesc";
+ } else {
+ long[] descriptorTimestamps = descriptorSessions.get(descriptor);
+ long sessionStart = descriptorTimestamps[4],
+ sessionEnd = descriptorTimestamps[5],
+ lastDescPubl = descriptorTimestamps[6];
+ long currentStatsEnd = sessionStart
+ + 48 * ((statusPublished - sessionStart) / 48 + 1);
+ if (sessionEnd <= currentStatsEnd) {
+ /* The current uptime session ends before the 24-hour statistics
+ * interval. */
+ reason = "lessthan24h";
+ } else if (currentStatsEnd > lastDescPubl) {
+ /* The current uptime session ended after the 24-hour statistics
+ * interval, but the bridge didn't publish a descriptor
+ * containing the statistics. */
+ reason = "publdelay";
+ } else {
+ /* There is some other reason why the bridge did not report
+ * statistics. */
+ reason = "other";
+ }
+ }
+ }
+ ebw.write(dateTimeFormat.format(statusPublished * HALF_HOUR) + ","
+ + fingerprint + "," + reported + "," + discarded + ","
+ + reason + "\n");
+ }
+ }
+ ebw.close();
+ }
+}
+
diff --git a/task-3261/AnalyzeStatsCoverage.java b/task-3261/AnalyzeStatsCoverage.java
deleted file mode 100644
index 4688bde..0000000
--- a/task-3261/AnalyzeStatsCoverage.java
+++ /dev/null
@@ -1,478 +0,0 @@
-import java.io.*;
-import java.text.*;
-import java.util.*;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.codec.binary.Hex;
-public class AnalyzeStatsCoverage {
- public static void main(String[] args) throws Exception {
- File inDirectory = new File("in");
- File tempDirectory = new File("temp");
- File outFile = new File("stats-coverage.csv");
-
- /* Extract relevant lines from extra-info descriptors in inDirectory
- * and write them to files tempDirectory/$date/$fingerprint-$date for
- * later processing by fingerprint and date. */
- SimpleDateFormat dateTimeFormat =
- new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
- dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (inDirectory.exists() && inDirectory.isDirectory()) {
- System.out.println("Parsing descriptors in '"
- + inDirectory.getAbsolutePath() + "'.");
- long started = System.currentTimeMillis();
- tempDirectory.mkdirs();
- Stack<File> dirs = new Stack<File>();
- SortedSet<File> files = new TreeSet<File>();
- dirs.add(inDirectory);
- while (!dirs.isEmpty()) {
- File file = dirs.pop();
- if (file.isDirectory()) {
- if (file.getName().equals("statuses")) {
- continue;
- }
- for (File f : file.listFiles()) {
- dirs.add(f);
- }
- } else {
- files.add(file);
- }
- }
- int totalFiles = files.size(), fileNumber = 0;
- for (File file : files) {
- if (++fileNumber % (totalFiles / 1000) == 0) {
- int numberLength = String.valueOf(totalFiles).length();
- long minutesLeft = (((System.currentTimeMillis() - started)
- * (totalFiles - fileNumber)) / fileNumber) / (60L * 1000L);
- System.out.printf("Parsed %" + numberLength + "d of %"
- + numberLength + "d descriptors (%3d %%) %d minutes left%n",
- fileNumber, totalFiles, (fileNumber * 100) / totalFiles,
- minutesLeft);
- }
- BufferedReader br = new BufferedReader(new FileReader(file));
- String line, fingerprint = null, publishedLine = null;
- SortedMap<String, SortedSet<String>> linesByDate =
- new TreeMap<String, SortedSet<String>>();
- while ((line = br.readLine()) != null) {
- if (line.startsWith("extra-info ")) {
- fingerprint = line.split(" ")[2];
- } else if (line.startsWith("write-history ") ||
- line.startsWith("read-history ")) {
- String[] parts = line.split(" ");
- if (parts.length < 6) {
- continue;
- }
- String historyEndDate = parts[1];
- long historyEndMillis = dateTimeFormat.parse(parts[1] + " "
- + parts[2]).getTime();
- long intervalLength = Long.parseLong(parts[3].substring(1));
- if (intervalLength != 900L) {
- System.out.println("Non-standard interval length in "
- + "line '" + line + "' in file "
- + file.getAbsolutePath() + ". Skipping this line.");
- continue;
- }
- int intervals = parts[5].split(",").length;
- long historyStartMillis = historyEndMillis
- - (intervals * intervalLength * 1000L);
- long currentMillis = historyStartMillis;
- String currentDate;
- while ((currentDate = dateFormat.format(currentMillis)).
- compareTo(historyEndDate) <= 0) {
- if (!linesByDate.containsKey(currentDate)) {
- linesByDate.put(currentDate, new TreeSet<String>());
- }
- linesByDate.get(currentDate).add(line);
- currentMillis += 24L * 60L * 60L * 1000L;
- }
- } else if (line.startsWith("dirreq-stats-end ") ||
- line.startsWith("entry-stats-end ") ||
- line.startsWith("exit-stats-end ") ||
- line.startsWith("cell-stats-end ") ||
- line.startsWith("conn-bi-direct ") ||
- line.startsWith("bridge-stats-end ")) {
- String[] parts = line.split(" ");
- if (parts.length < 5) {
- System.out.println("Malformed line '" + line + "' in "
- + "file " + file.getAbsolutePath() + ". Skipping "
- + "this line.");
- continue;
- }
- String statsEndDate = parts[1];
- long statsEndMillis = dateTimeFormat.parse(parts[1] + " "
- + parts[2]).getTime();
- long intervalLength = Long.parseLong(parts[3].substring(1));
- long statsStartMillis = statsEndMillis
- - intervalLength * 1000L;
- long currentMillis = statsStartMillis;
- String currentDate;
- while ((currentDate = dateFormat.format(currentMillis)).
- compareTo(statsEndDate) <= 0) {
- if (!linesByDate.containsKey(currentDate)) {
- linesByDate.put(currentDate, new TreeSet<String>());
- }
- linesByDate.get(currentDate).add(line);
- currentMillis += 24L * 60L * 60L * 1000L;
- }
- } else if (line.startsWith("published ")) {
- publishedLine = line;
- } else if (line.startsWith("geoip-start-time ")) {
- if (publishedLine == null) {
- System.out.println("Missing published line in file "
- + file.getAbsolutePath() + ". Skipping "
- + "geoip-start-time line.");
- continue;
- }
- String[] publishedParts = publishedLine.split(" ");
- if (publishedParts.length < 3) {
- System.out.println("Malformed line '" + publishedLine
- + "' in file " + file.getAbsolutePath() + ". "
- + "Skipping geoip-start-time line.");
- continue;
- }
- String[] parts = line.split(" ");
- if (parts.length < 3) {
- System.out.println("Malformed line '" + line + "' in "
- + "file " + file.getAbsolutePath() + ". Skipping "
- + "this line.");
- continue;
- }
- String statsEndDate = parts[1];
- long statsEndMillis = dateTimeFormat.parse(
- publishedParts[1] + " " + publishedParts[2]).getTime();
- long statsStartMillis = dateTimeFormat.parse(parts[1] + " "
- + parts[2]).getTime();
- long intervalLength = (statsEndMillis - statsStartMillis)
- / 1000L;
- String rewrittenLine = "geoip-stats-end "
- + publishedParts[1] + " " + publishedParts[2] + " ("
- + intervalLength + " s)";
- long currentMillis = statsStartMillis;
- String currentDate;
- while ((currentDate = dateFormat.format(currentMillis)).
- compareTo(statsEndDate) <= 0) {
- if (!linesByDate.containsKey(currentDate)) {
- linesByDate.put(currentDate, new TreeSet<String>());
- }
- linesByDate.get(currentDate).add(rewrittenLine);
- currentMillis += 24L * 60L * 60L * 1000L;
- }
- }
- }
- br.close();
- for (Map.Entry<String, SortedSet<String>> e :
- linesByDate.entrySet()) {
- String date = e.getKey();
- SortedSet<String> lines = e.getValue();
- File outputFile = new File(tempDirectory, date + "/"
- + fingerprint + "-" + date);
- if (outputFile.exists()) {
- br = new BufferedReader(new FileReader(outputFile));
- while ((line = br.readLine()) != null) {
- lines.add(line);
- }
- br.close();
- }
- outputFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- outputFile));
- for (String l : lines) {
- bw.write(l + "\n");
- }
- bw.close();
- }
- }
- }
-
- /* Parse bridge network statuses and append "running " lines to
- * files tempDirectory/$date/$fingerprint-$date for later processing
- * by fingerprint and date. */
- SimpleDateFormat statusFormat =
- new SimpleDateFormat("yyyyMMdd-HHmmss");
- statusFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- if (inDirectory.exists() && inDirectory.isDirectory()) {
- System.out.println("Parsing statuses in '"
- + inDirectory.getAbsolutePath() + "'.");
- long started = System.currentTimeMillis();
- tempDirectory.mkdirs();
- Stack<File> dirs = new Stack<File>();
- SortedSet<File> files = new TreeSet<File>();
- dirs.add(inDirectory);
- while (!dirs.isEmpty()) {
- File file = dirs.pop();
- if (file.isDirectory()) {
- if (file.getName().equals("extra-infos")) {
- continue;
- }
- for (File f : file.listFiles()) {
- dirs.add(f);
- }
- } else {
- files.add(file);
- }
- }
- int totalFiles = files.size(), fileNumber = 0;
- for (File file : files) {
- if (++fileNumber % (totalFiles / 1000) == 0) {
- int numberLength = String.valueOf(totalFiles).length();
- long minutesLeft = (((System.currentTimeMillis() - started)
- * (totalFiles - fileNumber)) / fileNumber) / (60L * 1000L);
- System.out.printf("Parsed %" + numberLength + "d of %"
- + numberLength + "d statuses (%3d %%) %d minutes left%n",
- fileNumber, totalFiles, (fileNumber * 100) / totalFiles,
- minutesLeft);
- }
- long statusPublishedMillis = statusFormat.parse(
- file.getName().substring(0, "YYYYMMdd-HHmmss".length())).
- getTime();
- SortedSet<String> statusPublishedDates = new TreeSet<String>();
- String statusPublishedString = dateTimeFormat.format(
- statusPublishedMillis);
- statusPublishedDates.add(dateFormat.format(
- statusPublishedMillis));
- statusPublishedDates.add(dateFormat.format(
- statusPublishedMillis + 15L * 60L * 1000L));
- BufferedReader br = new BufferedReader(new FileReader(file));
- String line, rLine = null;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("r ")) {
- rLine = line;
- } else if (line.startsWith("s ") && line.contains(" Running") &&
- rLine != null) {
- String[] parts = rLine.split(" ");
- if (parts.length != 9) {
- System.out.println("Illegal line '" + rLine + "' in "
- + file.getAbsolutePath() + ". Skipping this line.");
- continue;
- }
- String fingerprint = Hex.encodeHexString(Base64.decodeBase64(
- parts[2] + "=="));
- for (String date : statusPublishedDates) {
- File outputFile = new File(tempDirectory, date + "/"
- + fingerprint.toUpperCase() + "-" + date);
- outputFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- outputFile, true));
- bw.write("running " + statusPublishedString + "\n");
- bw.close();
- }
- }
- }
- }
- }
-
- /* Parse relevant lines by fingerprint and date. The result will be
- * how many bytes that relay or bridge read/wrote in total, and how
- * many bytes were included in the different reported statistics.
- * Other results are the number of seconds for which this relay or
- * bridge reported byte histories and other statistics, either based
- * on self-reported bandwidth histories or based on the Running flag
- * in bridge network statuses. */
- if (tempDirectory.exists() && tempDirectory.isDirectory()) {
- System.out.println("Evaluating previously parsed descriptors in '"
- + tempDirectory.getAbsolutePath() + "'.");
- BufferedWriter bw = new BufferedWriter(new FileWriter(outFile));
- bw.write("fingerprint,date,totalwritten,totalread,totalseconds,"
- + "totalrunning,dirreqwritten,dirreqread,dirreqseconds,"
- + "dirreqrunning,entrywritten,entryread,entryseconds,"
- + "entryrunning,exitwritten,exitread,exitseconds,exitrunning,"
- + "cellwritten,cellread,cellseconds,cellrunning,"
- + "connbidirectwritten,connbidirectread,connbidirectseconds,"
- + "connbidirectrunning,bridgewritten,bridgeread,bridgeseconds,"
- + "bridgerunning,geoipwritten,geoipread,geoipseconds,"
- + "geoiprunning\n");
- Stack<File> dirs = new Stack<File>();
- SortedSet<File> files = new TreeSet<File>();
- dirs.add(tempDirectory);
- while (!dirs.isEmpty()) {
- File file = dirs.pop();
- if (file.isDirectory()) {
- for (File f : file.listFiles()) {
- dirs.add(f);
- }
- } else {
- files.add(file);
- }
- }
- int totalFiles = files.size(), fileNumber = 0;
- for (File file : files) {
- if (++fileNumber % (totalFiles / 1000) == 0) {
- int numberLength = String.valueOf(totalFiles).length();
- System.out.printf("Evaluated %" + numberLength + "d of %"
- + numberLength + "d descriptors/days (%3d %%)%n",
- fileNumber, totalFiles, (fileNumber * 100) / totalFiles);
- }
- String fingerprint = file.getName().substring(0, 40);
- String date = file.getName().substring(41);
- long dateStartMillis = dateFormat.parse(date).getTime();
- long dateEndMillis = dateStartMillis + 24L * 60L * 60L * 1000L;
- long[] writeHistory = new long[96], readHistory = new long[96];
- boolean[] upBridge = new boolean[96],
- upStatus = new boolean[96],
- dirreqStats = new boolean[96],
- entryStats = new boolean[96],
- exitStats = new boolean[96],
- cellStats = new boolean[96],
- connBiDirectStats = new boolean[96],
- bridgeStats = new boolean[96],
- geoipStats = new boolean[96];
- BufferedReader br = new BufferedReader(new FileReader(file));
- String line;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("running ")) {
- long statusPublishedMillis = dateTimeFormat.parse(
- line.substring("running ".length())).getTime();
- int j = (int) ((statusPublishedMillis - dateStartMillis)
- / (900L * 1000L));
- for (int i = 0; i < 2; i++) {
- if (j + i >= 0 && j + i < 96) {
- upStatus[j + i] = true;
- }
- }
- } else if (line.startsWith("write-history ") ||
- line.startsWith("read-history ")) {
- long[] history = line.startsWith("write-history ")
- ? writeHistory : readHistory;
- String[] parts = line.split(" ");
- long historyEndMillis = dateTimeFormat.parse(parts[1] + " "
- + parts[2]).getTime();
- String[] historyValues = parts[5].split(",");
- long historyStartMillis = historyEndMillis
- - (historyValues.length * 900L * 1000L);
- long currentMillis = historyStartMillis;
- for (int i = 0; i < historyValues.length; i++) {
- if (currentMillis >= dateStartMillis &&
- currentMillis < dateEndMillis) {
- int j = (int) ((currentMillis - dateStartMillis)
- / (900L * 1000L));
- if (j < 0 || j >= 96) {
- System.out.println("Internal error when processing "
- + "line '" + line + "'. Index = " + j
- + ". Exiting.");
- System.exit(1);
- }
- history[j] = Long.parseLong(historyValues[i]);
- upBridge[j] = true;
- }
- currentMillis += 15L * 60L * 1000L;
- }
- } else if (line.startsWith("dirreq-stats-end ") ||
- line.startsWith("entry-stats-end ") ||
- line.startsWith("exit-stats-end ") ||
- line.startsWith("cell-stats-end ") ||
- line.startsWith("conn-bi-direct ") ||
- line.startsWith("bridge-stats-end ") ||
- line.startsWith("geoip-stats-end ")) {
- boolean[] stats = null;
- if (line.startsWith("dirreq-stats-end ")) {
- stats = dirreqStats;
- } else if (line.startsWith("entry-stats-end ")) {
- stats = entryStats;
- } else if (line.startsWith("exit-stats-end ")) {
- stats = exitStats;
- } else if (line.startsWith("cell-stats-end ")) {
- stats = cellStats;
- } else if (line.startsWith("conn-bi-direct ")) {
- stats = connBiDirectStats;
- } else if (line.startsWith("bridge-stats-end ")) {
- stats = bridgeStats;
- } else if (line.startsWith("geoip-stats-end ")) {
- stats = geoipStats;
- } else {
- System.out.println("Internal error when processing line '"
- + line + "'. Exiting.");
- System.exit(1);
- }
- String[] parts = line.split(" ");
- long statsEndMillis = dateTimeFormat.parse(parts[1] + " "
- + parts[2]).getTime();
- long intervalLength = Long.parseLong(parts[3].substring(1));
- long statsStartMillis = statsEndMillis
- - intervalLength * 1000L;
- long currentMillis = statsStartMillis;
- while (currentMillis < dateEndMillis) {
- if (currentMillis >= dateStartMillis) {
- int j = (int) ((currentMillis - dateStartMillis)
- / (900L * 1000L));
- if (j < 0 || j >= 96) {
- System.out.println("Internal error when processing "
- + "line '" + line + "'. Index = " + j
- + ". Exiting.");
- System.exit(1);
- }
- stats[j] = true;
- }
- currentMillis += 15L * 60L * 1000L;
- }
- }
- }
- br.close();
- bw.write(fingerprint + "," + date + ",");
- long totalWritten = 0L, totalRead = 0L, totalSeconds = 0L,
- totalRunning = 0L, dirreqWritten = 0L, dirreqRead = 0L,
- dirreqSeconds = 0L, dirreqRunning = 0L, entryWritten = 0L,
- entryRead = 0L, entrySeconds = 0L, entryRunning = 0L,
- exitWritten = 0L, exitRead = 0L, exitSeconds = 0L,
- exitRunning = 0L, cellWritten = 0L, cellRead = 0L,
- cellSeconds = 0L, cellRunning = 0L, connBiDirectWritten = 0L,
- connBiDirectRead = 0L, connBiDirectSeconds = 0L,
- connBiDirectRunning = 0L, bridgeWritten = 0L, bridgeRead = 0L,
- bridgeSeconds = 0L, bridgeRunning = 0L, geoipWritten = 0L,
- geoipRead = 0L, geoipSeconds = 0L, geoipRunning = 0L;
- for (int i = 0; i < 96; i++) {
- totalWritten += writeHistory[i];
- totalRead += readHistory[i];
- totalSeconds += upBridge[i] ? 900L : 0L;
- totalRunning += upStatus[i] ? 900L : 0L;
- dirreqWritten += dirreqStats[i] ? writeHistory[i] : 0L;
- dirreqRead += dirreqStats[i] ? readHistory[i] : 0L;
- dirreqSeconds += dirreqStats[i] && upBridge[i] ? 900L : 0L;
- dirreqRunning += dirreqStats[i] && upStatus[i] ? 900L : 0L;
- entryWritten += entryStats[i] ? writeHistory[i] : 0L;
- entryRead += entryStats[i] ? readHistory[i] : 0L;
- entrySeconds += entryStats[i] && upBridge[i] ? 900L : 0L;
- entryRunning += entryStats[i] && upStatus[i] ? 900L : 0L;
- exitWritten += exitStats[i] ? writeHistory[i] : 0L;
- exitRead += exitStats[i] ? readHistory[i] : 0L;
- exitSeconds += exitStats[i] && upBridge[i] ? 900L : 0L;
- exitRunning += exitStats[i] && upStatus[i] ? 900L : 0L;
- cellWritten += cellStats[i] ? writeHistory[i] : 0L;
- cellRead += cellStats[i] ? readHistory[i] : 0L;
- cellSeconds += cellStats[i] && upBridge[i] ? 900L : 0L;
- cellRunning += cellStats[i] && upStatus[i] ? 900L : 0L;
- connBiDirectWritten += connBiDirectStats[i] ? writeHistory[i]
- : 0L;
- connBiDirectRead += connBiDirectStats[i] ? readHistory[i]
- : 0L;
- connBiDirectSeconds += connBiDirectStats[i] && upBridge[i]
- ? 900L : 0L;
- connBiDirectRunning += connBiDirectStats[i] && upStatus[i]
- ? 900L : 0L;
- bridgeWritten += bridgeStats[i] ? writeHistory[i] : 0L;
- bridgeRead += bridgeStats[i] ? readHistory[i] : 0L;
- bridgeSeconds += bridgeStats[i] && upBridge[i] ? 900L : 0L;
- bridgeRunning += bridgeStats[i] && upStatus[i] ? 900L : 0L;
- geoipWritten += geoipStats[i] ? writeHistory[i] : 0L;
- geoipRead += geoipStats[i] ? readHistory[i] : 0L;
- geoipSeconds += geoipStats[i] && upBridge[i] ? 900L : 0L;
- geoipRunning += geoipStats[i] && upStatus[i] ? 900L : 0L;
- }
- bw.write(totalWritten + "," + totalRead + "," + totalSeconds + ","
- + totalRunning + "," + dirreqWritten + "," + dirreqRead + ","
- + dirreqSeconds + "," + dirreqRunning + "," + entryWritten
- + "," + entryRead + "," + entrySeconds + "," + entryRunning
- + "," + exitWritten + "," + exitRead + "," + exitSeconds + ","
- + exitRunning + "," + cellWritten + "," + cellRead + ","
- + cellSeconds + "," + cellRunning + "," + connBiDirectWritten
- + "," + connBiDirectRead + "," + connBiDirectSeconds + ","
- + connBiDirectRunning + "," + bridgeWritten + "," + bridgeRead
- + "," + bridgeSeconds + "," + bridgeRunning + ","
- + geoipWritten + "," + geoipRead + "," + geoipSeconds + ","
- + geoipRunning + "\n");
- }
- bw.close();
- }
- }
-}
-
diff --git a/task-3261/ExtractDescriptorParts.java b/task-3261/ExtractDescriptorParts.java
new file mode 100755
index 0000000..544022d
--- /dev/null
+++ b/task-3261/ExtractDescriptorParts.java
@@ -0,0 +1,172 @@
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.text.SimpleDateFormat;
+import java.util.Iterator;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeSet;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.torproject.descriptor.BridgeNetworkStatus;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorFile;
+import org.torproject.descriptor.DescriptorReader;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.ExtraInfoDescriptor;
+import org.torproject.descriptor.NetworkStatusEntry;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import org.torproject.descriptor.ServerDescriptor;
+
+/* Extract the relevant parts from bridge descriptors and consensuses that
+ * are required to answer what fraction of bridges are not reporting
+ * bridge usage statistics. */
+public class ExtractDescriptorParts {
+ public static void main(String[] args) throws Exception {
+
+ /* Define paths: we parse descriptor (tarballs) from in/, store the
+ * parse history to parse-history, write relevant parts per bridge to
+ * temp/, and write publication times of bridge network statuses to
+ * bridge-network-statuses. */
+ File inDirectory = new File("in");
+ File parseHistoryFile = new File("parse-history");
+ File tempDirectory = new File("temp");
+ File statusFile = new File("bridge-network-statuses");
+
+ /* Read descriptors. */
+ SimpleDateFormat dateTimeFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ DescriptorReader reader =
+ DescriptorSourceFactory.createDescriptorReader();
+ reader.addDirectory(inDirectory);
+ reader.setExcludeFiles(parseHistoryFile);
+ Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors();
+ while (descriptorFiles.hasNext()) {
+ DescriptorFile descriptorFile = descriptorFiles.next();
+ if (descriptorFile.getDescriptors() != null) {
+ for (Descriptor descriptor : descriptorFile.getDescriptors()) {
+
+ /* Extract bridge-stats and geoip-stats from bridge extra-info
+ * descriptors. */
+ if (descriptor instanceof ExtraInfoDescriptor) {
+ System.out.print("e");
+ SortedSet<String> lines = new TreeSet<String>();
+ ExtraInfoDescriptor extraInfoDescriptor =
+ (ExtraInfoDescriptor) descriptor;
+ if (extraInfoDescriptor.getBridgeStatsEndMillis() > 0) {
+ lines.add("bridge-stats " + dateTimeFormat.format(
+ extraInfoDescriptor.getBridgeStatsEndMillis()) + " "
+ + extraInfoDescriptor.getBridgeStatsIntervalLength()
+ + " " + (extraInfoDescriptor.getGeoipDbDigest() == null
+ ? "NA" : extraInfoDescriptor.getGeoipDbDigest()));
+ }
+ if (extraInfoDescriptor.getGeoipStartTimeMillis() > 0) {
+ long intervalLength =
+ (extraInfoDescriptor.getPublishedMillis()
+ - extraInfoDescriptor.getGeoipStartTimeMillis())
+ / 1000L;
+ String geoipStatsEnd = dateTimeFormat.format(
+ extraInfoDescriptor.getPublishedMillis());
+ lines.add("geoip-stats " + geoipStatsEnd + " "
+ + intervalLength + " "
+ + (extraInfoDescriptor.getGeoipDbDigest() == null
+ ? "NA" : extraInfoDescriptor.getGeoipDbDigest()));
+ }
+ if (!lines.isEmpty()) {
+ File outputFile = new File(tempDirectory,
+ extraInfoDescriptor.getFingerprint().toUpperCase());
+ outputFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ outputFile, true));
+ for (String l : lines) {
+ bw.write(l + "\n");
+ }
+ bw.close();
+ }
+
+ /* Extract all bridges with the Running flag from bridge network
+ * statuses. Also extract the status publication time. */
+ } else if (descriptor instanceof BridgeNetworkStatus) {
+ System.out.print("n");
+ BridgeNetworkStatus status = (BridgeNetworkStatus) descriptor;
+ String published = dateTimeFormat.format(
+ status.getPublishedMillis());
+ if (status.getStatusEntries() != null) {
+ for (NetworkStatusEntry entry :
+ status.getStatusEntries().values()) {
+ if (entry.getFlags().contains("Running")) {
+ File outputFile = new File(tempDirectory,
+ entry.getFingerprint().toUpperCase());
+ outputFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ outputFile, true));
+ String digest = entry.getDescriptor().toUpperCase();
+ bw.write("running-bridge " + published + " " + digest
+ + "\n");
+ bw.close();
+ }
+ }
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ statusFile, true));
+ bw.write(published + "\n");
+ bw.close();
+ }
+
+ /* Extract publication time, digest, uptime, and platform string
+ * from bridge server descriptors. */
+ } else if (descriptor instanceof ServerDescriptor) {
+ System.out.print("s");
+ ServerDescriptor serverDescriptor =
+ (ServerDescriptor) descriptor;
+ String published = dateTimeFormat.format(
+ serverDescriptor.getPublishedMillis());
+ String digest = descriptorFile.getFileName().substring(
+ descriptorFile.getFileName().lastIndexOf("/") + 1).
+ toUpperCase();
+ String uptime = serverDescriptor.getUptime() == null ? "-1"
+ : String.valueOf(serverDescriptor.getUptime());
+ String platform = serverDescriptor.getPlatform() == null
+ ? "NA" : serverDescriptor.getPlatform();
+ File outputFile = new File(tempDirectory,
+ serverDescriptor.getFingerprint().toUpperCase());
+ outputFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ outputFile, true));
+ bw.write("server-descriptor " + published + " "
+ + digest + " " + uptime + " " + platform + "\n");
+ bw.close();
+
+ /* Extract hashed fingerprints of all relays with the Running
+ * flag from relay network status consensuses. */
+ } else if (descriptor instanceof RelayNetworkStatusConsensus) {
+ System.out.print("r");
+ RelayNetworkStatusConsensus status =
+ (RelayNetworkStatusConsensus) descriptor;
+ if (status.getStatusEntries() != null) {
+ for (NetworkStatusEntry entry :
+ status.getStatusEntries().values()) {
+ if (entry.getFlags().contains("Running")) {
+ String hashedFingerprint = Hex.encodeHexString(
+ DigestUtils.sha(Hex.decodeHex(
+ entry.getFingerprint().toCharArray()))).
+ toUpperCase();
+ File outputFile = new File(tempDirectory,
+ hashedFingerprint);
+ outputFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ outputFile, true));
+ bw.write("running-relay " + dateTimeFormat.format(
+ status.getValidAfterMillis()) + "\n");
+ bw.close();
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
diff --git a/task-3261/README b/task-3261/README
old mode 100644
new mode 100755
index cb430ac..43ef208
--- a/task-3261/README
+++ b/task-3261/README
@@ -1,4 +1,36 @@
-$ javac -cp commons-codec-1.4.jar AnalyzeStatsCoverage.java
-$ java -cp commons-codec-1.4.jar.: -Xmx4g AnalyzeStatsCoverage
-$ R --slave -f stats-coverage.R
+What fraction of our bridges are not reporting usage statistics?
+================================================================
+
+Usage:
+
+1. Put metrics tarballs into a directory called in/. The best parsing
+ performance can be achieved by decompressing tarballs without
+ extracting them. The bridge-descriptors-* and consensuses-* tarballs
+ are required for this analysis.
+
+2. Clone metrics-lib.git, build descriptor.jar, and put it in this
+ directory.
+
+3. Download Apache Commons Codec and Compress and put the .jar files in
+ this directory.
+
+4. Parse descriptors and write all relevant parts to one file per bridge:
+ $ javac
+ -cp commons-codec-1.4.jar:commons-compress-1.3.jar:descriptor.jar
+ ExtractDescriptorParts.java
+ $ java
+ -cp commons-codec-1.4.jar:commons-compress-1.3.jar:descriptor.jar:.
+ ExtractDescriptorParts
+
+5. Analyze descriptors parts bridge by bridge and determine whether it
+ reported bridge stats at a given time, and if not, find out why not:
+ $ javac AnalyzeDescriptorParts.java
+ $ java AnalyzeDescriptorParts
+
+6. Aggregate daily statistics that can be plotted:
+ $ javac AggregateStats.java
+ $ java AggregateStats
+
+7. Plot results:
+ $ R --slave -f plot.R
diff --git a/task-3261/plot.R b/task-3261/plot.R
new file mode 100644
index 0000000..8a3808c
--- /dev/null
+++ b/task-3261/plot.R
@@ -0,0 +1,65 @@
+library(ggplot2)
+library(scales)
+library(reshape)
+a <- read.csv("aggregated.csv", stringsAsFactors = FALSE)
+
+e <- a
+e <- data.frame(date = as.Date(e$date), case = ifelse(
+ e$reported == "true", ifelse(e$discarded == "false", "case1", "case2"),
+ "case3"), bridges = e$bridges)
+e <- aggregate(list(bridges = e$bridges),
+ by = list(date = e$date, case = e$case), FUN = sum)
+e <- cast(e, date ~ case)
+sums <- e$case1 + e$case2 + e$case3
+e <- data.frame(date = e$date, case1 = e$case1 / sums,
+ case2 = e$case2 / sums, case3 = e$case3 / sums, stringsAsFactors = FALSE)
+e <- melt(e, "date")
+e <- data.frame(date = e$date, variable = ifelse(e$variable == "case1",
+ "reported and used", ifelse(e$variable == "case2",
+ "reported and discarded", "not reported")), value = e$value)
+ggplot(e, aes(x = as.Date(date), y = value)) +
+geom_line() +
+facet_grid(variable ~ .) +
+scale_x_date(name = "") +
+scale_y_continuous(name = "", labels = percent) +
+opts(title = "Fraction of bridge usage statistics that were...\n")
+ggsave("reported-bridge-statistics.png", width = 8, height = 6, dpi = 120)
+
+d <- a
+d <- d[d$reported == "false", ]
+d <- data.frame(date = d$date, reason = d$reason, value = d$bridges)
+d <- cast(d, date ~ reason)
+d <- data.frame(date = d$date, case1 = d$lessthan24h / sums,
+ case2 = d$publdelay / sums, case3 = d$other / sums)
+d <- melt(d, "date")
+d <- data.frame(date = d$date, variable = ifelse(d$variable == "case1",
+ "Less than 24h uptime", ifelse(d$variable == "case2",
+ "Publication delay", "Other reason")), value = d$value)
+ggplot(d, aes(x = as.Date(date), y = value)) +
+geom_line() +
+facet_grid(variable ~ .) +
+scale_x_date(name = "") +
+scale_y_continuous(name = "", labels = percent) +
+opts(title = "Reasons for bridges not reporting usage statistics\n")
+ggsave("bridge-statistics-nonreported.png", width = 8, height = 6,
+ dpi = 120)
+
+b <- a
+b <- b[b$discarded == "true", ]
+b <- data.frame(date = b$date, reason = b$reason, value = b$bridges)
+b <- cast(b, date ~ reason)
+b <- data.frame(date = b$date, case1 = b$geoip022 / sums,
+ case2 = b$nogeoipfile / sums, case3 = b$runasrelay / sums)
+b <- melt(b, "date")
+b <- data.frame(date = b$date, variable = ifelse(b$variable == "case1",
+ "0.2.2.x geoip-stats bug", ifelse(b$variable == "case2",
+ "missing geoip file", "Run as non-bridge relay")), value = b$value)
+ggplot(b, aes(x = as.Date(date), y = value)) +
+geom_line() +
+facet_grid(variable ~ .) +
+scale_x_date(name = "") +
+scale_y_continuous(name = "", labels = percent) +
+opts(title = "Reasons for discarding reported usage statistics\n")
+ggsave("bridge-statistics-discarded.png", width = 8, height = 6,
+ dpi = 120)
+
diff --git a/task-3261/stats-coverage.R b/task-3261/stats-coverage.R
deleted file mode 100644
index aef63f2..0000000
--- a/task-3261/stats-coverage.R
+++ /dev/null
@@ -1,25 +0,0 @@
-library(ggplot2)
-library(scales)
-b <- read.csv("stats-coverage.csv")
-b <- aggregate(list(
- totalwritten = b$totalwritten, totalseconds = b$totalseconds,
- totalrunning = b$totalrunning, bridgewritten = b$bridgewritten,
- bridgeseconds = b$bridgeseconds, bridgerunning = b$bridgerunning,
- geoipwritten = b$geoipwritten, geoipseconds = b$geoipseconds,
- geoiprunning = b$geoiprunning), by = list(date = as.Date(b$date)), sum)
-b <- rbind(data.frame(date = b$date, variable = "by written bytes",
- value = (b$bridgewritten + b$geoipwritten) / b$totalwritten),
- data.frame(date = b$date, variable = "by uptime (bandwidth history)",
- value = (b$bridgeseconds + b$geoipseconds) / b$totalseconds),
- data.frame(date = b$date, variable = "by uptime (Running flag)",
- value = (b$bridgerunning + b$geoiprunning) / b$totalrunning))
-b <- b[b$date >= as.Date("2010-10-01") & b$date < as.Date("2012-04-01"), ]
-ggplot(b, aes(x = date, y = value)) +
-geom_line() +
-facet_grid(variable ~ .) +
-scale_x_date(name = "") +
-scale_y_continuous(name = "", limits = c(0, 1), labels = percent) +
-scale_colour_hue(name = "") +
-opts(title = "Fraction of bridges reporting statistics\n")
-ggsave("stats-coverage-bridges.png", width = 8, height = 7, dpi = 72)
-
