[tor-commits] [metrics-tasks/master] Add code for 2680.
karsten at torproject.org
karsten at torproject.org
Fri Mar 11 13:51:51 UTC 2011
commit ee7df6a759f6788579f61403fb9771996ea5c988
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Fri Mar 11 14:51:12 2011 +0100
Add code for 2680.
---
task-2680/.gitignore | 6 +
task-2680/ProcessRelayConsensuses.java | 85 ++++++++
task-2680/ProcessSanitizedBridges.java | 327 ++++++++++++++++++++++++++++++++
task-2680/README | 145 ++++++++++++++
task-2680/verify.R | 27 +++
5 files changed, 590 insertions(+), 0 deletions(-)
diff --git a/task-2680/.gitignore b/task-2680/.gitignore
new file mode 100644
index 0000000..134e86d
--- /dev/null
+++ b/task-2680/.gitignore
@@ -0,0 +1,6 @@
+*.class
+*.csv
+bridge-descriptors/
+commons-codec-1.4.jar
+consensuses/
+
diff --git a/task-2680/ProcessRelayConsensuses.java b/task-2680/ProcessRelayConsensuses.java
new file mode 100644
index 0000000..44d9ce3
--- /dev/null
+++ b/task-2680/ProcessRelayConsensuses.java
@@ -0,0 +1,85 @@
+import java.io.*;
+import java.util.*;
+import org.apache.commons.codec.binary.*;
+import org.apache.commons.codec.digest.*;
+
+public class ProcessRelayConsensuses {
+ public static void main(String[] args) throws IOException {
+
+ /* Validate command-line arguments. */
+ if (args.length != 1 || !new File(args[0]).exists()) {
+ System.out.println("Usage: java ProcessRelayConsensuses <dir>");
+ System.exit(1);
+ }
+
+ /* Find all files that we should parse. Somewhat fragile, but should
+ * work. */
+ System.out.println("Creating list of files we should parse.");
+ SortedMap<String, File> consensuses = new TreeMap<String, File>();
+ Stack<File> files = new Stack<File>();
+ files.add(new File(args[0]));
+ while (!files.isEmpty()) {
+ File file = files.pop();
+ String filename = file.getName();
+ if (file.isDirectory()) {
+ files.addAll(Arrays.asList(file.listFiles()));
+ } else if (filename.endsWith("-consensus")) {
+ consensuses.put(filename, file);
+ }
+ }
+ System.out.println("We found " + consensuses.size()
+ + " consensuses.");
+
+ /* Parse consensuses. */
+ if (!consensuses.isEmpty()) {
+ System.out.println("Parsing consensuses.");
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ "relays.csv"));
+ bw.write("consensus,fingerprint\n");
+ int parsedConsensuses = 0, totalConsensuses = consensuses.size(),
+ writtenOutputLines = 1;
+ long started = System.currentTimeMillis();
+ for (File file : consensuses.values()) {
+ BufferedReader br = new BufferedReader(new FileReader(file));
+ String line, validAfter = null;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("valid-after ")) {
+ validAfter = line.substring("valid-after ".length());
+ } else if (line.startsWith("r ")) {
+ if (validAfter == null) {
+ System.out.println("Found an r line before the valid-after "
+ + "line in " + file.getName() + ". Please check. "
+ + "Exiting.");
+ System.exit(1);
+ }
+ String fingerprint = DigestUtils.shaHex(Base64.decodeBase64(
+ line.split(" ")[2] + "="));
+ bw.write(validAfter + "," + fingerprint + "\n");
+ writtenOutputLines++;
+ }
+ }
+ br.close();
+ parsedConsensuses++;
+ if (parsedConsensuses % (totalConsensuses / 10) == 0) {
+ double fractionDone = (double) (parsedConsensuses) /
+ (double) totalConsensuses;
+ double fractionLeft = 1.0D - fractionDone;
+ long now = System.currentTimeMillis();
+ double millisLeft = ((double) (now - started)) * fractionLeft /
+ fractionDone;
+ long secondsLeft = (long) millisLeft / 1000L;
+ System.out.println(" " + (parsedConsensuses / (totalConsensuses
+ / 10)) + "0% done, " + secondsLeft + " seconds left.");
+ }
+ }
+ bw.close();
+ System.out.println("Parsed " + parsedConsensuses + " consensuses "
+ + "and wrote " + writtenOutputLines + " lines to relays.csv.");
+ }
+
+ /* This is it. */
+ System.out.println("Terminating.");
+ }
+}
+
+
diff --git a/task-2680/ProcessSanitizedBridges.java b/task-2680/ProcessSanitizedBridges.java
new file mode 100644
index 0000000..1f0e00e
--- /dev/null
+++ b/task-2680/ProcessSanitizedBridges.java
@@ -0,0 +1,327 @@
+import java.io.*;
+import java.text.*;
+import java.util.*;
+import org.apache.commons.codec.binary.*;
+
+public class ProcessSanitizedBridges {
+ public static void main(String[] args) throws IOException,
+ ParseException {
+
+ /* Validate command-line arguments. */
+ if (args.length != 1 || !new File(args[0]).exists()) {
+ System.out.println("Usage: java ProcessSanitizedBridges <dir>");
+ System.exit(1);
+ }
+
+ /* Find all files that we should parse. Somewhat fragile, but should
+ * work. */
+ System.out.println("Creating list of files we should parse.");
+ SortedMap<String, File> statuses = new TreeMap<String, File>();
+ SortedMap<String, File> serverDescriptors =
+ new TreeMap<String, File>();
+ SortedMap<String, File> extraInfoDescriptors =
+ new TreeMap<String, File>();
+ Stack<File> files = new Stack<File>();
+ files.add(new File(args[0]));
+ while (!files.isEmpty()) {
+ File file = files.pop();
+ String path = file.getAbsolutePath();
+ String filename = file.getName();
+ if (file.isDirectory()) {
+ files.addAll(Arrays.asList(file.listFiles()));
+ } else if (path.contains("statuses")) {
+ statuses.put(filename, file);
+ } else if (path.contains("server-descriptors")) {
+ serverDescriptors.put(filename, file);
+ } else if (path.contains("extra-infos")) {
+ extraInfoDescriptors.put(filename, file);
+ }
+ }
+ System.out.println("We found\n " + statuses.size() + " statuses,\n "
+ + serverDescriptors.size() + " server descriptors, and\n "
+ + extraInfoDescriptors.size() + " extra-info descriptors.");
+
+ /* Parse statuses. */
+ if (!statuses.isEmpty()) {
+ System.out.println("Parsing statuses.");
+ List<String> knownFlags = new ArrayList<String>(Arrays.asList(
+ ("Authority,BadExit,BadDirectory,Exit,Fast,Guard,Named,Stable,"
+ + "Running,Valid,V2Dir").split(",")));
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ "statuses.csv"));
+ bw.write("status,fingerprint,descriptor,published,address,orport,"
+ + "dirport");
+ for (String knownFlag : knownFlags) {
+ bw.write("," + knownFlag.toLowerCase());
+ }
+ bw.write("\n");
+ int parsedStatuses = 0, totalStatuses = statuses.size(),
+ writtenOutputLines = 1;
+ long started = System.currentTimeMillis();
+ for (File file : statuses.values()) {
+ String filename = file.getName();
+ if (filename.length() != ("20110101-000703-"
+ + "4A0CCD2DDC7995083D73F5D667100C8A5831F16D").length()) {
+ System.out.println("Status filename has wrong length: '"
+ + filename + "' Please check. Exiting.");
+ System.exit(1);
+ }
+ String statusDateTime = filename.substring(0, 4) + "-"
+ + filename.substring(4, 6) + "-" + filename.substring(6, 8)
+ + " " + filename.substring(9, 11) + ":"
+ + filename.substring(11, 13) + ":"
+ + filename.substring(13, 15);
+ BufferedReader br = new BufferedReader(new FileReader(file));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("r ")) {
+ String[] parts = line.split(" ");
+ if (parts.length != 9) {
+ System.out.println("r line doesn't have the correct number "
+ + "of entries: '" + line + "'. Please check. Exiting.");
+ System.exit(1);
+ }
+ String fingerprint = Hex.encodeHexString(Base64.decodeBase64(
+ parts[2] + "="));
+ String descriptor = Hex.encodeHexString(Base64.decodeBase64(
+ parts[2] + "="));
+ String published = parts[4] + " " + parts[5];
+ String address = parts[6];
+ String orPort = parts[7];
+ String dirPort = parts[8];
+ bw.write(statusDateTime + "," + fingerprint + "," + descriptor
+ + "," + published + "," + address + "," + orPort + ","
+ + dirPort);
+ } else if (line.equals("s") || line.startsWith("s ")) {
+ String flags = line.substring(1);
+ for (String flag : knownFlags) {
+ if (flags.contains(" " + flag)) {
+ bw.write(",TRUE");
+ } else {
+ bw.write(",FALSE");
+ }
+ }
+ bw.write("\n");
+ writtenOutputLines++;
+ }
+ }
+ br.close();
+ parsedStatuses++;
+ if (parsedStatuses % (totalStatuses / 10) == 0) {
+ double fractionDone = (double) (parsedStatuses) /
+ (double) totalStatuses;
+ double fractionLeft = 1.0D - fractionDone;
+ long now = System.currentTimeMillis();
+ double millisLeft = ((double) (now - started)) * fractionLeft /
+ fractionDone;
+ long secondsLeft = (long) millisLeft / 1000L;
+ System.out.println(" " + (parsedStatuses / (totalStatuses
+ / 10)) + "0% done, " + secondsLeft + " seconds left.");
+ }
+ }
+ bw.close();
+ System.out.println("Parsed " + parsedStatuses + " statuses and "
+ + "wrote " + writtenOutputLines + " lines to statuses.csv.");
+ }
+
+ /* Parse server descriptors and extra-info descriptors. */
+ if (!serverDescriptors.isEmpty()) {
+ System.out.println("Parsing server descriptors and extra-info "
+ + "descriptors.");
+ List<String> knownCountries = new ArrayList<String>(Arrays.asList(
+ ("?? A1 A2 AD AE AF AG AI AL AM AN AO AP AQ AR AS AT AU AW AX "
+ + "AZ BA BB BD BE BF BG BH BI BJ BM BN BO BR BS BT BV BW BY BZ "
+ + "CA CD CF CG CH CI CK CL CM CN CO CR CS CU CV CY CZ DE DJ DK "
+ + "DM DO DZ EC EE EG ER ES ET EU FI FJ FK FM FO FR GA GB GD GE "
+ + "GF GG GH GI GL GM GN GP GQ GR GT GU GW GY HK HN HR HT HU ID "
+ + "IE IL IM IN IO IQ IR IS IT JE JM JO JP KE KG KH KI KM KN KP "
+ + "KR KW KY KZ LA LB LC LI LK LR LS LT LU LV LY MA MC MD ME MF "
+ + "MG MH MK ML MM MN MO MP MQ MR MS MT MU MV MW MX MY MZ NA NC "
+ + "NE NF NG NI NL NO NP NR NU NZ OM PA PE PF PG PH PK PL PM PR "
+ + "PS PT PW PY QA RE RO RS RU RW SA SB SC SD SE SG SH SI SJ SK "
+ + "SL SM SN SO SR ST SV SY SZ TC TD TG TH TJ TK TL TM TN TO TR "
+ + "TT TV TW TZ UA UG UM US UY UZ VA VC VE VG VI VN VU WF WS YE "
+ + "YT ZA ZM ZW").toLowerCase().split(" ")));
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ "descriptors.csv"));
+ bw.write("descriptor,fingerprint,published,address,orport,dirport,"
+ + "version,platform,uptime,bridgestatsend,bridgestatsseconds");
+ for (String country : knownCountries) {
+ bw.write("," + country);
+ }
+ bw.write(",bridgestatscountries,bridgestatstotal\n");
+ int parsedServerDescriptors = 0, parsedExtraInfoDescriptors = 0,
+ parsedGeoipStats = 0, skippedGeoipStats = 0,
+ parsedBridgeStats = 0,
+ totalServerDescriptors = serverDescriptors.size(),
+ writtenOutputLines = 1;
+ SimpleDateFormat timeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ timeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ long started = System.currentTimeMillis();
+ for (File file : serverDescriptors.values()) {
+ String filename = file.getName();
+ BufferedReader br = new BufferedReader(new FileReader(file));
+ String line, fingerprint = null, published = null, address = null,
+ orPort = null, dirPort = null, version = null,
+ platform = null, uptime = null, extraInfoDigest = null,
+ bridgeStatsEnd = null, bridgeStatsSeconds = null;
+ SortedMap<String, String> bridgeStatsIps =
+ new TreeMap<String, String>();
+ long bridgeStatsTotal = 0L;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("opt ")) {
+ line = line.substring(4);
+ }
+ if (line.startsWith("router ")) {
+ String[] parts = line.split(" ");
+ address = parts[2];
+ orPort = parts[3];
+ dirPort = parts[4];
+ } else if (line.startsWith("platform ")) {
+ version = line.split(" ")[2];
+ platform = line.substring(line.indexOf("on ")
+ + "on ".length());
+ if (platform.contains("Windows")) {
+ platform = "Windows";
+ } else if (platform.contains("Linux")) {
+ platform = "Linux";
+ } else if (platform.contains("Darwin")) {
+ platform = "Mac OS X";
+ } else if (platform.contains("BSD")) {
+ platform = "*BSD";
+ } else {
+ platform = "Other";
+ }
+ } else if (line.startsWith("published ")) {
+ String[] parts = line.split(" ");
+ published = parts[1] + " " + parts[2];
+ } else if (line.startsWith("fingerprint ")) {
+ fingerprint = line.substring("fingerprint".length()).
+ replaceAll(" ", "").toLowerCase();
+ } else if (line.startsWith("uptime ")) {
+ uptime = line.split(" ")[1];
+ } else if (line.startsWith("extra-info-digest ")) {
+ extraInfoDigest = line.substring("extra-info-digest ".
+ length()).toLowerCase();
+ if (extraInfoDescriptors.containsKey(extraInfoDigest)) {
+ parsedExtraInfoDescriptors++;
+ BufferedReader br2 = new BufferedReader(new FileReader(
+ extraInfoDescriptors.get(extraInfoDigest)));
+ String geoipStartTime = null, bridgeStatsEndLine = null;
+ while ((line = br2.readLine()) != null) {
+ if (line.startsWith("geoip-start-time ")) {
+ geoipStartTime = line.substring("geoip-start-time ".
+ length());
+ } else if (line.startsWith("geoip-client-origins ") &&
+ line.split(" ").length > 1 && published != null &&
+ geoipStartTime != null) {
+ if (version.startsWith("0.2.2.")) {
+ skippedGeoipStats++;
+ } else {
+ parsedGeoipStats++;
+ bridgeStatsEnd = published;
+ bridgeStatsSeconds = "" +
+ + (timeFormat.parse(published).getTime()
+ - timeFormat.parse(geoipStartTime).getTime())
+ / 1000L;
+ for (String pair : line.split(" ")[1].split(",")) {
+ String country = pair.substring(0, 2);
+ String ips = pair.substring(3);
+ bridgeStatsIps.put(country, ips);
+ bridgeStatsTotal += Long.parseLong(ips);
+ }
+ }
+ } else if (line.startsWith("bridge-stats-end ")) {
+ bridgeStatsEndLine = line;
+ } else if (line.startsWith("bridge-ips ") &&
+ line.length() > "bridge-ips ".length() &&
+ bridgeStatsEndLine != null) {
+ parsedBridgeStats++;
+ String[] parts = bridgeStatsEndLine.split(" ");
+ bridgeStatsEnd = parts[1] + " " + parts[2];
+ bridgeStatsSeconds = parts[3].substring(1);
+ for (String pair : line.split(" ")[1].split(",")) {
+ String country = pair.substring(0, 2);
+ String ips = pair.substring(3);
+ bridgeStatsIps.put(country, ips);
+ bridgeStatsTotal += Long.parseLong(ips);
+ }
+ }
+ }
+ br2.close();
+ }
+ }
+ }
+ br.close();
+ if (fingerprint == null || published == null || address == null ||
+ orPort == null || dirPort == null || version == null ||
+ platform == null || uptime == null) {
+ System.out.println("Server descriptor " + filename + " is "
+ + "missing critical information. Please check. Exiting.");
+ System.exit(1);
+ }
+ bw.write(filename + "," + fingerprint + "," + published + ","
+ + address + "," + orPort + "," + dirPort + "," + version + ","
+ + platform + "," + uptime);
+ if (bridgeStatsEnd != null) {
+ bw.write("," + bridgeStatsEnd + "," + bridgeStatsSeconds);
+ int bridgeStatsCountries = bridgeStatsIps.size();
+ for (String country : knownCountries) {
+ if (bridgeStatsIps.containsKey(country)) {
+ bw.write("," + bridgeStatsIps.remove(country));
+ } else {
+ bw.write(",0");
+ }
+ }
+ if (!bridgeStatsIps.isEmpty()) {
+ StringBuilder message = new StringBuilder();
+ for (String country : bridgeStatsIps.keySet()) {
+ message.append(", " + country);
+ }
+ System.out.println("Unknown " + (bridgeStatsIps.size() == 1 ?
+ "country" : "countries") + " " + message.toString().
+ substring(2) + " in extra-info descriptor "
+ + extraInfoDigest + ". Please check. Exiting.");
+ System.exit(1);
+ }
+ bw.write("," + bridgeStatsCountries + "," + bridgeStatsTotal
+ + "\n");
+ } else {
+ bw.write(",NA,NA");
+ for (String country : knownCountries) {
+ bw.write(",NA");
+ }
+ bw.write(",NA,NA\n");
+ }
+ writtenOutputLines++;
+ parsedServerDescriptors++;
+ if (parsedServerDescriptors % (totalServerDescriptors / 100)
+ == 0) {
+ double fractionDone = (double) (parsedServerDescriptors) /
+ (double) totalServerDescriptors;
+ double fractionLeft = 1.0D - fractionDone;
+ long now = System.currentTimeMillis();
+ double millisLeft = ((double) (now - started)) * fractionLeft /
+ fractionDone;
+ long secondsLeft = (long) millisLeft / 1000L;
+ System.out.println(" " + (parsedServerDescriptors /
+ (totalServerDescriptors / 100)) + "% done, " + secondsLeft
+ + " seconds left.");
+ }
+ }
+ bw.close();
+ System.out.println("Parsed " + parsedServerDescriptors + " server "
+ + "descriptors and " + parsedExtraInfoDescriptors
+ + " extra-info descriptors.\nParsed " + parsedGeoipStats
+ + " geoip-stats and " + parsedBridgeStats + " bridge-stats.\n"
+ + "Skipped " + skippedGeoipStats + " broken geoip-stats of "
+ + "0.2.2.x bridges.\nWrote " + writtenOutputLines + " to "
+ + "descriptors.csv.");
+ }
+
+ /* This is it. */
+ System.out.println("Terminating.");
+ }
+}
+
diff --git a/task-2680/README b/task-2680/README
new file mode 100644
index 0000000..a00856f
--- /dev/null
+++ b/task-2680/README
@@ -0,0 +1,145 @@
+This ticket contains Java and R code to
+
+ a) process bridge and relay data to convert them to a format that is more
+ useful for researchers and
+ b) verify that the output data files are valid.
+
+This README has a separate section for each Java or R code snippet.
+
+The Java applications produce three output formats containing bridge
+descriptors, bridge status lines, and hashed relay identities. The data
+formats are described below.
+
+--------------------------------------------------------------------------
+
+ProcessSanitizedBridges.java
+
+ - Download sanitized bridge descriptors from the metrics website, e.g.,
+ https://metrics.torproject.org/data/bridge-descriptors-2011-01.tar.bz2,
+ and extract them in a local directory, e.g., bridge-descriptors/.
+
+ - Download Apache Commons Codec 1.4 or higher and put in in this
+ directory.
+
+ - Compile the Java class, e.g.,
+ $ javac -cp commons-codec-1.4.jar ProcessSanitizedBridges.java
+
+ - Run the Java class, e.g.,
+ $ java -cp .:commons-codec-1.4.jar ProcessSanitizedBridges
+ bridge-descriptors/
+
+ - Once the Java application is done, you'll find the two files
+ statuses.csv and descriptors.csv in this directory.
+
+--------------------------------------------------------------------------
+
+ProcessRelayConsensuses.java
+
+ - Download v3 relay consensuses from the metrics website, e.g.,
+ https://metrics.torproject.org/data/consensuses-2011-01.tar.bz2, and
+ extract them in a local directory, e.g., consensuses/.
+
+ - Download Apache Commons Codec 1.4 or higher and put in in this
+ directory, unless you haven't already done this above for
+ ProcessSanitizedBridges.java.
+
+ - Compile the Java class, e.g.,
+ $ javac -cp commons-codec-1.4.jar ProcessRelayConsensuses.java
+
+ - Run the Java class, e.g.,
+ $ java -cp .:commons-codec-1.4.jar ProcessRelayConsensuses consensuses/
+
+ - Once the Java application is done, you'll find a file relays.csv in
+ this directory.
+
+--------------------------------------------------------------------------
+
+verify.R
+
+ - Run the R verification script like this:
+ $ R --slave -f verify.R
+
+--------------------------------------------------------------------------
+
+descriptors.csv
+
+The descriptors.csv file contains one line for each bridge descriptor that
+a bridge has published. This descriptor consists of fields coming from
+the bridge's server descriptor and the bridge's extra-info descriptor that
+was published at the same time.
+
+The columns in descriptors.csv are:
+
+ - descriptor: Hex-formatted descriptor identifier
+ - fingerprint: Hex-formatted SHA-1 hash of identity fingerprint
+ - published: ISO-formatted descriptor publication time
+ - address: Sanitized IPv4 address in dotted notation
+ - orport: OR port
+ - dirport: Dir port
+ - version: Tor version
+ - platform: Operating system family (Windows, Linux, etc.)
+ - uptime: Uptime in seconds
+ - bridgestatsend: ISO-formatted time when stats interval ended
+ - bridgestatsseconds: Stats interval length in seconds
+ - ??: Unique client IP addresses that could not be resolved
+ - a1: Unique client IP addresses from anonymous proxies
+ - a2: Unique client IP addresses from satellite providers
+ - ad: Unique client IP addresses from Andorra
+ - ae: Unique client IP addresses from the United Arab Emirates
+ - [...] See ISO 3166-1 alpha-2 country codes
+ - zw: Unique client IP addresses from Zimbabwe
+ - bridgestatscountries: Number of countries with non-zero unique IPs
+ - bridgestatstotal: Total number of unique IPs
+
+There are two sources for the bridgestats* and country-code columns,
+depending on Tor's version. Bridges running Tor version 0.2.1.x or
+earlier use dynamic stats intervals from a few hours to a few days.
+Bridges running early 0.2.2.x versions published faulty stats and are
+therefore removed from descriptors.csv. Bridges running 0.2.2.x or higher
+(except the faulty 0.2.2.x versions) collect stats in 24-hour intervals.
+
+--------------------------------------------------------------------------
+
+statuses.csv
+
+The statuses.csv file contains one line for every bridge that is
+referenced in a bridge network status. Note that if a bridge is running
+for, say, 12 hours, it will be contained in 24 half-hourly published
+statuses in that time and will be listed 24 times in statuses.csv.
+
+The columns in statuses.csv are:
+
+ - status: ISO-formatted status publication time
+ - fingerprint: Hex-formatted SHA-1 hash of identity fingerprint
+ - descriptor: Hex-formatted descriptor identifier
+ - published: ISO-formatted descriptor publication time
+ - address: Sanitized IPv4 address in dotted notation
+ - orport: OR port
+ - dirport: Dir port
+ - authority: TRUE if bridge has the Authority flag, FALSE otherwise
+ - badexit: TRUE if bridge has the BadExit flag, FALSE otherwise
+ - baddirectory: TRUE if bridge has the BadDirectory flag, FALSE otherwise
+ - exit: TRUE if bridge has the Exit flag, FALSE otherwise
+ - fast: TRUE if bridge has the Fast flag, FALSE otherwise
+ - guard: TRUE if bridge has the Guard flag, FALSE otherwise
+ - named: TRUE if bridge has the Named flag, FALSE otherwise
+ - stable: TRUE if bridge has the Stable flag, FALSE otherwise
+ - running: TRUE if bridge has the Running flag, FALSE otherwise
+ - valid: TRUE if bridge has the Valid flag, FALSE otherwise
+ - v2dir: TRUE if bridge has the V2Dir flag, FALSE otherwise
+
+--------------------------------------------------------------------------
+
+relays.csv
+
+The relays.csv file contains SHA-1 hashes of identity fingerprints of
+normal relays. If a bridge uses the same identity key that it also used
+as a relay, it might observe more users than it would observe as a pure
+bridge. Therefore, bridges that have been running as relays before should
+be excluded from bridge statistics.
+
+The columns in relays.csv are:
+
+ - consensus: ISO-formatted consensus publication time
+ - fingerprint: Hex-formatted SHA-1 hash of identity fingerprint
+
diff --git a/task-2680/verify.R b/task-2680/verify.R
new file mode 100644
index 0000000..63ef233
--- /dev/null
+++ b/task-2680/verify.R
@@ -0,0 +1,27 @@
+# Usage: R --slave -f verify.R
+
+if (file.exists("descriptors.csv")) {
+ cat("Verifying descriptors.csv. This may take a while.\n")
+ d <- read.csv("descriptors.csv", stringsAsFactors = FALSE)
+ cat(" ", length(na.omit(d$bridgestatsend)), "of", length(d$descriptor),
+ "descriptors contain bridge stats.\n")
+} else {
+ cat("descriptors.csv does not exist\n")
+}
+
+if (file.exists("statuses.csv")) {
+ cat("Verifying statuses.csv. This may take a while.\n")
+ s <- read.csv("statuses.csv", stringsAsFactors = FALSE)
+ cat(" ", length(s[s$running == TRUE, "running"]), "of",
+ length(s$running), "bridges contained in the statuses have the",
+ "Running flag.\n")
+} else {
+ cat("statuses.csv does not exist\n")
+}
+
+if (file.exists("relays.csv")) {
+ cat("Verifying relays.csv. This may take a while.\n")
+ r <- read.csv("relays.csv", stringsAsFactors = FALSE)
+ summary(as.POSIXct(r$consensus))
+}
+
More information about the tor-commits
mailing list