[or-cvs] r20617: {} Initial checkin of a script that tells you whether some IP a (in projects/archives/trunk: . exonerator)
kloesing at seul.org
kloesing at seul.org
Sat Sep 19 16:16:41 UTC 2009
Author: kloesing
Date: 2009-09-19 12:16:41 -0400 (Sat, 19 Sep 2009)
New Revision: 20617
Added:
projects/archives/trunk/exonerator/
projects/archives/trunk/exonerator/ExoneraTor.java
projects/archives/trunk/exonerator/HOWTO
projects/archives/trunk/exonerator/LICENSE
Log:
Initial checkin of a script that tells you whether some IP address was a Tor relay.
Added: projects/archives/trunk/exonerator/ExoneraTor.java
===================================================================
--- projects/archives/trunk/exonerator/ExoneraTor.java (rev 0)
+++ projects/archives/trunk/exonerator/ExoneraTor.java 2009-09-19 16:16:41 UTC (rev 20617)
@@ -0,0 +1,347 @@
+/* Copyright 2009 The Tor Project
+ * See LICENSE for licensing information */
+
+import java.io.*;
+import java.math.*;
+import java.text.*;
+import java.util.*;
+import org.bouncycastle.util.encoders.Base64;
+
+public final class ExoneraTor {
+
+ public static void main(final String[] args) throws Exception {
+
+ // check parameters
+ if (args.length < 4 || args.length > 5) {
+ System.err.println("\nUsage: java "
+ + ExoneraTor.class.getSimpleName()
+ + " <descriptor archive directory> <IP address in question> "
+ + "<timestamp, in UTC, formatted as YYYY-MM-DD hh:mm:ss> "
+ + "[<target address>[:<target port>]]\n");
+ return;
+ }
+ File archiveDirectory = new File(args[0]);
+ if (!archiveDirectory.exists() || !archiveDirectory.isDirectory()) {
+ System.err.println("\nDescriptor archive directory + "
+ + archiveDirectory.getAbsolutePath()
+ + " does not exist or is not a directory.\n");
+ return;
+ }
+ String relayIP = args[1];
+ String timestampStr = args[2] + " " + args[3];
+ SimpleDateFormat timeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ timeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ long timestamp = timeFormat.parse(timestampStr).getTime();
+ String target = null, targetIP = null, targetPort = null;
+ String[] targetIPParts = null;
+ if (args.length > 4) {
+ target = args[4];
+ if (target.contains(":")) {
+ targetIP = target.split(":")[0];
+ targetPort = target.split(":")[1];
+ } else {
+ targetIP = target;
+ }
+ targetIPParts = targetIP.replace(".", " ").split(" ");
+ }
+ String DELIMITER = "--------------------------------------------------"
+ + "-------------------------";
+ System.out.println("\nTrying to find out whether " + relayIP + " was "
+ + "running as a Tor relay at " + timestampStr
+ + (target != null ? " permitting exiting to " + target : "")
+ + "...\n\n" + DELIMITER);
+
+ // look for consensus files
+ long timestampTooOld = timestamp - 300 * 60 * 1000;
+ long timestampFrom = timestamp - 180 * 60 * 1000;
+ long timestampTooNew = timestamp + 120 * 60 * 1000;
+ Calendar calTooOld = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+ Calendar calFrom = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+ Calendar calTooNew = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+ calTooOld.setTimeInMillis(timestampTooOld);
+ calFrom.setTimeInMillis(timestampFrom);
+ calTooNew.setTimeInMillis(timestampTooNew);
+ System.out.printf("%nLooking for relevant consensuses between "
+ + "%tF %<tT and %s%n", calFrom, timestampStr);
+ SortedSet<File> tooOldConsensuses = new TreeSet<File>();
+ SortedSet<File> relevantConsensuses = new TreeSet<File>();
+ SortedSet<File> tooNewConsensuses = new TreeSet<File>();
+ Stack<File> directoriesLeftToParse = new Stack<File>();
+ directoriesLeftToParse.push(archiveDirectory);
+ SimpleDateFormat consensusTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd-HH:mm:ss");
+ consensusTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ while (!directoriesLeftToParse.isEmpty()) {
+ File directoryOrFile = directoriesLeftToParse.pop();
+ if (directoryOrFile.isDirectory()) {
+ for (File fileInDir : directoryOrFile.listFiles()) {
+ directoriesLeftToParse.push(fileInDir);
+ }
+ continue;
+ } else {
+ String filename = directoryOrFile.getName();
+ if (filename.endsWith("consensus")) {
+ long consensusTime = consensusTimeFormat.parse(
+ filename.substring(0, 19)).getTime();
+ if (consensusTime >= timestampTooOld &&
+ consensusTime < timestampFrom)
+ tooOldConsensuses.add(directoryOrFile);
+ else if (consensusTime >= timestampFrom &&
+ consensusTime <= timestamp)
+ relevantConsensuses.add(directoryOrFile);
+ else if (consensusTime > timestamp &&
+ consensusTime <= timestampTooNew)
+ tooNewConsensuses.add(directoryOrFile);
+ }
+ }
+ }
+ SortedSet<File> allConsensuses = new TreeSet<File>();
+ allConsensuses.addAll(tooOldConsensuses);
+ allConsensuses.addAll(relevantConsensuses);
+ allConsensuses.addAll(tooNewConsensuses);
+ if (allConsensuses.isEmpty()) {
+ System.out.println(" None found!\n\n" + DELIMITER + "\n\nResult is "
+ + "INDECISIVE!\n\nWe cannot make any statement about IP address "
+ + relayIP + "being a relay at " + timestampStr + " or not! We "
+ + "did not find any relevant consensuses preceding the given "
+ + "time. This either means that you did not download and "
+ + "extract the consensus archives preceding the hours before "
+ + "the given time, or (in rare cases) that the directory "
+ + "archives are missing the hours before the timestamp. Please "
+ + "check that your directory archives contain consensus files "
+ + "of the interval 5:00 hours before and 2:00 hours after the "
+ + "time you are looking for.\n");
+ return;
+ }
+ for (File f : relevantConsensuses)
+ System.out.println(" " + f.getAbsolutePath());
+
+ // parse consensuses to find descriptors belonging to the IP address
+ System.out.println("\nLooking for descriptor identifiers referenced "
+ + "in \"r \" lines in these consensuses containing IP address "
+ + relayIP + "...");
+ SortedSet<File> positiveConsensusesNoTarget = new TreeSet<File>();
+ Set<String> addressesInSameNetwork = new HashSet<String>();
+ SortedMap<String, Set<File>> relevantDescriptors =
+ new TreeMap<String, Set<File>>();
+ for (File consensus : allConsensuses) {
+ if (relevantConsensuses.contains(consensus))
+ System.out.println(" " + consensus.getAbsolutePath());
+ BufferedReader br = new BufferedReader(new FileReader(consensus));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (!line.startsWith("r "))
+ continue;
+ String[] parts = line.split(" ");
+ String address = parts[6];
+ if (address.equals(relayIP)) {
+ byte[] result = Base64.decode(parts[3] + "==");
+ String hex = new BigInteger(1, Base64.decode(parts[3] +
+ "==")).toString(16).substring(0, 40);
+ if (!relevantDescriptors.containsKey(hex))
+ relevantDescriptors.put(hex, new HashSet<File>());
+ relevantDescriptors.get(hex).add(consensus);
+ positiveConsensusesNoTarget.add(consensus);
+ if (relevantConsensuses.contains(consensus))
+ System.out.println(" \"" + line + "\" references "
+ + "descriptor " + hex);
+ } else {
+ if (relayIP.startsWith(address.substring(0,
+ address.lastIndexOf(".")))) {
+ addressesInSameNetwork.add(address);
+ }
+ }
+ }
+ br.close();
+ }
+ if (relevantDescriptors.isEmpty()) {
+ System.out.printf(" None found!\n\n" + DELIMITER + "\n\nResult is "
+ + "NEGATIVE with moderate certainty!\n\nWe did not find IP "
+ + "address " + relayIP + " in any of the consensuses that were "
+ + "published between %tF %<tT and %tF %<tT.\n\nA possible "
+ + "reason for false negatives is that the relay is using a "
+ + "different IP address when generating a descriptor than for "
+ + "exiting to the Internet. We hope to provide better checks "
+ + "for this case in the future.", calTooOld, calTooNew);
+ if (!addressesInSameNetwork.isEmpty()) {
+ System.out.println("\n\nThe following other IP addresses of Tor "
+ + "relays were found in the mentioned consensus files that "
+ + "are in the same /24 network and that could be related to "
+ + "IP address " + relayIP + ":");
+ for (String s : addressesInSameNetwork) {
+ System.out.println(" " + s);
+ }
+ }
+ System.out.println();
+ return;
+ }
+
+ // parse router descriptors to check exit policies
+ SortedSet<File> positiveConsensuses = new TreeSet<File>();
+ Set<String> missingDescriptors = new HashSet<String>();
+ if (target != null) {
+ System.out.println("\nChecking if referenced descriptors permit "
+ + "exiting to " + target + "...");
+ Set<String> descriptors = relevantDescriptors.keySet();
+ missingDescriptors.addAll(relevantDescriptors.keySet());
+ directoriesLeftToParse.clear();
+ directoriesLeftToParse.push(archiveDirectory);
+ while (!directoriesLeftToParse.isEmpty()) {
+ File directoryOrFile = directoriesLeftToParse.pop();
+ if (directoryOrFile.isDirectory()) {
+ for (File fileInDir : directoryOrFile.listFiles()) {
+ directoriesLeftToParse.push(fileInDir);
+ }
+ continue;
+ } else {
+ String filename = directoryOrFile.getName();
+ for (String descriptor : descriptors) {
+ if (filename.equals(descriptor)) {
+ missingDescriptors.remove(descriptor);
+ BufferedReader br = new BufferedReader(
+ new FileReader(directoryOrFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.startsWith("reject ") ||
+ line.startsWith("accept ")) {
+ boolean ruleAccept = line.split(" ")[0].equals("accept");
+ String ruleAddress = line.split(" ")[1].split(":")[0];
+ if (!ruleAddress.equals("*")) {
+ if (!ruleAddress.contains("/") &&
+ !ruleAddress.equals(targetIP))
+ continue; // IP address does not match
+ String[] ruleIPParts = ruleAddress.split("/")[0].
+ replace(".", " ").split(" ");
+ int ruleNetwork = Integer.parseInt(
+ ruleAddress.split("/")[1]);
+ for (int i = 0; i < 4; i++) {
+ if (ruleNetwork == 0) {
+ break;
+ } else if (ruleNetwork >= 8) {
+ if (ruleIPParts[i].equals(targetIPParts[i]))
+ ruleNetwork -= 8;
+ else
+ break;
+ } else {
+ int mask = 255 ^ 255 >>> ruleNetwork;
+ if ((Integer.parseInt(ruleIPParts[i]) & mask) ==
+ (Integer.parseInt(targetIPParts[i]) & mask))
+ ruleNetwork = 0;
+ break;
+ }
+ }
+ if (ruleNetwork > 0)
+ continue; // IP address does not match
+ }
+ String rulePort = line.split(" ")[1].split(":")[1];
+ if (targetPort == null && !ruleAccept &&
+ !rulePort.equals("*"))
+ continue; // with no port given, we only consider
+ // reject :* rules as matching
+ if (targetPort != null) {
+ if (!rulePort.equals("*") &&
+ !targetPort.equals(rulePort))
+ continue; // ports do not match
+ }
+ boolean relevantMatch = false;
+ for (File f : relevantDescriptors.get(descriptor))
+ if (relevantConsensuses.contains(f))
+ relevantMatch = true;
+ if (relevantMatch)
+ System.out.println(" "
+ + directoryOrFile.getAbsolutePath() + " "
+ + (ruleAccept ? "permits" : "does not permit")
+ + " exiting to " + target + " according to rule \""
+ + line + "\"");
+ if (ruleAccept)
+ positiveConsensuses.addAll(
+ relevantDescriptors.get(descriptor));
+ break;
+ }
+ }
+ br.close();
+ }
+ }
+ }
+ }
+ }
+
+ // print out result
+ Set<File> matches = (target != null) ? positiveConsensuses
+ : positiveConsensusesNoTarget;
+ if (matches.contains(relevantConsensuses.last())) {
+ System.out.println("\n" + DELIMITER + "\n\nResult is POSITIVE with "
+ + "high certainty!\n\nWe found one or more relays on IP address "
+ + relayIP
+ + (target != null ? " permitting exit to " + target : "")
+ + " in the most recent consensus preceding " + timestampStr
+ + " that clients were likely to know.\n");
+ return;
+ }
+ boolean inOtherRelevantConsensus = false, inTooOldConsensuses = false,
+ inTooNewConsensuses = false;
+ for (File f : matches)
+ if (relevantConsensuses.contains(f))
+ inOtherRelevantConsensus = true;
+ else if (tooOldConsensuses.contains(f))
+ inTooOldConsensuses = true;
+ else if (tooNewConsensuses.contains(f))
+ inTooNewConsensuses = true;
+ if (inOtherRelevantConsensus) {
+ System.out.println("\n" + DELIMITER + "\n\nResult is POSITIVE with "
+ + "moderate certainty!\n\nWe found one or more relays on IP "
+ + "address " + relayIP
+ + (target != null ? " permitting exit to " + target : "")
+ + ", but not in the consensus immediately preceding "
+ + timestampStr + ". A possible reason for the relay being "
+ + "missing in the last consensus preceding the given time might "
+ + "be that some of the directory authorities had difficulties "
+ + "connecting to the relay. However, clients might still have "
+ + "used the relay.");
+ } else {
+ System.out.println("Result is NEGATIVE with high certainty!\n\nWe "
+ + "did not find any relay on IP address " + relayIP
+ + (target != null ? " permitting exit to " + target : "")
+ + "in the consensuses 3:00 hours preceding " + timestampStr
+ + ".");
+ if (inTooOldConsensuses || inTooNewConsensuses) {
+ if (inTooOldConsensuses && !inTooNewConsensuses)
+ System.out.println("\nNote that we found a matching relay in "
+ + "consensuses that were published between 5:00 and 3:00 "
+ + "hours before " + timestampStr + ". ");
+ else if (!inTooOldConsensuses && inTooNewConsensuses)
+ System.out.println("\nNote that we found a matching relay in "
+ + "consensuses that were published up to 2:00 hours after "
+ + timestampStr + ". ");
+ else
+ System.out.println("\nNote that we found a matching relay in "
+ + "consensuses that were published between 5:00 and 3:00 "
+ + "hours before and in consensuses that were published up "
+ + "to 2:00 hours after " + timestampStr + ". ");
+ System.out.println("Make sure that the timestamp you provided is "
+ + "in the correct timezone: UTC (or GMT).");
+ }
+ }
+ if (target != null) {
+ if (positiveConsensuses.isEmpty() &&
+ !positiveConsensusesNoTarget.isEmpty())
+ System.out.println("\nNote that although the found relay(s) did "
+ + "not permit exiting to " + target + ", there have been one "
+ + "or more relays running at the given time.");
+ if (!missingDescriptors.isEmpty()) {
+ System.out.println("\nNote that not all referenced descriptors "
+ + "could be found. We cannot make any good statement about "
+ + "exit relays without these descriptors. Make sure you "
+ + "downloaded and extracted the server descriptors of the "
+ + "given time. (In rare cases it also happens that we are "
+ + "missing single descriptors in the archives.) The following "
+ + "descriptors are missing:");
+ for (String desc : missingDescriptors)
+ System.out.println(" " + desc);
+ }
+ }
+ System.out.println();
+ }
+}
+
Property changes on: projects/archives/trunk/exonerator/ExoneraTor.java
___________________________________________________________________
Added: svn:mergeinfo
+
Added: projects/archives/trunk/exonerator/HOWTO
===================================================================
--- projects/archives/trunk/exonerator/HOWTO (rev 0)
+++ projects/archives/trunk/exonerator/HOWTO 2009-09-19 16:16:41 UTC (rev 20617)
@@ -0,0 +1,110 @@
+ExoneraTor
+ or: a script that tells you whether some IP address was a Tor relay
+
+---------------------------------------------------------------------------
+
+Introduction:
+
+Some people have expressed the desire to learn whether a given IP address
+has been a Tor relay at a certain time. In addition to that, these people
+might want to know whether the IP address permitted exit to a given address
+and port.
+
+Answering these questions can be important for Tor relay operators to show
+to the authorities that an anonymous user might have conducted bad things
+with their IP address. Likewise, police investigators might be interested
+in the answer to these questions, too, in order to decide whether to
+proceed with their investigations or not.
+
+We can answer the above questions from looking at the descriptor archives
+that are available since late 2007 (or even beyond, but this script only
+works with the data format that was produced starting in October 2007).
+This script parses the directory archives to print out the answer whether
+a certain IP address was a Tor relay at a given time. The script further
+prints out all intermediate steps in answering this, so that users can
+confirm the correctness of the result themselves.
+
+---------------------------------------------------------------------------
+
+Quick Start:
+
+In order to run this script, you need to install and download the following
+software and data (please note that all instructions are written for Linux;
+commands for Windows or Mac OS X may vary):
+
+- Install Java 6 or higher.
+
+- Download the BouncyCastle provider that includes Base 64 decoding from
+ http://www.bouncycastle.org/download/bcprov-jdk16-143.jar and put it in
+ your working directory, e.g. /home/you/exonerator/ .
+
+- Copy the consensuses-* and server-descriptors-* files of the relevant
+ time from http://archive.torproject.org/tor-directory-authority-archive/
+ and extract them to a directory in your working directory, e.g.
+ /home/you/exonerator/data/ .
+
+ Note that all files are touched by the script at least once. You might
+ want to avoid putting the archives of more than, say, two months in that
+ directory for your evaluation. You may just temporarily move the
+ irrelevant archives away to another directory, e.g., data-tmp/ in your
+ working directory.
+
+ Also note that you only need the server-descriptors-* files if you want
+ to learn whether a given IP address permits exiting to a given target. If
+ you only want to learn whether that IP address was a Tor relay, you
+ don't need them.
+
+- Compile the (single) Java class using this command:
+
+ $ javac -cp bcprov-jdk16-143.jar ExoneraTor.java
+
+- Run the script, providing it with the parameters it needs:
+
+ java -cp .:bcprov-jdk16-143.jar ExoneraTor
+ <descriptor archive directory>
+ <IP address in question>
+ <timestamp, in UTC, formatted as YYYY-MM-DD hh:mm:ss>
+ [<target address>[:<target port>]]
+
+ Make sure that the timestamp is provided in UTC, which is similar to GMT,
+ and not in your local timezone! Otherwise, results will very likely be
+ wrong.
+
+ A sample invocation might be (without line break):
+
+ $ java -cp .:bcprov-jdk16-143.jar ExoneraTor data/ 209.17.171.104
+ 2009-08-15 16:05:00 209.85.129.104:80
+
+---------------------------------------------------------------------------
+
+Test cases:
+
+The following test cases work with the August 2009 archives and can be used
+to check whether this script works correctly (line breaks have been added
+only for formatting reasons here):
+
+- Positive result of echelon1+2 being a relay:
+
+ $ java -cp .:bcprov-jdk16-141.jar ExoneraTor data/ 209.17.171.104
+ 2009-08-15 16:05:00
+
+- Positive result of echelon1+2 exiting to google.com on any port
+
+ $ java -cp .:bcprov-jdk16-141.jar ExoneraTor data/ 209.17.171.104
+ 2009-08-15 16:05:00 209.85.129.104
+
+- Positive result of echelon1+2 exiting to google.com on port 80
+
+ $ java -cp .:bcprov-jdk16-141.jar ExoneraTor data/ 209.17.171.104
+ 2009-08-15 16:05:00 209.85.129.104:80
+
+- Negative result of echelon1+2 exiting to google.com, but not on port 25
+
+ $ java -cp .:bcprov-jdk16-141.jar ExoneraTor data/ 209.17.171.104
+ 2009-08-15 16:05:00 209.85.129.104:25
+
+- Negative result with IP address of echelon1+2 changed in the last octet
+
+ $ java -cp .:bcprov-jdk16-141.jar ExoneraTor data/ 209.17.171.50
+ 2009-08-15 16:05:00
+
Property changes on: projects/archives/trunk/exonerator/HOWTO
___________________________________________________________________
Added: svn:mergeinfo
+
Added: projects/archives/trunk/exonerator/LICENSE
===================================================================
--- projects/archives/trunk/exonerator/LICENSE (rev 0)
+++ projects/archives/trunk/exonerator/LICENSE 2009-09-19 16:16:41 UTC (rev 20617)
@@ -0,0 +1,30 @@
+Copyright 2009 The Tor Project
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+ copyright notice, this list of conditions and the following disclaimer
+ in the documentation and/or other materials provided with the
+ distribution.
+
+ * Neither the names of the copyright owners nor the names of its
+ contributors may be used to endorse or promote products derived from
+ this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
Property changes on: projects/archives/trunk/exonerator/LICENSE
___________________________________________________________________
Added: svn:mergeinfo
+
More information about the tor-commits
mailing list