Author: kloesing Date: 2011-05-18 20:15:43 +0000 (Wed, 18 May 2011) New Revision: 24768 Added: projects/archives/trunk/README Removed: projects/archives/trunk/bridge-desc-sanitizer/ConvertBridgeDescs.java projects/archives/trunk/bridge-desc-sanitizer/HOWTO projects/archives/trunk/bridge-desc-sanitizer/extract-bridges.sh projects/archives/trunk/exonerator/ExoneraTor.java projects/archives/trunk/exonerator/HOWTO projects/archives/trunk/exonerator/LICENSE projects/archives/trunk/exonerator/exonerator.py Log: Only leave a note that the code moved to Git. Added: projects/archives/trunk/README =================================================================== --- projects/archives/trunk/README (rev 0) +++ projects/archives/trunk/README 2011-05-18 20:15:43 UTC (rev 24768) @@ -0,0 +1,7 @@ +--------------------------------------------------------------------------- + + THIS REPOSITORY HAS MOVED TO GIT! + + git clone git://git.torproject.org/metrics-utils/ + +--------------------------------------------------------------------------- Deleted: projects/archives/trunk/bridge-desc-sanitizer/ConvertBridgeDescs.java =================================================================== --- projects/archives/trunk/bridge-desc-sanitizer/ConvertBridgeDescs.java 2011-05-17 22:47:52 UTC (rev 24767) +++ projects/archives/trunk/bridge-desc-sanitizer/ConvertBridgeDescs.java 2011-05-18 20:15:43 UTC (rev 24768) @@ -1,504 +0,0 @@ -import java.io.*; -import java.util.*; -import com.maxmind.geoip.*; -import org.apache.commons.codec.digest.*; -import org.apache.commons.codec.binary.*; - -public class ConvertBridgeDescs { - - public static void main(String[] args) throws Exception { - - /* If the following flag is set to true, don't write 127.0.0.1 for - * bridge IP addresses, but put replace IP addresses with - * H(IP address + bridge identity + secret)[:4] formatted as IP - * address. An example for the hash input would be: - * "12.34.56.78ABCDABCDABCDABCDABCDABCDABCDABCDABCDABCDpassword" - * (without quotes) */ - boolean hashIpAddresses = false; - String secret = "password"; - - long started = System.currentTimeMillis(); - - if (args.length < 5) { - System.err.println("Usage: java " - + ConvertBridgeDescs.class.getSimpleName() - + " <input directory> <geoip.txt file> <YYYY> <MM> " - + "<output directory>"); - System.exit(1); - } - File inDir = new File(args[0]); - File geoipFile = new File(args[1]); - LookupService cl = new LookupService(geoipFile, - LookupService.GEOIP_MEMORY_CACHE); - Set<String> unresolved = new HashSet<String>(); - unresolved.add("--"); - unresolved.add("a1"); - unresolved.add("a2"); - unresolved.add("eu"); - unresolved.add("ap"); - String year = args[2]; - String month = args[3]; - int yearInt = Integer.parseInt(year); - int monthInt = Integer.parseInt(month); - File outDir = new File(args[4] + File.separator - + "bridge-descriptors-" + year + "-" + month); - outDir.mkdirs(); - - SortedSet<File> statuses = new TreeSet<File>(); - Set<File> descriptors = new HashSet<File>(); - Set<File> extrainfos = new HashSet<File>(); - - System.out.println("Checking files in " + inDir.getAbsolutePath() - + "..."); - Stack<File> directoriesLeftToParse = new Stack<File>(); - directoriesLeftToParse.push(inDir); - String currentYearAndMonth = "from-tonga-" + year + "-" + month; - String previousYearAndMonth = "from-tonga-" + (monthInt == 1 ? - "" + (yearInt - 1) + "-12" : - year + "-" + (monthInt < 11 ? "0" : "") + (monthInt - 1)); - String nextYearAndMonth = "from-tonga-" + (monthInt == 12 ? - "" + (yearInt + 1) + "-01" : - year + "-" + (monthInt < 9 ? "0" : "") + (monthInt + 1)); - while (!directoriesLeftToParse.isEmpty()) { - File directoryOrFile = directoriesLeftToParse.pop(); - String filename = directoryOrFile.getName(); - if (directoryOrFile.isDirectory()) { - if (/* base directory */ - filename.equals(inDir.getName()) || - /* current month */ - filename.startsWith(currentYearAndMonth) || - /* last days of previous month */ - (filename.startsWith(previousYearAndMonth) - && Integer.parseInt(filename.substring(19, 21)) > 24) || - /* first days of next month */ - (filename.startsWith(nextYearAndMonth) - && Integer.parseInt(filename.substring(19, 21)) < 6)) { - for (File fileInDir : directoryOrFile.listFiles()) { - directoriesLeftToParse.push(fileInDir); - } - } - continue; - } - if (filename.startsWith("cached-extrainfo")) { - extrainfos.add(directoryOrFile); - } else if (filename.equals("bridge-descriptors")) { - descriptors.add(directoryOrFile); - } else if (filename.equals("networkstatus-bridges")) { - statuses.add(directoryOrFile); - } - } - - int days = ((extrainfos.size() / 2 + descriptors.size() - + statuses.size()) + 3 * 24) / (3 * 48); - System.out.println("Found " + extrainfos.size() - + " cached-extrainfo[.new] files, " + descriptors.size() - + " bridge-descriptors files, and " + statuses.size() - + " networkstatus-bridges files, covering approximately " + days - + " days."); - - System.out.print("Parsing server descriptors to find out country " - + "codes of bridges in extra-info descriptors"); - Map<String, String> bridgeCountries = new HashMap<String, String>(); - int parsed = 0; - for (File file : descriptors) { - if (parsed++ > descriptors.size() / days) { - System.out.print("."); - parsed = 0; - } - BufferedReader br = new BufferedReader(new FileReader(file)); - String line = null, routerLine = null; - while ((line = br.readLine()) != null) { - if (line.startsWith("router ")) { - routerLine = line; - } else if (line.startsWith("opt extra-info-digest ")) { - String extraInfoDigest = line.split(" ")[2]; - String countryCode = cl.getCountry(routerLine.split(" ")[2]). - getCode(); - if (bridgeCountries.containsKey(extraInfoDigest) && - !bridgeCountries.get(extraInfoDigest). - equals(countryCode)) { - System.out.println("Mapping already contains extra-info " - + "digest " + extraInfoDigest + " with different " - + "country. Exiting."); - System.exit(1); - } - bridgeCountries.put(extraInfoDigest, countryCode); - } - } - } - System.out.println("Mapping contains " + bridgeCountries.size() - + " entries."); - - System.out.print("Parsing extra-info descriptors"); - String[] hex = new String[] { "0", "1", "2", "3", "4", "5", "6", "7", - "8", "9", "a", "b", "c", "d", "e", "f" }; - for (String x : hex) - for (String y : hex) - new File(outDir + File.separator + "extra-infos" + File.separator - + x + File.separator + y).mkdirs(); - int writtenExtrainfos = 0; - Map<String, String> extrainfoMapping = new HashMap<String, String>(); - parsed = 0; - for (File file : extrainfos) { - if (parsed++ > extrainfos.size() / days) { - System.out.print("."); - parsed = 0; - } - FileInputStream fis = new FileInputStream(file); - BufferedInputStream bis = new BufferedInputStream(fis); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] data = new byte[1024]; - while ((len = bis.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - bis.close(); - byte[] allData = baos.toByteArray(); - int startDescriptorIndex = -1, endDescriptorIndex = -1; - String asciiString = new String(allData, "US-ASCII"); - BufferedReader br = new BufferedReader(new StringReader( - asciiString)); - String line = null; - StringBuilder scrubbed = null; - boolean skipSignature = false; - boolean skipDescriptor = false; - while ((line = br.readLine()) != null) { - if (skipSignature && !line.equals("-----END SIGNATURE-----")) { - continue; - } else if (line.startsWith("extra-info ")) { - endDescriptorIndex = startDescriptorIndex = - asciiString.indexOf(line, startDescriptorIndex + 1); - scrubbed = new StringBuilder(DigestUtils.shaHex(Hex.decodeHex( - line.split(" ")[2].toCharArray())).toUpperCase() + "\n"); - } else if (line.startsWith("published ") - || line.startsWith("write-history ") - || line.startsWith("read-history ") - || line.startsWith("geoip-start-time ") - || line.startsWith("geoip-client-origins ") - || line.startsWith("bridge-stats-end ") - || line.startsWith("bridge-ips ")) { - scrubbed.append(line + "\n"); - } else if (line.startsWith("router-signature")) { - if (skipDescriptor) { - System.out.println("Skipping!"); - skipDescriptor = false; - } else { - endDescriptorIndex = asciiString.indexOf(line, - endDescriptorIndex + 1) + line.length() + 1; - byte[] forDigest = new byte[endDescriptorIndex - - startDescriptorIndex]; - System.arraycopy(allData, startDescriptorIndex, forDigest, 0, - endDescriptorIndex - startDescriptorIndex); - String originalHash = DigestUtils.shaHex(forDigest); - String countryCode = "ZZ"; - if (bridgeCountries.containsKey(originalHash.toUpperCase())) { - countryCode = bridgeCountries.get(originalHash.toUpperCase()); - } - String scrubbedDesc = "extra-info Unnamed" + countryCode + " " - + scrubbed.toString(); - String scrubbedHash = DigestUtils.shaHex(scrubbedDesc); - if (extrainfoMapping.containsKey(originalHash) && - !extrainfoMapping.get(originalHash).equals(scrubbedHash)) { - System.out.println("We already have an extra-info mapping " - + "from " + originalHash + " to " - + extrainfoMapping.get(originalHash) + ", but we now " - + "want to add a mapping to " + scrubbedHash - + ". Exiting"); - System.exit(1); - } - extrainfoMapping.put(originalHash, scrubbedHash); - File out = new File(outDir + File.separator + "extra-infos" - + File.separator + scrubbedHash.charAt(0) + File.separator - + scrubbedHash.charAt(1) + File.separator + scrubbedHash); - if (!out.exists()) { - BufferedWriter bw = new BufferedWriter(new FileWriter(out)); - bw.write(scrubbedDesc); - bw.close(); - writtenExtrainfos++; - } - } - } else if (line.equals("-----BEGIN SIGNATURE-----")) { - skipSignature = true; - } else if (line.equals("-----END SIGNATURE-----")) { - skipSignature = false; - } else if (line.startsWith("dirreq-") || line.startsWith("cell-") - || line.startsWith("exit-")) { - continue; - } else { - System.out.println("Unrecognized line '" + line + "'. Skipping"); - skipDescriptor = true; - } - } - br.close(); - } - System.out.println("\nWrote " + writtenExtrainfos - + " extra-info descriptors."); - - System.out.print("Parsing server descriptors"); - for (String x : hex) - for (String y : hex) - new File(outDir + File.separator + "server-descriptors" - + File.separator + x + File.separator + y).mkdirs(); - int writtenDescriptors = 0; - Map<String, String> descriptorMapping = new HashMap<String, String>(); - int found = 0, notfound = 0; - parsed = 0; - String haveExtraInfo = null; - for (File file : descriptors) { - if (parsed++ > descriptors.size() / days) { - System.out.print("."); - parsed = 0; - } - FileInputStream fis = new FileInputStream(file); - BufferedInputStream bis = new BufferedInputStream(fis); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] data = new byte[1024]; - while ((len = bis.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - bis.close(); - byte[] allData = baos.toByteArray(); - int startDescriptorIndex = -1, endDescriptorIndex = -1; - String asciiString = new String(allData, "US-ASCII"); - BufferedReader br = new BufferedReader(new StringReader( - asciiString)); - String line = null, country = null, originalAddress = null, - ipAddress = "127.0.0.1", routerLinePartOne = null, - routerLinePartTwo = null; - StringBuilder scrubbed = null; - boolean skipCrypto = false, contactWritten = false; - while ((line = br.readLine()) != null) { - if (skipCrypto && !line.startsWith("-----END ")) { - continue; - } else if (line.startsWith("router ")) { - endDescriptorIndex = startDescriptorIndex = - asciiString.indexOf(line, startDescriptorIndex + 1); - country = cl.getCountry(line.split(" ")[2]).getCode(). - toLowerCase(); - if (unresolved.contains(country)) { - country = "zz"; - } - originalAddress = line.split(" ")[2]; - scrubbed = new StringBuilder(); - routerLinePartOne = "router Unnamed" + country.toUpperCase(); - routerLinePartTwo = line.split(" ")[3] + " " - + line.split(" ")[4] + " " + line.split(" ")[5] + "\n"; - contactWritten = false; - haveExtraInfo = null; - } else if (line.startsWith("opt fingerprint ")) { - scrubbed.append("opt fingerprint"); - String fingerprint = DigestUtils.shaHex(Hex.decodeHex( - line.substring(16).replaceAll(" ", "").toCharArray())). - toUpperCase(); - for (int i = 0; i < fingerprint.length() / 4; i++) - scrubbed.append(" " + fingerprint.substring(4 * i, - 4 * (i + 1))); - scrubbed.append("\n"); - if (hashIpAddresses) { - byte[] hashedOctets = DigestUtils.sha(originalAddress - + line.substring(16).replaceAll(" ", "") + secret); - String hashedIp = ""; - for (int i = 0; i < 4; i++) { - hashedIp += "." + ((int) hashedOctets[i] + 256) % 256; - } - ipAddress = hashedIp.substring(1); - } - } else if (line.startsWith("contact ")) { - scrubbed.append("contact somebody at example dot " + country - + "\n"); - contactWritten = true; - } else if (line.startsWith("router-signature")) { - endDescriptorIndex = asciiString.indexOf(line, - endDescriptorIndex + 1) + line.length() + 1; - byte[] forDigest = new byte[endDescriptorIndex - - startDescriptorIndex]; - System.arraycopy(allData, startDescriptorIndex, forDigest, 0, - endDescriptorIndex - startDescriptorIndex); - String originalHash = DigestUtils.shaHex(forDigest); - String scrubbedDesc = routerLinePartOne + " " + ipAddress - + " " + routerLinePartTwo + scrubbed.toString(); - String scrubbedHash = DigestUtils.shaHex(scrubbedDesc); - if (descriptorMapping.containsKey(originalHash) && - !descriptorMapping.get(originalHash).equals(scrubbedHash)) { - System.out.println("We already have a descriptor mapping " - + "from " + originalHash + " to " - + descriptorMapping.get(originalHash) + ", but we now " - + "want to add a mapping to " + scrubbedHash - + ". Exiting"); - System.exit(1); - } - descriptorMapping.put(originalHash, scrubbedHash); - if (haveExtraInfo != null) { - File out = new File(outDir + File.separator - + "server-descriptors" + File.separator - + scrubbedHash.charAt(0) + File.separator - + scrubbedHash.charAt(1) + File.separator + scrubbedHash); - if (!out.exists()) { - BufferedWriter bw2 = new BufferedWriter(new FileWriter(out)); - bw2.write(scrubbedDesc); - bw2.close(); - writtenDescriptors++; - } - } - } else if (line.startsWith("opt extra-info-digest ")) { - String originalExtraInfo = line.split(" ")[2].toLowerCase(); - if (!extrainfoMapping.containsKey(originalExtraInfo)) { - notfound++; - haveExtraInfo = "0000000000000000000000000000000000000000"; - } else { - found++; - haveExtraInfo = extrainfoMapping.get(originalExtraInfo). - toUpperCase(); - } - scrubbed.append("opt extra-info-digest " + haveExtraInfo - + "\n"); - } else if (line.startsWith("reject ") - || line.startsWith("accept ")) { - if (!contactWritten) { - scrubbed.append("contact nobody at example dot " + country - + "\n"); - contactWritten = true; - } - scrubbed.append(line + "\n"); - } else if (line.startsWith("platform ") - || line.startsWith("opt protocols ") - || line.startsWith("published ") - || line.startsWith("uptime ") - || line.startsWith("bandwidth ") - || line.startsWith("opt hibernating ") - || line.equals("opt hidden-service-dir") - || line.equals("opt caches-extra-info") - || line.equals("opt allow-single-hop-exits")) { - scrubbed.append(line + "\n"); - } else if (line.startsWith("family ")) { - StringBuilder familyLine = new StringBuilder("family"); - for (String s : line.substring(7).split(" ")) { - if (s.startsWith("$")) { - familyLine.append(" $" + DigestUtils.shaHex(Hex.decodeHex( - s.substring(1).toCharArray())).toUpperCase()); - } else { - familyLine.append(" Unnamed"); - } - } - scrubbed.append(familyLine.toString() + "\n"); - } else if (line.startsWith("@purpose ")) { - continue; - } else if (line.startsWith("-----BEGIN ") - || line.equals("onion-key") || line.equals("signing-key")) { - skipCrypto = true; - } else if (line.startsWith("-----END ")) { - skipCrypto = false; - } else { - System.out.println("Unrecognized line '" + line + "'. Exiting"); - System.exit(1); - } - } - br.close(); - } - System.out.println("\nWrote " + writtenDescriptors - + " bridge descriptors. While parsing, we found that we parsed " - + found + " extra-info identifiers before, but are missing " - + notfound + ". (The number of missing identifiers should be " - + "significantly smaller.)"); - - System.out.print("Parsing network statuses"); - parsed = notfound = found = 0; - for (File file : statuses) { - if (parsed++ > statuses.size() / days) { - System.out.print("."); - parsed = 0; - } - if (!file.getParent().substring(file.getParent(). - indexOf("from-tonga-")).startsWith(currentYearAndMonth)) { - continue; - } - BufferedReader br = new BufferedReader(new FileReader(file)); - String line = null; - StringBuilder scrubbed = new StringBuilder(); - while ((line = br.readLine()) != null) { - if (line.startsWith("r ")) { - String[] parts = line.split(" "); - String bridgeIdentity = parts[2] + "=="; - String hashedBridgeIdentity = Base64.encodeBase64String( - DigestUtils.sha(Base64.decodeBase64(bridgeIdentity))). - substring(0, 27); - String descIdentifier = parts[3] + "=="; - String hexDescIdentifier = Hex.encodeHexString( - Base64.decodeBase64(descIdentifier)); - String replacementDescIdentifier = null; - if (!descriptorMapping.containsKey(hexDescIdentifier)) { - notfound++; - replacementDescIdentifier = "AAAAAAAAAAAAAAAAAAAAAAAAAAA"; - } else { - found++; - String refDesc = descriptorMapping.get(hexDescIdentifier). - toLowerCase(); - File descriptorFile = new File(outDir + File.separator - + "server-descriptors" + File.separator - + refDesc.charAt(0) + File.separator + refDesc.charAt(1) - + File.separator + refDesc); - if (!descriptorFile.exists()) { - System.out.println("Descriptor file '" - + descriptorFile.getAbsolutePath() + "' does not exist."); - System.exit(1); - } - replacementDescIdentifier = Base64.encodeBase64String( - Hex.decodeHex(descriptorMapping.get(hexDescIdentifier). - toCharArray())).substring(0, 27); - } - String country = cl.getCountry(parts[6]).getCode(). - toLowerCase(); - if (unresolved.contains(country)) { - country = "zz"; - } - String ipAddress = "127.0.0.1"; - if (hashIpAddresses) { - byte[] hashedOctets = DigestUtils.sha(parts[6] - + Hex.encodeHexString(Base64.decodeBase64( - bridgeIdentity)).toUpperCase() + secret); - String hashedIp = ""; - for (int i = 0; i < 4; i++) { - hashedIp += "." + ((int) hashedOctets[i] + 256) % 256; - } - ipAddress = hashedIp.substring(1); - } - scrubbed.append("r Unnamed" + country.toUpperCase() + " " - + hashedBridgeIdentity - + " " + replacementDescIdentifier + " " + parts[4] + " " - + parts[5] + " " + ipAddress + " " + parts[7] + " " - + parts[8] + "\n"); - } else if (line.startsWith("s ")) { - scrubbed.append(line + "\n"); - } else { - System.out.println("Unknown line: " + line); - System.exit(1); - } - } - String timeString = file.getParent().substring(file.getParent(). - indexOf("from-tonga-") + 11); - String[] date = timeString.substring(0, 10).split("-"); - String time = timeString.substring(11, 17); - File dir = new File(outDir + File.separator + "statuses" - + File.separator + date[2] + File.separator); - dir.mkdirs(); - File out = new File(dir.getAbsolutePath() + File.separator + date[0] - + date[1] + date[2] + "-" + time + "-" - + "4A0CCD2DDC7995083D73F5D667100C8A5831F16D"); - if (!out.exists()) { - BufferedWriter bw3 = new BufferedWriter(new FileWriter(out)); - bw3.write(scrubbed.toString()); - bw3.close(); - } - } - System.out.println("\nWhile parsing, we found that we parsed " - + found + " bridge descriptors before, but are missing " - + notfound + ". (The number of missing identifiers should be " - + "significantly smaller.)"); - - long finished = System.currentTimeMillis(); - System.out.println("Processing took " + ((finished - started) / 1000) - + " seconds."); - } -} - Deleted: projects/archives/trunk/bridge-desc-sanitizer/HOWTO =================================================================== --- projects/archives/trunk/bridge-desc-sanitizer/HOWTO 2011-05-17 22:47:52 UTC (rev 24767) +++ projects/archives/trunk/bridge-desc-sanitizer/HOWTO 2011-05-18 20:15:43 UTC (rev 24768) @@ -1,138 +0,0 @@ -Bridge descriptor sanitizer - ---------------------------------------------------------------------------- - - THIS REPOSITORY HAS MOVED TO GIT! - - git clone git://git.torproject.org/metrics-utils/ - ---------------------------------------------------------------------------- - -Introduction: - -The bridge authority Tonga maintains a list of bridges in order to serve -bridge addresses and descriptors to its clients. Every half hour, Tonga -takes a snapshot of the known bridge descriptors and copies them to -byblos for later statistical analysis. As a guiding principle, the Tor -project makes all data that it uses for statistical analysis available to -the interested public, in order to maximize transparency towards the -community. However, the bridge descriptors contain the IP addresses and -other contact information of bridges that must not be made public, or the -purpose of bridges as non-public entry points into the Tor network would -be obsolete. This script takes the half-hourly snapshots as input, removes -all possibly sensitive information from the descriptors, and puts out the -sanitized bridge descriptors that are safe to be published. - ---------------------------------------------------------------------------- - -Processing steps: - -The following steps are taken to remove all potentially sensitive -information from the bridge descriptors while keeping them useful for -statistical analysis. - -1. Replace the bridge identity with its SHA1 value - - Clients can request a bridge's current descriptor by sending its - identity string to the bridge authority. This is a feature to make - bridges on dynamic IP addresses useful. Therefore, the original - identities (and anything that could be used to derive them) need to be - removed from the descriptors. The bridge identity is replaced with its - SHA1 hash value. The idea is to have a consistent replacement that - remains stable over months or even years (without keeping a secret for a - keyed hash function). - -2. Remove all cryptographic keys and signatures - - It would be straightforward to learn about the bridge identity from the - bridge's public key. Replacing keys by newly generated ones seemed to be - unnecessary (and would involve keeping a state over months/years), so - that all cryptographic objects have simply been removed. - -3. Replace IP address with 127.0.0.1 - - Of course, the IP address needs to be removed, too. However, the IP - address is resolved to a country code first and the result written to - the contact line as "somebody at example dot de" for Germany, etc. The - ports are kept unchanged though. - -4. Replace contact information - - If there is contact information in a descriptor, the contact line is - changed to "somebody at ...". If there is none, a contact line is added - saying "nobody at ..." in order to put in the country code. If the - bridge's IP address cannot be resolved to a country, the unassigned - country code "zz" is written to the contact line. - -5. Replace nickname with UnnamedCC - - The bridge nicknames might give hints on the location of the bridge if - chosen without care; e.g. a bridge nickname might be very similar to the - operators' relay nicknames which might be located on adjacent IP - addresses. All bridge nicknames are therefore replaced with the string - UnnamedCC with CC being the upper-case country code. - -6. Replace references to descriptors - - Changing anything in the server descriptors or extra-info descriptors - invalidates the references from network statuses or server descriptors, - respectively. All references are replaced with the new hashes of - referenced descriptors, if available. In case of missing descriptors, - references are replaced with all zeros (or 'A's in base 64 encoding). - -Note that these processing steps only prevent people from learning about -new bridge locations. People who already know a bridge identity or location -can easily learn more about this bridge from the sanitized descriptors. -This is useful for statistical analysis, e.g. to filter out bridges that -have been running as relays before. - ---------------------------------------------------------------------------- - -Quick Start: - -The following steps are necessary to process the half-hourly snapshots as -collected by moria: - -- Install Java 5 or higher. - -- Download Apache Commons Codec 1.4 or higher for Base 64 and hex encoding - from http://commons.apache.org/codec/ and place the .jar (in the - following assumed to be commons-codec-1.4.jar) in the same directory as - this HOWTO file. - -- Download MaxMind GeoIP Java library from http://geolite.maxmind.com/ - download/geoip/api/java/ and generate a JAR file as described in the - README file. Place the resulting maxmindgeoip.jar in the same directory - as this HOWTO file. - -- Copy the half-hourly snapshots named from-tonga-YYYY-MM-DDThhmmssZ.tar.gz - in a directory called data/ in the same directory as this HOWTO file. - -- Run ./extract-bridges.sh to extract the half-hourly snapshots in data/ - to separate directories in the newly created subdirectory in/ . - -- Put the binary MaxMind GeoIP database file that shall be used for - resolving IP addresses to country codes in the same directory as this - HOWTO file. Either the free or the commercial version of the database - can be used. For the archives provided by The Tor Project, the first - available commercial version of the subsequent month is used. - -- Compile the Java class using - - $ javac -cp commons-codec-1.4.jar:maxmindgeoip.jar - ConvertBridgeDescs.java - -- Run the script, providing it with the parameters it needs: - - java -cp .:commons-codec-1.4.jar:maxmindgeoip.jar ConvertBridgeDescs - <input directory> <geoip database file> <YYYY> <MM> - <output directory> - - Note that YYYY and MM specify the month that shall be processed. The other - descriptors in the input directory are ignored. - - A sample invocation might be: - - $ java -cp .:commons-codec-1.4.jar:maxmindgeoip.jar ConvertBridgeDescs - in/ GeoIP-106_20081101.dat 2008 10 out/ - Deleted: projects/archives/trunk/bridge-desc-sanitizer/extract-bridges.sh =================================================================== --- projects/archives/trunk/bridge-desc-sanitizer/extract-bridges.sh 2011-05-17 22:47:52 UTC (rev 24767) +++ projects/archives/trunk/bridge-desc-sanitizer/extract-bridges.sh 2011-05-18 20:15:43 UTC (rev 24768) @@ -1,8 +0,0 @@ -#!/bin/bash -mkdir "in/" -for i in `ls data/ | cut -c 1-29` -do -mkdir "in/"$i -tar -C "in/"$i -xf "data/"$i".tar.gz" -done - Deleted: projects/archives/trunk/exonerator/ExoneraTor.java =================================================================== --- projects/archives/trunk/exonerator/ExoneraTor.java 2011-05-17 22:47:52 UTC (rev 24767) +++ projects/archives/trunk/exonerator/ExoneraTor.java 2011-05-18 20:15:43 UTC (rev 24768) @@ -1,404 +0,0 @@ -/* Copyright 2009 The Tor Project - * See LICENSE for licensing information */ - -import java.io.*; -import java.math.*; -import java.text.*; -import java.util.*; -import org.bouncycastle.util.encoders.Base64; - -public final class ExoneraTor { - - public static void main(final String[] args) throws Exception { - - // check parameters - if (args.length < 4 || args.length > 5) { - System.err.println("\nUsage: java " - + ExoneraTor.class.getSimpleName() - + " <descriptor archive directory> <IP address in question> " - + "<timestamp, in UTC, formatted as YYYY-MM-DD hh:mm:ss> " - + "[<target address>[:<target port>]]\n"); - return; - } - File archiveDirectory = new File(args[0]); - if (!archiveDirectory.exists() || !archiveDirectory.isDirectory()) { - System.err.println("\nDescriptor archive directory + " - + archiveDirectory.getAbsolutePath() - + " does not exist or is not a directory.\n"); - return; - } - String relayIP = args[1]; - String timestampStr = args[2] + " " + args[3]; - SimpleDateFormat timeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - timeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - long timestamp = timeFormat.parse(timestampStr).getTime(); - String target = null, targetIP = null, targetPort = null; - String[] targetIPParts = null; - if (args.length > 4) { - target = args[4]; - if (target.contains(":")) { - targetIP = target.split(":")[0]; - targetPort = target.split(":")[1]; - } else { - targetIP = target; - } - targetIPParts = targetIP.replace(".", " ").split(" "); - } - String DELIMITER = "--------------------------------------------------" - + "-------------------------"; - System.out.println("\nTrying to find out whether " + relayIP + " was " - + "running as a Tor relay at " + timestampStr - + (target != null ? " permitting exiting to " + target : "") - + "...\n\n" + DELIMITER); - - // check that we have the required archives - long timestampTooOld = timestamp - 300 * 60 * 1000; - long timestampFrom = timestamp - 180 * 60 * 1000; - long timestampTooNew = timestamp + 120 * 60 * 1000; - Calendar calTooOld = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - Calendar calFrom = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - Calendar calTooNew = Calendar.getInstance(TimeZone.getTimeZone("UTC")); - calTooOld.setTimeInMillis(timestampTooOld); - calFrom.setTimeInMillis(timestampFrom); - calTooNew.setTimeInMillis(timestampTooNew); - System.out.printf("%nChecking that relevant archives between " - + "%tF %<tT and %tF %<tT are available...%n", calTooOld, - calTooNew); - SortedSet<String> requiredDirs = new TreeSet<String>(); - requiredDirs.add(String.format("consensuses-%tY-%<tm", calTooOld)); - requiredDirs.add(String.format("consensuses-%tY-%<tm", calTooNew)); - if (target != null) { - requiredDirs.add(String.format("server-descriptors-%tY-%<tm", - calTooOld)); - requiredDirs.add(String.format("server-descriptors-%tY-%<tm", - calTooNew)); - } - SortedSet<File> consensusDirs = new TreeSet<File>(); - SortedSet<File> descriptorsDirs = new TreeSet<File>(); - Stack<File> directoriesLeftToParse = new Stack<File>(); - directoriesLeftToParse.push(archiveDirectory); - while (!directoriesLeftToParse.isEmpty()) { - File directoryOrFile = directoriesLeftToParse.pop(); - if (directoryOrFile.getName().startsWith("consensuses-")) { - if (requiredDirs.contains(directoryOrFile.getName())) { - requiredDirs.remove(directoryOrFile.getName()); - consensusDirs.add(directoryOrFile); - } - } else if (directoryOrFile.getName().startsWith( - "server-descriptors-")) { - if (requiredDirs.contains(directoryOrFile.getName())) { - requiredDirs.remove(directoryOrFile.getName()); - descriptorsDirs.add(directoryOrFile); - } - } else { - for (File fileInDir : directoryOrFile.listFiles()) - if (fileInDir.isDirectory()) - directoriesLeftToParse.push(fileInDir); - } - } - for (File dir : consensusDirs) - System.out.println(" " + dir.getAbsolutePath()); - for (File dir : descriptorsDirs) - System.out.println(" " + dir.getAbsolutePath()); - if (!requiredDirs.isEmpty()) { - System.out.println("\nWe are missing consensuses and/or server " - + "descriptors. Please download these archives and extract them " - + "to your data directory. Be sure NOT to rename the extracted " - + "directories or the contained files."); - for (String dir : requiredDirs) - System.out.println(" " + dir + ".tar.bz2"); - return; - } - - // look for consensus files - System.out.printf("%nLooking for relevant consensuses between " - + "%tF %<tT and %s...%n", calFrom, timestampStr); - SortedSet<File> tooOldConsensuses = new TreeSet<File>(); - SortedSet<File> relevantConsensuses = new TreeSet<File>(); - SortedSet<File> tooNewConsensuses = new TreeSet<File>(); - directoriesLeftToParse.clear(); - for (File consensusDir : consensusDirs) - directoriesLeftToParse.push(consensusDir); - SimpleDateFormat consensusTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd-HH-mm-ss"); - consensusTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - while (!directoriesLeftToParse.isEmpty()) { - File directoryOrFile = directoriesLeftToParse.pop(); - if (directoryOrFile.isDirectory()) { - for (File fileInDir : directoryOrFile.listFiles()) { - directoriesLeftToParse.push(fileInDir); - } - continue; - } else { - String filename = directoryOrFile.getName(); - if (filename.endsWith("consensus")) { - long consensusTime = consensusTimeFormat.parse( - filename.substring(0, 19)).getTime(); - if (consensusTime >= timestampTooOld && - consensusTime < timestampFrom) - tooOldConsensuses.add(directoryOrFile); - else if (consensusTime >= timestampFrom && - consensusTime <= timestamp) - relevantConsensuses.add(directoryOrFile); - else if (consensusTime > timestamp && - consensusTime <= timestampTooNew) - tooNewConsensuses.add(directoryOrFile); - } - } - } - SortedSet<File> allConsensuses = new TreeSet<File>(); - allConsensuses.addAll(tooOldConsensuses); - allConsensuses.addAll(relevantConsensuses); - allConsensuses.addAll(tooNewConsensuses); - if (allConsensuses.isEmpty()) { - System.out.println(" None found!\n\n" + DELIMITER + "\n\nResult is " - + "INDECISIVE!\n\nWe cannot make any statement about IP address " - + relayIP + " being a relay at " + timestampStr + " or not! We " - + "did not find any relevant consensuses preceding the given " - + "time. This either means that you did not download and " - + "extract the consensus archives preceding the hours before " - + "the given time, or (in rare cases) that the directory " - + "archives are missing the hours before the timestamp. Please " - + "check that your directory archives contain consensus files " - + "of the interval 5:00 hours before and 2:00 hours after the " - + "time you are looking for.\n"); - return; - } - for (File f : relevantConsensuses) - System.out.println(" " + f.getAbsolutePath()); - - // parse consensuses to find descriptors belonging to the IP address - System.out.println("\nLooking for descriptor identifiers referenced " - + "in \"r \" lines in these consensuses containing IP address " - + relayIP + "..."); - SortedSet<File> positiveConsensusesNoTarget = new TreeSet<File>(); - Set<String> addressesInSameNetwork = new HashSet<String>(); - SortedMap<String, Set<File>> relevantDescriptors = - new TreeMap<String, Set<File>>(); - for (File consensus : allConsensuses) { - if (relevantConsensuses.contains(consensus)) - System.out.println(" " + consensus.getAbsolutePath()); - BufferedReader br = new BufferedReader(new FileReader(consensus)); - String line; - while ((line = br.readLine()) != null) { - if (!line.startsWith("r ")) - continue; - String[] parts = line.split(" "); - String address = parts[6]; - if (address.equals(relayIP)) { - byte[] result = Base64.decode(parts[3] + "=="); - String hex = String.format("%040x", new BigInteger(1, - Base64.decode(parts[3] + "=="))); - if (!relevantDescriptors.containsKey(hex)) - relevantDescriptors.put(hex, new HashSet<File>()); - relevantDescriptors.get(hex).add(consensus); - positiveConsensusesNoTarget.add(consensus); - if (relevantConsensuses.contains(consensus)) - System.out.println(" \"" + line + "\" references " - + "descriptor " + hex); - } else { - if (relayIP.startsWith(address.substring(0, - address.lastIndexOf(".")))) { - addressesInSameNetwork.add(address); - } - } - } - br.close(); - } - if (relevantDescriptors.isEmpty()) { - System.out.printf(" None found!\n\n" + DELIMITER + "\n\nResult is " - + "NEGATIVE with moderate certainty!\n\nWe did not find IP " - + "address " + relayIP + " in any of the consensuses that were " - + "published between %tF %<tT and %tF %<tT.\n\nA possible " - + "reason for false negatives is that the relay is using a " - + "different IP address when generating a descriptor than for " - + "exiting to the Internet. We hope to provide better checks " - + "for this case in the future.", calTooOld, calTooNew); - if (!addressesInSameNetwork.isEmpty()) { - System.out.println("\n\nThe following other IP addresses of Tor " - + "relays were found in the mentioned consensus files that " - + "are in the same /24 network and that could be related to " - + "IP address " + relayIP + ":"); - for (String s : addressesInSameNetwork) { - System.out.println(" " + s); - } - } - System.out.println(); - return; - } - - // parse router descriptors to check exit policies - SortedSet<File> positiveConsensuses = new TreeSet<File>(); - Set<String> missingDescriptors = new HashSet<String>(); - if (target != null) { - System.out.println("\nChecking if referenced descriptors permit " - + "exiting to " + target + "..."); - Set<String> descriptors = relevantDescriptors.keySet(); - missingDescriptors.addAll(relevantDescriptors.keySet()); - directoriesLeftToParse.clear(); - for (File descriptorsDir : descriptorsDirs) - directoriesLeftToParse.push(descriptorsDir); - while (!directoriesLeftToParse.isEmpty()) { - File directoryOrFile = directoriesLeftToParse.pop(); - if (directoryOrFile.isDirectory()) { - for (File fileInDir : directoryOrFile.listFiles()) { - directoriesLeftToParse.push(fileInDir); - } - continue; - } else { - String filename = directoryOrFile.getName(); - for (String descriptor : descriptors) { - if (filename.equals(descriptor)) { - missingDescriptors.remove(descriptor); - BufferedReader br = new BufferedReader( - new FileReader(directoryOrFile)); - String line; - while ((line = br.readLine()) != null) { - if (line.startsWith("reject ") || - line.startsWith("accept ")) { - boolean ruleAccept = line.split(" ")[0].equals("accept"); - String ruleAddress = line.split(" ")[1].split(":")[0]; - if (!ruleAddress.equals("*")) { - if (!ruleAddress.contains("/") && - !ruleAddress.equals(targetIP)) - continue; // IP address does not match - String[] ruleIPParts = ruleAddress.split("/")[0]. - replace(".", " ").split(" "); - int ruleNetwork = ruleAddress.contains("/") ? - Integer.parseInt(ruleAddress.split("/")[1]) : 32; - for (int i = 0; i < 4; i++) { - if (ruleNetwork == 0) { - break; - } else if (ruleNetwork >= 8) { - if (ruleIPParts[i].equals(targetIPParts[i])) - ruleNetwork -= 8; - else - break; - } else { - int mask = 255 ^ 255 >>> ruleNetwork; - if ((Integer.parseInt(ruleIPParts[i]) & mask) == - (Integer.parseInt(targetIPParts[i]) & mask)) - ruleNetwork = 0; - break; - } - } - if (ruleNetwork > 0) - continue; // IP address does not match - } - String rulePort = line.split(" ")[1].split(":")[1]; - if (targetPort == null && !ruleAccept && - !rulePort.equals("*")) - continue; // with no port given, we only consider - // reject :* rules as matching - if (targetPort != null) { - if (!rulePort.equals("*") && - !targetPort.equals(rulePort)) - continue; // ports do not match - } - boolean relevantMatch = false; - for (File f : relevantDescriptors.get(descriptor)) - if (relevantConsensuses.contains(f)) - relevantMatch = true; - if (relevantMatch) - System.out.println(" " - + directoryOrFile.getAbsolutePath() + " " - + (ruleAccept ? "permits" : "does not permit") - + " exiting to " + target + " according to rule \"" - + line + "\""); - if (ruleAccept) - positiveConsensuses.addAll( - relevantDescriptors.get(descriptor)); - break; - } - } - br.close(); - } - } - } - } - } - - // print out result - Set<File> matches = (target != null) ? positiveConsensuses - : positiveConsensusesNoTarget; - if (matches.contains(relevantConsensuses.last())) { - System.out.println("\n" + DELIMITER + "\n\nResult is POSITIVE with " - + "high certainty!\n\nWe found one or more relays on IP address " - + relayIP - + (target != null ? " permitting exit to " + target : "") - + " in the most recent consensus preceding " + timestampStr - + " that clients were likely to know.\n"); - return; - } - boolean resultIndecisive = target != null - && !missingDescriptors.isEmpty(); - if (resultIndecisive) { - System.out.println("\n" + DELIMITER + "\n\nResult is INDECISIVE!\n\n" - + "At least one referenced descriptor could not be found. This " - + "is a rare case, but one that (apparently) happens. We cannot " - + "make any good statement about exit relays without these " - + "descriptors. The following descriptors are missing:"); - for (String desc : missingDescriptors) - System.out.println(" " + desc); - } - boolean inOtherRelevantConsensus = false, inTooOldConsensuses = false, - inTooNewConsensuses = false; - for (File f : matches) - if (relevantConsensuses.contains(f)) - inOtherRelevantConsensus = true; - else if (tooOldConsensuses.contains(f)) - inTooOldConsensuses = true; - else if (tooNewConsensuses.contains(f)) - inTooNewConsensuses = true; - if (inOtherRelevantConsensus) { - if (!resultIndecisive) - System.out.println("\n" + DELIMITER + "\n\nResult is POSITIVE " - + "with moderate certainty!"); - System.out.println("\nWe found one or more relays on IP address " - + relayIP - + (target != null ? " permitting exit to " + target : "") - + ", but not in the consensus immediately preceding " - + timestampStr + ". A possible reason for the relay being " - + "missing in the last consensus preceding the given time might " - + "be that some of the directory authorities had difficulties " - + "connecting to the relay. However, clients might still have " - + "used the relay."); - } else { - if (!resultIndecisive) - System.out.println("\n" + DELIMITER + "\n\nResult is NEGATIVE " - + "with high certainty!"); - System.out.println("\nWe did not find any relay on IP address " - + relayIP - + (target != null ? " permitting exit to " + target : "") - + " in the consensuses 3:00 hours preceding " + timestampStr - + "."); - if (inTooOldConsensuses || inTooNewConsensuses) { - if (inTooOldConsensuses && !inTooNewConsensuses) - System.out.println("\nNote that we found a matching relay in " - + "consensuses that were published between 5:00 and 3:00 " - + "hours before " + timestampStr + "."); - else if (!inTooOldConsensuses && inTooNewConsensuses) - System.out.println("\nNote that we found a matching relay in " - + "consensuses that were published up to 2:00 hours after " - + timestampStr + "."); - else - System.out.println("\nNote that we found a matching relay in " - + "consensuses that were published between 5:00 and 3:00 " - + "hours before and in consensuses that were published up " - + "to 2:00 hours after " + timestampStr + "."); - System.out.println("Make sure that the timestamp you provided is " - + "in the correct timezone: UTC (or GMT)."); - } - } - if (target != null) { - if (positiveConsensuses.isEmpty() && - !positiveConsensusesNoTarget.isEmpty()) - System.out.println("\nNote that although the found relay(s) did " - + "not permit exiting to " + target + ", there have been one " - + "or more relays running at the given time."); - } - System.out.println(); - } -} - Deleted: projects/archives/trunk/exonerator/HOWTO =================================================================== --- projects/archives/trunk/exonerator/HOWTO 2011-05-17 22:47:52 UTC (rev 24767) +++ projects/archives/trunk/exonerator/HOWTO 2011-05-18 20:15:43 UTC (rev 24768) @@ -1,165 +0,0 @@ -ExoneraTor - or: a script that tells you whether some IP address was a Tor relay - ---------------------------------------------------------------------------- - - THIS REPOSITORY HAS MOVED TO GIT! - - git clone git://git.torproject.org/metrics-utils/ - ---------------------------------------------------------------------------- - -Introduction: - -Some people have expressed the desire to learn whether a given IP address -has been a Tor relay at a certain time. In addition to that, these people -might want to know whether the IP address permitted exit to a given address -and port. - -Answering these questions can be important for Tor relay operators to show -to the authorities that an anonymous user might have conducted bad things -with their IP address. Likewise, police investigators might be interested -in the answer to these questions, too, in order to decide whether to -proceed with their investigations or not. - -We can answer the above questions from looking at the descriptor archives -that are available since late 2007 (or even beyond, but this script only -works with the data format that was produced starting in October 2007). -This script parses the directory archives to print out the answer whether -a certain IP address was a Tor relay at a given time. The script further -prints out all intermediate steps in answering this, so that users can -confirm the correctness of the result themselves. - -This script is available in two versions written in Python and in Java with -equivalent functionality. - ---------------------------------------------------------------------------- - -Python Quick Start: - -In order to run the Python version of this script, you need to install and -download the following software and data (please note that all instructions -are written for Linux; commands for Windows or Mac OS X may vary): - -- Install Python 2.6.2 or higher. (Previous Python versions might work, - too, but have not been tested.) - -- Install the Python module IPy 0.62 or higher either from - http://pypi.python.org/pypi/IPy/ or using "apt-get install python-ipy" on - Debian-based systems. - -- Download the v3 consensuses and server descriptors of the relevant time - from http://metrics.torproject.org/data.html and extract them to a - directory in your working directory, e.g. /home/you/exonerator/data/ . - Don't rename the extracted directories or any of the contained files, or - the script won't find the contained descriptors. - - Note that you only need the server descriptors if you want to learn - whether a given IP address permits exiting to a given target. If you - only want to learn whether that IP address was a Tor relay, you don't - need them. - -- Run the script, providing it with the parameters it needs: - - python exonerator.py [--archive=<descriptor archive directory>] - <IP address in question> - <timestamp, in UTC, formatted as YYYY-MM-DD hh:mm:ss> - [<target address>[:<target port>]] - - The --archive option defaults to data/ . In the following examples, it is - assumed that this default applies. - - Make sure that the timestamp is provided in UTC, which is equivalent to - GMT, and not in your local timezone! Otherwise, results will very likely - be wrong. - - A sample invocation might be: - - $ python exonerator.py 209.17.171.104 2009-08-15 16:05:00 - 209.85.129.104:80 - ---------------------------------------------------------------------------- - -Java Quick Start: - -In order to run the Java version of this script, you need to install and -download the following software and data (please note that all instructions -are written for Linux; commands for Windows or Mac OS X may vary): - -- Install Java 6 or higher. - -- Download the BouncyCastle provider that includes Base 64 decoding from - http://www.bouncycastle.org/download/bcprov-jdk16-143.jar and put it in - your working directory, e.g. /home/you/exonerator/ . - -- Download the v3 consensuses and server descriptors of the relevant time - from http://metrics.torproject.org/data.html and extract them to a - directory in your working directory, e.g. /home/you/exonerator/data/ . - Don't rename the extracted directories or any of the contained files, or - the script won't find the contained descriptors. - - Note that you only need the server descriptors if you want to learn - whether a given IP address permits exiting to a given target. If you - only want to learn whether that IP address was a Tor relay, you don't - need them. - -- Compile the (single) Java class using this command: - - $ javac -cp bcprov-jdk16-143.jar ExoneraTor.java - -- Run the script, providing it with the parameters it needs: - - java -cp .:bcprov-jdk16-143.jar ExoneraTor - <descriptor archive directory> - <IP address in question> - <timestamp, in UTC, formatted as YYYY-MM-DD hh:mm:ss> - [<target address>[:<target port>]] - - Make sure that the timestamp is provided in UTC, which is equivalent to - GMT, and not in your local timezone! Otherwise, results will very likely - be wrong. - - A sample invocation might be: - - $ java -cp .:bcprov-jdk16-143.jar ExoneraTor data/ 209.17.171.104 \ - 2009-08-15 16:05:00 209.85.129.104:80 - ---------------------------------------------------------------------------- - -Test cases: - -The following test cases work with the August 2009 archives and can be used -to check whether this script works correctly: - -- Positive result of echelon1+2 being a relay: - - $ python exonerator.py 209.17.171.104 2009-08-15 16:05:00 - $ java -cp .:bcprov-jdk16-143.jar ExoneraTor data/ 209.17.171.104 \ - 2009-08-15 16:05:00 - -- Positive result of echelon1+2 exiting to google.com on any port - - $ python exonerator.py 209.17.171.104 2009-08-15 16:05:00 209.85.129.104 - $ java -cp .:bcprov-jdk16-143.jar ExoneraTor data/ 209.17.171.104 \ - 2009-08-15 16:05:00 209.85.129.104 - -- Positive result of echelon1+2 exiting to google.com on port 80 - - $ python exonerator.py 209.17.171.104 2009-08-15 16:05:00 \ - 209.85.129.104:80 - $ java -cp .:bcprov-jdk16-143.jar ExoneraTor data/ 209.17.171.104 \ - 2009-08-15 16:05:00 209.85.129.104:80 - -- Negative result of echelon1+2 exiting to google.com, but not on port 25 - - $ python exonerator.py 209.17.171.104 2009-08-15 16:05:00 \ - 209.85.129.104:25 - $ java -cp .:bcprov-jdk16-143.jar ExoneraTor data/ 209.17.171.104 \ - 2009-08-15 16:05:00 209.85.129.104:25 - -- Negative result with IP address of echelon1+2 changed in the last octet - - $ python exonerator.py 209.17.171.50 2009-08-15 16:05:00 - $ java -cp .:bcprov-jdk16-143.jar ExoneraTor data/ 209.17.171.50 \ - 2009-08-15 16:05:00 - Deleted: projects/archives/trunk/exonerator/LICENSE =================================================================== --- projects/archives/trunk/exonerator/LICENSE 2011-05-17 22:47:52 UTC (rev 24767) +++ projects/archives/trunk/exonerator/LICENSE 2011-05-18 20:15:43 UTC (rev 24768) @@ -1,30 +0,0 @@ -Copyright 2009 The Tor Project - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - * Neither the names of the copyright owners nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - Deleted: projects/archives/trunk/exonerator/exonerator.py =================================================================== --- projects/archives/trunk/exonerator/exonerator.py 2011-05-17 22:47:52 UTC (rev 24767) +++ projects/archives/trunk/exonerator/exonerator.py 2011-05-18 20:15:43 UTC (rev 24768) @@ -1,370 +0,0 @@ -#!/usr/bin/env python -# Copyright 2009 The Tor Project -- see LICENSE for licensing information - -import binascii -import os -import sys -import time -import calendar -from optparse import OptionParser -from IPy import IP - -USAGE = "usage: %prog [options] <IP address in question> " \ - "<timestamp, in UTC, formatted as YYYY-MM-DD hh:mm:ss> " \ - "[<target address>[:<target port>]]" -DELIMITER = "-" * 75 - -if __name__ == '__main__': - # check parameters - parser = OptionParser(usage=USAGE) - parser.add_option("-a", "--archive", dest="archive", default="data/", - help="descriptor archive directory") - (options, args) = parser.parse_args() - if len(args) not in (3, 4): - parser.error("incorrect number of arguments") - if not os.path.isdir(options.archive): - parser.error("descriptor archive directory %s does not exist or " \ - "is not a directory." % \ - os.path.abspath(options.archive)) - archiveDirectory = os.path.dirname(options.archive) - try: - relayIP = IP(args[0]) - except ValueError: - parser.error("invalid IP address in question: '%s'" % args[0]) - timestampStr = "%s %s" % (args[1], args[2]) - try: - timestamp = time.strptime(timestampStr + " UTC", "%Y-%m-%d %H:%M:%S %Z") - except ValueError: - parser.error("incorrect time format: '%s'" % timestampStr) - # if a target is given, parse address and possibly port part of it - target = None - targetIP = None - targetPort = None - if len(args) == 4: - target = args[3] - targetParts = target.split(":") - try: - targetIP = IP(targetParts[0]) - except ValueError: - parser.error("invalid target IP address in: '%s'" % args[3]) - if len(targetParts) > 2: - parser.error("invalid target format: '%s'" % args[3]) - if len(targetParts) > 1: - try: - targetPortTest = int(targetParts[1]) - except ValueError: - parser.error("invalid target port number in: '%s'" % \ - args[3]) - if targetPortTest not in range(1, 65535): - parser.error("invalid target port number in: '%s'" % \ - args[3]) - targetPort = targetParts[1] - - targetHelpStr = "" - if target: - targetHelpStr = " permitting exiting to %s" % target - print "\nTrying to find out whether %s was running a Tor relay at " \ - "%s%s...\n\n%s\n" % (relayIP, timestampStr, targetHelpStr, - DELIMITER) - - # check that we have the required archives - timestampTooOld = time.gmtime(calendar.timegm(timestamp) - 300 * 60) - timestampFrom = time.gmtime(calendar.timegm(timestamp) - 180 * 60) - timestampTooNew = time.gmtime(calendar.timegm(timestamp) + 120 * 60) - timestampTooOldStr = time.strftime("%Y-%m-%d %H:%M:%S", - timestampTooOld) - timestampFromStr = time.strftime("%Y-%m-%d %H:%M:%S", timestampFrom) - timestampTooNewStr = time.strftime("%Y-%m-%d %H:%M:%S", - timestampTooNew) - print "\nChecking that relevant archives between %s and %s are " \ - "available..." % (timestampTooOldStr, timestampTooNewStr) - - requiredDirs = set() - requiredDirs.add(time.strftime("consensuses-%Y-%m", timestampTooOld)) - requiredDirs.add(time.strftime("consensuses-%Y-%m", timestampTooNew)) - if target: - requiredDirs.add(time.strftime("server-descriptors-%Y-%m", - timestampTooOld)) - requiredDirs.add(time.strftime("server-descriptors-%Y-%m", - timestampTooNew)) - - consensusDirs = list() - descriptorsDirs = list() - directoriesLeftToParse = list() - directoriesLeftToParse.append(archiveDirectory) - - while directoriesLeftToParse: - directoryOrFile = directoriesLeftToParse.pop() - basename = os.path.basename(directoryOrFile) - if basename.startswith("consensuses-"): - if basename in requiredDirs: - requiredDirs.remove(basename) - consensusDirs.append(directoryOrFile) - elif basename.startswith("server-descriptors-"): - if basename in requiredDirs: - requiredDirs.remove(basename) - descriptorsDirs.append(directoryOrFile) - else: - for filename in os.listdir(directoryOrFile): - entry = "%s/%s" % (directoryOrFile, filename) - if os.path.isdir(entry): - directoriesLeftToParse.append(entry) - - consensusDirs.sort() - for consensusDir in consensusDirs: - print " %s" % consensusDir - descriptorsDirs.sort() - for descriptorsDir in descriptorsDirs: - print " %s" % descriptorsDir - - if requiredDirs: - print "\nWe are missing consensuses and/or server descriptors. " \ - "Please download these archives and extract them to your " \ - "data directory. Be sure NOT to rename the extracted " \ - "directories or the contained files." - for requiredDir in sorted(requiredDirs): - print " %s.tar.bz2" % requiredDir - sys.exit() - - # look for consensus files - print "\nLooking for relevant consensuses between %s and %s..." % \ - (timestampFromStr, timestampStr) - tooOldConsensuses = set() - relevantConsensuses = set() - tooNewConsensuses = set() - directoriesLeftToParse = list(consensusDirs) - while directoriesLeftToParse: - directoryOrFile = directoriesLeftToParse.pop() - if os.path.isdir(directoryOrFile): - for filename in os.listdir(directoryOrFile): - entry = "%s/%s" % (directoryOrFile, filename) - directoriesLeftToParse.append(entry) - else: - basename = os.path.basename(directoryOrFile) - if (basename.endswith("consensus")): - consensusTime = time.strptime(basename[0:19], - "%Y-%m-%d-%H-%M-%S") - if consensusTime >= timestampTooOld and \ - consensusTime < timestampFrom: - tooOldConsensuses.add(directoryOrFile) - elif consensusTime >= timestampFrom and \ - consensusTime <= timestamp: - relevantConsensuses.add(directoryOrFile) - elif consensusTime > timestamp and \ - consensusTime <= timestampTooNew: - tooNewConsensuses.add(directoryOrFile) - allConsensuses = set() - allConsensuses.update(tooOldConsensuses) - allConsensuses.update(relevantConsensuses) - allConsensuses.update(tooNewConsensuses) - if not allConsensuses: - print " None found!\n\n%s\n\nResult is INDECISIVE!\n\nWe " \ - "cannot make any statement about IP address %s being a " \ - "relay at %s or not! We did not find any relevant " \ - "consensuses preceding the given time. This either means " \ - "that you did not download and extract the consensus " \ - "archives preceding the hours before the given time, or " \ - "(in rare cases) that the directory archives are missing " \ - "the hours before the timestamp. Please check that your " \ - "directory archives contain consensus files of the " \ - "interval 5:00 hours before and 2:00 hours after the time " \ - "you are looking for.\n" % (DELIMITER, relayIP, timestampStr) - sys.exit() - for consensus in sorted(relevantConsensuses): - print " %s" % consensus - - # parse consensuses to find descriptors belonging to the IP address - print "\nLooking for descriptor identifiers referenced in \"r \" " \ - "lines in these consensuses containing IP address %s..." % \ - relayIP - positiveConsensusesNoTarget = set() - addressesInSameNetwork = set() - relevantDescriptors = dict() - for consensus in allConsensuses: - if consensus in relevantConsensuses: - print " %s" % consensus - consensusFile = open(consensus, "r") - line = consensusFile.readline() - while line: - if line.startswith("r "): - address = IP(line.split(" ")[6]) - if address == relayIP: - hexDesc = binascii.b2a_hex(binascii.a2b_base64( - line.split(" ")[3] + "==")) - if hexDesc not in relevantDescriptors.keys(): - relevantDescriptors[hexDesc] = set() - relevantDescriptors[hexDesc].add(consensus) - positiveConsensusesNoTarget.add(consensus) - if consensus in relevantConsensuses: - print " \"%s\" references descriptor %s" % \ - (line.rstrip(), hexDesc) - elif relayIP.overlaps(IP("%s/24" % address, - make_net=True)): - addressesInSameNetwork.add(address) - line = consensusFile.readline() - consensusFile.close() - if not relevantDescriptors: - print " None found!\n\n%s\n\nResult is NEGATIVE with moderate " \ - "certainty!\n\nWe did not find IP address %s in any of " \ - "the consensuses that were published between %s and " \ - "%s.\n\nA possible reason for false negatives is that the " \ - "relay is using a different IP address when generating a " \ - "descriptor than for exiting to the Internet. We hope to " \ - "provide better checks for this case in the future." % \ - (DELIMITER, relayIP, timestampTooOldStr, timestampTooNewStr) - if addressesInSameNetwork: - print "\nThe following other IP addresses of Tor relays " \ - "were found in the mentioned consensus files that are " \ - "in the same /24 network and that could be related to " \ - "IP address %s:" % relayIP - for addr in addressesInSameNetwork: - print " %s" % addr - print "" - sys.exit() - - # parse router descriptors to check exit policies - positiveConsensuses = set() - missingDescriptors = set() - if target: - print "\nChecking if referenced descriptors permit exiting to " \ - "%s..." % target - descriptors = relevantDescriptors.keys() - for desc in descriptors: - missingDescriptors.add(desc) - directoriesLeftToParse = list(descriptorsDirs) - while directoriesLeftToParse: - directoryOrFile = directoriesLeftToParse.pop() - if os.path.isdir(directoryOrFile): - for filename in os.listdir(directoryOrFile): - entry = "%s/%s" % (directoryOrFile, filename) - directoriesLeftToParse.append(entry) - else: - basename = os.path.basename(directoryOrFile) - for descriptor in descriptors: - if basename == descriptor: - missingDescriptors.remove(descriptor) - descriptorFile = open(directoryOrFile, "r") - line = descriptorFile.readline() - while line: - if line.startswith("reject ") or \ - line.startswith("accept "): - ruleAccept = line.split()[0] == "accept" - ruleAddress = line.split()[1].split(":")[0] - if ruleAddress != "*" and not \ - IP(ruleAddress).overlaps(targetIP): - # IP address does not match - line = descriptorFile.readline() - continue - rulePort = line.split()[1].split(":")[1] - if not targetPort and not ruleAccept and \ - rulePort != "*": - # with no port given, we only consider - # reject :* rules as matching - line = descriptorFile.readline() - continue - if targetPort and rulePort != "*" and \ - targetPort != rulePort: - # ports do not match - line = descriptorFile.readline() - continue - relevantMatch = False - for f in relevantDescriptors.get( - descriptor): - if f in relevantConsensuses: - relevantMatch = True - if relevantMatch: - if ruleAccept: - print " %s permits exiting to " \ - "%s according to rule " \ - "\"%s\"" % (directoryOrFile, - target, line.rstrip()) - else: - print " %s does not permit " \ - "exiting to %s according " \ - "to rule \"%s\"" % \ - (directoryOrFile, - target, line.rstrip()) - if ruleAccept: - for consensus in \ - relevantDescriptors.get( - descriptor): - positiveConsensuses.add(consensus) - break - line = descriptorFile.readline() - descriptorFile.close() - - # print out result - matches = None - if target: - matches = positiveConsensuses - else: - matches = positiveConsensusesNoTarget - lastConsensus = sorted(relevantConsensuses)[len(relevantConsensuses)-1] - if lastConsensus in matches: - print "\n%s\n\nResult is POSITIVE with high certainty!\n\nWe " \ - "found one or more relays on IP address %s%s in the most " \ - "recent consensus preceding %s that clients were likely " \ - "to know.\n" % (DELIMITER, relayIP, targetHelpStr, - timestampStr) - sys.exit() - resultIndecisive = target and len(missingDescriptors) > 0 - if resultIndecisive: - print "\n%s\n\nResult is INDECISIVE!\n\nAt least one " \ - "referenced descriptor could not be found. This is a rare " \ - "case, but one that (apparently) happens. We cannot make " \ - "any good statement about exit relays without these " \ - "descriptors. The following descriptors are missing:" % \ - DELIMITER - for desc in missingDescriptors: - print " %s" % desc - inOtherRelevantConsensus = False - inTooOldConsensuses = False - inTooNewConsensuses = False - for f in matches: - if f in relevantConsensuses: - inOtherRelevantConsensus = True - elif f in tooOldConsensuses: - inTooOldConsensuses = True - elif f in tooNewConsensuses: - inTooNewConsensuses = True - if inOtherRelevantConsensus: - if not resultIndecisive: - print "\n%s\n\nResult is POSITIVE with moderate certainty!" % \ - DELIMITER - print "\nWe found one or more relays on IP address %s%s, but " \ - "not in the consensus immediately preceding %s. A " \ - "possible reason for the relay being missing in the last " \ - "consensus preceding the given time might be that some of " \ - "the directory authorities had difficulties connecting to " \ - "the relay. However, clients might still have used the " \ - "relay." % (relayIP, targetHelpStr, timestampStr) - else: - if not resultIndecisive: - print "\n%s\n\nResult is NEGATIVE with high certainty!" % \ - DELIMITER - print "\nWe did not find any relay on IP address %s%s in the " \ - "consensuses 3:00 hours preceding %s." % (relayIP, - targetHelpStr, timestampStr) - if inTooOldConsensuses or inTooNewConsensuses: - if inTooOldConsensuses and not inTooNewConsensuses: - print "\nNote that we found a matching relay in " \ - "consensuses that were published between 5:00 and " \ - "3:00 hours before %s." % timestampStr - elif not inTooOldConsensuses and inTooNewConsensuses: - print "\nNote that we found a matching relay in " \ - "consensuses that were published up to 2:00 hours " \ - "after %s." % timestampStr - else: - print "\nNote that we found a matching relay in " \ - "consensuses that were published between 5:00 and " \ - "3:00 hours before and in consensuses that were " \ - "published up to 2:00 hours after %s." % timestampStr - print "Make sure that the timestamp you provided is in the " \ - "correct timezone: UTC (or GMT)." - if target: - if not positiveConsensuses and positiveConsensusesNoTarget: - print "\nNote that although the found relay(s) did not " \ - "permit exiting to %s there have been one or more " \ - "relays running at the given time." % target - print "" -