[or-cvs] r20631: {projects} Check that we have the required archives available more effi (projects/archives/trunk/exonerator)

[kloesing at seul.org]

Author: kloesing
Date: 2009-09-21 14:05:51 -0400 (Mon, 21 Sep 2009)
New Revision: 20631

Modified:
   projects/archives/trunk/exonerator/ExoneraTor.java
   projects/archives/trunk/exonerator/HOWTO
Log:
Check that we have the required archives available more efficiently.
Suggested by Sebastian.


Modified: projects/archives/trunk/exonerator/ExoneraTor.java
===================================================================
--- projects/archives/trunk/exonerator/ExoneraTor.java	2009-09-21 17:36:57 UTC (rev 20630)
+++ projects/archives/trunk/exonerator/ExoneraTor.java	2009-09-21 18:05:51 UTC (rev 20631)
@@ -52,7 +52,7 @@
         + (target != null ? " permitting exiting to " + target : "")
         + "...\n\n" + DELIMITER);
 
-    // look for consensus files
+    // check that we have the required archives
     long timestampTooOld = timestamp - 300 * 60 * 1000;
     long timestampFrom = timestamp - 180 * 60 * 1000;
     long timestampTooNew = timestamp + 120 * 60 * 1000;
@@ -62,13 +62,64 @@
     calTooOld.setTimeInMillis(timestampTooOld);
     calFrom.setTimeInMillis(timestampFrom);
     calTooNew.setTimeInMillis(timestampTooNew);
+    System.out.printf("%nChecking that relevant archives between "
+        + "%tF %<tT and %tF %<tT are available...%n", calTooOld,
+        calTooNew);
+    SortedSet<String> requiredDirs = new TreeSet<String>();
+    requiredDirs.add(String.format("consensuses-%tY-%<tm", calTooOld));
+    requiredDirs.add(String.format("consensuses-%tY-%<tm", calTooNew));
+    if (target != null) {
+      requiredDirs.add(String.format("server-descriptors-%tY-%<tm",
+          calTooOld));
+      requiredDirs.add(String.format("server-descriptors-%tY-%<tm",
+          calTooNew));
+    }
+    SortedSet<File> consensusDirs = new TreeSet<File>();
+    SortedSet<File> descriptorsDirs = new TreeSet<File>();
+    Stack<File> directoriesLeftToParse = new Stack<File>();
+    directoriesLeftToParse.push(archiveDirectory);
+    while (!directoriesLeftToParse.isEmpty()) {
+      File directoryOrFile = directoriesLeftToParse.pop();
+      if (directoryOrFile.getName().startsWith("consensuses-")) {
+        if (requiredDirs.contains(directoryOrFile.getName())) {
+          requiredDirs.remove(directoryOrFile.getName());
+          consensusDirs.add(directoryOrFile);
+        }
+      } else if (directoryOrFile.getName().startsWith(
+          "server-descriptors-")) {
+        if (requiredDirs.contains(directoryOrFile.getName())) {
+          requiredDirs.remove(directoryOrFile.getName());
+          descriptorsDirs.add(directoryOrFile);
+        }
+      } else {
+        for (File fileInDir : directoryOrFile.listFiles())
+          if (fileInDir.isDirectory())
+            directoriesLeftToParse.push(fileInDir);
+      }
+    }
+    for (File dir : consensusDirs)
+      System.out.println("  " + dir.getAbsolutePath());
+    for (File dir : descriptorsDirs)
+      System.out.println("  " + dir.getAbsolutePath());
+    if (!requiredDirs.isEmpty()) {
+      System.out.println("\nWe are missing consensuses and/or server "
+          + "descriptors. Please download these archives and extract them "
+          + "to your data directory. Be sure NOT to rename the extracted "
+          + "directories or the contained files.");
+      for (String dir : requiredDirs)
+        System.out.println("  " + dir + ".tar.bz2");
+      return;
+    }
+
+    // look for consensus files
     System.out.printf("%nLooking for relevant consensuses between "
-        + "%tF %<tT and %s%n", calFrom, timestampStr);
+        + "%tF %<tT and %s...%n", calFrom, timestampStr);
     SortedSet<File> tooOldConsensuses = new TreeSet<File>();
     SortedSet<File> relevantConsensuses = new TreeSet<File>();
     SortedSet<File> tooNewConsensuses = new TreeSet<File>();
-    Stack<File> directoriesLeftToParse = new Stack<File>();
-    directoriesLeftToParse.push(archiveDirectory);
+    directoriesLeftToParse.clear();
+    for (File consensusDir : consensusDirs)
+      directoriesLeftToParse.push(consensusDir);
     SimpleDateFormat consensusTimeFormat = new SimpleDateFormat(
         "yyyy-MM-dd-HH:mm:ss");
     consensusTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -186,7 +237,8 @@
       Set<String> descriptors = relevantDescriptors.keySet();
       missingDescriptors.addAll(relevantDescriptors.keySet());
       directoriesLeftToParse.clear();
-      directoriesLeftToParse.push(archiveDirectory);
+      for (File descriptorsDir : descriptorsDirs)
+        directoriesLeftToParse.push(descriptorsDir);
       while (!directoriesLeftToParse.isEmpty()) {
         File directoryOrFile = directoriesLeftToParse.pop();
         if (directoryOrFile.isDirectory()) {

Modified: projects/archives/trunk/exonerator/HOWTO
===================================================================
--- projects/archives/trunk/exonerator/HOWTO	2009-09-21 17:36:57 UTC (rev 20630)
+++ projects/archives/trunk/exonerator/HOWTO	2009-09-21 18:05:51 UTC (rev 20631)
@@ -41,16 +41,12 @@
 - Copy the consensuses-* and server-descriptors-* files of the relevant
   time from http://archive.torproject.org/tor-directory-authority-archive/
   and extract them to a directory in your working directory, e.g.
-  /home/you/exonerator/data/ .
+  /home/you/exonerator/data/ . Don't rename the extracted directories or
+  any of the contained files, or the script won't find the contained
+  descriptors.
 
-  Note that all files are touched by the script at least once. You might
-  want to avoid putting the archives of more than, say, two months in that
-  directory for your evaluation. You may just temporarily move the
-  irrelevant archives away to another directory, e.g., data-tmp/ in your
-  working directory.
-
-  Also note that you only need the server-descriptors-* files if you want
-  to learn whether a given IP address permits exiting to a given target. If
+  Note that you only need the server-descriptors-* files if you want to
+  learn whether a given IP address permits exiting to a given target. If
   you only want to learn whether that IP address was a Tor relay, you
   don't need them.
 
@@ -80,8 +76,7 @@
 Test cases:
 
 The following test cases work with the August 2009 archives and can be used
-to check whether this script works correctly (line breaks have been added
-only for formatting reasons here):
+to check whether this script works correctly:
 
 - Positive result of echelon1+2 being a relay: