[or-cvs] [ernie/master 3/5] Add ExoneraTor servlet.

karsten at torproject.org karsten at torproject.org
Wed May 19 17:14:36 UTC 2010


Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Wed, 19 May 2010 14:57:27 +0200
Subject: Add ExoneraTor servlet.
Commit: 580aa2f5e26054017ae97eb7df46ae84bb11792d

---
 etc/web.xml                                        |   24 +
 src/org/torproject/ernie/web/ConsensusServlet.java |   86 +++
 .../torproject/ernie/web/ExoneraTorServlet.java    |  789 ++++++++++++++++++++
 .../ernie/web/ServerDescriptorServlet.java         |   86 +++
 4 files changed, 985 insertions(+), 0 deletions(-)
 create mode 100644 src/org/torproject/ernie/web/ConsensusServlet.java
 create mode 100644 src/org/torproject/ernie/web/ExoneraTorServlet.java
 create mode 100644 src/org/torproject/ernie/web/ServerDescriptorServlet.java

diff --git a/etc/web.xml b/etc/web.xml
index 67e0822..885a632 100644
--- a/etc/web.xml
+++ b/etc/web.xml
@@ -18,5 +18,29 @@
     <servlet-name>Image</servlet-name>
     <url-pattern>/graphs/*</url-pattern>
   </servlet-mapping>-->
+  <servlet>
+    <servlet-name>ExoneraTor</servlet-name>
+    <servlet-class>org.torproject.ernie.web.ExoneraTorServlet</servlet-class>
+  </servlet>
+  <servlet-mapping>
+    <servlet-name>ExoneraTor</servlet-name>
+    <url-pattern>/exonerator.html</url-pattern>
+  </servlet-mapping>
+  <servlet>
+    <servlet-name>ServerDescriptor</servlet-name>
+    <servlet-class>org.torproject.ernie.web.ServerDescriptorServlet</servlet-class>
+  </servlet>
+  <servlet-mapping>
+    <servlet-name>ServerDescriptor</servlet-name>
+    <url-pattern>/serverdesc</url-pattern>
+  </servlet-mapping>
+  <servlet>
+    <servlet-name>Consensus</servlet-name>
+    <servlet-class>org.torproject.ernie.web.ConsensusServlet</servlet-class>
+  </servlet>
+  <servlet-mapping>
+    <servlet-name>Consensus</servlet-name>
+    <url-pattern>/consensus</url-pattern>
+  </servlet-mapping>
 </web-app>
 
diff --git a/src/org/torproject/ernie/web/ConsensusServlet.java b/src/org/torproject/ernie/web/ConsensusServlet.java
new file mode 100644
index 0000000..a95df12
--- /dev/null
+++ b/src/org/torproject/ernie/web/ConsensusServlet.java
@@ -0,0 +1,86 @@
+package org.torproject.ernie.web;
+
+import javax.servlet.*;
+import javax.servlet.http.*;
+import java.io.*;
+import java.math.*;
+import java.text.*;
+import java.util.*;
+
+public class ConsensusServlet extends HttpServlet {
+
+  public void doGet(HttpServletRequest request,
+      HttpServletResponse response) throws IOException,
+      ServletException {
+
+    String validAfterParameter = request.getParameter("valid-after");
+
+    /* Check if we have a descriptors directory. */
+    // TODO make this configurable!
+    File archiveDirectory = new File("/srv/metrics.torproject.org/archives");
+    if (!archiveDirectory.exists() || !archiveDirectory.isDirectory()) {
+      /* Oops, we don't have any descriptors to serve. */
+// TODO change to internal server error
+      response.setStatus(HttpServletResponse.SC_NOT_FOUND);
+      return;
+    }
+
+    /* Check valid-after parameter. */
+    if (validAfterParameter == null ||
+        validAfterParameter.length() < "yyyy-MM-dd-HH-mm-ss".length()) {
+// TODO is there something like "wrong parameter"?
+      response.setStatus(HttpServletResponse.SC_NOT_FOUND);
+      return;
+    }
+    SimpleDateFormat timeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd-HH-mm-ss");
+    timeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    Date parsedTimestamp = null;
+    try {
+      parsedTimestamp = timeFormat.parse(validAfterParameter);
+    } catch (ParseException e) {
+// TODO is there something like "wrong parameter"?
+      response.setStatus(HttpServletResponse.SC_NOT_FOUND);
+      return;
+    }
+    if (parsedTimestamp == null) {
+// TODO is there something like "wrong parameter"?
+      response.setStatus(HttpServletResponse.SC_NOT_FOUND);
+      return;
+    }
+    String consensusFilename = archiveDirectory.getAbsolutePath()
+        + "/consensuses-" + validAfterParameter.substring(0, 4) + "-"
+        + validAfterParameter.substring(5, 7) + "/"
+        + validAfterParameter.substring(8, 10) + "/"
+        + validAfterParameter + "-consensus";
+    File consensusFile = new File(consensusFilename);
+
+    if (!consensusFile.exists()) {
+      response.setStatus(HttpServletResponse.SC_NOT_FOUND);
+      return;
+    }
+
+    /* Read file from disk and write it to response. */
+    BufferedInputStream input = null;
+    BufferedOutputStream output = null;
+    try {
+      response.setContentType("text/plain");
+      response.setHeader("Content-Length", String.valueOf(
+          consensusFile.length()));
+      response.setHeader("Content-Disposition",
+          "inline; filename=\"" + consensusFile.getName() + "\"");
+      input = new BufferedInputStream(new FileInputStream(consensusFile),
+          1024);
+      output = new BufferedOutputStream(response.getOutputStream(), 1024);
+      byte[] buffer = new byte[1024];
+      int length;
+      while ((length = input.read(buffer)) > 0) {
+          output.write(buffer, 0, length);
+      }
+    } finally {
+      output.close();
+      input.close();
+    }
+  }
+}
+
diff --git a/src/org/torproject/ernie/web/ExoneraTorServlet.java b/src/org/torproject/ernie/web/ExoneraTorServlet.java
new file mode 100644
index 0000000..c1bfea1
--- /dev/null
+++ b/src/org/torproject/ernie/web/ExoneraTorServlet.java
@@ -0,0 +1,789 @@
+package org.torproject.ernie.web;
+
+import javax.servlet.*;
+import javax.servlet.http.*;
+import java.io.*;
+import java.math.*;
+import java.text.*;
+import java.util.*;
+import java.util.regex.*;
+
+import org.apache.commons.codec.binary.*;
+
+public class ExoneraTorServlet extends HttpServlet {
+
+  private void writeHeader(PrintWriter out) throws IOException {
+    out.println("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 "
+          + "Transitional//EN\"\n"
+        + "\"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\n"
+        + "<html xmlns=\"http://www.w3.org/1999/xhtml\">\n"
+        + "  <head>\n"
+        + "    <meta content=\"text/html; charset=ISO-8859-1\"\n"
+        + "          http-equiv=\"content-type\" />\n"
+        + "    <title>ExoneraTor</title>\n"
+        + "    <meta http-equiv=Content-Type content=\"text/html; "
+          + "charset=iso-8859-1\">\n"
+        + "    <link href=\"http://www.torproject.org/"
+          + "stylesheet-ltr.css\" type=text/css rel=stylesheet>\n"
+        + "    <link href=\"http://www.torproject.org/favicon.ico\" "
+          + "type=image/x-icon rel=\"shortcut icon\">\n"
+        + "  </head>\n"
+        + "  <body>\n"
+        + "    <div class=\"center\">\n"
+        + "      <table class=\"banner\" border=\"0\" cellpadding=\"0\" "
+          + "cellspacing=\"0\" summary=\"\">\n"
+        + "        <tr>\n"
+        + "          <td class=\"banner-left\"><a "
+          + "href=\"https://www.torproject.org/\"><img "
+          + "src=\"http://www.torproject.org/images/top-left.png\" "
+          + "alt=\"Click to go to home page\" width=\"193\" "
+          + "height=\"79\"></a></td>\n"
+        + "          <td class=\"banner-middle\"></td>\n"
+        + "          <td class=\"banner-right\"></td>\n"
+        + "        </tr>\n"
+        + "      </table>\n"
+        + "      <div class=\"main-column\" style=\"margin:5; "
+          + "Padding:0;\">\n"
+        + "        <h2>ExoneraTor</h2>\n"
+        + "        <h3>or: a website that tells you whether some IP "
+          + "address was a Tor relay</h3>\n"
+        + "        <p>ExoneraTor tells you whether there was a Tor relay "
+          + "running on a given IP address at a given time. ExoneraTor "
+          + "can further find out whether this relay permitted exiting "
+          + "to a given server and/or TCP port. ExoneraTor learns about "
+          + "these facts from parsing the public relay lists and relay "
+          + "descriptors that are collected from the Tor directory "
+          + "authorities.\n"
+        + "        <br/>\n"
+        + "        <p><font color=\"red\"><b>Notice:</b> Note that the "
+          + "information you are providing below may be leaked to anyone "
+          + "who can read the network traffic between you and this web "
+          + "server or who has access to this web server. If you need to "
+          + "keep the IP addresses and incident times confidential, you "
+          + "should download the <a href=\"https://svn.torproject.org/"
+            + "svn/projects/archives/trunk/exonerator/\">Java or Python "
+            + "version of ExoneraTor</a> and run it on your local "
+            + "machine.</font></p>\n"
+        + "        <br/>\n");
+  }
+
+  private void writeFooter(PrintWriter out) throws IOException {
+    out.println("        <br/>\n"
+        + "      </div>\n"
+        + "    </div>\n"
+        + "    <div class=\"bottom\" id=\"bottom\">\n"
+        + "      <p>\"Tor\" and the \"Onion Logo\" are <a "
+          + "href=\"https://www.torproject.org/trademark-faq.html.en\">"
+          + "registered trademarks</a> of The Tor Project, Inc.</p>\n"
+        + "    </div>\n"
+        + "  </body>\n"
+        + "</html>");
+    out.close();
+  }
+
+  // TODO make this configurable!
+  public final String ARCHIVES_DIRECTORY = "/home/karsten/archives";
+
+  private static final boolean TEST_MODE = false;
+
+  public void doGet(HttpServletRequest request,
+      HttpServletResponse response) throws IOException,
+      ServletException {
+
+    /* Get print writer and start writing response. We're wrapping the
+     * PrintWriter, because we want it to auto-flush as soon as we have
+     * written a line. */
+    //PrintWriter out = new PrintWriter(response.getWriter(), true);
+    PrintWriter out = response.getWriter();
+    writeHeader(out);
+
+    SortedSet<File> consensusDirectories = new TreeSet<File>();
+    SortedSet<File> serverDescriptorDirectories = new TreeSet<File>();
+
+    /* Check if we have a descriptors directory. */
+    File archiveDirectory = new File("/srv/metrics.torproject.org/archives");
+    if (!archiveDirectory.exists() || !archiveDirectory.isDirectory()) {
+      /* Leave sets with consensus and server descriptor directories
+         empty. */
+      return;
+    }
+    for (File dir : archiveDirectory.listFiles()) {
+      if (dir.getName().startsWith("consensuses-")) {
+        consensusDirectories.add(dir);
+      } else if (dir.getName().startsWith("server-descriptors-")) {
+        serverDescriptorDirectories.add(dir);
+      }
+    }
+
+    if (consensusDirectories.isEmpty()) {
+      out.println("<p><font color=\"red\"><b>Warning: </b></font>This "
+          + "server doesn't have any relay lists available. If this "
+          + "problem persists, please "
+          + "<a href=\"mailto:tor-assistants at freehaven.net\">let us "
+          + "know</a>!</p>\n");
+      writeFooter(out);
+      return;
+    }
+    String firstDay = consensusDirectories.first().getName().
+        substring("consensuses-".length()) + "-" + new TreeSet<File>(
+        Arrays.asList(consensusDirectories.first().listFiles())).
+        first().getName();
+    String lastDay = consensusDirectories.last().getName().
+        substring("consensuses-".length()) + "-" + new TreeSet<File>(
+        Arrays.asList(consensusDirectories.last().listFiles())).
+        last().getName();
+
+    out.println("<a id=\"relay\"/><h3>Was there a Tor relay running on "
+        + "this IP address?</h3>");
+
+    /* Parse IP parameter. */
+    Pattern ipAddressPattern = Pattern.compile(
+        "^([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." +
+        "([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." +
+        "([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." +
+        "([01]?\\d\\d?|2[0-4]\\d|25[0-5])$");
+    String ipParameter = request.getParameter("ip");
+    String relayIP = "", ipWarning = "";
+    if (ipParameter != null && ipParameter.length() > 0) {
+      Matcher ipParameterMatcher = ipAddressPattern.matcher(ipParameter);
+      if (ipParameterMatcher.matches()) {
+        String[] ipParts = ipParameter.split("\\.");
+        relayIP = Integer.parseInt(ipParts[0]) + "."
+            + Integer.parseInt(ipParts[1]) + "."
+            + Integer.parseInt(ipParts[2]) + "."
+            + Integer.parseInt(ipParts[3]);
+      } else {
+        ipWarning = "\"" + (ipParameter.length() > 20 ?
+            ipParameter.substring(0, 20) + "[...]" :
+            ipParameter) + "\" is not a valid IP address.";
+      }
+    }
+
+    /* Parse timestamp parameter. */
+    String timestampParameter = request.getParameter("timestamp");
+    long timestamp = 0L;
+    String timestampStr = "", timestampWarning = "";
+    SimpleDateFormat parseTimeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd HH:mm");
+    parseTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    if (timestampParameter != null && timestampParameter.length() > 0) {
+      try {
+        Date parsedTimestamp = parseTimeFormat.parse(timestampParameter);
+        if (timestampParameter.compareTo(firstDay) >= 0 &&
+            timestampParameter.compareTo(lastDay) <= 0) {
+          timestamp = parsedTimestamp.getTime();
+          timestampStr = parseTimeFormat.format(timestamp);
+        } else {
+          timestampWarning = "Please pick a value between \"" + firstDay
+              + " 03:00\" and \"" + lastDay + " 21:00\".";
+        }
+      } catch (ParseException e) {
+        /* We have no way to handle this exception, other than leaving
+           timestampStr at "". */
+        timestampWarning = "\"" + (timestampParameter.length() > 20 ?
+            timestampParameter.substring(0, 20) + "[...]" :
+            timestampParameter) + "\" is not a valid timestamp.";
+      }
+    }
+
+    /* If either IP address or timestamp is provided, the other one must
+     * be provided, too. */
+    if (relayIP.length() < 1 && timestampStr.length() > 0 &&
+        ipWarning.length() < 1) {
+      ipWarning = "Please provide an IP address.";
+    }
+    if (relayIP.length() > 0 && timestampStr.length() < 1 &&
+        timestampWarning.length() < 1) {
+      timestampWarning = "Please provide a timestamp.";
+    }
+
+    /* Parse target IP parameter. */
+    String targetIP = "", targetPort = "", target = "";
+    String[] targetIPParts = null;
+    String targetAddrParameter = request.getParameter("targetaddr");
+    String targetAddrWarning = "";
+    if (targetAddrParameter != null && targetAddrParameter.length() > 0) {
+      Matcher targetAddrParameterMatcher =
+          ipAddressPattern.matcher(targetAddrParameter);
+      if (targetAddrParameterMatcher.matches()) {
+        String[] targetAddrParts = targetAddrParameter.split("\\.");
+        targetIP = Integer.parseInt(targetAddrParts[0]) + "."
+            + Integer.parseInt(targetAddrParts[1]) + "."
+            + Integer.parseInt(targetAddrParts[2]) + "."
+            + Integer.parseInt(targetAddrParts[3]);
+        target = targetIP;
+        targetIPParts = targetIP.split("\\.");
+      } else {
+        targetAddrWarning = "\"" + (targetAddrParameter.length() > 20 ?
+            timestampParameter.substring(0, 20) + "[...]" :
+            timestampParameter) + "\" is not a valid IP address.";
+      }
+    }
+
+    /* Parse target port parameter. */
+    String targetPortParameter = request.getParameter("targetport");
+    String targetPortWarning = "";
+    if (targetPortParameter != null && targetPortParameter.length() > 0) {
+      Pattern targetPortPattern = Pattern.compile("\\d+");
+      if (targetPortParameter.length() < 5 &&
+          targetPortPattern.matcher(targetPortParameter).matches() &&
+          !targetPortParameter.equals("0") &&
+          Integer.parseInt(targetPortParameter) < 65536) {
+        targetPort = targetPortParameter;
+        if (target != null) {
+          target += ":" + targetPort;
+        } else {
+          target = targetPort;
+        }
+      } else {
+        targetPortWarning = "\"" + (targetPortParameter.length() > 8 ?
+            targetPortParameter.substring(0, 8) + "[...]" :
+            targetPortParameter) + "\" is not a valid TCP port.";
+      }
+    }
+
+    /* If target port is provided, a target address must be provided,
+     * too. */
+    if (targetPort.length() > 0 && targetIP.length() < 1 &&
+        targetAddrWarning.length() < 1) {
+      targetAddrWarning = "Please provide an IP address.";
+    }
+
+    /* Write form with IP address and timestamp. */
+    out.println("        <form action=\"exonerator.html#relay\">\n"
+        + "          <input type=\"hidden\" name=\"targetaddr\" "
+        + (targetIP.length() > 0 ? " value=\"" + targetIP + "\"" : "")
+        + "/>\n"
+        + "          <input type=\"hidden\" name=\"targetPort\""
+        + (targetPort.length() > 0 ? " value=\"" + targetPort + "\"" : "")
+        + "/>\n"
+        + "          <table>\n"
+        + "            <tr>\n"
+        + "              <td align=\"right\">IP address in question:</td>\n"
+        + "              <td><input type=\"text\" name=\"ip\""
+          + (relayIP.length() > 0 ? " value=\"" + relayIP + "\"" :
+            (TEST_MODE ? " value=\"209.17.171.104\"" : ""))
+          + "\"/>"
+          + (ipWarning.length() > 0 ? "<br/><font color=\"red\">"
+          + ipWarning + "</font>" : "")
+        + "</td>\n"
+        + "              <td><i>(Ex.: 1.2.3.4)</i></td>\n"
+        + "            </tr>\n"
+        + "            <tr>\n"
+        + "              <td align=\"right\">Timestamp, in UTC:</td>\n"
+        + "              <td><input type=\"text\" name=\"timestamp\""
+          + (timestampStr.length() > 0 ? " value=\"" + timestampStr + "\"" :
+             (TEST_MODE ? " value=\"2009-08-15 16:05\"" : ""))
+          + "\"/>"
+          + (timestampWarning.length() > 0 ? "<br/><font color=\"red\">"
+              + timestampWarning + "</font>" : "")
+        + "</td>\n"
+        + "              <td><i>(Ex.: 2010-01-01 12:00)</i></td>\n"
+        + "            </tr>\n"
+        + "            <tr>\n"
+        + "              <td/>\n"
+        + "              <td>\n"
+        + "                <input type=\"submit\">\n"
+        + "                <input type=\"reset\">\n"
+        + "              </td>\n"
+        + "              <td/>\n"
+        + "            </tr>\n"
+        + "          </table>\n"
+        + "        </form>\n");
+
+    if (relayIP.length() < 1 || timestampStr.length() < 1) {
+      writeFooter(out);
+      return;
+    }
+
+    /* Look up relevant consensuses. */
+    long timestampTooOld = timestamp - 15L * 60L * 60L * 1000L;
+    long timestampFrom = timestamp - 3L * 60L * 60L * 1000L;
+    long timestampTooNew = timestamp + 12L * 60L * 60L * 1000L;
+    Calendar calTooOld = Calendar.getInstance(
+        TimeZone.getTimeZone("UTC"));
+    Calendar calFrom = Calendar.getInstance(
+        TimeZone.getTimeZone("UTC"));
+    Calendar calTooNew = Calendar.getInstance(
+        TimeZone.getTimeZone("UTC"));
+    calTooOld.setTimeInMillis(timestampTooOld);
+    calFrom.setTimeInMillis(timestampFrom);
+    calTooNew.setTimeInMillis(timestampTooNew);
+    out.printf("<p>Looking up IP address %s in the relay lists published "
+        + "between %tF %tR and %s. "
+        + "Clients could have used any of these relay lists to "
+        + "select relays for their paths and build circuits using them. "
+        + "You may follow the links to relay lists and relay descriptors "
+        + "to grep for the lines printed below and confirm that results "
+        + "are correct.<br/>", relayIP, calFrom, calFrom, timestampStr);
+    SimpleDateFormat consensusTimeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd-HH-mm-ss");
+    consensusTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    String fromTime = consensusTimeFormat.format(
+        new Date(timestampTooOld));
+    String fromDay = fromTime.substring(0, "yyyy-MM-dd".length());
+    String fromMonth = fromDay.substring(0, "yyyy-MM".length());
+    String toTime = consensusTimeFormat.format(new Date(timestampTooNew));
+    String toDay = toTime.substring(0, "yyyy-MM-dd".length());
+    String toMonth = toDay.substring(0, "yyyy-MM".length());
+    SortedSet<File> tooOldConsensuses = new TreeSet<File>();
+    SortedSet<File> relevantConsensuses = new TreeSet<File>();
+    SortedSet<File> tooNewConsensuses = new TreeSet<File>();
+    for (File consensusMonth : consensusDirectories) {
+      String month = consensusMonth.getName().substring(
+          "consensuses-".length());
+      if (month.compareTo(fromMonth) < 0 ||
+          month.compareTo(toMonth) > 0) {
+        continue;
+      }
+      for (File consensusDay : consensusMonth.listFiles()) {
+        String day = month + "-" + consensusDay.getName();
+        if (day.compareTo(fromDay) < 0 ||
+            day.compareTo(toDay) > 0) {
+          continue;
+        }
+        for (File consensusFile : consensusDay.listFiles()) {
+          String time = consensusFile.getName().substring(0,
+              "yyyy-MM-dd-HH-mm-ss".length());
+          if (time.compareTo(fromTime) < 0 ||
+              time.compareTo(toTime) > 0) {
+            continue;
+          }
+          Date consensusDate = null;
+          try {
+            consensusDate = consensusTimeFormat.parse(time);
+          } catch (ParseException e) {
+            /* This should never happen. If it does, it's a bug. */
+            throw new RuntimeException(e);
+          }
+          long consensusTime = consensusDate.getTime();
+          if (consensusTime >= timestampTooOld &&
+              consensusTime < timestampFrom)
+            tooOldConsensuses.add(consensusFile);
+          else if (consensusTime >= timestampFrom &&
+                   consensusTime <= timestamp)
+            relevantConsensuses.add(consensusFile);
+          else if (consensusTime > timestamp &&
+                   consensusTime <= timestampTooNew)
+            tooNewConsensuses.add(consensusFile);
+        }
+      }
+    }
+    SortedSet<File> allConsensuses = new TreeSet<File>();
+    allConsensuses.addAll(tooOldConsensuses);
+    allConsensuses.addAll(relevantConsensuses);
+    allConsensuses.addAll(tooNewConsensuses);
+    if (allConsensuses.isEmpty()) {
+      out.println("        <p>No relay lists found!</p>\n"
+          + "        <p>Result is INDECISIVE!</p>\n"
+          + "        <p>We cannot make any statement whether there was "
+          + "a Tor relay running on IP address "
+          + relayIP + " at " + timestampStr + "! We "
+          + "did not find any relevant relay lists preceding the given "
+          + "time. If you think this is an error on our side, please "
+          + "<a href=\"mailto:tor-assistants at freehaven.net\">contact "
+          + "us</a>!</p>\n");
+      writeFooter(out);
+      return;
+    }
+
+    // parse consensuses to find descriptors belonging to the IP address
+    SortedSet<File> positiveConsensusesNoTarget = new TreeSet<File>();
+    Set<String> addressesInSameNetwork = new HashSet<String>();
+    SortedMap<String, Set<File>> relevantDescriptors =
+        new TreeMap<String, Set<File>>();
+    SimpleDateFormat validAfterTimeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd HH:mm:ss");
+    validAfterTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    for (File consensus : allConsensuses) {
+      if (relevantConsensuses.contains(consensus)) {
+        String validAfterString = consensus.getName().substring(0,
+            "yyyy-MM-dd-HH-mm-ss".length());
+        Date validAfterDate = null;
+        try {
+          validAfterDate = consensusTimeFormat.parse(validAfterString);
+        } catch (ParseException e) {
+          /* This should never happen. If it does, it's a bug. */
+          throw new RuntimeException(e);
+        }
+        long validAfterTime = validAfterDate.getTime();
+        String validAfterDatetime = validAfterTimeFormat.format(
+            validAfterTime);
+        out.println("        <br/><tt>valid-after <b>"
+            + "<a href=\"consensus?valid-after="
+            + validAfterString + "\" target=\"_blank\">"
+            + validAfterDatetime + "</b></a></tt><br/>");
+      }
+      BufferedReader br = new BufferedReader(new FileReader(consensus));
+      String line;
+      while ((line = br.readLine()) != null) {
+        if (!line.startsWith("r "))
+          continue;
+        String[] parts = line.split(" ");
+        String address = parts[6];
+        if (address.equals(relayIP)) {
+          String hex = String.format("%040x", new BigInteger(1,
+              Base64.decodeBase64(parts[3] + "==")));
+          if (!relevantDescriptors.containsKey(hex))
+            relevantDescriptors.put(hex, new HashSet<File>());
+          relevantDescriptors.get(hex).add(consensus);
+          positiveConsensusesNoTarget.add(consensus);
+          if (relevantConsensuses.contains(consensus)) {
+            out.println("    <tt>r " + parts[1] + " " + parts[2] + " "
+                + "<a href=\"serverdesc?desc-id=" + hex + "\" "
+                + "target=\"_blank\">" + parts[3] + "</a> " + parts[4]
+                + " " + parts[5] + " <b>" + parts[6] + "</b> " + parts[7]
+                + " " + parts[8] + "</tt><br/>");
+          }
+        } else {
+          if (relayIP.startsWith(address.substring(0,
+              address.lastIndexOf(".")))) {
+            addressesInSameNetwork.add(address);
+          }
+        }
+      }
+      br.close();
+    }
+    if (relevantDescriptors.isEmpty()) {
+      out.printf("        <p>None found!</p>\n"
+          + "        <p>Result is NEGATIVE with moderate certainty!</p>\n"
+          + "        <p>We did not find IP "
+          + "address " + relayIP + " in any of the relay lists that were "
+          + "published between %tF %tR and %tF %tR.\n\nA possible "
+          + "reason for false negatives is that the relay is using a "
+          + "different IP address when generating a descriptor than for "
+          + "exiting to the Internet. We hope to provide better checks "
+          + "for this case in the future.</p>\n", calTooOld, calTooOld,
+          calTooNew, calTooNew);
+      if (!addressesInSameNetwork.isEmpty()) {
+        out.println("        <p>The following other IP addresses of Tor "
+            + "relays were found in the mentioned relay lists that "
+            + "are in the same /24 network and that could be related to "
+            + "IP address " + relayIP + ":</p>\n");
+        for (String s : addressesInSameNetwork) {
+          out.println("        <p>" + s + "</p>\n");
+        }
+      }
+      writeFooter(out);
+      return;
+    }
+
+    // print out result
+    Set<File> matches = positiveConsensusesNoTarget;
+    if (matches.contains(relevantConsensuses.last())) {
+      out.println("        <p>Result is POSITIVE with high certainty!"
+            + "</p>\n"
+          + "        <p>We found one or more relays on IP address "
+          + relayIP
+          + " in the most recent relay list preceding " + timestampStr
+          + " that clients were likely to know.</p>\n");
+    } else {
+      boolean inOtherRelevantConsensus = false,
+          inTooOldConsensuses = false,
+          inTooNewConsensuses = false;
+      for (File f : matches)
+        if (relevantConsensuses.contains(f))
+          inOtherRelevantConsensus = true;
+        else if (tooOldConsensuses.contains(f))
+          inTooOldConsensuses = true;
+        else if (tooNewConsensuses.contains(f))
+          inTooNewConsensuses = true;
+      if (inOtherRelevantConsensus) {
+        out.println("        <p>Result is POSITIVE "
+            + "with moderate certainty!</p>\n");
+        out.println("<p>We found one or more relays on IP address "
+            + relayIP
+            + ", but not in the relay list immediately preceding "
+            + timestampStr + ". A possible reason for the relay being "
+            + "missing in the last relay list preceding the given time might "
+            + "be that some of the directory authorities had difficulties "
+            + "connecting to the relay. However, clients might still have "
+            + "used the relay.</p>\n");
+      } else {
+        out.println("        <p>Result is NEGATIVE "
+            + "with high certainty!</p>\n");
+        out.println("        <p>We did not find any relay on IP address "
+            + relayIP
+            + " in the relay lists 3 hours preceding " + timestampStr
+            + ".</p>\n");
+        if (inTooOldConsensuses || inTooNewConsensuses) {
+          if (inTooOldConsensuses && !inTooNewConsensuses) {
+            out.println("        <p>Note that we found a matching relay in "
+                + "relay lists that were published between 5 and 3 "
+                + "hours before " + timestampStr + ".</p>\n");
+          } else if (!inTooOldConsensuses && inTooNewConsensuses) {
+            out.println("        <p>Note that we found a matching relay in "
+                + "relay lists that were published up to 2 hours after "
+                + timestampStr + ".</p>\n");
+          } else {
+            out.println("        <p>Note that we found a matching relay in "
+                + "relay lists that were published between 5 and 3 "
+                + "hours before and in relay lists that were published up "
+                + "to 2 hours after " + timestampStr + ".</p>\n");
+          }
+          out.println("<p>Make sure that the timestamp you provided is "
+              + "in the correct timezone: UTC (or GMT).</p>");
+        }
+        writeFooter(out);
+        return;
+      }
+    }
+
+    /* Second part: target */
+    out.println("<br/><a id=\"exit\"/><h3>Was this relay configured to "
+        + "permit exiting to a given target?</h3>");
+
+    if (serverDescriptorDirectories.isEmpty()) {
+      out.println("<p><font color=\"red\"><b>Warning: </b></font>This "
+          + "server doesn't have any relay descriptors available. If "
+          + "this problem persists, please "
+          + "<a href=\"mailto:tor-assistants at freehaven.net\">let us "
+          + "know</a>!</p>\n");
+      writeFooter(out);
+      return;
+    }
+
+    out.println("        <form action=\"exonerator.html#exit\">\n"
+        + "              <input type=\"hidden\" name=\"timestamp\"\n"
+        + "                         value=\"" + timestampStr + "\"/>\n"
+        + "              <input type=\"hidden\" name=\"ip\" "
+          + "value=\"" + relayIP + "\"/>\n"
+        + "          <table>\n"
+        + "            <tr>\n"
+        + "              <td align=\"right\">Target address:</td>\n"
+        + "              <td><input type=\"text\" name=\"targetaddr\""
+          + (targetIP.length() > 0 ? " value=\"" + targetIP + "\"" :
+             (TEST_MODE ? " value=\"209.85.129.104\"" : ""))
+          + "\"/>"
+          + (targetAddrWarning.length() > 0 ? "<br/><font color=\"red\">"
+              + targetAddrWarning + "</font>" : "")
+        + "</td>\n"
+        + "              <td><i>(Ex.: 4.3.2.1)</i></td>\n"
+        + "            </tr>\n"
+        + "            <tr>\n"
+        + "              <td align=\"right\">Target port:</td>\n"
+        + "              <td><input type=\"text\" name=\"targetport\""
+          + (targetPort.length() > 0 ? " value=\"" + targetPort + "\"" :
+             (TEST_MODE ? " value=\"80\"" : ""))
+          + "/>"
+          + (targetPortWarning.length() > 0 ? "<br/><font color=\"red\">"
+              + targetPortWarning + "</font>" : "")
+        + "</td>\n"
+        + "              <td><i>(Ex.: 80)</i></td>\n"
+        + "            </tr>\n"
+        + "            <tr>\n"
+        + "              <td/>\n"
+        + "              <td>\n"
+        + "                <input type=\"submit\">\n"
+        + "                <input type=\"reset\">\n"
+        + "              </td>\n"
+        + "              <td/>\n"
+        + "            </tr>\n"
+        + "          </table>\n"
+        + "        </form>\n");
+
+    if (targetIP.length() < 1) {
+      writeFooter(out);
+      return;
+    }
+
+    // parse router descriptors to check exit policies
+    out.println("<p>Searching the relay descriptors published by the relay "
+        + "on IP address " + relayIP + " to find out whether this relay "
+        + "permitted exiting to " + target + ". You may follow the links "
+        + "above to the relay descriptors and grep them for the lines "
+        + "printed below to confirm that results are correct.</p>");
+    SortedSet<File> positiveConsensuses = new TreeSet<File>();
+    Set<String> missingDescriptors = new HashSet<String>();
+    Set<String> descriptors = relevantDescriptors.keySet();
+    for (String descriptor : descriptors) {
+      for (File directory : serverDescriptorDirectories) {
+        File subDirectory = new File(directory.getAbsolutePath() + "/"
+            + descriptor.substring(0, 1) + "/"
+            + descriptor.substring(1, 2));
+        if (subDirectory.exists()) {
+          File descriptorFile = new File(subDirectory.getAbsolutePath()
+              + "/" + descriptor);
+          if (!descriptorFile.exists()) {
+            continue;
+          }
+          missingDescriptors.remove(descriptor);
+          BufferedReader br = new BufferedReader(
+              new FileReader(descriptorFile));
+          String line, routerLine = null, publishedLine = null;
+          StringBuilder acceptRejectLines = new StringBuilder();
+          boolean foundMatch = false;
+          while ((line = br.readLine()) != null) {
+            if (line.startsWith("router ")) {
+              routerLine = line;
+            } else if (line.startsWith("published ")) {
+              publishedLine = line;
+            } else if (line.startsWith("reject ") ||
+                line.startsWith("accept ")) {
+              if (foundMatch) {
+                out.println("<tt> " + line + "</tt><br/>");
+                continue;
+              }
+              boolean ruleAccept = line.split(" ")[0].equals("accept");
+              String ruleAddress = line.split(" ")[1].split(":")[0];
+              if (!ruleAddress.equals("*")) {
+                if (!ruleAddress.contains("/") &&
+                    !ruleAddress.equals(targetIP)) {
+                  acceptRejectLines.append("<tt> " + line + "</tt><br/>\n");
+                  continue; // IP address does not match
+                }
+                String[] ruleIPParts = ruleAddress.split("/")[0].
+                    split("\\.");
+                int ruleNetwork = ruleAddress.contains("/") ?
+                    Integer.parseInt(ruleAddress.split("/")[1]) : 32;
+                for (int i = 0; i < 4; i++) {
+                  if (ruleNetwork == 0) {
+                    break;
+                  } else if (ruleNetwork >= 8) {
+                    if (ruleIPParts[i].equals(targetIPParts[i])) {
+                      ruleNetwork -= 8;
+                    } else {
+                      break;
+                    }
+                  } else {
+                    int mask = 255 ^ 255 >>> ruleNetwork;
+                    if ((Integer.parseInt(ruleIPParts[i]) & mask) ==
+                        (Integer.parseInt(targetIPParts[i]) & mask)) {
+                      ruleNetwork = 0;
+                    }
+                    break;
+                  }
+                }
+                if (ruleNetwork > 0) {
+                  acceptRejectLines.append("<tt> " + line + "</tt><br/>\n");
+                  continue; // IP address does not match
+                }
+              }
+              String rulePort = line.split(" ")[1].split(":")[1];
+              if (targetPort.length() < 1 && !ruleAccept &&
+                  !rulePort.equals("*")) {
+                acceptRejectLines.append("<tt> " + line + "</tt><br/>\n");
+                continue; // with no port given, we only consider
+                          // reject :* rules as matching
+              }
+              if (targetPort.length() > 0) {
+                if (!rulePort.equals("*") &&
+                    !targetPort.equals(rulePort)) {
+                  acceptRejectLines.append("<tt> " + line + "</tt><br/>\n");
+                  continue; // ports do not match
+                }
+              }
+              boolean relevantMatch = false;
+              for (File f : relevantDescriptors.get(descriptor))
+                if (relevantConsensuses.contains(f))
+                  relevantMatch = true;
+              if (relevantMatch) {
+                String[] routerParts = routerLine.split(" ");
+                out.println("<br/><tt>" + routerParts[0] + " "
+                    + routerParts[1] + " <b>" + routerParts[2] + "</b> "
+                    + routerParts[3] + " " + routerParts[4] + " "
+                    + routerParts[5] + "</tt><br/>");
+                String[] publishedParts = publishedLine.split(" ");
+                out.println("<tt>" + publishedParts[0] + " <b>"
+                    + publishedParts[1] + " " + publishedParts[2]
+                    + "</b></tt><br/>");
+                out.println(acceptRejectLines.toString());
+                out.println("<tt><b>" + line + "</b></tt><br/>");
+                foundMatch = true;
+              }
+              if (ruleAccept) {
+                positiveConsensuses.addAll(
+                    relevantDescriptors.get(descriptor));
+              }
+            }
+          }
+          br.close();
+        }
+      }
+    }
+
+    // print out result
+// TODO don't repeat results from above
+    matches = positiveConsensuses;
+    if (matches.contains(relevantConsensuses.last())) {
+      out.println("        <p>Result is POSITIVE with high certainty!</p>\n"
+          + "        <p>We found one or more relays on IP address "
+          + relayIP + " permitting exit to " + target
+          + " in the most recent relay list preceding " + timestampStr
+          + " that clients were likely to know.</p>\n");
+      writeFooter(out);
+      return;
+    }
+    boolean resultIndecisive = target.length() > 0
+        && !missingDescriptors.isEmpty();
+    if (resultIndecisive) {
+      out.println("        <p>Result is INDECISIVE!</p>\n"
+          + "        <p>At least one referenced descriptor could not be found. This "
+          + "is a rare case, but one that (apparently) happens. We cannot "
+          + "make any good statement about exit relays without these "
+          + "descriptors. The following descriptors are missing:</p>");
+      for (String desc : missingDescriptors)
+        out.println("        <p>" + desc + "</p>\n");
+    }
+    boolean inOtherRelevantConsensus = false, inTooOldConsensuses = false,
+        inTooNewConsensuses = false;
+    for (File f : matches)
+      if (relevantConsensuses.contains(f))
+        inOtherRelevantConsensus = true;
+      else if (tooOldConsensuses.contains(f))
+        inTooOldConsensuses = true;
+      else if (tooNewConsensuses.contains(f))
+        inTooNewConsensuses = true;
+    if (inOtherRelevantConsensus) {
+      if (!resultIndecisive) {
+        out.println("        <p>Result is POSITIVE "
+            + "with moderate certainty!</p>\n");
+      }
+      out.println("<p>We found one or more relays on IP address "
+          + relayIP + " permitting exit to " + target
+          + ", but not in the relay list immediately preceding "
+          + timestampStr + ". A possible reason for the relay being "
+          + "missing in the last relay list preceding the given time might "
+          + "be that some of the directory authorities had difficulties "
+          + "connecting to the relay. However, clients might still have "
+          + "used the relay.</p>\n");
+    } else {
+      if (!resultIndecisive) {
+        out.println("        <p>Result is NEGATIVE "
+            + "with high certainty!</p>\n");
+      }
+      out.println("        <p>We did not find any relay on IP address "
+          + relayIP + " permitting exit to " + target
+          + " in the relay list 3 hours preceding " + timestampStr
+          + ".</p>\n");
+      if (inTooOldConsensuses || inTooNewConsensuses) {
+        if (inTooOldConsensuses && !inTooNewConsensuses)
+          out.println("        <p>Note that we found a matching relay in "
+              + "relay lists that were published between 5 and 3 "
+              + "hours before " + timestampStr + ".</p>\n");
+        else if (!inTooOldConsensuses && inTooNewConsensuses)
+          out.println("        <p>Note that we found a matching relay in "
+              + "relay lists that were published up to 2 hours after "
+              + timestampStr + ".</p>\n");
+        else
+          out.println("        <p>Note that we found a matching relay in "
+              + "relay lists that were published between 5 and 3 "
+              + "hours before and in relay lists that were published up "
+              + "to 2 hours after " + timestampStr + ".</p>\n");
+        out.println("<p>Make sure that the timestamp you provided is "
+            + "in the correct timezone: UTC (or GMT).</p>");
+      }
+    }
+    if (target != null) {
+      if (positiveConsensuses.isEmpty() &&
+          !positiveConsensusesNoTarget.isEmpty())
+        out.println("        <p>Note that although the found relay(s) did "
+            + "not permit exiting to " + target + ", there have been one "
+            + "or more relays running at the given time.</p>");
+    }
+
+    /* Finish writing response. */
+    writeFooter(out);
+  }
+}
+
diff --git a/src/org/torproject/ernie/web/ServerDescriptorServlet.java b/src/org/torproject/ernie/web/ServerDescriptorServlet.java
new file mode 100644
index 0000000..056ce9c
--- /dev/null
+++ b/src/org/torproject/ernie/web/ServerDescriptorServlet.java
@@ -0,0 +1,86 @@
+package org.torproject.ernie.web;
+
+import javax.servlet.*;
+import javax.servlet.http.*;
+import java.io.*;
+import java.math.*;
+import java.text.*;
+import java.util.*;
+import java.util.regex.*;
+
+public class ServerDescriptorServlet extends HttpServlet {
+
+  public void doGet(HttpServletRequest request,
+      HttpServletResponse response) throws IOException,
+      ServletException {
+
+    String descIdParameter = request.getParameter("desc-id");
+
+    /* Check if we have a descriptors directory. */
+    // TODO make this configurable!
+    File archiveDirectory = new File("/srv/metrics.torproject.org/archives");
+    if (!archiveDirectory.exists() || !archiveDirectory.isDirectory()) {
+      /* Oops, we don't have any descriptors to serve. */
+// TODO change to internal server error
+      response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+      return;
+    }
+
+    /* Check desc-id parameter. */
+    if (descIdParameter == null || descIdParameter.length() < 4) {
+// TODO is there something like "wrong parameter"?
+      response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
+      return;
+    }
+    Pattern descIdPattern = Pattern.compile("^[0-9a-f]+$");
+    Matcher descIdMatcher = descIdPattern.matcher(descIdParameter);
+    if (!descIdMatcher.matches()) {
+// TODO is there something like "wrong parameter"?
+      response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
+      return;
+    }
+
+    for (File directory : archiveDirectory.listFiles()) {
+      if (!directory.isDirectory() ||
+          !directory.getName().startsWith("server-descriptors-")) {
+        continue;
+      }
+      File subDirectory = new File(directory.getAbsolutePath() + "/"
+          + descIdParameter.substring(0, 1) + "/"
+          + descIdParameter.substring(1, 2));
+      if (subDirectory.exists()) {
+        for (File serverDescriptorFile : subDirectory.listFiles()) {
+          if (!serverDescriptorFile.getName().startsWith(descIdParameter)) {
+            continue;
+          }
+
+          /* Found it! Read file from disk and write it to response. */
+          BufferedInputStream input = null;
+          BufferedOutputStream output = null;
+          try {
+            response.setContentType("text/plain");
+            response.setHeader("Content-Length", String.valueOf(
+                serverDescriptorFile.length()));
+            response.setHeader("Content-Disposition",
+                "inline; filename=\"" + serverDescriptorFile.getName() + "\"");
+            input = new BufferedInputStream(new FileInputStream(
+                serverDescriptorFile), 1024);
+            output = new BufferedOutputStream(response.getOutputStream(), 1024);
+            byte[] buffer = new byte[1024];
+            int length;
+            while ((length = input.read(buffer)) > 0) {
+                output.write(buffer, 0, length);
+            }
+          } finally {
+            output.close();
+            input.close();
+          }
+        }
+      }
+    }
+
+    /* Not found. */
+    response.setStatus(HttpServletResponse.SC_NOT_FOUND);
+  }
+}
+
-- 
1.6.5




More information about the tor-commits mailing list