commit 0a43ae9eba5aa70d4321cb3caf2b6dfeee6494f9 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Sun Dec 1 07:04:05 2013 +0100 Excise ExoneraTor to make it a service of its own. --- config.template | 7 - db/exonerator.sql | 361 ------- etc/context.xml.template | 10 - etc/web.xml | 22 - .../ernie/status/exonerator/ConsensusServlet.java | 124 --- .../exonerator/ExoneraTorDatabaseImporter.java | 619 ----------- .../ernie/status/exonerator/ExoneraTorServlet.java | 1140 +------------------- .../status/exonerator/ServerDescriptorServlet.java | 132 --- .../status/relaysearch/RelaySearchServlet.java | 7 +- web/WEB-INF/exonerator.jsp | 45 + web/robots.txt | 2 - 11 files changed, 55 insertions(+), 2414 deletions(-) diff --git a/config.template b/config.template index 0491431..8f0789b 100644 --- a/config.template +++ b/config.template @@ -45,10 +45,3 @@ ## Relative path to directory to import torperf results from #TorperfDirectory in/torperf/ # -## JDBC string for ExoneraTor database -#ExoneraTorDatabaseJdbc jdbc:postgresql://localhost/exonerator?user=metrics&password=password -# -## Relative path to directory where to find descriptors to import into the -## ExoneraTor database -#ExoneraTorImportDirectory exonerator-import/ - diff --git a/db/exonerator.sql b/db/exonerator.sql deleted file mode 100755 index fd58531..0000000 --- a/db/exonerator.sql +++ /dev/null @@ -1,361 +0,0 @@ --- Copyright 2011 The Tor Project --- See LICENSE for licensing information - --- The descriptor table holds server descriptors that we use for display --- purposes and to parse exit policies. -CREATE TABLE descriptor ( - - -- The 40-character lower-case hex string identifies a descriptor - -- uniquely and is used to join statusentry and this table. - descriptor CHARACTER(40) NOT NULL PRIMARY KEY, - - -- The raw descriptor string is used for display purposes and to check - -- whether the relay allowed exiting to a given target or not. - rawdescriptor BYTEA NOT NULL -); - --- The consensus table stores network status consensuses to be looked up --- by valid-after time and displayed upon request. A second purpose is --- to learn quickly whether the database contains status entries for a --- given day or not. -CREATE TABLE consensus ( - - -- The unique valid-after time of the consensus. - validafter TIMESTAMP WITHOUT TIME ZONE NOT NULL PRIMARY KEY, - - -- The raw consensus string for display purposes only. - rawconsensus BYTEA NOT NULL -); - --- The statusentry table stores network status consensus entries listing --- a relay as running at a certain point in time. Only relays with the --- Running flag shall be inserted into this table. If a relay advertises --- more than one IP address, there is a distinct entry for each address in --- this table. If a relay advertises more than one TCP port on the same --- IP address, there is only a single entry in this table. -CREATE TABLE statusentry ( - - -- The valid-after time of the consensus that contains this entry. - validafter TIMESTAMP WITHOUT TIME ZONE NOT NULL, - - -- The 40-character lower-case hex string uniquely identifying the - -- relay. - fingerprint CHARACTER(40) NOT NULL, - - -- The 40-character lower-case hex string that identifies the server - -- descriptor published by the relay. - descriptor CHARACTER(40) NOT NULL, - - -- The most significant 3 bytes of the relay's onion routing IPv4 - -- address in lower-case hex notation, or null if the relay's onion - -- routing address in this status entry is IPv6. The purpose is to - -- quickly reduce query results for relays in the same /24 network. - oraddress24 CHARACTER(6), - - -- The most significant 6 bytes of the relay's onion routing IPv6 - -- address in lower-case hex notation, or null if the relay's onion - -- routing address in this status entry is IPv4. The purpose is to - -- quickly reduce query results for relays in the same /48 network. - oraddress48 CHARACTER(12), - - -- The relay's onion routing address. Can be an IPv4 or an IPv6 - -- address. If a relay advertises more than one address, there are - -- multiple entries in this table for the same status entry. - oraddress INET NOT NULL, - - -- The raw status entry string as contained in the network status - -- consensus for display purposes only. - rawstatusentry BYTEA NOT NULL, - - -- A status entry is uniquely identified by its valid-after time, relay - -- fingerprint, and onion routing address. - CONSTRAINT statusentry_pkey - PRIMARY KEY (validafter, fingerprint, oraddress) -); - --- The index on the exact onion routing address and on the valid-after --- date is used to speed up ExoneraTor's query for status entries. -CREATE INDEX statusentry_oraddress_validafterdate - ON statusentry (oraddress, DATE(validafter)); - --- The index on the most significant 3 bytes of the relay's onion routing --- address and on the valid-after date is used to speed up queries for --- other relays in the same /24 network. -CREATE INDEX statusentry_oraddress24_validafterdate - ON statusentry (oraddress24, DATE(validafter)); - --- The index on the most significant 6 bytes of the relay's onion routing --- address and on the valid-after date is used to speed up queries for --- other relays in the same /48 network. -CREATE INDEX statusentry_oraddress48_validafterdate - ON statusentry (oraddress48, DATE(validafter)); - --- The exitlistentry table stores the results of the active testing, --- DNS-based exit list for exit nodes. An entry in this table means that --- a relay was scanned at a given time and found to be exiting to the --- Internet from a given IP address. This IP address can be different --- from the relay's onion routing address if the relay uses more than one --- IP addresses. -CREATE TABLE exitlistentry ( - - -- The 40-character lower-case hex string identifying the relay. - fingerprint CHARACTER(40) NOT NULL, - - -- The most significant 3 bytes of the relay's exit IPv4 address in - -- lower-case hex notation, or null if the relay's exit address in this - -- entry is IPv6. The purpose is to quickly reduce query results for - -- relays exiting from the same /24 network. - exitaddress24 CHARACTER(6), - - -- The IP address that the relay uses for exiting to the Internet. If - -- the relay uses more than one IP address, there are multiple entries - -- in this table. - exitaddress INET NOT NULL, - - -- The time when the relay was scanned to find out its exit IP - -- address(es). - scanned TIMESTAMP WITHOUT TIME ZONE NOT NULL, - - -- The raw exit list entry containing all scan results for a given relay - -- for display purposes. - rawexitlistentry BYTEA NOT NULL, - - -- An exit list entry is uniquely identified by its scan time, relay - -- fingerprint, and exit address. - CONSTRAINT exitlistentry_pkey - PRIMARY KEY (scanned, fingerprint, exitaddress) -); - --- The index on the exact exit address and on the valid-after date is used --- to speed up ExoneraTor's query for status entries referencing exit list --- entries. -CREATE INDEX exitlistentry_exitaddress_scanneddate - ON exitlistentry (exitaddress, DATE(scanned)); - --- The index on the most significant 3 bytes of the relay's exit address --- and on the valid-after date is used to speed up queries for other --- relays in the same /24 network. -CREATE INDEX exitlistentry_exitaddress24_scanneddate - ON exitlistentry (exitaddress24, DATE(scanned)); - --- Create the plpgsql language, so that we can use it below. -CREATE LANGUAGE plpgsql; - --- Insert a server descriptor into the descriptor table. Before doing so, --- check that there is no descriptor with the same descriptor identifier --- in the table yet. Return 1 if the descriptor was inserted, 0 --- otherwise. -CREATE OR REPLACE FUNCTION insert_descriptor ( - insert_descriptor CHARACTER(40), - insert_rawdescriptor BYTEA) - RETURNS INTEGER AS $$ - BEGIN - -- Look up if the descriptor is already contained in the descriptor - -- table. - IF (SELECT COUNT(*) - FROM descriptor - WHERE descriptor = insert_descriptor) = 0 THEN - -- Insert the descriptor and remember the new descriptorid to update - -- the foreign key in statusentry. - INSERT INTO descriptor (descriptor, rawdescriptor) - VALUES (insert_descriptor, insert_rawdescriptor); - -- Return 1 for a successfully inserted descriptor. - RETURN 1; - ELSE - -- Return 0 because we didn't change anything. - RETURN 0; - END IF; - END; -$$ LANGUAGE 'plpgsql'; - --- Insert a status entry into the statusentry table. First check that --- this status entry isn't contained in the table yet. It's okay to --- insert the same status entry multiple times for different IP addresses --- though. Return 1 if it was inserted, 0 otherwise. -CREATE OR REPLACE FUNCTION insert_statusentry ( - insert_validafter TIMESTAMP WITHOUT TIME ZONE, - insert_fingerprint CHARACTER(40), - insert_descriptor CHARACTER(40), - insert_oraddress24 CHARACTER(6), - insert_oraddress48 CHARACTER(12), - insert_oraddress TEXT, - insert_rawstatusentry BYTEA) - RETURNS INTEGER AS $$ - BEGIN - -- Look up if the status entry is already contained in the statusentry - -- table. - IF (SELECT COUNT(*) - FROM statusentry - WHERE validafter = insert_validafter - AND fingerprint = insert_fingerprint - AND oraddress = insert_oraddress::INET) = 0 THEN - -- Insert the status entry. - INSERT INTO statusentry (validafter, fingerprint, descriptor, - oraddress24, oraddress48, oraddress, rawstatusentry) - VALUES (insert_validafter, insert_fingerprint, - insert_descriptor, insert_oraddress24, insert_oraddress48, - insert_oraddress::INET, insert_rawstatusentry); - -- Return 1 for a successfully inserted status entry. - RETURN 1; - ELSE - -- Return 0 because we already had this status entry. - RETURN 0; - END IF; - END; -$$ LANGUAGE 'plpgsql'; - --- Insert a consensus into the consensus table. Check that the same --- consensus has not been imported before. Return 1 if it was inserted, 0 --- otherwise. -CREATE OR REPLACE FUNCTION insert_consensus ( - insert_validafter TIMESTAMP WITHOUT TIME ZONE, - insert_rawconsensus BYTEA) - RETURNS INTEGER AS $$ - BEGIN - -- Look up if the consensus is already contained in the consensus - -- table. - IF (SELECT COUNT(*) - FROM consensus - WHERE validafter = insert_validafter) = 0 THEN - -- Insert the consensus. - INSERT INTO consensus (validafter, rawconsensus) - VALUES (insert_validafter, insert_rawconsensus); - -- Return 1 for a successful insert operation. - RETURN 1; - ELSE - -- Return 0 for not inserting the consensus. - RETURN 0; - END IF; - END; -$$ LANGUAGE 'plpgsql'; - --- Insert an exit list entry into the exitlistentry table. Check that --- this entry hasn't been inserted before. It's okay to insert the same --- exit list entry multiple times for different exit addresses. Return 1 --- if the entry was inserted, 0 otherwise. -CREATE OR REPLACE FUNCTION insert_exitlistentry ( - insert_fingerprint CHARACTER(40), - insert_exitaddress24 CHARACTER(6), - insert_exitaddress TEXT, - insert_scanned TIMESTAMP WITHOUT TIME ZONE, - insert_rawexitlistentry BYTEA) - RETURNS INTEGER AS $$ - BEGIN - IF (SELECT COUNT(*) - FROM exitlistentry - WHERE fingerprint = insert_fingerprint - AND exitaddress = insert_exitaddress::INET - AND scanned = insert_scanned) = 0 THEN - -- This exit list entry is not in the database yet. Add it. - INSERT INTO exitlistentry (fingerprint, exitaddress24, exitaddress, - scanned, rawexitlistentry) - VALUES (insert_fingerprint, insert_exitaddress24, - insert_exitaddress::INET, insert_scanned, - insert_rawexitlistentry); - -- Return 1 for a successfully inserted exit list entry. - RETURN 1; - ELSE - -- Return 0 to show that we didn't add anything. - RETURN 0; - END IF; - END; -$$ LANGUAGE 'plpgsql'; - --- Search for status entries with the given IP address as onion routing --- address, plus status entries of relays having an exit list entry with --- the given IP address as exit address. -CREATE OR REPLACE FUNCTION search_statusentries_by_address_date ( - select_address TEXT, - select_date DATE) - RETURNS TABLE(rawstatusentry BYTEA, - descriptor CHARACTER(40), - validafter TIMESTAMP WITHOUT TIME ZONE, - fingerprint CHARACTER(40), - oraddress TEXT, - exitaddress TEXT, - scanned TIMESTAMP WITHOUT TIME ZONE) AS $$ - -- The first select finds all status entries of relays with the given - -- IP address as onion routing address. - SELECT rawstatusentry, - descriptor, - validafter, - fingerprint, - HOST(oraddress), - NULL, - NULL - FROM statusentry - WHERE oraddress = $1::INET - AND DATE(validafter) >= $2 - 1 - AND DATE(validafter) <= $2 + 1 - UNION - -- The second select finds status entries of relays having an exit list - -- entry with the provided IP address as the exit address. - SELECT statusentry.rawstatusentry, - statusentry.descriptor, - statusentry.validafter, - statusentry.fingerprint, - HOST(statusentry.oraddress), - HOST(exitlistentry.exitaddress), - -- Pick only the last scan result that took place in the 24 hours - -- before the valid-after time. - MAX(exitlistentry.scanned) - FROM statusentry - JOIN exitlistentry - ON statusentry.fingerprint = exitlistentry.fingerprint - WHERE exitlistentry.exitaddress = $1::INET - -- Focus on a time period from 1 day before and 1 day after the - -- given date. Also include a second day before the given date - -- for exit lists, because it can take up to 24 hours to scan a - -- relay again. We shouldn't miss exit list entries here. - AND DATE(exitlistentry.scanned) >= $2 - 2 - AND DATE(exitlistentry.scanned) <= $2 + 1 - AND DATE(statusentry.validafter) >= $2 - 1 - AND DATE(statusentry.validafter) <= $2 + 1 - -- Consider only exit list scans that took place in the 24 hours - -- before the relay was listed in a consensus. - AND statusentry.validafter >= exitlistentry.scanned - AND statusentry.validafter - exitlistentry.scanned <= - '1 day'::INTERVAL - GROUP BY 1, 2, 3, 4, 5, 6 - ORDER BY 3, 4, 6; -$$ LANGUAGE SQL; - --- Look up all IPv4 OR and exit addresses in the /24 network of a given --- address to suggest other addresses the user may be looking for. -CREATE OR REPLACE FUNCTION search_addresses_in_same_24 ( - select_address24 CHARACTER(6), - select_date DATE) - RETURNS TABLE(addresstext TEXT, - addressinet INET) AS $$ - SELECT HOST(oraddress), - oraddress - FROM statusentry - WHERE oraddress24 = $1 - AND DATE(validafter) >= $2 - 1 - AND DATE(validafter) <= $2 + 1 - UNION - SELECT HOST(exitaddress), - exitaddress - FROM exitlistentry - WHERE exitaddress24 = $1 - AND DATE(scanned) >= $2 - 2 - AND DATE(scanned) <= $2 + 1 - ORDER BY 2; -$$ LANGUAGE SQL; - --- Look up all IPv6 OR addresses in the /48 network of a given address to --- suggest other addresses the user may be looking for. -CREATE OR REPLACE FUNCTION search_addresses_in_same_48 ( - select_address48 CHARACTER(12), - select_date DATE) - RETURNS TABLE(addresstext TEXT, - addressinet INET) AS $$ - SELECT HOST(oraddress), - oraddress - FROM statusentry - WHERE oraddress48 = $1 - AND DATE(validafter) >= $2 - 1 - AND DATE(validafter) <= $2 + 1 - ORDER BY 2; -$$ LANGUAGE SQL; - diff --git a/etc/context.xml.template b/etc/context.xml.template index 152f1de..00f14fe 100644 --- a/etc/context.xml.template +++ b/etc/context.xml.template @@ -1,14 +1,4 @@ <Context cookies="false"> - <Resource name="jdbc/exonerator" - type="javax.sql.DataSource" - auth="Container" - username="metrics" - password="password" - driverClassName="org.postgresql.Driver" - url="jdbc:postgresql://localhost/exonerator" - maxActive="8" - maxIdle="4" - maxWait="15000"/> <Resource name="jdbc/tordir" type="javax.sql.DataSource" auth="Container" diff --git a/etc/web.xml b/etc/web.xml index e499eca..9b4f23a 100644 --- a/etc/web.xml +++ b/etc/web.xml @@ -245,28 +245,6 @@ </servlet-mapping> <servlet> - <servlet-name>ServerDescriptor</servlet-name> - <servlet-class> - org.torproject.ernie.status.exonerator.ServerDescriptorServlet - </servlet-class> - </servlet> - <servlet-mapping> - <servlet-name>ServerDescriptor</servlet-name> - <url-pattern>/serverdesc</url-pattern> - </servlet-mapping> - - <servlet> - <servlet-name>Consensus</servlet-name> - <servlet-class> - org.torproject.ernie.status.exonerator.ConsensusServlet - </servlet-class> - </servlet> - <servlet-mapping> - <servlet-name>Consensus</servlet-name> - <url-pattern>/consensus</url-pattern> - </servlet-mapping> - - <servlet> <servlet-name>ConsensusHealthServlet</servlet-name> <servlet-class> org.torproject.ernie.status.doctor.ConsensusHealthServlet diff --git a/src/org/torproject/ernie/status/exonerator/ConsensusServlet.java b/src/org/torproject/ernie/status/exonerator/ConsensusServlet.java deleted file mode 100644 index f7ed381..0000000 --- a/src/org/torproject/ernie/status/exonerator/ConsensusServlet.java +++ /dev/null @@ -1,124 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.status.exonerator; - -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.TimeZone; -import java.util.logging.Level; -import java.util.logging.Logger; - -import javax.naming.Context; -import javax.naming.InitialContext; -import javax.naming.NamingException; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.sql.DataSource; - -public class ConsensusServlet extends HttpServlet { - - private static final long serialVersionUID = 3147332016303032164L; - - private DataSource ds; - - private Logger logger; - - public void init() { - - /* Initialize logger. */ - this.logger = Logger.getLogger(ConsensusServlet.class.toString()); - - /* Look up data source. */ - try { - Context cxt = new InitialContext(); - this.ds = (DataSource) cxt.lookup("java:comp/env/jdbc/exonerator"); - this.logger.info("Successfully looked up data source."); - } catch (NamingException e) { - this.logger.log(Level.WARNING, "Could not look up data source", e); - } - } - - public void doGet(HttpServletRequest request, - HttpServletResponse response) throws IOException, - ServletException { - - /* Check valid-after parameter. */ - String validAfterParameter = request.getParameter("valid-after"); - if (validAfterParameter == null || - validAfterParameter.length() != "yyyy-MM-dd-HH-mm-ss".length()) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - SimpleDateFormat parameterFormat = new SimpleDateFormat( - "yyyy-MM-dd-HH-mm-ss"); - parameterFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - long parsedTimestamp = -1L; - try { - parsedTimestamp = parameterFormat.parse(validAfterParameter). - getTime(); - } catch (ParseException e) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - if (parsedTimestamp < 0L) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - - /* Look up consensus in the database. */ - SimpleDateFormat databaseFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - databaseFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String databaseParameter = databaseFormat.format(parsedTimestamp); - byte[] rawDescriptor = null; - try { - long requestedConnection = System.currentTimeMillis(); - Connection conn = this.ds.getConnection(); - Statement statement = conn.createStatement(); - String query = "SELECT rawconsensus FROM consensus " - + "WHERE validafter = '" + databaseParameter + "'"; - ResultSet rs = statement.executeQuery(query); - if (rs.next()) { - rawDescriptor = rs.getBytes(1); - } - rs.close(); - statement.close(); - conn.close(); - this.logger.info("Returned a database connection to the pool after " - + (System.currentTimeMillis() - requestedConnection) - + " millis."); - } catch (SQLException e) { - response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - return; - } - - /* Write response. */ - if (rawDescriptor == null) { - response.sendError(HttpServletResponse.SC_NOT_FOUND); - return; - } - try { - response.setContentType("text/plain"); - response.setHeader("Content-Length", String.valueOf( - rawDescriptor.length)); - response.setHeader("Content-Disposition", "inline; filename=\"" - + validAfterParameter + "-consensus\""); - BufferedOutputStream output = new BufferedOutputStream( - response.getOutputStream()); - output.write(rawDescriptor); - output.flush(); - output.close(); - } finally { - /* Nothing to do here. */ - } - } -} - diff --git a/src/org/torproject/ernie/status/exonerator/ExoneraTorDatabaseImporter.java b/src/org/torproject/ernie/status/exonerator/ExoneraTorDatabaseImporter.java deleted file mode 100644 index d89288f..0000000 --- a/src/org/torproject/ernie/status/exonerator/ExoneraTorDatabaseImporter.java +++ /dev/null @@ -1,619 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.status.exonerator; - -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.StringReader; -import java.io.UnsupportedEncodingException; -import java.sql.CallableStatement; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.sql.Types; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.Stack; -import java.util.TimeZone; - -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; - -/* Import Tor descriptors into the ExoneraTor database. */ -public class ExoneraTorDatabaseImporter { - - /* Main function controlling the parsing process. */ - public static void main(String[] args) { - readConfiguration(); - openDatabaseConnection(); - prepareDatabaseStatements(); - createLockFile(); - readImportHistoryToMemory(); - parseDescriptors(); - writeImportHistoryToDisk(); - closeDatabaseConnection(); - deleteLockFile(); - } - - /* JDBC string of the ExoneraTor database. */ - private static String jdbcString; - - /* Directory from which to import descriptors. */ - private static String importDirString; - - /* Learn JDBC string and directory to parse descriptors from. */ - private static void readConfiguration() { - File configFile = new File("config"); - if (!configFile.exists()) { - System.err.println("Could not find config file. Exiting."); - System.exit(1); - } - String line = null; - try { - BufferedReader br = new BufferedReader(new FileReader(configFile)); - while ((line = br.readLine()) != null) { - if (line.startsWith("#") || line.length() < 1) { - continue; - } else if (line.startsWith("ExoneraTorDatabaseJdbc")) { - jdbcString = line.split(" ")[1]; - } else if (line.startsWith("ExoneraTorImportDirectory")) { - importDirString = line.split(" ")[1]; - } else { - /* Ignore unrecognized configuration keys. */ - } - } - br.close(); - } catch (IOException e) { - System.err.println("Could not parse config file. Exiting."); - System.exit(1); - } - } - - /* Database connection. */ - private static Connection connection; - - /* Open a database connection using the JDBC string in the config. */ - private static void openDatabaseConnection() { - try { - connection = DriverManager.getConnection(jdbcString); - } catch (SQLException e) { - System.out.println("Could not connect to database. Exiting."); - System.exit(1); - } - } - - /* Callable statements to import data into the database. */ - private static CallableStatement insertDescriptorStatement; - private static CallableStatement insertStatusentryStatement; - private static CallableStatement insertConsensusStatement; - private static CallableStatement insertExitlistentryStatement; - - /* Prepare statements for importing data into the database. */ - private static void prepareDatabaseStatements() { - try { - insertDescriptorStatement = connection.prepareCall( - "{call insert_descriptor(?, ?)}"); - insertStatusentryStatement = connection.prepareCall( - "{call insert_statusentry(?, ?, ?, ?, ?, ?, ?)}"); - insertConsensusStatement = connection.prepareCall( - "{call insert_consensus(?, ?)}"); - insertExitlistentryStatement = connection.prepareCall( - "{call insert_exitlistentry(?, ?, ?, ?, ?)}"); - } catch (SQLException e) { - System.out.println("Could not prepare callable statements to " - + "import data into the database. Exiting."); - System.exit(1); - } - } - - /* Create a local lock file to prevent other instances of this import - * tool to run concurrently. */ - private static void createLockFile() { - File lockFile = new File("exonerator-lock"); - try { - if (lockFile.exists()) { - BufferedReader br = new BufferedReader(new FileReader(lockFile)); - long runStarted = Long.parseLong(br.readLine()); - br.close(); - if (System.currentTimeMillis() - runStarted - < 6L * 60L * 60L * 1000L) { - System.out.println("File 'exonerator-lock' is less than 6 " - + "hours old. Exiting."); - System.exit(1); - } else { - System.out.println("File 'exonerator-lock' is at least 6 hours " - + "old. Overwriting and executing anyway."); - } - } - BufferedWriter bw = new BufferedWriter(new FileWriter( - "exonerator-lock")); - bw.append(String.valueOf(System.currentTimeMillis()) + "\n"); - bw.close(); - } catch (IOException e) { - System.out.println("Could not create 'exonerator-lock' file. " - + "Exiting."); - System.exit(1); - } - } - - /* Last and next parse histories containing paths of parsed files and - * last modified times. */ - private static Map<String, Long> - lastImportHistory = new HashMap<String, Long>(), - nextImportHistory = new HashMap<String, Long>(); - - /* Read stats/exonerator-import-history file from disk and remember - * locally when files were last parsed. */ - private static void readImportHistoryToMemory() { - File parseHistoryFile = new File("stats", - "exonerator-import-history"); - if (parseHistoryFile.exists()) { - try { - BufferedReader br = new BufferedReader(new FileReader( - parseHistoryFile)); - String line = null; - int lineNumber = 0; - while ((line = br.readLine()) != null) { - lineNumber++; - String[] parts = line.split(","); - if (parts.length != 2) { - System.out.println("File 'stats/exonerator-import-history' " - + "contains a corrupt entry in line " + lineNumber - + ". Ignoring parse history file entirely."); - lastImportHistory.clear(); - br.close(); - return; - } - long lastModified = Long.parseLong(parts[0]); - String filename = parts[1]; - lastImportHistory.put(filename, lastModified); - } - br.close(); - } catch (IOException e) { - System.out.println("Could not read import history. Ignoring."); - lastImportHistory.clear(); - } - } - } - - /* Parse descriptors in the import directory and its subdirectories. */ - private static void parseDescriptors() { - File file = new File(importDirString); - if (!file.exists()) { - System.out.println("File or directory " + importDirString + " does " - + "not exist. Exiting."); - return; - } - Stack<File> files = new Stack<File>(); - files.add(file); - while (!files.isEmpty()) { - file = files.pop(); - if (file.isDirectory()) { - for (File f : file.listFiles()) { - files.add(f); - } - } else { - parseFile(file); - } - } - } - - /* Import a file if it wasn't imported before, and add it to the import - * history for the next execution. */ - private static void parseFile(File file) { - long lastModified = file.lastModified(); - String filename = file.getName(); - nextImportHistory.put(filename, lastModified); - if (!lastImportHistory.containsKey(filename) || - lastImportHistory.get(filename) < lastModified) { - try { - FileInputStream fis = new FileInputStream(file); - BufferedInputStream bis = new BufferedInputStream(fis); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] bytes = new byte[1024]; - while ((len = bis.read(bytes, 0, 1024)) >= 0) { - baos.write(bytes, 0, len); - } - bis.close(); - byte[] allBytes = baos.toByteArray(); - splitFile(file, allBytes); - } catch (IOException e) { - System.out.println("Could not read '" + file + "' to memory. " - + "Skipping."); - nextImportHistory.remove(filename); - } - } - } - - /* Detect what descriptor type is contained in a file and split it to - * parse the single descriptors. */ - private static void splitFile(File file, byte[] bytes) { - try { - String asciiString = new String(bytes, "US-ASCII"); - BufferedReader br = new BufferedReader(new StringReader( - asciiString)); - String line = br.readLine(); - while (line != null && line.startsWith("@")) { - line = br.readLine(); - } - if (line == null) { - return; - } - br.close(); - String startToken = null; - if (line.startsWith("router ")) { - startToken = "router "; - } else if (line.equals("network-status-version 3")) { - startToken = "network-status-version 3"; - } else if (line.startsWith("Downloaded ") || - line.startsWith("ExitNode ")) { - startToken = "ExitNode "; - } else { - System.out.println("Unknown descriptor type in file '" + file - + "'. Ignoring."); - return; - } - String splitToken = "\n" + startToken; - int length = bytes.length, start = asciiString.indexOf(startToken); - while (start < length) { - int end = asciiString.indexOf(splitToken, start); - if (end < 0) { - end = length; - } else { - end += 1; - } - byte[] descBytes = new byte[end - start]; - System.arraycopy(bytes, start, descBytes, 0, end - start); - if (startToken.equals("router ")) { - parseServerDescriptor(file, descBytes); - } else if (startToken.equals("network-status-version 3")) { - parseConsensus(file, descBytes); - } else if (startToken.equals("ExitNode ")) { - parseExitList(file, descBytes); - } - start = end; - } - } catch (IOException e) { - System.out.println("Could not parse descriptor '" + file + "'. " - + "Skipping."); - } - } - - /* Date format to parse UTC timestamps. */ - private static SimpleDateFormat parseFormat; - static { - parseFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - parseFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - } - - /* Parse a single server descriptor. */ - private static void parseServerDescriptor(File file, byte[] bytes) { - String ascii = ""; - try { - ascii = new String(bytes, "US-ASCII"); - } catch (UnsupportedEncodingException e) { - /* We know that US-ASCII is a supported encoding. */ - } - String startToken = "router "; - String sigToken = "\nrouter-signature\n"; - int start = ascii.indexOf(startToken); - int sig = ascii.indexOf(sigToken) + sigToken.length(); - String descriptor = null; - if (start >= 0 || sig >= 0 || sig > start) { - byte[] forDigest = new byte[sig - start]; - System.arraycopy(bytes, start, forDigest, 0, sig - start); - descriptor = DigestUtils.shaHex(forDigest); - } - if (descriptor == null) { - System.out.println("Could not calculate descriptor digest. " - + "Skipping."); - return; - } - importDescriptor(descriptor, bytes); - } - - /* Import a single server descriptor into the database. */ - private static void importDescriptor(String descriptor, - byte[] rawDescriptor) { - try { - insertDescriptorStatement.clearParameters(); - insertDescriptorStatement.setString(1, descriptor); - insertDescriptorStatement.setBytes(2, rawDescriptor); - insertDescriptorStatement.execute(); - } catch (SQLException e) { - System.out.println("Could not import descriptor into the " - + "database. Exiting."); - System.exit(1); - } - } - - /* Parse a consensus. */ - private static void parseConsensus(File file, byte[] bytes) { - try { - BufferedReader br = new BufferedReader(new StringReader(new String( - bytes, "US-ASCII"))); - String line, fingerprint = null, descriptor = null; - Set<String> orAddresses = new HashSet<String>(); - long validAfterMillis = -1L; - StringBuilder rawStatusentryBuilder = null; - boolean isRunning = false; - while ((line = br.readLine()) != null) { - if (line.startsWith("vote-status ") && - !line.equals("vote-status consensus")) { - System.out.println("File '" + file + "' contains network status " - + "*votes*, not network status *consensuses*. Skipping."); - return; - } else if (line.startsWith("valid-after ")) { - String validAfterTime = line.substring("valid-after ".length()); - try { - validAfterMillis = parseFormat.parse(validAfterTime). - getTime(); - } catch (ParseException e) { - System.out.println("Could not parse valid-after timestamp in " - + "'" + file + "'. Skipping."); - return; - } - importConsensus(validAfterMillis, bytes); - } else if (line.startsWith("r ") || - line.equals("directory-footer")) { - if (isRunning) { - byte[] rawStatusentry = rawStatusentryBuilder.toString(). - getBytes(); - importStatusentry(validAfterMillis, fingerprint, descriptor, - orAddresses, rawStatusentry); - orAddresses = new HashSet<String>(); - } - if (line.equals("directory-footer")) { - return; - } - rawStatusentryBuilder = new StringBuilder(line + "\n"); - String[] parts = line.split(" "); - if (parts.length < 9) { - System.out.println("Could not parse r line '" + line - + "'. Skipping."); - return; - } - fingerprint = Hex.encodeHexString(Base64.decodeBase64(parts[2] - + "=")).toLowerCase(); - descriptor = Hex.encodeHexString(Base64.decodeBase64(parts[3] - + "=")).toLowerCase(); - orAddresses.add(parts[6]); - } else if (line.startsWith("a ")) { - rawStatusentryBuilder.append(line + "\n"); - orAddresses.add(line.substring("a ".length(), - line.lastIndexOf(":"))); - } else if (line.startsWith("s ") || line.equals("s")) { - rawStatusentryBuilder.append(line + "\n"); - isRunning = line.contains(" Running"); - } else if (rawStatusentryBuilder != null) { - rawStatusentryBuilder.append(line + "\n"); - } - } - } catch (IOException e) { - System.out.println("Could not parse consensus. Skipping."); - return; - } - } - - /* UTC calendar for importing timestamps into the database. */ - private static Calendar calendarUTC = Calendar.getInstance( - TimeZone.getTimeZone("UTC")); - - /* Import a status entry with one or more OR addresses into the - * database. */ - private static void importStatusentry(long validAfterMillis, - String fingerprint, String descriptor, Set<String> orAddresses, - byte[] rawStatusentry) { - try { - for (String orAddress : orAddresses) { - insertStatusentryStatement.clearParameters(); - insertStatusentryStatement.setTimestamp(1, - new Timestamp(validAfterMillis), calendarUTC); - insertStatusentryStatement.setString(2, fingerprint); - insertStatusentryStatement.setString(3, descriptor); - if (!orAddress.contains(":")) { - String[] addressParts = orAddress.split("\\."); - byte[] address24Bytes = new byte[3]; - address24Bytes[0] = (byte) Integer.parseInt(addressParts[0]); - address24Bytes[1] = (byte) Integer.parseInt(addressParts[1]); - address24Bytes[2] = (byte) Integer.parseInt(addressParts[2]); - String orAddress24 = Hex.encodeHexString(address24Bytes); - insertStatusentryStatement.setString(4, orAddress24); - insertStatusentryStatement.setNull(5, Types.VARCHAR); - insertStatusentryStatement.setString(6, orAddress); - } else { - StringBuilder addressHex = new StringBuilder(); - int start = orAddress.startsWith("[::") ? 2 : 1; - int end = orAddress.length() - - (orAddress.endsWith("::]") ? 2 : 1); - String[] parts = orAddress.substring(start, end).split(":", -1); - for (int i = 0; i < parts.length; i++) { - String part = parts[i]; - if (part.length() == 0) { - addressHex.append("x"); - } else if (part.length() <= 4) { - addressHex.append(String.format("%4s", part)); - } else { - addressHex = null; - break; - } - } - String orAddress48 = null; - if (addressHex != null) { - String addressHexString = addressHex.toString(); - addressHexString = addressHexString.replaceFirst("x", - String.format("%" + (33 - addressHexString.length()) - + "s", "0")); - if (!addressHexString.contains("x") && - addressHexString.length() == 32) { - orAddress48 = addressHexString.replaceAll(" ", "0"). - toLowerCase().substring(0, 12); - } - } - if (orAddress48 != null) { - insertStatusentryStatement.setNull(4, Types.VARCHAR); - insertStatusentryStatement.setString(5, orAddress48); - insertStatusentryStatement.setString(6, - orAddress.replaceAll("[\\[\\]]", "")); - } else { - System.err.println("Could not import status entry with IPv6 " - + "address '" + orAddress + "'. Exiting."); - System.exit(1); - } - } - insertStatusentryStatement.setBytes(7, rawStatusentry); - insertStatusentryStatement.execute(); - } - } catch (SQLException e) { - System.out.println("Could not import status entry. Exiting."); - System.exit(1); - } - } - - /* Import a consensus into the database. */ - private static void importConsensus(long validAfterMillis, - byte[] rawConsensus) { - try { - insertConsensusStatement.clearParameters(); - insertConsensusStatement.setTimestamp(1, - new Timestamp(validAfterMillis), calendarUTC); - insertConsensusStatement.setBytes(2, rawConsensus); - insertConsensusStatement.execute(); - } catch (SQLException e) { - System.out.println("Could not import consensus. Exiting."); - System.exit(1); - } - } - - /* Parse an exit list. */ - private static void parseExitList(File file, byte[] bytes) { - try { - BufferedReader br = new BufferedReader(new StringReader(new String( - bytes, "US-ASCII"))); - String fingerprint = null; - Set<String> exitAddressLines = new HashSet<String>(); - StringBuilder rawExitlistentryBuilder = new StringBuilder(); - while (true) { - String line = br.readLine(); - if ((line == null || line.startsWith("ExitNode ")) && - fingerprint != null) { - for (String exitAddressLine : exitAddressLines) { - String[] parts = exitAddressLine.split(" "); - String exitAddress = parts[1]; - /* TODO Extend the following code for IPv6 once the exit list - * format supports it. */ - String[] exitAddressParts = exitAddress.split("\\."); - byte[] exitAddress24Bytes = new byte[3]; - exitAddress24Bytes[0] = (byte) Integer.parseInt( - exitAddressParts[0]); - exitAddress24Bytes[1] = (byte) Integer.parseInt( - exitAddressParts[1]); - exitAddress24Bytes[2] = (byte) Integer.parseInt( - exitAddressParts[2]); - String exitAddress24 = Hex.encodeHexString( - exitAddress24Bytes); - String scannedTime = parts[2] + " " + parts[3]; - long scannedMillis = -1L; - try { - scannedMillis = parseFormat.parse(scannedTime).getTime(); - } catch (ParseException e) { - System.out.println("Could not parse timestamp in " - + "'" + file + "'. Skipping."); - return; - } - byte[] rawExitlistentry = rawExitlistentryBuilder.toString(). - getBytes(); - importExitlistentry(fingerprint, exitAddress24, exitAddress, - scannedMillis, rawExitlistentry); - } - exitAddressLines.clear(); - rawExitlistentryBuilder = new StringBuilder(); - } - if (line == null) { - break; - } - rawExitlistentryBuilder.append(line + "\n"); - if (line.startsWith("ExitNode ")) { - fingerprint = line.substring("ExitNode ".length()). - toLowerCase(); - } else if (line.startsWith("ExitAddress ")) { - exitAddressLines.add(line); - } - } - br.close(); - } catch (IOException e) { - System.out.println("Could not parse exit list. Skipping."); - return; - } - } - - /* Import an exit list entry into the database. */ - private static void importExitlistentry(String fingerprint, - String exitAddress24, String exitAddress, long scannedMillis, - byte[] rawExitlistentry) { - try { - insertExitlistentryStatement.clearParameters(); - insertExitlistentryStatement.setString(1, fingerprint); - insertExitlistentryStatement.setString(2, exitAddress24); - insertExitlistentryStatement.setString(3, exitAddress); - insertExitlistentryStatement.setTimestamp(4, - new Timestamp(scannedMillis), calendarUTC); - insertExitlistentryStatement.setBytes(5, rawExitlistentry); - insertExitlistentryStatement.execute(); - } catch (SQLException e) { - System.out.println("Could not import exit list entry. Exiting."); - System.exit(1); - } - } - - /* Write parse history from memory to disk for the next execution. */ - private static void writeImportHistoryToDisk() { - File parseHistoryFile = new File("stats/exonerator-import-history"); - parseHistoryFile.getParentFile().mkdirs(); - try { - BufferedWriter bw = new BufferedWriter(new FileWriter( - parseHistoryFile)); - for (Map.Entry<String, Long> historyEntry : - nextImportHistory.entrySet()) { - bw.write(String.valueOf(historyEntry.getValue()) + "," - + historyEntry.getKey() + "\n"); - } - bw.close(); - } catch (IOException e) { - System.out.println("File 'stats/exonerator-import-history' could " - + "not be written. Ignoring."); - } - } - - /* Close the database connection. */ - private static void closeDatabaseConnection() { - try { - connection.close(); - } catch (SQLException e) { - System.out.println("Could not close database connection. " - + "Ignoring."); - } - } - - /* Delete the exonerator-lock file to allow the next executing of this - * tool. */ - private static void deleteLockFile() { - new File("exonerator-lock").delete(); - } -} - diff --git a/src/org/torproject/ernie/status/exonerator/ExoneraTorServlet.java b/src/org/torproject/ernie/status/exonerator/ExoneraTorServlet.java index 9d296fc..d37b9a8 100644 --- a/src/org/torproject/ernie/status/exonerator/ExoneraTorServlet.java +++ b/src/org/torproject/ernie/status/exonerator/ExoneraTorServlet.java @@ -2,1153 +2,23 @@ * See LICENSE for licensing information */ package org.torproject.ernie.status.exonerator; -import java.io.BufferedReader; import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringReader; -import java.sql.CallableStatement; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TimeZone; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import javax.naming.Context; -import javax.naming.InitialContext; -import javax.naming.NamingException; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import javax.sql.DataSource; - -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.lang.StringEscapeUtils; public class ExoneraTorServlet extends HttpServlet { - private static final long serialVersionUID = 1370088989739567509L; - - private DataSource ds; - - private Logger logger; - - public void init() { - - /* Initialize logger. */ - this.logger = Logger.getLogger(ExoneraTorServlet.class.toString()); - - /* Look up data source. */ - try { - Context cxt = new InitialContext(); - this.ds = (DataSource) cxt.lookup("java:comp/env/jdbc/exonerator"); - this.logger.info("Successfully looked up data source."); - } catch (NamingException e) { - this.logger.log(Level.WARNING, "Could not look up data source", e); - } - } - - private void writeHeader(PrintWriter out) throws IOException { - out.println("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.0 " - + "Transitional//EN\">\n" - + "<html>\n" - + " <head>\n" - + " <title>Tor Metrics Portal: ExoneraTor</title>\n" - + " <meta http-equiv=\"content-type\" content=\"text/html; " - + "charset=ISO-8859-1\">\n" - + " <link href=\"/css/stylesheet-ltr.css\" type=\"text/css\" " - + "rel=\"stylesheet\">\n" - + " <link href=\"/images/favicon.ico\" " - + "type=\"image/x-icon\" rel=\"shortcut icon\">\n" - + " </head>\n" - + " <body>\n" - + " <div class=\"center\">\n" - + " <table class=\"banner\" border=\"0\" cellpadding=\"0\" " - + "cellspacing=\"0\" summary=\"\">\n" - + " <tr>\n" - + " <td class=\"banner-left\"><a " - + "href=\"/index.html\"><img src=\"/images/top-left.png\" " - + "alt=\"Click to go to home page\" width=\"193\" " - + "height=\"79\"></a></td>\n" - + " <td class=\"banner-middle\">\n" - + " <a href=\"/\">Home</a>\n" - + " <a href=\"graphs.html\">Graphs</a>\n" - + " <a href=\"research.html\">Research</a>\n" - + " <a href=\"status.html\">Status</a>\n" - + " <br>\n" - + " <font size=\"2\">\n" - + " <a class=\"current\">ExoneraTor</a>\n" - + " <a href=\"relay-search.html\">Relay Search</a>\n" - + " <a href=\"consensus-health.html\">Consensus " - + "Health</a>\n" - + " </font>\n" - + " </td>\n" - + " <td class=\"banner-right\"></td>\n" - + " </tr>\n" - + " </table>\n" - + " <div class=\"main-column\" style=\"margin:5; " - + "Padding:0;\">\n" - + " <h2>ExoneraTor</h2>\n" - + " <h3>or: a website that tells you whether a given IP " - + "address was a Tor relay</h3>\n" - + " <br>\n" - + " <p>Just because you see an Internet connection from a " - + "particular IP address does not mean you know <i>who</i> " - + "originated the traffic. Tor anonymizes Internet traffic by " - + "\"<a href=\"https://www.torproject.org/about/overview" - + "#thesolution\">onion routing</a>,\" sending packets " - + "through a series of encrypted hops before they reach their " - + "destination. Therefore, if you see traffic from a Tor node, " - + "you may be seeing traffic that originated from someone " - + "using Tor, rather than from the node operator itself. The " - + "Tor Project and Tor node operators have no records of the " - + "traffic that passes over the network, but we do maintain " - + "current and historical records of which IP addresses are " - + "part of the Tor network.</p>\n" - + " <br>\n" - + " <p>ExoneraTor tells you whether there was a Tor relay " - + "running on a given IP address at a given time. ExoneraTor " - + "can further indicate whether this relay permitted exiting " - + "to a given server and/or TCP port. ExoneraTor learns these " - + "facts by parsing the public relay lists and relay " - + "descriptors that are collected from the Tor directory " - + "authorities and the exit lists collected by TorDNSEL. By " - + "inputting an IP address and time, you can determine whether " - + "that IP was then a part of the Tor network.</p>\n" - + " <br>\n" - + " <p><font color=\"red\"><b>Notice:</b> Note that the " - + "information you are providing below may be visible to " - + "anyone who can read the network traffic between you and " - + "this web server or who has access to this web " - + "server.</font></p>\n" - + " <br>\n"); - } - - private void writeFooter(PrintWriter out) throws IOException { - out.println(" <br>\n" - + " </div>\n" - + " </div>\n" - + " <div class=\"bottom\" id=\"bottom\">\n" - + " <p>This material is supported in part by the National " - + "Science Foundation under Grant No. CNS-0959138. Any " - + "opinions, finding, and conclusions or recommendations " - + "expressed in this material are those of the author(s) and " - + "do not necessarily reflect the views of the National " - + "Science Foundation.</p>\n" - + " <p>\"Tor\" and the \"Onion Logo\" are <a " - + "href=\"https://www.torproject.org/docs/trademark-faq.html.en" - + "\">registered trademarks</a> of The Tor Project, Inc.</p>\n" - + " <p>Data on this site is freely available under a <a " - + "href=\"http://creativecommons.org/publicdomain/zero/1.0/\">" - + "CC0 no copyright declaration</a>: To the extent possible " - + "under law, the Tor Project has waived all copyright and " - + "related or neighboring rights in the data. Graphs are " - + "licensed under a <a " - + "href=\"http://creativecommons.org/licenses/by/3.0/us/\">" - + "Creative Commons Attribution 3.0 United States " - + "License</a>.</p>\n" - + " </div>\n" - + " </body>\n" - + "</html>"); - out.close(); - } + private static final long serialVersionUID = -6227541092325776626L; public void doGet(HttpServletRequest request, - HttpServletResponse response) throws IOException, - ServletException { - - /* Start writing response. */ - PrintWriter out = response.getWriter(); - writeHeader(out); - - /* Open a database connection that we'll use to handle the whole - * request. */ - Connection conn = null; - long requestedConnection = System.currentTimeMillis(); - try { - conn = this.ds.getConnection(); - } catch (SQLException e) { - out.println("<p><font color=\"red\"><b>Warning: </b></font>Unable " - + "to connect to the database. If this problem persists, " - + "please <a href=\"mailto:tor-assistants@torproject.org\">let " - + "us know</a>!</p>\n"); - writeFooter(out); - return; - } - - /* Look up first and last consensus in the database. */ - long firstValidAfter = -1L, lastValidAfter = -1L; - try { - Statement statement = conn.createStatement(); - String query = "SELECT MIN(validafter) AS first, " - + "MAX(validafter) AS last FROM consensus"; - ResultSet rs = statement.executeQuery(query); - if (rs.next()) { - firstValidAfter = rs.getTimestamp(1).getTime(); - lastValidAfter = rs.getTimestamp(2).getTime(); - } - rs.close(); - statement.close(); - } catch (SQLException e) { - /* Looks like we don't have any consensuses. */ - } - if (firstValidAfter < 0L || lastValidAfter < 0L) { - out.println("<p><font color=\"red\"><b>Warning: </b></font>This " - + "server doesn't have any relay lists available. If this " - + "problem persists, please " - + "<a href=\"mailto:tor-assistants@torproject.org\">let us " - + "know</a>!</p>\n"); - writeFooter(out); - try { - conn.close(); - this.logger.info("Returned a database connection to the pool " - + "after " + (System.currentTimeMillis() - - requestedConnection) + " millis."); - } catch (SQLException e) { - } - return; - } - - out.println("<a name=\"relay\"></a><h3>Was there a Tor relay running " - + "on this IP address?</h3>"); - - /* Parse IP parameter. */ - Pattern ipv4AddressPattern = Pattern.compile( - "^([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." + - "([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." + - "([01]?\\d\\d?|2[0-4]\\d|25[0-5])\\." + - "([01]?\\d\\d?|2[0-4]\\d|25[0-5])$"); - Pattern ipv6AddressPattern = Pattern.compile( - "^\\[?[0-9a-fA-F:]{3,39}\\]?$"); - String ipParameter = request.getParameter("ip"); - String relayIP = "", ipWarning = ""; - if (ipParameter != null && ipParameter.length() > 0) { - if (ipv4AddressPattern.matcher(ipParameter).matches()) { - String[] ipParts = ipParameter.split("\\."); - relayIP = Integer.parseInt(ipParts[0]) + "." - + Integer.parseInt(ipParts[1]) + "." - + Integer.parseInt(ipParts[2]) + "." - + Integer.parseInt(ipParts[3]); - } else if (ipv6AddressPattern.matcher(ipParameter).matches()) { - if (ipParameter.startsWith("[") && ipParameter.endsWith("]")) { - ipParameter = ipParameter.substring(1, - ipParameter.length() - 1); - } - StringBuilder addressHex = new StringBuilder(); - int start = ipParameter.startsWith("::") ? 1 : 0; - int end = ipParameter.length() - - (ipParameter.endsWith("::") ? 1 : 0); - String[] parts = ipParameter.substring(start, end).split(":", -1); - for (int i = 0; i < parts.length; i++) { - String part = parts[i]; - if (part.length() == 0) { - addressHex.append("x"); - } else if (part.length() <= 4) { - addressHex.append(String.format("%4s", part)); - } else { - addressHex = null; - break; - } - } - if (addressHex != null) { - String addressHexString = addressHex.toString(); - addressHexString = addressHexString.replaceFirst("x", - String.format("%" + (33 - addressHexString.length()) + "s", - "0")); - if (!addressHexString.contains("x") && - addressHexString.length() == 32) { - relayIP = ipParameter.toLowerCase(); - } - } - if (relayIP.length() < 1) { - ipWarning = "\"" + (ipParameter.length() > 40 ? - StringEscapeUtils.escapeHtml(ipParameter.substring(0, 40)) - + "[...]" : StringEscapeUtils.escapeHtml(ipParameter)) - + "\" is not a valid IP address."; - } - } else { - ipWarning = "\"" + (ipParameter.length() > 20 ? - StringEscapeUtils.escapeHtml(ipParameter.substring(0, 20)) - + "[...]" : StringEscapeUtils.escapeHtml(ipParameter)) - + "\" is not a valid IP address."; - } - } - - /* Parse timestamp parameter. */ - String timestampParameter = request.getParameter("timestamp"); - long timestamp = 0L; - boolean timestampIsDate = false; - String timestampStr = "", timestampWarning = ""; - SimpleDateFormat shortDateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm"); - shortDateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - if (timestampParameter != null && timestampParameter.length() > 0) { - try { - if (timestampParameter.split(" ").length == 1) { - timestamp = dateFormat.parse(timestampParameter).getTime(); - timestampStr = dateFormat.format(timestamp); - timestampIsDate = true; - } else { - timestamp = shortDateTimeFormat.parse(timestampParameter). - getTime(); - timestampStr = shortDateTimeFormat.format(timestamp); - } - if (timestamp < firstValidAfter || timestamp > lastValidAfter) { - timestampWarning = "Please pick a date or timestamp between \"" - + shortDateTimeFormat.format(firstValidAfter) + "\" and \"" - + shortDateTimeFormat.format(lastValidAfter) + "\"."; - timestamp = 0L; - } - } catch (ParseException e) { - /* We have no way to handle this exception, other than leaving - timestampStr at "". */ - timestampWarning = "\"" + (timestampParameter.length() > 20 ? - StringEscapeUtils.escapeHtml(timestampParameter. - substring(0, 20)) + "[...]" : - StringEscapeUtils.escapeHtml(timestampParameter)) - + "\" is not a valid date or timestamp."; - } - } - - /* If either IP address or timestamp is provided, the other one must - * be provided, too. */ - if (relayIP.length() < 1 && timestampStr.length() > 0 && - ipWarning.length() < 1) { - ipWarning = "Please provide an IP address."; - } - if (relayIP.length() > 0 && timestamp < 1 && - timestampWarning.length() < 1) { - timestampWarning = "Please provide a date or timestamp."; - } - - /* Parse target IP parameter. */ - String targetIP = "", targetPort = "", target = ""; - String[] targetIPParts = null; - String targetAddrParameter = request.getParameter("targetaddr"); - String targetAddrWarning = ""; - if (targetAddrParameter != null && targetAddrParameter.length() > 0) { - Matcher targetAddrParameterMatcher = - ipv4AddressPattern.matcher(targetAddrParameter); - if (targetAddrParameterMatcher.matches()) { - String[] targetAddrParts = targetAddrParameter.split("\\."); - targetIP = Integer.parseInt(targetAddrParts[0]) + "." - + Integer.parseInt(targetAddrParts[1]) + "." - + Integer.parseInt(targetAddrParts[2]) + "." - + Integer.parseInt(targetAddrParts[3]); - target = targetIP; - targetIPParts = targetIP.split("\\."); - } else { - targetAddrWarning = "\"" + (targetAddrParameter.length() > 20 ? - StringEscapeUtils.escapeHtml(targetAddrParameter.substring( - 0, 20)) + "[...]" : StringEscapeUtils.escapeHtml( - targetAddrParameter)) + "\" is not a valid IP address."; - } - } - - /* Parse target port parameter. */ - String targetPortParameter = request.getParameter("targetport"); - String targetPortWarning = ""; - if (targetPortParameter != null && targetPortParameter.length() > 0) { - Pattern targetPortPattern = Pattern.compile("\\d+"); - if (targetPortParameter.length() < 5 && - targetPortPattern.matcher(targetPortParameter).matches() && - !targetPortParameter.equals("0") && - Integer.parseInt(targetPortParameter) < 65536) { - targetPort = targetPortParameter; - if (target != null) { - target += ":" + targetPort; - } else { - target = targetPort; - } - } else { - targetPortWarning = "\"" + (targetPortParameter.length() > 8 ? - StringEscapeUtils.escapeHtml(targetPortParameter. - substring(0, 8)) + "[...]" : - StringEscapeUtils.escapeHtml(targetPortParameter)) - + "\" is not a valid TCP port."; - } - } - - /* If target port is provided, a target address must be provided, - * too. */ - /* TODO Relax this requirement. */ - if (targetPort.length() > 0 && targetIP.length() < 1 && - targetAddrWarning.length() < 1) { - targetAddrWarning = "Please provide an IP address."; - } - - /* Write form with IP address and timestamp. */ - out.println(" <form action=\"#relay\">\n" - + " <input type=\"hidden\" name=\"targetaddr\" " - + (targetIP.length() > 0 ? " value=\"" + targetIP + "\"" : "") - + ">\n" - + " <input type=\"hidden\" name=\"targetPort\"" - + (targetPort.length() > 0 ? " value=\"" + targetPort + "\"" : "") - + ">\n" - + " <table>\n" - + " <tr>\n" - + " <td align=\"right\">IP address in question:" - + "</td>\n" - + " <td><input type=\"text\" name=\"ip\" size=\"30\"" - + (relayIP.length() > 0 ? " value=\"" + relayIP + "\"" - : "") - + ">" - + (ipWarning.length() > 0 ? "<br><font color=\"red\">" - + ipWarning + "</font>" : "") - + "</td>\n" - + " <td><i>(Ex.: 86.59.21.38 or " - + "2001:858:2:2:aabb:0:563b:1526)</i></td>\n" - + " </tr>\n" - + " <tr>\n" - + " <td align=\"right\">Date or timestamp, in " - + "UTC:</td>\n" - + " <td><input type=\"text\" name=\"timestamp\"" - + " size=\"30\"" - + (timestampStr.length() > 0 ? " value=\"" + timestampStr + "\"" - : "") - + ">" - + (timestampWarning.length() > 0 ? "<br><font color=\"red\">" - + timestampWarning + "</font>" : "") - + "</td>\n" - + " <td><i>(Ex.: 2010-01-01 or 2010-01-01 12:00)" - + "</i></td>\n" - + " </tr>\n" - + " <tr>\n" - + " <td></td>\n" - + " <td>\n" - + " <input type=\"submit\">\n" - + " <input type=\"reset\">\n" - + " </td>\n" - + " <td></td>\n" - + " </tr>\n" - + " </table>\n" - + " </form>\n"); - - if (relayIP.length() < 1 || timestamp < 1) { - writeFooter(out); - try { - conn.close(); - this.logger.info("Returned a database connection to the pool " - + "after " + (System.currentTimeMillis() - - requestedConnection) + " millis."); - } catch (SQLException e) { - } - return; - } - - out.printf("<p>Looking up IP address %s in the relay lists " - + "published ", relayIP); - long timestampFrom, timestampTo; - if (timestampIsDate) { - /* If we only have a date, consider all consensuses published on the - * given date, plus the ones published 3 hours before the given date - * and until 23:59:59. */ - timestampFrom = timestamp - 3L * 60L * 60L * 1000L; - timestampTo = timestamp + (24L * 60L * 60L - 1L) * 1000L; - out.printf("on %s", timestampStr); - } else { - /* If we have an exact timestamp, consider the consensuses published - * in the 3 hours preceding the UTC timestamp. */ - timestampFrom = timestamp - 3L * 60L * 60L * 1000L; - timestampTo = timestamp; - out.printf("between %s and %s UTC", - shortDateTimeFormat.format(timestampFrom), - shortDateTimeFormat.format(timestampTo)); - } - /* If we don't find any relays in the given time interval, also look - * at consensuses published 12 hours before and 12 hours after the - * interval, in case the user got the "UTC" bit wrong. */ - long timestampTooOld = timestampFrom - 12L * 60L * 60L * 1000L; - long timestampTooNew = timestampTo + 12L * 60L * 60L * 1000L; - out.print(" as well as in the relevant exit lists. Clients could " - + "have selected any of these relays to build circuits. " - + "You may follow the links to relay lists and relay descriptors " - + "to grep for the lines printed below and confirm that results " - + "are correct.<br>"); - SimpleDateFormat validAfterTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - validAfterTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String fromValidAfter = validAfterTimeFormat.format(timestampTooOld); - String toValidAfter = validAfterTimeFormat.format(timestampTooNew); - SortedSet<Long> tooOldConsensuses = new TreeSet<Long>(); - SortedSet<Long> relevantConsensuses = new TreeSet<Long>(); - SortedSet<Long> tooNewConsensuses = new TreeSet<Long>(); - try { - Statement statement = conn.createStatement(); - String query = "SELECT validafter FROM consensus " - + "WHERE validafter >= '" + fromValidAfter - + "' AND validafter <= '" + toValidAfter + "'"; - ResultSet rs = statement.executeQuery(query); - while (rs.next()) { - long consensusTime = rs.getTimestamp(1).getTime(); - if (consensusTime < timestampFrom) { - tooOldConsensuses.add(consensusTime); - } else if (consensusTime > timestampTo) { - tooNewConsensuses.add(consensusTime); - } else { - relevantConsensuses.add(consensusTime); - } - } - rs.close(); - statement.close(); - } catch (SQLException e) { - /* Looks like we don't have any consensuses in the requested - * interval. */ - } - SortedSet<Long> allConsensuses = new TreeSet<Long>(); - allConsensuses.addAll(tooOldConsensuses); - allConsensuses.addAll(relevantConsensuses); - allConsensuses.addAll(tooNewConsensuses); - if (allConsensuses.isEmpty()) { - out.println(" <p>No relay lists found!</p>\n" - + " <p>Result is INDECISIVE!</p>\n" - + " <p>We cannot make any statement whether there was " - + "a Tor relay running on IP address " + relayIP - + (timestampIsDate ? " on " : " at ") + timestampStr + "! We " - + "did not find any relevant relay lists at the given time. If " - + "you think this is an error on our side, please " - + "<a href=\"mailto:tor-assistants@torproject.org\">contact " - + "us</a>!</p>\n"); - writeFooter(out); - try { - conn.close(); - this.logger.info("Returned a database connection to the pool " - + "after " + (System.currentTimeMillis() - - requestedConnection) + " millis."); - } catch (SQLException e) { - } - return; - } - - /* Search for status entries with the given IP address as onion - * routing address, plus status entries of relays having an exit list - * entry with the given IP address as exit address. */ - SortedMap<Long, SortedMap<String, String>> statusEntries = - new TreeMap<Long, SortedMap<String, String>>(); - SortedSet<Long> positiveConsensusesNoTarget = new TreeSet<Long>(); - SortedMap<String, Set<Long>> relevantDescriptors = - new TreeMap<String, Set<Long>>(); - try { - CallableStatement cs = conn.prepareCall( - "{call search_statusentries_by_address_date(?, ?)}"); - cs.setString(1, relayIP); - cs.setDate(2, new java.sql.Date(timestamp)); - ResultSet rs = cs.executeQuery(); - while (rs.next()) { - byte[] rawstatusentry = rs.getBytes(1); - String descriptor = rs.getString(2); - long validafter = rs.getTimestamp(3).getTime(); - positiveConsensusesNoTarget.add(validafter); - if (!relevantDescriptors.containsKey(descriptor)) { - relevantDescriptors.put(descriptor, new HashSet<Long>()); - } - relevantDescriptors.get(descriptor).add(validafter); - String fingerprint = rs.getString(4); - String exitaddress = rs.getString(6); - StringBuilder html = new StringBuilder(); - for (String line : new String(rawstatusentry).split("\n")) { - if (line.startsWith("r ")) { - String[] parts = line.split(" "); - boolean orAddressMatches = parts[6].equals(relayIP); - html.append("r " + parts[1] + " " + parts[2] + " " - + "<a href=\"serverdesc?desc-id=" + descriptor + "\" " - + "target=\"_blank\">" + parts[3] + "</a> " + parts[4] - + " " + parts[5] + " " + (orAddressMatches ? "<b>" : "") - + parts[6] + (orAddressMatches ? "</b>" : "") + " " - + parts[7] + " " + parts[8] + "\n"); - } else if (line.startsWith("a ") && - line.toLowerCase().contains(relayIP)) { - String address = line.substring("a ".length(), - line.lastIndexOf(":")); - String port = line.substring(line.lastIndexOf(":")); - html.append("a <b>" + address + "</b>" + port + "\n"); - } - } - if (exitaddress != null && exitaddress.length() > 0) { - long scanned = rs.getTimestamp(7).getTime(); - html.append(" [ExitAddress <b>" + exitaddress - + "</b> " + validAfterTimeFormat.format(scanned) + "]\n"); - } - if (!statusEntries.containsKey(validafter)) { - statusEntries.put(validafter, new TreeMap<String, String>()); - } - statusEntries.get(validafter).put(fingerprint, html.toString()); - } - rs.close(); - cs.close(); - } catch (SQLException e) { - /* Nothing found. */ - } - - /* Print out what we found. */ - SimpleDateFormat validAfterUrlFormat = new SimpleDateFormat( - "yyyy-MM-dd-HH-mm-ss"); - validAfterUrlFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - out.print("<pre><code>"); - for (long consensus : allConsensuses) { - if (relevantConsensuses.contains(consensus)) { - String validAfterDatetime = validAfterTimeFormat.format( - consensus); - String validAfterString = validAfterUrlFormat.format(consensus); - out.print("valid-after <b>" - + "<a href=\"consensus?valid-after=" - + validAfterString + "\" target=\"_blank\">" - + validAfterDatetime + "</b></a>\n"); - if (statusEntries.containsKey(consensus)) { - for (String htmlString : - statusEntries.get(consensus).values()) { - out.print(htmlString); - } - } - out.print("\n"); - } - } - out.print("</code></pre>"); - if (relevantDescriptors.isEmpty()) { - out.printf(" <p>None found!</p>\n" - + " <p>Result is NEGATIVE with high certainty!</p>\n" - + " <p>We did not find IP " - + "address " + relayIP + " in any of the relay or exit lists " - + "that were published between %s and %s.</p>\n", - dateFormat.format(timestampTooOld), - dateFormat.format(timestampTooNew)); - /* Run another query to find out if there are relays running on - * other IP addresses in the same /24 or /48 network and tell the - * user about it. */ - List<String> addressesInSameNetwork = new ArrayList<String>(); - if (!relayIP.contains(":")) { - String[] relayIPParts = relayIP.split("\\."); - byte[] address24Bytes = new byte[3]; - address24Bytes[0] = (byte) Integer.parseInt(relayIPParts[0]); - address24Bytes[1] = (byte) Integer.parseInt(relayIPParts[1]); - address24Bytes[2] = (byte) Integer.parseInt(relayIPParts[2]); - String address24 = Hex.encodeHexString(address24Bytes); - try { - CallableStatement cs = conn.prepareCall( - "{call search_addresses_in_same_24 (?, ?)}"); - cs.setString(1, address24); - cs.setDate(2, new java.sql.Date(timestamp)); - ResultSet rs = cs.executeQuery(); - while (rs.next()) { - String address = rs.getString(1); - if (!addressesInSameNetwork.contains(address)) { - addressesInSameNetwork.add(address); - } - } - rs.close(); - cs.close(); - } catch (SQLException e) { - /* No other addresses in the same /24 found. */ - } - } else { - StringBuilder addressHex = new StringBuilder(); - int start = relayIP.startsWith("::") ? 1 : 0; - int end = relayIP.length() - (relayIP.endsWith("::") ? 1 : 0); - String[] parts = relayIP.substring(start, end).split(":", -1); - for (int i = 0; i < parts.length; i++) { - String part = parts[i]; - if (part.length() == 0) { - addressHex.append("x"); - } else if (part.length() <= 4) { - addressHex.append(String.format("%4s", part)); - } else { - addressHex = null; - break; - } - } - String address48 = null; - if (addressHex != null) { - String addressHexString = addressHex.toString(); - addressHexString = addressHexString.replaceFirst("x", - String.format("%" + (33 - addressHexString.length()) - + "s", "0")); - if (!addressHexString.contains("x") && - addressHexString.length() == 32) { - address48 = addressHexString.replaceAll(" ", "0"). - toLowerCase().substring(0, 12); - } - } - if (address48 != null) { - try { - CallableStatement cs = conn.prepareCall( - "{call search_addresses_in_same_48 (?, ?)}"); - cs.setString(1, address48); - cs.setDate(2, new java.sql.Date(timestamp)); - ResultSet rs = cs.executeQuery(); - while (rs.next()) { - String address = rs.getString(1); - if (!addressesInSameNetwork.contains(address)) { - addressesInSameNetwork.add(address); - } - } - rs.close(); - cs.close(); - } catch (SQLException e) { - /* No other addresses in the same /48 found. */ - } - } - } - if (!addressesInSameNetwork.isEmpty()) { - if (!relayIP.contains(":")) { - out.print(" <p>The following other IP addresses of Tor " - + "relays in the same /24 network were found in relay " - + "and/or exit lists around the time that could be related " - + "to IP address " + relayIP + ":</p>\n"); - } else { - out.print(" <p>The following other IP addresses of Tor " - + "relays in the same /48 network were found in relay " - + "lists around the time that could be related to IP " - + "address " + relayIP + ":</p>\n"); - } - out.print(" <ul>\n"); - for (String s : addressesInSameNetwork) { - out.print(" <li>" + s + "</li>\n"); - } - out.print(" </ul>\n"); - } - writeFooter(out); - try { - conn.close(); - this.logger.info("Returned a database connection to the pool " - + "after " + (System.currentTimeMillis() - - requestedConnection) + " millis."); - } catch (SQLException e) { - } - return; - } - - /* Print out result. */ - boolean inMostRelevantConsensuses = false, - inOtherRelevantConsensus = false, - inTooOldConsensuses = false, - inTooNewConsensuses = false; - for (long match : positiveConsensusesNoTarget) { - if (timestampIsDate && - dateFormat.format(match).equals(timestampStr)) { - inMostRelevantConsensuses = true; - } else if (!timestampIsDate && - match == relevantConsensuses.last()) { - inMostRelevantConsensuses = true; - } else if (relevantConsensuses.contains(match)) { - inOtherRelevantConsensus = true; - } else if (tooOldConsensuses.contains(match)) { - inTooOldConsensuses = true; - } else if (tooNewConsensuses.contains(match)) { - inTooNewConsensuses = true; - } - } - if (inMostRelevantConsensuses) { - out.print(" <p>Result is POSITIVE with high certainty!" - + "</p>\n" - + " <p>We found one or more relays on IP address " - + relayIP + " in "); - if (timestampIsDate) { - out.print("relay list published on " + timestampStr); - } else { - out.print("the most recent relay list preceding " + timestampStr); - } - out.print(" that clients were likely to know.</p>\n"); - } else { - if (inOtherRelevantConsensus) { - out.println(" <p>Result is POSITIVE " - + "with moderate certainty!</p>\n"); - out.println("<p>We found one or more relays on IP address " - + relayIP + ", but not in "); - if (timestampIsDate) { - out.print("a relay list published on " + timestampStr); - } else { - out.print("the most recent relay list preceding " - + timestampStr); - } - out.print(". A possible reason for the relay being missing in a " - + "relay list might be that some of the directory " - + "authorities had difficulties connecting to the relay. " - + "However, clients might still have used the relay.</p>\n"); - } else { - out.println(" <p>Result is NEGATIVE " - + "with high certainty!</p>\n"); - out.println(" <p>We did not find any relay on IP address " - + relayIP - + " in the relay lists 3 hours preceding " + timestampStr - + ".</p>\n"); - if (inTooOldConsensuses || inTooNewConsensuses) { - if (inTooOldConsensuses && !inTooNewConsensuses) { - out.println(" <p>Note that we found a matching relay " - + "in relay lists that were published between 15 and 3 " - + "hours before " + timestampStr + ".</p>\n"); - } else if (!inTooOldConsensuses && inTooNewConsensuses) { - out.println(" <p>Note that we found a matching relay " - + "in relay lists that were published up to 12 hours " - + "after " + timestampStr + ".</p>\n"); - } else { - out.println(" <p>Note that we found a matching relay " - + "in relay lists that were published between 15 and 3 " - + "hours before and in relay lists that were published " - + "up to 12 hours after " + timestampStr + ".</p>\n"); - } - if (timestampIsDate) { - out.println("<p>Be sure to try out the previous/next day or " - + "provide an exact timestamp in UTC.</p>"); - } else { - out.println("<p>Make sure that the timestamp you " - + "provided is correctly converted to the UTC " - + "timezone.</p>"); - } - } - /* We didn't find any descriptor. No need to look up targets. */ - writeFooter(out); - try { - conn.close(); - this.logger.info("Returned a database connection to the pool " - + "after " + (System.currentTimeMillis() - - requestedConnection) + " millis."); - } catch (SQLException e) { - } - return; - } - } - - /* Looking up targets for IPv6 is not supported yet. */ - if (relayIP.contains(":")) { - writeFooter(out); - return; - } - - /* Second part: target */ - out.println("<br><a name=\"exit\"></a><h3>Was this relay configured " - + "to permit exiting to a given target?</h3>"); - - out.println(" <form action=\"#exit\">\n" - + " <input type=\"hidden\" name=\"timestamp\"\n" - + " value=\"" + timestampStr + "\">\n" - + " <input type=\"hidden\" name=\"ip\" " - + "value=\"" + relayIP + "\">\n" - + " <table>\n" - + " <tr>\n" - + " <td align=\"right\">Target address:</td>\n" - + " <td><input type=\"text\" name=\"targetaddr\"" - + (targetIP.length() > 0 ? " value=\"" + targetIP + "\"" : "") - + "\">" - + (targetAddrWarning.length() > 0 ? "<br><font color=\"red\">" - + targetAddrWarning + "</font>" : "") - + "</td>\n" - + " <td><i>(Ex.: 4.3.2.1)</i></td>\n" - + " </tr>\n" - + " <tr>\n" - + " <td align=\"right\">Target port:</td>\n" - + " <td><input type=\"text\" name=\"targetport\"" - + (targetPort.length() > 0 ? " value=\"" + targetPort + "\"" - : "") - + ">" - + (targetPortWarning.length() > 0 ? "<br><font color=\"red\">" - + targetPortWarning + "</font>" : "") - + "</td>\n" - + " <td><i>(Ex.: 80)</i></td>\n" - + " </tr>\n" - + " <tr>\n" - + " <td></td>\n" - + " <td>\n" - + " <input type=\"submit\">\n" - + " <input type=\"reset\">\n" - + " </td>\n" - + " <td></td>\n" - + " </tr>\n" - + " </table>\n" - + " </form>\n"); - - if (targetIP.length() < 1) { - writeFooter(out); - try { - conn.close(); - this.logger.info("Returned a database connection to the pool " - + "after " + (System.currentTimeMillis() - - requestedConnection) + " millis."); - } catch (SQLException e) { - } - return; - } - - /* Parse router descriptors to check exit policies. */ - out.println("<p>Searching the relay descriptors published by the " - + "relay on IP address " + relayIP + " to find out whether this " - + "relay permitted exiting to " + target + ". You may follow the " - + "links above to the relay descriptors and grep them for the " - + "lines printed below to confirm that results are correct.</p>"); - SortedSet<Long> positiveConsensuses = new TreeSet<Long>(); - Set<String> missingDescriptors = new HashSet<String>(); - Set<String> descriptors = relevantDescriptors.keySet(); - for (String descriptor : descriptors) { - byte[] rawDescriptor = null; - try { - String query = "SELECT rawdescriptor FROM descriptor " - + "WHERE descriptor = '" + descriptor + "'"; - Statement statement = conn.createStatement(); - ResultSet rs = statement.executeQuery(query); - if (rs.next()) { - rawDescriptor = rs.getBytes(1); - } - rs.close(); - statement.close(); - } catch (SQLException e) { - /* Consider this descriptors as 'missing'. */ - continue; - } - if (rawDescriptor != null && rawDescriptor.length > 0) { - missingDescriptors.remove(descriptor); - String rawDescriptorString = new String(rawDescriptor, - "US-ASCII"); - try { - BufferedReader br = new BufferedReader( - new StringReader(rawDescriptorString)); - String line = null, routerLine = null, publishedLine = null; - StringBuilder acceptRejectLines = new StringBuilder(); - boolean foundMatch = false; - while ((line = br.readLine()) != null) { - if (line.startsWith("router ")) { - routerLine = line; - } else if (line.startsWith("published ")) { - publishedLine = line; - } else if (line.startsWith("reject ") || - line.startsWith("accept ")) { - if (foundMatch) { - out.println(line); - continue; - } - boolean ruleAccept = line.split(" ")[0].equals("accept"); - String ruleAddress = line.split(" ")[1].split(":")[0]; - if (!ruleAddress.equals("*")) { - if (!ruleAddress.contains("/") && - !ruleAddress.equals(targetIP)) { - /* IP address does not match. */ - acceptRejectLines.append(line + "\n"); - continue; - } - String[] ruleIPParts = ruleAddress.split("/")[0]. - split("\\."); - int ruleNetwork = ruleAddress.contains("/") ? - Integer.parseInt(ruleAddress.split("/")[1]) : 32; - for (int i = 0; i < 4; i++) { - if (ruleNetwork == 0) { - break; - } else if (ruleNetwork >= 8) { - if (ruleIPParts[i].equals(targetIPParts[i])) { - ruleNetwork -= 8; - } else { - break; - } - } else { - int mask = 255 ^ 255 >>> ruleNetwork; - if ((Integer.parseInt(ruleIPParts[i]) & mask) == - (Integer.parseInt(targetIPParts[i]) & mask)) { - ruleNetwork = 0; - } - break; - } - } - if (ruleNetwork > 0) { - /* IP address does not match. */ - acceptRejectLines.append(line + "\n"); - continue; - } - } - String rulePort = line.split(" ")[1].split(":")[1]; - if (targetPort.length() < 1 && !ruleAccept && - !rulePort.equals("*")) { - /* With no port given, we only consider reject :* rules as - matching. */ - acceptRejectLines.append(line + "\n"); - continue; - } - if (targetPort.length() > 0 && !rulePort.equals("*") && - rulePort.contains("-")) { - int fromPort = Integer.parseInt(rulePort.split("-")[0]); - int toPort = Integer.parseInt(rulePort.split("-")[1]); - int targetPortInt = Integer.parseInt(targetPort); - if (targetPortInt < fromPort || - targetPortInt > toPort) { - /* Port not contained in interval. */ - continue; - } - } - if (targetPort.length() > 0) { - if (!rulePort.equals("*") && - !rulePort.contains("-") && - !targetPort.equals(rulePort)) { - /* Ports do not match. */ - acceptRejectLines.append(line + "\n"); - continue; - } - } - boolean relevantMatch = false; - for (long match : relevantDescriptors.get(descriptor)) { - if (relevantConsensuses.contains(match)) { - relevantMatch = true; - } - } - if (relevantMatch) { - String[] routerParts = routerLine.split(" "); - out.println("<pre><code>" + routerParts[0] + " " - + routerParts[1] + " <b>" + routerParts[2] + "</b> " - + routerParts[3] + " " + routerParts[4] + " " - + routerParts[5]); - String[] publishedParts = publishedLine.split(" "); - out.println(publishedParts[0] + " <b>" - + publishedParts[1] + " " + publishedParts[2] - + "</b>"); - out.print(acceptRejectLines.toString()); - out.println("<b>" + line + "</b>"); - foundMatch = true; - } - if (ruleAccept) { - positiveConsensuses.addAll( - relevantDescriptors.get(descriptor)); - } - } - } - br.close(); - if (foundMatch) { - out.println("</code></pre>"); - } - } catch (IOException e) { - /* Could not read descriptor string. */ - continue; - } - } - } + HttpServletResponse response) throws IOException, ServletException { - /* Print out result. */ - inMostRelevantConsensuses = false; - inOtherRelevantConsensus = false; - inTooOldConsensuses = false; - inTooNewConsensuses = false; - for (long match : positiveConsensuses) { - if (timestampIsDate && - dateFormat.format(match).equals(timestampStr)) { - inMostRelevantConsensuses = true; - } else if (!timestampIsDate && match == relevantConsensuses.last()) { - inMostRelevantConsensuses = true; - } else if (relevantConsensuses.contains(match)) { - inOtherRelevantConsensus = true; - } else if (tooOldConsensuses.contains(match)) { - inTooOldConsensuses = true; - } else if (tooNewConsensuses.contains(match)) { - inTooNewConsensuses = true; - } - } - if (inMostRelevantConsensuses) { - out.print(" <p>Result is POSITIVE with high certainty!" - + "</p>\n" - + " <p>We found one or more relays on IP address " - + relayIP + " permitting exit to " + target + " in "); - if (timestampIsDate) { - out.print("relay list published on " + timestampStr); - } else { - out.print("the most recent relay list preceding " + timestampStr); - } - out.print(" that clients were likely to know.</p>\n"); - writeFooter(out); - try { - conn.close(); - this.logger.info("Returned a database connection to the pool " - + "after " + (System.currentTimeMillis() - - requestedConnection) + " millis."); - } catch (SQLException e) { - } - return; - } - boolean resultIndecisive = target.length() > 0 - && !missingDescriptors.isEmpty(); - if (resultIndecisive) { - out.println(" <p>Result is INDECISIVE!</p>\n" - + " <p>At least one referenced descriptor could not be " - + "found. This is a rare case, but one that (apparently) " - + "happens. We cannot make any good statement about exit " - + "relays without these descriptors. The following descriptors " - + "are missing:</p>"); - for (String desc : missingDescriptors) - out.println(" <p>" + desc + "</p>\n"); - } - if (inOtherRelevantConsensus) { - if (!resultIndecisive) { - out.println(" <p>Result is POSITIVE " - + "with moderate certainty!</p>\n"); - } - out.println("<p>We found one or more relays on IP address " - + relayIP + " permitting exit to " + target + ", but not in "); - if (timestampIsDate) { - out.print("a relay list published on " + timestampStr); - } else { - out.print("the most recent relay list preceding " + timestampStr); - } - out.print(". A possible reason for the relay being missing in a " - + "relay list might be that some of the directory authorities " - + "had difficulties connecting to the relay. However, clients " - + "might still have used the relay.</p>\n"); - } else { - if (!resultIndecisive) { - out.println(" <p>Result is NEGATIVE " - + "with high certainty!</p>\n"); - } - out.println(" <p>We did not find any relay on IP address " - + relayIP + " permitting exit to " + target - + " in the relay list 3 hours preceding " + timestampStr - + ".</p>\n"); - if (inTooOldConsensuses || inTooNewConsensuses) { - if (inTooOldConsensuses && !inTooNewConsensuses) { - out.println(" <p>Note that we found a matching relay in " - + "relay lists that were published between 15 and 3 " - + "hours before " + timestampStr + ".</p>\n"); - } else if (!inTooOldConsensuses && inTooNewConsensuses) { - out.println(" <p>Note that we found a matching relay in " - + "relay lists that were published up to 12 hours after " - + timestampStr + ".</p>\n"); - } else { - out.println(" <p>Note that we found a matching relay in " - + "relay lists that were published between 15 and 3 " - + "hours before and in relay lists that were published up " - + "to 12 hours after " + timestampStr + ".</p>\n"); - } - if (timestampIsDate) { - out.println("<p>Be sure to try out the previous/next day or " - + "provide an exact timestamp in UTC.</p>"); - } else { - out.println("<p>Make sure that the timestamp you provided is " - + "correctly converted to the UTC timezone.</p>"); - } - } - } - if (target != null) { - if (positiveConsensuses.isEmpty() && - !positiveConsensusesNoTarget.isEmpty()) { - out.println(" <p>Note that although the found relay(s) did " - + "not permit exiting to " + target + ", there have been one " - + "or more relays running at the given time.</p>"); - } - } - try { - conn.close(); - this.logger.info("Returned a database connection to the pool " - + "after " + (System.currentTimeMillis() - - requestedConnection) + " millis."); - } catch (SQLException e) { - } - writeFooter(out); + /* Forward the request to the JSP that does all the hard work. */ + request.getRequestDispatcher("WEB-INF/exonerator.jsp").forward( + request, response); } } diff --git a/src/org/torproject/ernie/status/exonerator/ServerDescriptorServlet.java b/src/org/torproject/ernie/status/exonerator/ServerDescriptorServlet.java deleted file mode 100644 index f94611e..0000000 --- a/src/org/torproject/ernie/status/exonerator/ServerDescriptorServlet.java +++ /dev/null @@ -1,132 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.ernie.status.exonerator; - -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import javax.naming.Context; -import javax.naming.InitialContext; -import javax.naming.NamingException; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.sql.DataSource; - -public class ServerDescriptorServlet extends HttpServlet { - - private static final long serialVersionUID = -7935883442750583462L; - - private DataSource ds; - - private Logger logger; - - public void init() { - - /* Initialize logger. */ - this.logger = Logger.getLogger( - ServerDescriptorServlet.class.toString()); - - /* Look up data source. */ - try { - Context cxt = new InitialContext(); - this.ds = (DataSource) cxt.lookup("java:comp/env/jdbc/exonerator"); - this.logger.info("Successfully looked up data source."); - } catch (NamingException e) { - this.logger.log(Level.WARNING, "Could not look up data source", e); - } - } - - public void doGet(HttpServletRequest request, - HttpServletResponse response) throws IOException, - ServletException { - - /* Read desc-id parameter. */ - String descIdParameter = request.getParameter("desc-id"); - - /* See if we were given a desc-id parameter. If so, look up this - * descriptor and return it. */ - List<byte[]> rawDescriptors = new ArrayList<byte[]>(); - String filename = null; - if (descIdParameter != null) { - if (descIdParameter.length() < 8 || - descIdParameter.length() > 40) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - String descId = descIdParameter.toLowerCase(); - Pattern descIdPattern = Pattern.compile("^[0-9a-f]+$"); - Matcher descIdMatcher = descIdPattern.matcher(descId); - if (!descIdMatcher.matches()) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - - /* Look up descriptor in the database. */ - try { - long requestedConnection = System.currentTimeMillis(); - Connection conn = ds.getConnection(); - Statement statement = conn.createStatement(); - String query = "SELECT descriptor, rawdescriptor FROM descriptor " - + "WHERE descriptor LIKE '" + descId + "%'"; - ResultSet rs = statement.executeQuery(query); - if (rs.next()) { - filename = rs.getString(1); - rawDescriptors.add(rs.getBytes(2)); - } - rs.close(); - statement.close(); - conn.close(); - this.logger.info("Returned a database connection to the pool " - + "after " + (System.currentTimeMillis() - - requestedConnection) + " millis."); - } catch (SQLException e) { - response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - return; - } - - /* Return an error if no desc-id parameter was given. */ - } else { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - - /* Write response. */ - if (rawDescriptors.size() == 0) { - response.sendError(HttpServletResponse.SC_NOT_FOUND); - return; - } - try { - response.setContentType("text/plain"); - int responseLength = 0; - for (byte[] rawDescriptor : rawDescriptors) { - responseLength += rawDescriptor.length; - } - response.setHeader("Content-Length", String.valueOf( - responseLength)); - response.setHeader("Content-Disposition", "inline; filename=\"" - + filename + "\""); - BufferedOutputStream output = new BufferedOutputStream( - response.getOutputStream()); - for (byte[] rawDescriptor : rawDescriptors) { - output.write(rawDescriptor); - } - output.flush(); - output.close(); - } finally { - /* Nothing to do here. */ - } - } -} - diff --git a/src/org/torproject/ernie/status/relaysearch/RelaySearchServlet.java b/src/org/torproject/ernie/status/relaysearch/RelaySearchServlet.java index cd5c4c1..b7c8291 100644 --- a/src/org/torproject/ernie/status/relaysearch/RelaySearchServlet.java +++ b/src/org/torproject/ernie/status/relaysearch/RelaySearchServlet.java @@ -467,7 +467,8 @@ public class RelaySearchServlet extends HttpServlet { + fingerprint); if (!rawValidAfterLines.containsKey(validAfter)) { rawValidAfterLines.put(validAfter, "<tt>valid-after " - + "<a href=\"consensus?valid-after=" + + "<a href=\"https://exonerator.torproject.org/" + + "consensus?valid-after=" + validAfter.replaceAll(":", "-").replaceAll(" ", "-") + "\" target=\"_blank\">" + validAfter + "</a></tt><br>"); } @@ -488,7 +489,9 @@ public class RelaySearchServlet extends HttpServlet { new BigInteger(1, Base64.decodeBase64(parts[3] + "=="))); rawStatusEntryBuilder.append("<tt>r " + parts[1] + " " - + parts[2] + " <a href=\"serverdesc?desc-id=" + + parts[2] + " <a href=\"" + + "https://exonerator.torproject.org/" + + "serverdesc?desc-id=" + descriptorBase64 + "\" target=\"_blank\">" + parts[3] + "</a> " + parts[4] + " " + parts[5] + " " + parts[6] + " " + parts[7] + " " + parts[8] + "</tt><br>"); diff --git a/web/WEB-INF/exonerator.jsp b/web/WEB-INF/exonerator.jsp new file mode 100644 index 0000000..0eefe99 --- /dev/null +++ b/web/WEB-INF/exonerator.jsp @@ -0,0 +1,45 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> +<html> +<head> + <title>Tor Metrics Portal: ExoneraTor</title> + <meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"> + <link href="/css/stylesheet-ltr.css" type="text/css" rel="stylesheet"> + <link href="/images/favicon.ico" type="image/x-icon" rel="shortcut icon"> +</head> +<body> + <div class="center"> + <%@ include file="banner.jsp"%> + <div class="main-column" style="margin:5; Padding:0;"> + <h2>ExoneraTor</h2> + <h3>or: a website that tells you whether a given IP address was a Tor + relay</h3> + <br> + <p>Just because you see an Internet connection from a particular IP + address does not mean you know <i>who</i> originated the traffic. Tor + anonymizes Internet traffic by "<a href="https://www.torproject.org/about/overview#thesolution">onion + routing</a>," sending packets through a series of encrypted hops + before they reach their destination. Therefore, if you see traffic + from a Tor node, you may be seeing traffic that originated from + someone using Tor, rather than from the node operator itself. The Tor + Project and Tor node operators have no records of the traffic that + passes over the network, but we do maintain current and historical + records of which IP addresses are part of the Tor network.</p> + <br> + <p>ExoneraTor tells you whether there was a Tor relay running on a + given IP address at a given time. ExoneraTor can further indicate + whether this relay permitted exiting to a given server and/or TCP + port. ExoneraTor learns these facts by parsing the public relay lists + and relay descriptors that are collected from the Tor directory + authorities and the exit lists collected by TorDNSEL. By inputting an + IP address and time, you can determine whether that IP was then a part + of the Tor network.</p> + <br> + <p><font color="red"><b>Notice:</b> This service has moved to: + <a href="https://exonerator.torproject.org/">https://exonerator.torproject.org/</a></font></p> + </div> + </div> + <div class="bottom" id="bottom"> + <%@ include file="footer.jsp"%> + </div> +</body> +</html> diff --git a/web/robots.txt b/web/robots.txt index f3ffac3..c59aca1 100644 --- a/web/robots.txt +++ b/web/robots.txt @@ -1,7 +1,5 @@ User-agent: * Disallow: /relay.html Disallow: /csv/ -Disallow: /serverdesc -Disallow: /consensus Disallow: /consensus-health.html