commit 34e793e7c318d5bd3836c5ae2c22abc4becd8ee7 Author: Karsten Loesing karsten.loesing@gmx.net Date: Wed Aug 16 10:31:24 2017 +0200

Provide and use query results via query.json.

This prepares splitting up ExoneraTor into front-end and back-end.

Note that some code duplication between ExoneraTorServlet and QueryServlet was deemed acceptable, because ExoneraTorServlet will be moved to metrics-web in the medium term anyway. --- build.xml | 1 + .../torproject/exonerator/ExoneraTorServlet.java | 383 ++-------------- .../org/torproject/exonerator/QueryResponse.java | 3 + .../org/torproject/exonerator/QueryServlet.java | 502 +++++++++++++++++++++ src/main/webapp/web.xml | 11 + 5 files changed, 554 insertions(+), 346 deletions(-)

diff --git a/build.xml b/build.xml index 359ae20..eb39e7e 100644 --- a/build.xml +++ b/build.xml @@ -108,6 +108,7 @@ <lib dir="${libs}"> <include name="commons-codec-1.10.jar"/> <include name="commons-lang-2.6.jar"/> + <include name="gson-2.4.jar" /> <include name="postgresql-9.4.1212.jar"/> </lib> <classes dir="${classes}"/> diff --git a/src/main/java/org/torproject/exonerator/ExoneraTorServlet.java b/src/main/java/org/torproject/exonerator/ExoneraTorServlet.java index b7d0497..1fb7073 100644 --- a/src/main/java/org/torproject/exonerator/ExoneraTorServlet.java +++ b/src/main/java/org/torproject/exonerator/ExoneraTorServlet.java @@ -3,71 +3,60 @@

package org.torproject.exonerator;

-import org.apache.commons.codec.binary.Hex; +import com.google.gson.Gson; import org.apache.commons.lang.StringEscapeUtils;

import java.io.IOException; +import java.io.InputStreamReader; import java.io.PrintWriter; -import java.sql.CallableStatement; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; +import java.net.MalformedURLException; +import java.net.URL; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; -import java.util.Calendar; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.ResourceBundle; import java.util.SortedMap; -import java.util.SortedSet; import java.util.TimeZone; import java.util.TreeMap; -import java.util.TreeSet; -import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern;

-import javax.naming.Context; -import javax.naming.InitialContext; -import javax.naming.NamingException; +import javax.servlet.ServletConfig; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import javax.sql.DataSource;

public class ExoneraTorServlet extends HttpServlet {

private static final long serialVersionUID = 1370088989739567509L;

- private DataSource ds; - private Logger logger;

+ private String exoneraTorHost = "https://exonerator.torproject.org"; + + /* Don't accept query responses with versions lower than this. */ + private static final String firstRecognizedVersion = "1.0"; + + /* Don't accept query responses with this version or higher. */ + private static final String firstUnrecognizedVersion = "2.0"; + private List<String> availableLanguages = Arrays.asList("de", "en", "fr", "ro", "sv");

private SortedMap<String, String> availableLanguageNames;

@Override - public void init() { + public void init(ServletConfig config) throws ServletException { + super.init(config);

/* Initialize logger. */ this.logger = Logger.getLogger(ExoneraTorServlet.class.toString());

- /* Look up data source. */ - try { - Context cxt = new InitialContext(); - this.ds = (DataSource) cxt.lookup("java:comp/env/jdbc/exonerator"); - this.logger.info("Successfully looked up data source."); - } catch (NamingException e) { - this.logger.log(Level.WARNING, "Could not look up data source", e); - } - this.availableLanguageNames = new TreeMap<>(); for (String locale : this.availableLanguages) { ResourceBundle rb = ResourceBundle.getBundle("ExoneraTor", @@ -298,50 +287,6 @@ public class ExoneraTorServlet extends HttpServlet { return relayIp; }

- private String convertIpV4ToHex(String relayIp) { - String[] relayIpParts = relayIp.split("\."); - byte[] address24Bytes = new byte[4]; - for (int i = 0; i < address24Bytes.length; i++) { - address24Bytes[i] = (byte) Integer.parseInt(relayIpParts[i]); - } - String address24 = Hex.encodeHexString(address24Bytes); - return address24; - } - - private String convertIpV6ToHex(String relayIp) { - if (relayIp.startsWith("[") && relayIp.endsWith("]")) { - relayIp = relayIp.substring(1, relayIp.length() - 1); - } - StringBuilder addressHex = new StringBuilder(); - int start = relayIp.startsWith("::") ? 1 : 0; - int end = relayIp.length() - (relayIp.endsWith("::") ? 1 : 0); - String[] parts = relayIp.substring(start, end).split(":", -1); - for (int i = 0; i < parts.length; i++) { - String part = parts[i]; - if (part.length() == 0) { - addressHex.append("x"); - } else if (part.length() <= 4) { - addressHex.append(String.format("%4s", part)); - } else { - addressHex = null; - break; - } - } - String address48 = null; - if (addressHex != null) { - String addressHexString = addressHex.toString(); - addressHexString = addressHexString.replaceFirst("x", - String.format("%" + (33 - addressHexString.length()) - + "s", "0")); - if (!addressHexString.contains("x") - && addressHexString.length() == 32) { - address48 = addressHexString.replaceAll(" ", "0") - .toLowerCase(); - } - } - return address48; - } - private String parseTimestampParameter( String passedTimestampParameter) { String timestampStr = ""; @@ -361,289 +306,35 @@ public class ExoneraTorServlet extends HttpServlet { return timestampStr; }

- /* Helper methods for querying the database. */ + /* Helper method for fetching a query response via URL. */

private QueryResponse queryDatabase(String relayIp, String timestampStr) { - - QueryResponse response = null; - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - SimpleDateFormat validAfterTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - validAfterTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - long timestamp = 0L; - if (timestampStr != null && timestampStr.length() > 0) { - try { - timestamp = dateFormat.parse(timestampStr).getTime(); - } catch (ParseException e) { - /* Already checked in parseTimestamp(). */ - } - } - - /* Only query the database if we received valid user input. */ - if (!"".equals(relayIp) && !"".equals(timestampStr)) { - - /* Open a database connection that we'll use to handle the whole - * request. */ - long requestedConnection = System.currentTimeMillis(); - Connection conn = this.connectToDatabase(); - if (null != conn) { - - response = new QueryResponse(); - response.queryAddress = relayIp; - response.queryDate = timestampStr; - - /* Look up first and last date in the database. */ - long[] firstAndLastDates = this.queryFirstAndLastDatesFromDatabase( - conn); - if (null != firstAndLastDates) { - response.firstDateInDatabase = dateFormat.format( - firstAndLastDates[0]); - response.lastDateInDatabase = dateFormat.format(firstAndLastDates[1]); - - /* Consider all consensuses published on or within a day of the given - * date. */ - long timestampFrom = timestamp - 24L * 60L * 60L * 1000L; - long timestampTo = timestamp + 2 * 24L * 60L * 60L * 1000L - 1L; - String fromValidAfter = validAfterTimeFormat.format(timestampFrom); - String toValidAfter = validAfterTimeFormat.format(timestampTo); - SortedSet<Long> relevantConsensuses = - this.queryKnownConsensusValidAfterTimes(conn, fromValidAfter, - toValidAfter); - if (null != relevantConsensuses && !relevantConsensuses.isEmpty()) { - response.relevantStatuses = true; - - /* Search for status entries with the given IP address as onion - * routing address, plus status entries of relays having an exit - * list entry with the given IP address as exit address. */ - List<QueryResponse.Match> matches = this.queryStatusEntries(conn, - relayIp, timestamp, validAfterTimeFormat); - if (!matches.isEmpty()) { - response.matches = matches.toArray(new QueryResponse.Match[0]); - - /* If we didn't find anything, run another query to find out if - * there are relays running on other IP addresses in the same /24 or - * /48 network and tell the user about it. */ - } else { - if (!relayIp.contains(":")) { - String address24 = this.convertIpV4ToHex(relayIp) - .substring(0, 6); - if (address24 != null) { - response.nearbyAddresses = this.queryAddressesInSame24(conn, - address24, timestamp).toArray(new String[0]); - } - } else { - String address48 = this.convertIpV6ToHex(relayIp) - .substring(0, 12); - if (address48 != null) { - response.nearbyAddresses = this.queryAddressesInSame48(conn, - address48, timestamp).toArray(new String[0]); - } - } - } - } - } - - /* Close the database connection. */ - this.closeDatabaseConnection(conn, requestedConnection); + QueryResponse response; + try (InputStreamReader isr = new InputStreamReader(new URL( + this.exoneraTorHost + "/query.json?ip=" + relayIp + "&timestamp=" + + timestampStr).openStream())) { + Gson gson = new Gson(); + response = gson.fromJson(isr, QueryResponse.class); + if (null == response || null == response.version) { + logger.warning("Response is either empty or does not contain " + + "version information."); + response = null; + } else if (response.version.compareTo(firstRecognizedVersion) < 0 + || response.version.compareTo(firstUnrecognizedVersion) >= 0) { + logger.warning("Response has either an older or a newer version (" + + response.version + ") than we can handle (" + + firstRecognizedVersion + " <= x < " + firstUnrecognizedVersion + + ")."); + response = null; } + } catch (IOException | RuntimeException e) { + /* This could be anything, but the effect is that we don't have a query + * response to process further. */ + response = null; } return response; }

- private Connection connectToDatabase() { - Connection conn = null; - try { - conn = this.ds.getConnection(); - } catch (SQLException e) { - this.logger.log(Level.WARNING, "Couldn't connect: " + e.getMessage(), e); - } - return conn; - } - - private long[] queryFirstAndLastDatesFromDatabase(Connection conn) { - long[] firstAndLastDates = null; - try { - Statement statement = conn.createStatement(); - String query = "SELECT DATE(MIN(validafter)) AS first, " - + "DATE(MAX(validafter)) AS last FROM statusentry"; - ResultSet rs = statement.executeQuery(query); - if (rs.next()) { - Calendar utcCalendar = Calendar.getInstance( - TimeZone.getTimeZone("UTC")); - firstAndLastDates = new long[] { - rs.getTimestamp(1, utcCalendar).getTime(), - rs.getTimestamp(2, utcCalendar).getTime() - }; - } - rs.close(); - statement.close(); - } catch (SQLException e) { - /* Looks like we don't have any consensuses. */ - firstAndLastDates = null; - } - return firstAndLastDates; - } - - private SortedSet<Long> queryKnownConsensusValidAfterTimes( - Connection conn, String fromValidAfter, String toValidAfter) { - SortedSet<Long> relevantConsensuses = new TreeSet<>(); - try { - Statement statement = conn.createStatement(); - String query = "SELECT DISTINCT validafter FROM statusentry " - + "WHERE validafter >= '" + fromValidAfter - + "' AND validafter <= '" + toValidAfter + "'"; - ResultSet rs = statement.executeQuery(query); - while (rs.next()) { - long consensusTime = rs.getTimestamp(1).getTime(); - relevantConsensuses.add(consensusTime); - } - rs.close(); - statement.close(); - } catch (SQLException e) { - /* Looks like we don't have any consensuses in the requested - * interval. */ - relevantConsensuses = null; - } - return relevantConsensuses; - } - - private List<QueryResponse.Match> queryStatusEntries(Connection conn, - String relayIp, long timestamp, - SimpleDateFormat validAfterTimeFormat) { - List<QueryResponse.Match> matches = new ArrayList<>(); - String addressHex = !relayIp.contains(":") - ? this.convertIpV4ToHex(relayIp) : this.convertIpV6ToHex(relayIp); - if (addressHex == null) { - return null; - } - String address24Or48Hex = !relayIp.contains(":") - ? addressHex.substring(0, 6) : addressHex.substring(0, 12); - try { - CallableStatement cs; - if (!relayIp.contains(":")) { - cs = conn.prepareCall("{call search_by_address24_date(?, ?)}"); - } else { - cs = conn.prepareCall("{call search_by_address48_date(?, ?)}"); - } - cs.setString(1, address24Or48Hex); - Calendar utcCalendar = Calendar.getInstance( - TimeZone.getTimeZone("UTC")); - cs.setDate(2, new java.sql.Date(timestamp), utcCalendar); - ResultSet rs = cs.executeQuery(); - while (rs.next()) { - byte[] rawstatusentry = rs.getBytes(1); - SortedSet<String> addresses = new TreeSet<>(); - SortedSet<String> addressesHex = new TreeSet<>(); - String nickname = null; - Boolean exit = null; - for (String line : new String(rawstatusentry).split("\n")) { - if (line.startsWith("r ")) { - String[] parts = line.split(" "); - nickname = parts[1]; - addresses.add(parts[6]); - addressesHex.add(this.convertIpV4ToHex(parts[6])); - } else if (line.startsWith("a ")) { - String address = line.substring("a ".length(), - line.lastIndexOf(":")); - addresses.add(address); - String orAddressHex = !address.contains(":") - ? this.convertIpV4ToHex(address) - : this.convertIpV6ToHex(address); - addressesHex.add(orAddressHex); - } else if (line.startsWith("p ")) { - exit = !line.equals("p reject 1-65535"); - } - } - String exitaddress = rs.getString(4); - if (exitaddress != null && exitaddress.length() > 0) { - addresses.add(exitaddress); - addressesHex.add(this.convertIpV4ToHex(exitaddress)); - } - if (!addressesHex.contains(addressHex)) { - continue; - } - long validafter = rs.getTimestamp(2, utcCalendar).getTime(); - String validAfterString = validAfterTimeFormat.format(validafter); - String fingerprint = rs.getString(3).toUpperCase(); - QueryResponse.Match match = new QueryResponse.Match(); - match.timestamp = validAfterString; - match.addresses = addresses.toArray(new String[0]); - match.fingerprint = fingerprint; - match.nickname = nickname; - match.exit = exit; - matches.add(match); - } - rs.close(); - cs.close(); - } catch (SQLException e) { - /* Nothing found. */ - matches.clear(); - } - return matches; - } - - private List<String> queryAddressesInSame24(Connection conn, - String address24, long timestamp) { - List<String> addressesInSameNetwork = new ArrayList<>(); - try { - CallableStatement cs = conn.prepareCall( - "{call search_addresses_in_same_24 (?, ?)}"); - cs.setString(1, address24); - cs.setDate(2, new java.sql.Date(timestamp)); - ResultSet rs = cs.executeQuery(); - while (rs.next()) { - String address = rs.getString(1); - if (!addressesInSameNetwork.contains(address)) { - addressesInSameNetwork.add(address); - } - } - rs.close(); - cs.close(); - } catch (SQLException e) { - /* No other addresses in the same /24 found. */ - addressesInSameNetwork = null; - } - return addressesInSameNetwork; - } - - private List<String> queryAddressesInSame48(Connection conn, - String address48, long timestamp) { - List<String> addressesInSameNetwork = new ArrayList<>(); - try { - CallableStatement cs = conn.prepareCall( - "{call search_addresses_in_same_48 (?, ?)}"); - cs.setString(1, address48); - cs.setDate(2, new java.sql.Date(timestamp)); - ResultSet rs = cs.executeQuery(); - while (rs.next()) { - String address = rs.getString(1); - if (!addressesInSameNetwork.contains(address)) { - addressesInSameNetwork.add(address); - } - } - rs.close(); - cs.close(); - } catch (SQLException e) { - /* No other addresses in the same /48 found. */ - addressesInSameNetwork = null; - } - return addressesInSameNetwork; - } - - private void closeDatabaseConnection(Connection conn, - long requestedConnection) { - try { - conn.close(); - this.logger.info("Returned a database connection to the pool " - + "after " + (System.currentTimeMillis() - - requestedConnection) + " millis."); - } catch (SQLException e) { - this.logger.log(Level.WARNING, "Couldn't close: " + e.getMessage(), e); - } - return; - } - /* Helper methods for writing the response. */

private void writeHeader(PrintWriter out, ResourceBundle rb, String langStr) diff --git a/src/main/java/org/torproject/exonerator/QueryResponse.java b/src/main/java/org/torproject/exonerator/QueryResponse.java index d86efb0..45dd017 100644 --- a/src/main/java/org/torproject/exonerator/QueryResponse.java +++ b/src/main/java/org/torproject/exonerator/QueryResponse.java @@ -8,6 +8,9 @@ import com.google.gson.annotations.SerializedName; /** Query response from the ExoneraTor database. */ public class QueryResponse {

+ /** Version of this response format. */ + String version = "1.0"; + /** Query IP address passed in the request; never <code>null</code>. */ @SerializedName("query_address") String queryAddress; diff --git a/src/main/java/org/torproject/exonerator/QueryServlet.java b/src/main/java/org/torproject/exonerator/QueryServlet.java new file mode 100644 index 0000000..873a53a --- /dev/null +++ b/src/main/java/org/torproject/exonerator/QueryServlet.java @@ -0,0 +1,502 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.exonerator; + +import com.google.gson.Gson; +import org.apache.commons.codec.binary.Hex; + +import java.io.IOException; +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeSet; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Pattern; + +import javax.naming.Context; +import javax.naming.InitialContext; +import javax.naming.NamingException; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.sql.DataSource; + +public class QueryServlet extends HttpServlet { + + private static final long serialVersionUID = 7109011659099295183L; + + private Logger logger; + + private DataSource ds; + + @Override + public void init() { + + /* Initialize logger. */ + this.logger = Logger.getLogger(QueryServlet.class.toString()); + + /* Look up data source. */ + try { + Context cxt = new InitialContext(); + this.ds = (DataSource) cxt.lookup("java:comp/env/jdbc/exonerator"); + this.logger.info("Successfully looked up data source."); + } catch (NamingException e) { + this.logger.log(Level.WARNING, "Could not look up data source", e); + } + } + + @Override + public void doGet(HttpServletRequest request, + HttpServletResponse response) throws IOException, + ServletException { + + /* Parse ip parameter. */ + String ipParameter = request.getParameter("ip"); + if (null == ipParameter) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST, + "Missing ip parameter."); + return; + } + String relayIp = this.parseIpParameter(ipParameter); + if (null == relayIp) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST, + "Invalid ip parameter."); + return; + } + + /* Parse timestamp parameter. */ + String timestampParameter = request.getParameter("timestamp"); + if (null == timestampParameter) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST, + "Missing timestamp parameter."); + return; + } + Long timestamp = this.parseTimestampParameter(timestampParameter); + if (null == timestamp) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST, + "Invalid timestamp parameter."); + return; + } + + /* Query the database. */ + QueryResponse queryResponse = this.queryDatabase(relayIp, timestamp); + if (null == queryResponse) { + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, + "Database error."); + return; + } + + /* Format the query response. */ + Gson gson = new Gson(); + String formattedResponse = gson.toJson(queryResponse); + + /* Write the response. */ + response.setContentType("application/json"); + response.setCharacterEncoding("utf-8"); + response.getWriter().write(formattedResponse); + } + + /* Helper methods for handling the request. */ + + private String parseIpParameter(String passedIpParameter) { + String relayIp = null; + if (passedIpParameter != null && passedIpParameter.length() > 0) { + String ipParameter = passedIpParameter.trim(); + Pattern ipv4AddressPattern = Pattern.compile( + "^([01]?\d\d?|2[0-4]\d|25[0-5])\." + + "([01]?\d\d?|2[0-4]\d|25[0-5])\." + + "([01]?\d\d?|2[0-4]\d|25[0-5])\." + + "([01]?\d\d?|2[0-4]\d|25[0-5])$"); + Pattern ipv6AddressPattern = Pattern.compile( + "^\[?[0-9a-fA-F:]{3,39}\]?$"); + if (ipv4AddressPattern.matcher(ipParameter).matches()) { + String[] ipParts = ipParameter.split("\."); + relayIp = Integer.parseInt(ipParts[0]) + "." + + Integer.parseInt(ipParts[1]) + "." + + Integer.parseInt(ipParts[2]) + "." + + Integer.parseInt(ipParts[3]); + } else if (ipv6AddressPattern.matcher(ipParameter).matches()) { + if (ipParameter.startsWith("[") && ipParameter.endsWith("]")) { + ipParameter = ipParameter.substring(1, + ipParameter.length() - 1); + } + StringBuilder addressHex = new StringBuilder(); + int start = ipParameter.startsWith("::") ? 1 : 0; + int end = ipParameter.length() + - (ipParameter.endsWith("::") ? 1 : 0); + String[] parts = ipParameter.substring(start, end).split(":", -1); + for (int i = 0; i < parts.length; i++) { + String part = parts[i]; + if (part.length() == 0) { + addressHex.append("x"); + } else if (part.length() <= 4) { + addressHex.append(String.format("%4s", part)); + } else { + addressHex = null; + break; + } + } + if (addressHex != null) { + String addressHexString = addressHex.toString(); + addressHexString = addressHexString.replaceFirst("x", + String.format("%" + (33 - addressHexString.length()) + "s", + "0")); + if (!addressHexString.contains("x") + && addressHexString.length() == 32) { + relayIp = ipParameter.toLowerCase(); + } + } + } + } else { + relayIp = ""; + } + return relayIp; + } + + private String convertIpV4ToHex(String relayIp) { + String[] relayIpParts = relayIp.split("\."); + byte[] address24Bytes = new byte[4]; + for (int i = 0; i < address24Bytes.length; i++) { + address24Bytes[i] = (byte) Integer.parseInt(relayIpParts[i]); + } + String address24 = Hex.encodeHexString(address24Bytes); + return address24; + } + + private String convertIpV6ToHex(String relayIp) { + if (relayIp.startsWith("[") && relayIp.endsWith("]")) { + relayIp = relayIp.substring(1, relayIp.length() - 1); + } + StringBuilder addressHex = new StringBuilder(); + int start = relayIp.startsWith("::") ? 1 : 0; + int end = relayIp.length() - (relayIp.endsWith("::") ? 1 : 0); + String[] parts = relayIp.substring(start, end).split(":", -1); + for (int i = 0; i < parts.length; i++) { + String part = parts[i]; + if (part.length() == 0) { + addressHex.append("x"); + } else if (part.length() <= 4) { + addressHex.append(String.format("%4s", part)); + } else { + addressHex = null; + break; + } + } + String address48 = null; + if (addressHex != null) { + String addressHexString = addressHex.toString(); + addressHexString = addressHexString.replaceFirst("x", + String.format("%" + (33 - addressHexString.length()) + + "s", "0")); + if (!addressHexString.contains("x") + && addressHexString.length() == 32) { + address48 = addressHexString.replaceAll(" ", "0") + .toLowerCase(); + } + } + return address48; + } + + private Long parseTimestampParameter( + String passedTimestampParameter) { + Long timestamp = null; + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + dateFormat.setLenient(false); + if (passedTimestampParameter != null + && passedTimestampParameter.length() > 0) { + String timestampParameter = passedTimestampParameter.trim(); + try { + timestamp = dateFormat.parse(timestampParameter).getTime(); + } catch (ParseException e) { + timestamp = null; + } + } + return timestamp; + } + + /* Helper methods for querying the database. */ + + private QueryResponse queryDatabase(String relayIp, long timestamp) { + + QueryResponse response = null; + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + SimpleDateFormat validAfterTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + validAfterTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + + /* Open a database connection that we'll use to handle the whole + * request. */ + long requestedConnection = System.currentTimeMillis(); + Connection conn = this.connectToDatabase(); + if (null != conn) { + + response = new QueryResponse(); + response.queryAddress = relayIp; + response.queryDate = dateFormat.format(timestamp); + + /* Look up first and last date in the database. */ + long[] firstAndLastDates = this.queryFirstAndLastDatesFromDatabase( + conn); + if (null != firstAndLastDates) { + response.firstDateInDatabase = dateFormat.format( + firstAndLastDates[0]); + response.lastDateInDatabase = dateFormat.format(firstAndLastDates[1]); + + /* Consider all consensuses published on or within a day of the given + * date. */ + long timestampFrom = timestamp - 24L * 60L * 60L * 1000L; + long timestampTo = timestamp + 2 * 24L * 60L * 60L * 1000L - 1L; + String fromValidAfter = validAfterTimeFormat.format(timestampFrom); + String toValidAfter = validAfterTimeFormat.format(timestampTo); + SortedSet<Long> relevantConsensuses = + this.queryKnownConsensusValidAfterTimes(conn, fromValidAfter, + toValidAfter); + if (null != relevantConsensuses && !relevantConsensuses.isEmpty()) { + response.relevantStatuses = true; + + /* Search for status entries with the given IP address as onion + * routing address, plus status entries of relays having an exit + * list entry with the given IP address as exit address. */ + List<QueryResponse.Match> matches = this.queryStatusEntries(conn, + relayIp, timestamp, validAfterTimeFormat); + if (!matches.isEmpty()) { + response.matches = matches.toArray(new QueryResponse.Match[0]); + + /* If we didn't find anything, run another query to find out if + * there are relays running on other IP addresses in the same /24 or + * /48 network and tell the user about it. */ + } else { + if (!relayIp.contains(":")) { + String address24 = this.convertIpV4ToHex(relayIp) + .substring(0, 6); + if (address24 != null) { + response.nearbyAddresses = this.queryAddressesInSame24(conn, + address24, timestamp).toArray(new String[0]); + } + } else { + String address48 = this.convertIpV6ToHex(relayIp) + .substring(0, 12); + if (address48 != null) { + response.nearbyAddresses = this.queryAddressesInSame48(conn, + address48, timestamp).toArray(new String[0]); + } + } + } + } + } + + /* Close the database connection. */ + this.closeDatabaseConnection(conn, requestedConnection); + } + return response; + } + + private Connection connectToDatabase() { + Connection conn = null; + try { + conn = this.ds.getConnection(); + } catch (SQLException e) { + this.logger.log(Level.WARNING, "Couldn't connect: " + e.getMessage(), e); + } + return conn; + } + + private long[] queryFirstAndLastDatesFromDatabase(Connection conn) { + long[] firstAndLastDates = null; + try { + Statement statement = conn.createStatement(); + String query = "SELECT DATE(MIN(validafter)) AS first, " + + "DATE(MAX(validafter)) AS last FROM statusentry"; + ResultSet rs = statement.executeQuery(query); + if (rs.next()) { + Calendar utcCalendar = Calendar.getInstance( + TimeZone.getTimeZone("UTC")); + firstAndLastDates = new long[] { + rs.getTimestamp(1, utcCalendar).getTime(), + rs.getTimestamp(2, utcCalendar).getTime() + }; + } + rs.close(); + statement.close(); + } catch (SQLException e) { + /* Looks like we don't have any consensuses. */ + firstAndLastDates = null; + } + return firstAndLastDates; + } + + private SortedSet<Long> queryKnownConsensusValidAfterTimes( + Connection conn, String fromValidAfter, String toValidAfter) { + SortedSet<Long> relevantConsensuses = new TreeSet<>(); + try { + Statement statement = conn.createStatement(); + String query = "SELECT DISTINCT validafter FROM statusentry " + + "WHERE validafter >= '" + fromValidAfter + + "' AND validafter <= '" + toValidAfter + "'"; + ResultSet rs = statement.executeQuery(query); + while (rs.next()) { + long consensusTime = rs.getTimestamp(1).getTime(); + relevantConsensuses.add(consensusTime); + } + rs.close(); + statement.close(); + } catch (SQLException e) { + /* Looks like we don't have any consensuses in the requested + * interval. */ + relevantConsensuses = null; + } + return relevantConsensuses; + } + + private List<QueryResponse.Match> queryStatusEntries(Connection conn, + String relayIp, long timestamp, + SimpleDateFormat validAfterTimeFormat) { + List<QueryResponse.Match> matches = new ArrayList<>(); + String addressHex = !relayIp.contains(":") + ? this.convertIpV4ToHex(relayIp) : this.convertIpV6ToHex(relayIp); + if (addressHex == null) { + return null; + } + String address24Or48Hex = !relayIp.contains(":") + ? addressHex.substring(0, 6) : addressHex.substring(0, 12); + try { + CallableStatement cs; + if (!relayIp.contains(":")) { + cs = conn.prepareCall("{call search_by_address24_date(?, ?)}"); + } else { + cs = conn.prepareCall("{call search_by_address48_date(?, ?)}"); + } + cs.setString(1, address24Or48Hex); + Calendar utcCalendar = Calendar.getInstance( + TimeZone.getTimeZone("UTC")); + cs.setDate(2, new java.sql.Date(timestamp), utcCalendar); + ResultSet rs = cs.executeQuery(); + while (rs.next()) { + byte[] rawstatusentry = rs.getBytes(1); + SortedSet<String> addresses = new TreeSet<>(); + SortedSet<String> addressesHex = new TreeSet<>(); + String nickname = null; + Boolean exit = null; + for (String line : new String(rawstatusentry).split("\n")) { + if (line.startsWith("r ")) { + String[] parts = line.split(" "); + nickname = parts[1]; + addresses.add(parts[6]); + addressesHex.add(this.convertIpV4ToHex(parts[6])); + } else if (line.startsWith("a ")) { + String address = line.substring("a ".length(), + line.lastIndexOf(":")); + addresses.add(address); + String orAddressHex = !address.contains(":") + ? this.convertIpV4ToHex(address) + : this.convertIpV6ToHex(address); + addressesHex.add(orAddressHex); + } else if (line.startsWith("p ")) { + exit = !line.equals("p reject 1-65535"); + } + } + String exitaddress = rs.getString(4); + if (exitaddress != null && exitaddress.length() > 0) { + addresses.add(exitaddress); + addressesHex.add(this.convertIpV4ToHex(exitaddress)); + } + if (!addressesHex.contains(addressHex)) { + continue; + } + long validafter = rs.getTimestamp(2, utcCalendar).getTime(); + String validAfterString = validAfterTimeFormat.format(validafter); + String fingerprint = rs.getString(3).toUpperCase(); + QueryResponse.Match match = new QueryResponse.Match(); + match.timestamp = validAfterString; + match.addresses = addresses.toArray(new String[0]); + match.fingerprint = fingerprint; + match.nickname = nickname; + match.exit = exit; + matches.add(match); + } + rs.close(); + cs.close(); + } catch (SQLException e) { + /* Nothing found. */ + matches.clear(); + } + return matches; + } + + private List<String> queryAddressesInSame24(Connection conn, + String address24, long timestamp) { + List<String> addressesInSameNetwork = new ArrayList<>(); + try { + CallableStatement cs = conn.prepareCall( + "{call search_addresses_in_same_24 (?, ?)}"); + cs.setString(1, address24); + cs.setDate(2, new java.sql.Date(timestamp)); + ResultSet rs = cs.executeQuery(); + while (rs.next()) { + String address = rs.getString(1); + if (!addressesInSameNetwork.contains(address)) { + addressesInSameNetwork.add(address); + } + } + rs.close(); + cs.close(); + } catch (SQLException e) { + /* No other addresses in the same /24 found. */ + addressesInSameNetwork = null; + } + return addressesInSameNetwork; + } + + private List<String> queryAddressesInSame48(Connection conn, + String address48, long timestamp) { + List<String> addressesInSameNetwork = new ArrayList<>(); + try { + CallableStatement cs = conn.prepareCall( + "{call search_addresses_in_same_48 (?, ?)}"); + cs.setString(1, address48); + cs.setDate(2, new java.sql.Date(timestamp)); + ResultSet rs = cs.executeQuery(); + while (rs.next()) { + String address = rs.getString(1); + if (!addressesInSameNetwork.contains(address)) { + addressesInSameNetwork.add(address); + } + } + rs.close(); + cs.close(); + } catch (SQLException e) { + /* No other addresses in the same /48 found. */ + addressesInSameNetwork = null; + } + return addressesInSameNetwork; + } + + private void closeDatabaseConnection(Connection conn, + long requestedConnection) { + try { + conn.close(); + this.logger.info("Returned a database connection to the pool " + + "after " + (System.currentTimeMillis() + - requestedConnection) + " millis."); + } catch (SQLException e) { + this.logger.log(Level.WARNING, "Couldn't close: " + e.getMessage(), e); + } + return; + } + +} + diff --git a/src/main/webapp/web.xml b/src/main/webapp/web.xml index ce12489..d838059 100644 --- a/src/main/webapp/web.xml +++ b/src/main/webapp/web.xml @@ -17,6 +17,17 @@ <url-pattern>/index.html</url-pattern> </servlet-mapping>

+ <servlet> + <servlet-name>Query</servlet-name> + <servlet-class> + org.torproject.exonerator.QueryServlet + </servlet-class> + </servlet> + <servlet-mapping> + <servlet-name>Query</servlet-name> + <url-pattern>/query.json</url-pattern> + </servlet-mapping> + <welcome-file-list> <welcome-file>index.html</welcome-file> </welcome-file-list>