[tor-commits] [exonerator/master] Remove raw descriptors and two unused servlets.
karsten at torproject.org
karsten at torproject.org
Mon Aug 31 17:10:20 UTC 2015
commit 2c0fb225dae7f7bc3625de8906f7902c61ded8ea
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Mon Aug 31 08:55:33 2015 +0200
Remove raw descriptors and two unused servlets.
We stopped processing server descriptors and linking raw descriptors
in the latest version of ExoneraTor. The idea was to simplify the
service and make it useful for the average user who doesn't care about
this level of detail. And it's a good opportunity to prevent the
database from growing even larger.
Implements #16595.
---
db/exonerator.sql | 82 +-----------
etc/web.xml | 22 ----
.../torproject/exonerator/ConsensusServlet.java | 124 ------------------
.../exonerator/ExoneraTorDatabaseImporter.java | 72 +----------
.../torproject/exonerator/ExoneraTorServlet.java | 4 +-
.../exonerator/ServerDescriptorServlet.java | 132 --------------------
web/robots.txt | 4 -
7 files changed, 8 insertions(+), 432 deletions(-)
diff --git a/db/exonerator.sql b/db/exonerator.sql
index fd58531..fda9071 100755
--- a/db/exonerator.sql
+++ b/db/exonerator.sql
@@ -1,32 +1,6 @@
-- Copyright 2011 The Tor Project
-- See LICENSE for licensing information
--- The descriptor table holds server descriptors that we use for display
--- purposes and to parse exit policies.
-CREATE TABLE descriptor (
-
- -- The 40-character lower-case hex string identifies a descriptor
- -- uniquely and is used to join statusentry and this table.
- descriptor CHARACTER(40) NOT NULL PRIMARY KEY,
-
- -- The raw descriptor string is used for display purposes and to check
- -- whether the relay allowed exiting to a given target or not.
- rawdescriptor BYTEA NOT NULL
-);
-
--- The consensus table stores network status consensuses to be looked up
--- by valid-after time and displayed upon request. A second purpose is
--- to learn quickly whether the database contains status entries for a
--- given day or not.
-CREATE TABLE consensus (
-
- -- The unique valid-after time of the consensus.
- validafter TIMESTAMP WITHOUT TIME ZONE NOT NULL PRIMARY KEY,
-
- -- The raw consensus string for display purposes only.
- rawconsensus BYTEA NOT NULL
-);
-
-- The statusentry table stores network status consensus entries listing
-- a relay as running at a certain point in time. Only relays with the
-- Running flag shall be inserted into this table. If a relay advertises
@@ -73,6 +47,10 @@ CREATE TABLE statusentry (
PRIMARY KEY (validafter, fingerprint, oraddress)
);
+-- The index on the valid-after time is used to return first and last
+-- date in the database and known valid-after times in a given interval.
+CREATE INDEX statusentry_validafter ON statusentry (validafter);
+
-- The index on the exact onion routing address and on the valid-after
-- date is used to speed up ExoneraTor's query for status entries.
CREATE INDEX statusentry_oraddress_validafterdate
@@ -141,33 +119,6 @@ CREATE INDEX exitlistentry_exitaddress24_scanneddate
-- Create the plpgsql language, so that we can use it below.
CREATE LANGUAGE plpgsql;
--- Insert a server descriptor into the descriptor table. Before doing so,
--- check that there is no descriptor with the same descriptor identifier
--- in the table yet. Return 1 if the descriptor was inserted, 0
--- otherwise.
-CREATE OR REPLACE FUNCTION insert_descriptor (
- insert_descriptor CHARACTER(40),
- insert_rawdescriptor BYTEA)
- RETURNS INTEGER AS $$
- BEGIN
- -- Look up if the descriptor is already contained in the descriptor
- -- table.
- IF (SELECT COUNT(*)
- FROM descriptor
- WHERE descriptor = insert_descriptor) = 0 THEN
- -- Insert the descriptor and remember the new descriptorid to update
- -- the foreign key in statusentry.
- INSERT INTO descriptor (descriptor, rawdescriptor)
- VALUES (insert_descriptor, insert_rawdescriptor);
- -- Return 1 for a successfully inserted descriptor.
- RETURN 1;
- ELSE
- -- Return 0 because we didn't change anything.
- RETURN 0;
- END IF;
- END;
-$$ LANGUAGE 'plpgsql';
-
-- Insert a status entry into the statusentry table. First check that
-- this status entry isn't contained in the table yet. It's okay to
-- insert the same status entry multiple times for different IP addresses
@@ -204,31 +155,6 @@ CREATE OR REPLACE FUNCTION insert_statusentry (
END;
$$ LANGUAGE 'plpgsql';
--- Insert a consensus into the consensus table. Check that the same
--- consensus has not been imported before. Return 1 if it was inserted, 0
--- otherwise.
-CREATE OR REPLACE FUNCTION insert_consensus (
- insert_validafter TIMESTAMP WITHOUT TIME ZONE,
- insert_rawconsensus BYTEA)
- RETURNS INTEGER AS $$
- BEGIN
- -- Look up if the consensus is already contained in the consensus
- -- table.
- IF (SELECT COUNT(*)
- FROM consensus
- WHERE validafter = insert_validafter) = 0 THEN
- -- Insert the consensus.
- INSERT INTO consensus (validafter, rawconsensus)
- VALUES (insert_validafter, insert_rawconsensus);
- -- Return 1 for a successful insert operation.
- RETURN 1;
- ELSE
- -- Return 0 for not inserting the consensus.
- RETURN 0;
- END IF;
- END;
-$$ LANGUAGE 'plpgsql';
-
-- Insert an exit list entry into the exitlistentry table. Check that
-- this entry hasn't been inserted before. It's okay to insert the same
-- exit list entry multiple times for different exit addresses. Return 1
diff --git a/etc/web.xml b/etc/web.xml
index 812448a..ce12489 100644
--- a/etc/web.xml
+++ b/etc/web.xml
@@ -17,28 +17,6 @@
<url-pattern>/index.html</url-pattern>
</servlet-mapping>
- <servlet>
- <servlet-name>ServerDescriptor</servlet-name>
- <servlet-class>
- org.torproject.exonerator.ServerDescriptorServlet
- </servlet-class>
- </servlet>
- <servlet-mapping>
- <servlet-name>ServerDescriptor</servlet-name>
- <url-pattern>/serverdesc</url-pattern>
- </servlet-mapping>
-
- <servlet>
- <servlet-name>Consensus</servlet-name>
- <servlet-class>
- org.torproject.exonerator.ConsensusServlet
- </servlet-class>
- </servlet>
- <servlet-mapping>
- <servlet-name>Consensus</servlet-name>
- <url-pattern>/consensus</url-pattern>
- </servlet-mapping>
-
<welcome-file-list>
<welcome-file>index.html</welcome-file>
</welcome-file-list>
diff --git a/src/org/torproject/exonerator/ConsensusServlet.java b/src/org/torproject/exonerator/ConsensusServlet.java
deleted file mode 100644
index 5bc4e57..0000000
--- a/src/org/torproject/exonerator/ConsensusServlet.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.exonerator;
-
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.TimeZone;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.naming.Context;
-import javax.naming.InitialContext;
-import javax.naming.NamingException;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.sql.DataSource;
-
-public class ConsensusServlet extends HttpServlet {
-
- private static final long serialVersionUID = 3147332016303032164L;
-
- private DataSource ds;
-
- private Logger logger;
-
- public void init() {
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(ConsensusServlet.class.toString());
-
- /* Look up data source. */
- try {
- Context cxt = new InitialContext();
- this.ds = (DataSource) cxt.lookup("java:comp/env/jdbc/exonerator");
- this.logger.info("Successfully looked up data source.");
- } catch (NamingException e) {
- this.logger.log(Level.WARNING, "Could not look up data source", e);
- }
- }
-
- public void doGet(HttpServletRequest request,
- HttpServletResponse response) throws IOException,
- ServletException {
-
- /* Check valid-after parameter. */
- String validAfterParameter = request.getParameter("valid-after");
- if (validAfterParameter == null ||
- validAfterParameter.length() != "yyyy-MM-dd-HH-mm-ss".length()) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- SimpleDateFormat parameterFormat = new SimpleDateFormat(
- "yyyy-MM-dd-HH-mm-ss");
- parameterFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- long parsedTimestamp = -1L;
- try {
- parsedTimestamp = parameterFormat.parse(validAfterParameter).
- getTime();
- } catch (ParseException e) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- if (parsedTimestamp < 0L) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
-
- /* Look up consensus in the database. */
- SimpleDateFormat databaseFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- databaseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String databaseParameter = databaseFormat.format(parsedTimestamp);
- byte[] rawDescriptor = null;
- try {
- long requestedConnection = System.currentTimeMillis();
- Connection conn = this.ds.getConnection();
- Statement statement = conn.createStatement();
- String query = "SELECT rawconsensus FROM consensus "
- + "WHERE validafter = '" + databaseParameter + "'";
- ResultSet rs = statement.executeQuery(query);
- if (rs.next()) {
- rawDescriptor = rs.getBytes(1);
- }
- rs.close();
- statement.close();
- conn.close();
- this.logger.info("Returned a database connection to the pool after "
- + (System.currentTimeMillis() - requestedConnection)
- + " millis.");
- } catch (SQLException e) {
- response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
- return;
- }
-
- /* Write response. */
- if (rawDescriptor == null) {
- response.sendError(HttpServletResponse.SC_NOT_FOUND);
- return;
- }
- try {
- response.setContentType("text/plain");
- response.setHeader("Content-Length", String.valueOf(
- rawDescriptor.length));
- response.setHeader("Content-Disposition", "inline; filename=\""
- + validAfterParameter + "-consensus\"");
- BufferedOutputStream output = new BufferedOutputStream(
- response.getOutputStream());
- output.write(rawDescriptor);
- output.flush();
- output.close();
- } finally {
- /* Nothing to do here. */
- }
- }
-}
-
diff --git a/src/org/torproject/exonerator/ExoneraTorDatabaseImporter.java b/src/org/torproject/exonerator/ExoneraTorDatabaseImporter.java
index 6111888..41751ca 100644
--- a/src/org/torproject/exonerator/ExoneraTorDatabaseImporter.java
+++ b/src/org/torproject/exonerator/ExoneraTorDatabaseImporter.java
@@ -100,20 +100,14 @@ public class ExoneraTorDatabaseImporter {
}
/* Callable statements to import data into the database. */
- private static CallableStatement insertDescriptorStatement;
private static CallableStatement insertStatusentryStatement;
- private static CallableStatement insertConsensusStatement;
private static CallableStatement insertExitlistentryStatement;
/* Prepare statements for importing data into the database. */
private static void prepareDatabaseStatements() {
try {
- insertDescriptorStatement = connection.prepareCall(
- "{call insert_descriptor(?, ?)}");
insertStatusentryStatement = connection.prepareCall(
"{call insert_statusentry(?, ?, ?, ?, ?, ?, ?)}");
- insertConsensusStatement = connection.prepareCall(
- "{call insert_consensus(?, ?)}");
insertExitlistentryStatement = connection.prepareCall(
"{call insert_exitlistentry(?, ?, ?, ?, ?)}");
} catch (SQLException e) {
@@ -159,7 +153,6 @@ public class ExoneraTorDatabaseImporter {
DescriptorSourceFactory.createDescriptorCollector();
collector.collectDescriptors("https://collector.torproject.org",
new String[] { "/recent/relay-descriptors/consensuses/",
- "/recent/relay-descriptors/server-descriptors/",
"/recent/exit-lists/" }, 0L, new File(importDirString), true);
}
@@ -269,9 +262,7 @@ public class ExoneraTorDatabaseImporter {
}
br.close();
String startToken = null;
- if (line.startsWith("router ")) {
- startToken = "router ";
- } else if (line.equals("network-status-version 3")) {
+ if (line.equals("network-status-version 3")) {
startToken = "network-status-version 3";
} else if (line.startsWith("Downloaded ") ||
line.startsWith("ExitNode ")) {
@@ -292,9 +283,7 @@ public class ExoneraTorDatabaseImporter {
}
byte[] descBytes = new byte[end - start];
System.arraycopy(bytes, start, descBytes, 0, end - start);
- if (startToken.equals("router ")) {
- parseServerDescriptor(file, descBytes);
- } else if (startToken.equals("network-status-version 3")) {
+ if (startToken.equals("network-status-version 3")) {
parseConsensus(file, descBytes);
} else if (startToken.equals("ExitNode ")) {
parseExitList(file, descBytes);
@@ -314,47 +303,6 @@ public class ExoneraTorDatabaseImporter {
parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
}
- /* Parse a single server descriptor. */
- private static void parseServerDescriptor(File file, byte[] bytes) {
- String ascii = "";
- try {
- ascii = new String(bytes, "US-ASCII");
- } catch (UnsupportedEncodingException e) {
- /* We know that US-ASCII is a supported encoding. */
- }
- String startToken = "router ";
- String sigToken = "\nrouter-signature\n";
- int start = ascii.indexOf(startToken);
- int sig = ascii.indexOf(sigToken) + sigToken.length();
- String descriptor = null;
- if (start >= 0 || sig >= 0 || sig > start) {
- byte[] forDigest = new byte[sig - start];
- System.arraycopy(bytes, start, forDigest, 0, sig - start);
- descriptor = DigestUtils.shaHex(forDigest);
- }
- if (descriptor == null) {
- System.out.println("Could not calculate descriptor digest. "
- + "Skipping.");
- return;
- }
- importDescriptor(descriptor, bytes);
- }
-
- /* Import a single server descriptor into the database. */
- private static void importDescriptor(String descriptor,
- byte[] rawDescriptor) {
- try {
- insertDescriptorStatement.clearParameters();
- insertDescriptorStatement.setString(1, descriptor);
- insertDescriptorStatement.setBytes(2, rawDescriptor);
- insertDescriptorStatement.execute();
- } catch (SQLException e) {
- System.out.println("Could not import descriptor into the "
- + "database. Exiting.");
- System.exit(1);
- }
- }
-
/* Parse a consensus. */
private static void parseConsensus(File file, byte[] bytes) {
try {
@@ -381,7 +329,6 @@ public class ExoneraTorDatabaseImporter {
+ "'" + file + "'. Skipping.");
return;
}
- importConsensus(validAfterMillis, bytes);
} else if (line.startsWith("r ") ||
line.equals("directory-footer")) {
if (isRunning) {
@@ -498,21 +445,6 @@ public class ExoneraTorDatabaseImporter {
}
}
- /* Import a consensus into the database. */
- private static void importConsensus(long validAfterMillis,
- byte[] rawConsensus) {
- try {
- insertConsensusStatement.clearParameters();
- insertConsensusStatement.setTimestamp(1,
- new Timestamp(validAfterMillis), calendarUTC);
- insertConsensusStatement.setBytes(2, rawConsensus);
- insertConsensusStatement.execute();
- } catch (SQLException e) {
- System.out.println("Could not import consensus. Exiting.");
- System.exit(1);
- }
- }
-
/* Parse an exit list. */
private static void parseExitList(File file, byte[] bytes) {
try {
diff --git a/src/org/torproject/exonerator/ExoneraTorServlet.java b/src/org/torproject/exonerator/ExoneraTorServlet.java
index 76da657..a0c798f 100644
--- a/src/org/torproject/exonerator/ExoneraTorServlet.java
+++ b/src/org/torproject/exonerator/ExoneraTorServlet.java
@@ -350,7 +350,7 @@ public class ExoneraTorServlet extends HttpServlet {
try {
Statement statement = conn.createStatement();
String query = "SELECT DATE(MIN(validafter)) AS first, "
- + "DATE(MAX(validafter)) AS last FROM consensus";
+ + "DATE(MAX(validafter)) AS last FROM statusentry";
ResultSet rs = statement.executeQuery(query);
if (rs.next()) {
Calendar utcCalendar = Calendar.getInstance(
@@ -374,7 +374,7 @@ public class ExoneraTorServlet extends HttpServlet {
SortedSet<Long> relevantConsensuses = new TreeSet<Long>();
try {
Statement statement = conn.createStatement();
- String query = "SELECT validafter FROM consensus "
+ String query = "SELECT DISTINCT validafter FROM statusentry "
+ "WHERE validafter >= '" + fromValidAfter
+ "' AND validafter <= '" + toValidAfter + "'";
ResultSet rs = statement.executeQuery(query);
diff --git a/src/org/torproject/exonerator/ServerDescriptorServlet.java b/src/org/torproject/exonerator/ServerDescriptorServlet.java
deleted file mode 100644
index 91a15df..0000000
--- a/src/org/torproject/exonerator/ServerDescriptorServlet.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.exonerator;
-
-import java.io.BufferedOutputStream;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import javax.naming.Context;
-import javax.naming.InitialContext;
-import javax.naming.NamingException;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.sql.DataSource;
-
-public class ServerDescriptorServlet extends HttpServlet {
-
- private static final long serialVersionUID = -7935883442750583462L;
-
- private DataSource ds;
-
- private Logger logger;
-
- public void init() {
-
- /* Initialize logger. */
- this.logger = Logger.getLogger(
- ServerDescriptorServlet.class.toString());
-
- /* Look up data source. */
- try {
- Context cxt = new InitialContext();
- this.ds = (DataSource) cxt.lookup("java:comp/env/jdbc/exonerator");
- this.logger.info("Successfully looked up data source.");
- } catch (NamingException e) {
- this.logger.log(Level.WARNING, "Could not look up data source", e);
- }
- }
-
- public void doGet(HttpServletRequest request,
- HttpServletResponse response) throws IOException,
- ServletException {
-
- /* Read desc-id parameter. */
- String descIdParameter = request.getParameter("desc-id");
-
- /* See if we were given a desc-id parameter. If so, look up this
- * descriptor and return it. */
- List<byte[]> rawDescriptors = new ArrayList<byte[]>();
- String filename = null;
- if (descIdParameter != null) {
- if (descIdParameter.length() < 8 ||
- descIdParameter.length() > 40) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- String descId = descIdParameter.toLowerCase();
- Pattern descIdPattern = Pattern.compile("^[0-9a-f]+$");
- Matcher descIdMatcher = descIdPattern.matcher(descId);
- if (!descIdMatcher.matches()) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
-
- /* Look up descriptor in the database. */
- try {
- long requestedConnection = System.currentTimeMillis();
- Connection conn = ds.getConnection();
- Statement statement = conn.createStatement();
- String query = "SELECT descriptor, rawdescriptor FROM descriptor "
- + "WHERE descriptor LIKE '" + descId + "%'";
- ResultSet rs = statement.executeQuery(query);
- if (rs.next()) {
- filename = rs.getString(1);
- rawDescriptors.add(rs.getBytes(2));
- }
- rs.close();
- statement.close();
- conn.close();
- this.logger.info("Returned a database connection to the pool "
- + "after " + (System.currentTimeMillis()
- - requestedConnection) + " millis.");
- } catch (SQLException e) {
- response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
- return;
- }
-
- /* Return an error if no desc-id parameter was given. */
- } else {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
-
- /* Write response. */
- if (rawDescriptors.size() == 0) {
- response.sendError(HttpServletResponse.SC_NOT_FOUND);
- return;
- }
- try {
- response.setContentType("text/plain");
- int responseLength = 0;
- for (byte[] rawDescriptor : rawDescriptors) {
- responseLength += rawDescriptor.length;
- }
- response.setHeader("Content-Length", String.valueOf(
- responseLength));
- response.setHeader("Content-Disposition", "inline; filename=\""
- + filename + "\"");
- BufferedOutputStream output = new BufferedOutputStream(
- response.getOutputStream());
- for (byte[] rawDescriptor : rawDescriptors) {
- output.write(rawDescriptor);
- }
- output.flush();
- output.close();
- } finally {
- /* Nothing to do here. */
- }
- }
-}
-
diff --git a/web/robots.txt b/web/robots.txt
deleted file mode 100644
index 418cff8..0000000
--- a/web/robots.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-User-agent: *
-Disallow: /serverdesc
-Disallow: /consensus
-
More information about the tor-commits
mailing list