April 2020 - tor-commits - lists.torproject.org

[exonerator/release] remove unused parameters
by karsten＠torproject.org 13 Apr '20

13 Apr '20

commit eb3ef031b5482265055ad6f66e4a051e3d392dbc Author: tordinge <tordinge(a)aprivatesub.net> Date: Mon Feb 24 20:42:04 2020 -0500 remove unused parameters --- .../java/org/torproject/metrics/exonerator/ExoneraTorServlet.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java index 06fa0f5..b3f5b59 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java +++ b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java @@ -138,7 +138,7 @@ public class ExoneraTorServlet extends HttpServlet { /* Start writing response. */ StringWriter so = new StringWriter(); PrintWriter out = new PrintWriter(so); - this.writeHeader(out, rb, langStr); + this.writeHeader(out); /* Obtain the current request URI for relative links and the configured * base URL for absolute links like the printed permanent link. If no base @@ -321,7 +321,7 @@ public class ExoneraTorServlet extends HttpServlet { /* Helper methods for writing the response. */ - private void writeHeader(PrintWriter out, ResourceBundle rb, String langStr) { + private void writeHeader(PrintWriter out) { out.printf(" <div class=\"container\">\n"); }

1 0

[exonerator/release] use faster lastIndexOf(char) instead of lastIndexOf(String)
by karsten＠torproject.org 13 Apr '20

13 Apr '20

commit 639b9ebd2f545c3bd42fb756b76bd3c4328c462a Author: tordinge <tordinge(a)aprivatesub.net> Date: Mon Feb 24 20:47:11 2020 -0500 use faster lastIndexOf(char) instead of lastIndexOf(String) --- .../org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java index 1e5f3bb..3dbb119 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java +++ b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java @@ -244,7 +244,7 @@ public class ExoneraTorDatabaseImporter { orAddresses.add(entry.getAddress()); for (String orAddressAndPort : entry.getOrAddresses()) { orAddresses.add(orAddressAndPort.substring(0, - orAddressAndPort.lastIndexOf(":"))); + orAddressAndPort.lastIndexOf(':'))); } importStatusentry(validAfter, fingerprintBase64, nickname, exit, orAddresses);

1 0

[exonerator/release] Simplify logging configuration.
by karsten＠torproject.org 13 Apr '20

13 Apr '20

commit 5a7c19700f740ff88c1a1ae05b2383851d8cf9c7 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Apr 8 08:52:45 2020 +0200 Simplify logging configuration. Implements #33549. --- CHANGELOG.md | 1 + build.xml | 2 +- src/build | 2 +- .../exonerator/ExoneraTorDatabaseImporter.java | 2 +- .../metrics/exonerator/ExoneraTorServlet.java | 4 +- .../metrics/exonerator/QueryResponse.java | 3 +- .../metrics/exonerator/QueryServlet.java | 17 +++---- .../torproject/metrics/exonerator/ServerMain.java | 7 +-- src/main/resources/logback.xml | 57 ---------------------- 9 files changed, 20 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca10ec2..d22bc6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ * Minor changes - Improve logging of the database importer. + - Simplify logging configuration. # Changes in version 4.3.0 - 2020-01-15 diff --git a/build.xml b/build.xml index cdc919c..419e828 100644 --- a/build.xml +++ b/build.xml @@ -108,7 +108,7 @@ </archives> </restrict> <exclude name="**/Test*.class" /> - <zipfileset dir="${resources}" + <zipfileset dir="${buildresources}" prefix="" includes="logback.xml" /> <zipfileset dir="${resources}/web" diff --git a/src/build b/src/build index 264e498..b5e1a2d 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 264e498f54a20f7d299daaf2533d043f880e6a8b +Subproject commit b5e1a2d7b29e58cc0645f068a1ebf4377bf9d8b8 diff --git a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java index 53e58f7..b4d68e5 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java +++ b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java @@ -47,7 +47,7 @@ import java.util.TreeMap; /* Import Tor descriptors into the ExoneraTor database. */ public class ExoneraTorDatabaseImporter { - private static Logger logger + private static final Logger logger = LoggerFactory.getLogger(ExoneraTorDatabaseImporter.class); /** Main function controlling the parsing process. */ diff --git a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java index b3f5b59..59535ea 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java +++ b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java @@ -36,7 +36,8 @@ public class ExoneraTorServlet extends HttpServlet { private static final long serialVersionUID = 1370088989739567509L; - private Logger logger; + private static final Logger logger + = LoggerFactory.getLogger(ExoneraTorServlet.class); private String exoneraTorHost = System.getProperty("exonerator.url", "https://exonerator.torproject.org"); @@ -49,7 +50,6 @@ public class ExoneraTorServlet extends HttpServlet { @Override public void init(ServletConfig config) throws ServletException { super.init(config); - this.logger = LoggerFactory.getLogger(ExoneraTorServlet.class); this.availableLanguageNames = new TreeMap<>(); for (String locale : this.availableLanguages) { ResourceBundle rb = ResourceBundle.getBundle("ExoneraTor", diff --git a/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java b/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java index aee8dbf..14f2fa9 100644 --- a/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java +++ b/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java @@ -18,7 +18,8 @@ import java.util.SortedSet; /** Query response from the ExoneraTor database. */ public class QueryResponse { - private static Logger logger = LoggerFactory.getLogger(QueryResponse.class); + private static final Logger logger + = LoggerFactory.getLogger(QueryResponse.class); /* Actual version implemented by this class. */ private static final String VERSION = "1.0"; diff --git a/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java b/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java index ee61e28..dda63da 100644 --- a/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java +++ b/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java @@ -46,7 +46,8 @@ public class QueryServlet extends HttpServlet { private static final long serialVersionUID = 7109011659099295183L; - private Logger logger; + private static final Logger logger + = LoggerFactory.getLogger(QueryServlet.class); private DataSource ds; @@ -56,15 +57,13 @@ public class QueryServlet extends HttpServlet { @Override public void init() { - this.logger = LoggerFactory.getLogger(QueryServlet.class); - /* Look up data source. */ try { Context cxt = new InitialContext(); this.ds = (DataSource) cxt.lookup("java:comp/env/jdbc/exonerator"); - this.logger.info("Successfully looked up data source."); + logger.info("Successfully looked up data source."); } catch (NamingException e) { - this.logger.warn("Could not look up data source", e); + logger.warn("Could not look up data source", e); } } @@ -341,17 +340,17 @@ public class QueryServlet extends HttpServlet { } } } catch (SQLException e) { - this.logger.warn("Result set error. Returning 'null'.", e); + logger.warn("Result set error. Returning 'null'.", e); return null; } - this.logger.info("Returned a database connection to the pool after {}.", + logger.info("Returned a database connection to the pool after {}.", Duration.between(requestedConnection, Instant.now())); } catch (SQLException e) { - this.logger.warn("Callable statement error. Returning 'null'.", e); + logger.warn("Callable statement error. Returning 'null'.", e); return null; } } catch (Throwable e) { // Catch all problems left. - this.logger.warn("Database error. Returning 'null'.", e); + logger.warn("Database error. Returning 'null'.", e); return null; } diff --git a/src/main/java/org/torproject/metrics/exonerator/ServerMain.java b/src/main/java/org/torproject/metrics/exonerator/ServerMain.java index 0663e0f..1819a3c 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ServerMain.java +++ b/src/main/java/org/torproject/metrics/exonerator/ServerMain.java @@ -11,20 +11,21 @@ import org.slf4j.LoggerFactory; public class ServerMain { - private static final Logger log = LoggerFactory.getLogger(ServerMain.class); + private static final Logger logger + = LoggerFactory.getLogger(ServerMain.class); /** Starts the web server listening for incoming client connections. */ public static void main(String[] args) { try { Resource jettyXml = Resource.newSystemResource("jetty.xml"); - log.info("Reading configuration from '{}'.", jettyXml); + logger.info("Reading configuration from '{}'.", jettyXml); XmlConfiguration configuration = new XmlConfiguration(jettyXml.getInputStream()); Server server = (Server) configuration.configure(); server.start(); server.join(); } catch (Exception ex) { - log.error("Exiting, because of: {}.", ex.getMessage(), ex); + logger.error("Exiting, because of: {}.", ex.getMessage(), ex); System.exit(1); } } diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml deleted file mode 100644 index c29c057..0000000 --- a/src/main/resources/logback.xml +++ /dev/null @@ -1,57 +0,0 @@ -<configuration debug="false"> - -  - <property name="logfile-base" value="${LOGBASE}/exonerator-" /> - -  - <property name="fileall-logname" value="${logfile-base}all" /> - <property name="fileerr-logname" value="${logfile-base}err" /> - <property name="filestatistics-logname" value="${logfile-base}statistics" /> - -  - <property name="utc-date-pattern" value="%date{ISO8601, UTC}" /> - -  - <appender name="FILEALL" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${fileall-logname}.log</file> - <encoder> - <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> - </encoder> - <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> -  - <FileNamePattern>${fileall-logname}.%d{yyyy-MM-dd}.%i.log</FileNamePattern> - <maxHistory>10</maxHistory> - <timeBasedFileNamingAndTriggeringPolicy - class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> -  - <maxFileSize>1MB</maxFileSize> - </timeBasedFileNamingAndTriggeringPolicy> - </rollingPolicy> - </appender> - - <appender name="FILEERR" class="ch.qos.logback.core.FileAppender"> - <file>${fileerr-logname}.log</file> - <encoder> - <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> - </encoder> - -  - <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> - <level>ERROR</level> - </filter> - </appender> - -  - <logger name="org.torproject" > - <appender-ref ref="FILEERR" /> - </logger> - - <logger name="org.eclipse" level="INFO" /> - <logger name="org.apache" level="INFO" /> - - <root level="INFO"> - <appender-ref ref="FILEALL" /> - </root> - -</configuration> -

1 0

[exonerator/release] Improve logging of database importer.
by karsten＠torproject.org 13 Apr '20

13 Apr '20

commit 54ad288276bd2313ff81a71406f027801e51fa55 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Mar 15 14:31:10 2020 +0100 Improve logging of database importer. This patch focuses on providing more context in log messages and unify the use of log levels. It also adds counters and timestamps as far as they are being used in log messages. And in very few cases it contains minor changes like using try-with-resource when we're touching that code anyway. But most other refactorings are left as future work. Fixes the first part of #24542. The second part will be to improve logging of the web server parts. --- CHANGELOG.md | 3 + .../exonerator/ExoneraTorDatabaseImporter.java | 239 ++++++++++++++++----- 2 files changed, 186 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c87922..ca10ec2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changes in version 4.?.? - 2020-0?-?? + * Minor changes + - Improve logging of the database importer. + # Changes in version 4.3.0 - 2020-01-15 diff --git a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java index 3dbb119..53e58f7 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java +++ b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java @@ -11,6 +11,7 @@ import org.torproject.descriptor.ExitList; import org.torproject.descriptor.ExitList.Entry; import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.UnparseableDescriptor; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Base64; @@ -25,7 +26,9 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.io.LineNumberReader; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.sql.CallableStatement; import java.sql.Connection; @@ -49,46 +52,65 @@ public class ExoneraTorDatabaseImporter { /** Main function controlling the parsing process. */ public static void main(String[] args) { + logger.info("Starting ExoneraTor database importer."); readConfiguration(); openDatabaseConnection(); prepareDatabaseStatements(); createLockFile(); + logger.info("Fetching descriptors from CollecTor."); fetchDescriptors(); + logger.info("Importing descriptors into the database."); readImportHistoryToMemory(); parseDescriptors(); writeImportHistoryToDisk(); closeDatabaseConnection(); deleteLockFile(); + logger.info("Terminating ExoneraTor database importer."); } - /* JDBC string of the ExoneraTor database. */ + /* JDBC string of the ExoneraTor database. Never include this in a log + * message, because it likely contains the database password! */ private static String jdbcString; /* Directory from which to import descriptors. */ - private static String importDirString; + private static File importDirectory; /* Learn JDBC string and directory to parse descriptors from. */ private static void readConfiguration() { File configFile = new File("config"); if (!configFile.exists()) { - logger.error("Could not find config file. Exiting."); + logger.error("Could not find configuration file {}. Make sure that this " + + "file exists. Exiting.", configFile.getAbsoluteFile()); System.exit(1); } - String line; - try { - BufferedReader br = new BufferedReader(new FileReader(configFile)); + try (BufferedReader br = new BufferedReader(new FileReader(configFile))) { + String line; while ((line = br.readLine()) != null) { if (line.startsWith("ExoneraTorDatabaseJdbc")) { jdbcString = line.split(" ")[1]; } else if (line.startsWith("ExoneraTorImportDirectory")) { - importDirString = line.split(" ")[1]; + importDirectory = new File(line.split(" ")[1]); } } - br.close(); } catch (IOException e) { - logger.error("Could not parse config file. Exiting.", e); + logger.error("Caught an I/O exception while reading configuration file " + + "{}. Make sure that this file is readable. Exiting.", + configFile.getAbsoluteFile(), e); + System.exit(1); + } catch (ArrayIndexOutOfBoundsException e) { + logger.error("Found invalid entry in configuration file {} containing " + + "fewer than 2 space-separated parts. Fix that line. Exiting.", + configFile.getAbsoluteFile()); + System.exit(1); + } + if (null == jdbcString || null == importDirectory) { + logger.error("Missing at least one mandatory line in configuration file " + + "{}. Be sure to configure ExoneraTorDatabaseJdbc and " + + "ExoneraTorImportDirectory. Exiting.", + configFile.getAbsoluteFile()); System.exit(1); } + logger.debug("Read configuration file {}.", configFile.getAbsoluteFile()); } /* Database connection. */ @@ -99,9 +121,12 @@ public class ExoneraTorDatabaseImporter { try { connection = DriverManager.getConnection(jdbcString); } catch (SQLException e) { - logger.error("Could not connect to database. Exiting.", e); + logger.error("Caught an SQL exception while connecting to the database. " + + "Make sure that the database exists and that the configured JDBC " + + "string is correct.", e); System.exit(1); } + logger.debug("Connected to the database."); } /* Callable statements to import data into the database. */ @@ -117,8 +142,10 @@ public class ExoneraTorDatabaseImporter { insertExitlistentryStatement = connection.prepareCall( "{call insert_exitlistentry_exitaddress(?, ?, ?, ?)}"); } catch (SQLException e) { - logger.warn("Could not prepare callable statements to " - + "import data into the database. Exiting.", e); + logger.error("Caught an SQL exception while preparing callable " + + "statements for importing data into the database. Make sure that " + + "the configured database user has permissions to insert data. Also " + + "make sure that the database uses the correct database schema.", e); System.exit(1); } } @@ -127,30 +154,40 @@ public class ExoneraTorDatabaseImporter { * tool to run concurrently. */ private static void createLockFile() { File lockFile = new File("exonerator-lock"); - try { - if (lockFile.exists()) { - BufferedReader br = new BufferedReader(new FileReader(lockFile)); + if (lockFile.exists()) { + try (BufferedReader br = new BufferedReader(new FileReader(lockFile))) { Instant runStarted = Instant.ofEpochMilli(Long.parseLong( br.readLine())); - br.close(); if (runStarted.plus(Duration.ofHours(6L)) .compareTo(Instant.now()) >= 0) { - logger.warn("File 'exonerator-lock' is less than 6 " - + "hours old. Exiting."); + logger.error("Lock file {} is less than 6 hours old. Either make " + + "sure that there are no other ExoneraTor database importers " + + "running and manually delete that file, or wait until the file " + + "is 6 hours old when it will be overwritten automatically. " + + "Exiting.", lockFile.getAbsoluteFile()); System.exit(1); } else { - logger.warn("File 'exonerator-lock' is at least 6 hours old." - + " Overwriting and executing anyway."); + logger.warn("Lock file {} is at least 6 hours old. Overwriting and " + + "continuing with the database import.", + lockFile.getAbsoluteFile()); } + } catch (IOException e) { + logger.error("Caught an I/O exception when reading existing lock file " + + "{}. Make sure that this file is readable. Exiting.", + lockFile.getAbsoluteFile(), e); + System.exit(1); } - BufferedWriter bw = new BufferedWriter(new FileWriter( - "exonerator-lock")); + } + try (BufferedWriter bw = new BufferedWriter(new FileWriter(lockFile))) { bw.append(String.valueOf(System.currentTimeMillis())).append("\n"); - bw.close(); } catch (IOException e) { - logger.warn("Could not create 'exonerator-lock' file. Exiting."); + logger.error("Caught an I/O exception when creating lock file {}. Make " + + "sure that the parent directory exists and that the user running " + + "the ExoneraTor database importer has permissions to create the " + + "lock file. Exiting.", lockFile.getAbsoluteFile(), e); System.exit(1); } + logger.debug("Created lock file {}.", lockFile.getAbsoluteFile()); } /* Fetch recent descriptors from CollecTor. */ @@ -159,7 +196,7 @@ public class ExoneraTorDatabaseImporter { DescriptorSourceFactory.createDescriptorCollector(); collector.collectDescriptors("https://collector.torproject.org", new String[] { "/recent/relay-descriptors/consensuses/", - "/recent/exit-lists/" }, 0L, new File(importDirString), true); + "/recent/exit-lists/" }, 0L, importDirectory, true); } /* Last and next parse histories containing paths of parsed files and @@ -168,34 +205,55 @@ public class ExoneraTorDatabaseImporter { private static SortedMap<String, Long> nextImportHistory = new TreeMap<>(); + /* Parse history file. */ + private static File parseHistoryFile = new File("stats", + "exonerator-import-history"); + /* Read stats/exonerator-import-history file from disk and remember * locally when files were last parsed. */ private static void readImportHistoryToMemory() { - File parseHistoryFile = new File("stats", - "exonerator-import-history"); if (parseHistoryFile.exists()) { - try (BufferedReader br = new BufferedReader(new FileReader( - parseHistoryFile))) { + try (LineNumberReader lnr = new LineNumberReader(new FileReader( + parseHistoryFile))) { String line; - int lineNumber = 0; - while ((line = br.readLine()) != null) { - lineNumber++; + while ((line = lnr.readLine()) != null) { + Long lastModified = null; + String filename = null; String[] parts = line.split(","); - if (parts.length != 2) { - logger.warn("File 'stats/exonerator-import-history' " - + "contains a corrupt entry in line {}. " - + "Ignoring parse history file entirely.", lineNumber); + if (parts.length == 2) { + try { + lastModified = Long.parseLong(parts[0]); + filename = parts[1]; + } catch (NumberFormatException e) { + /* Handle below. */ + } + } + if (null != lastModified && null != filename) { + lastImportHistory.put(filename, lastModified); + } else { + logger.warn("Read a corrupt entry in line {} of parse history file " + + "{}. Ignoring the parse history file entirely and moving on " + + "by parsing all descriptors in {}.", + lnr.getLineNumber(), parseHistoryFile.getAbsoluteFile(), + importDirectory.getAbsoluteFile()); lastImportHistory.clear(); return; } - long lastModified = Long.parseLong(parts[0]); - String filename = parts[1]; - lastImportHistory.put(filename, lastModified); } } catch (IOException e) { - logger.warn("Could not read import history. Ignoring.", e); + logger.warn("Caught an I/O exception while reading parse history file " + + "{}. Ignoring the parse history file entirely and moving on " + + "by parsing all descriptors in {}.", + parseHistoryFile.getAbsoluteFile(), + importDirectory.getAbsoluteFile(), e); lastImportHistory.clear(); + return; } + logger.debug("Read parse history file {} and extracted {} entries.", + parseHistoryFile.getAbsoluteFile(), lastImportHistory.size()); + } else { + logger.debug("Not reading parse history file {}, because it does not yet " + + "exist.", parseHistoryFile.getAbsoluteFile()); } } @@ -205,23 +263,48 @@ public class ExoneraTorDatabaseImporter { DescriptorSourceFactory.createDescriptorReader(); descriptorReader.setMaxDescriptorsInQueue(20); descriptorReader.setExcludedFiles(lastImportHistory); + int parsedConsensuses = 0; + int parsedExitLists = 0; + int unparseableDescriptors = 0; for (Descriptor descriptor : descriptorReader.readDescriptors( - new File(importDirString))) { + importDirectory)) { if (descriptor instanceof RelayNetworkStatusConsensus) { parseConsensus((RelayNetworkStatusConsensus) descriptor); + parsedConsensuses++; } else if (descriptor instanceof ExitList) { parseExitList((ExitList) descriptor); + parsedExitLists++; + } else if (descriptor instanceof UnparseableDescriptor) { + logger.debug("Found descriptor in {} to be unparseable. Check the " + + "descriptor parse exception and/or descriptor file for details. " + + "Skipping.", + descriptor.getDescriptorFile().getAbsoluteFile(), + ((UnparseableDescriptor) descriptor).getDescriptorParseException()); + unparseableDescriptors++; } } + if (unparseableDescriptors > 0) { + logger.warn("Found {} descriptors in {} to be unparseable and skipped " + + "them. Check the debug-level logs and/or descriptor files for " + + "details. If this happened due to a bug in the parsing code, " + + "manually delete the parse history file {} and run the database " + + "importer again. Continuing.", unparseableDescriptors, + importDirectory.getAbsoluteFile(), + parseHistoryFile.getAbsoluteFile()); + } nextImportHistory.putAll( descriptorReader.getExcludedFiles()); nextImportHistory.putAll(descriptorReader.getParsedFiles()); + logger.debug("Read {} consensuses and {} exit lists from {}.", + parsedConsensuses, parsedExitLists, importDirectory.getAbsoluteFile()); } /* Parse a consensus. */ private static void parseConsensus(RelayNetworkStatusConsensus consensus) { + Instant beforeParsingConsensus = Instant.now(); LocalDateTime validAfter = LocalDateTime.ofInstant(Instant.ofEpochMilli( consensus.getValidAfterMillis()), ZoneOffset.UTC); + int importedStatusEntries = 0; for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { if (entry.getFlags().contains("Running")) { String fingerprintBase64 = null; @@ -230,8 +313,10 @@ public class ExoneraTorDatabaseImporter { Hex.decodeHex(entry.getFingerprint().toCharArray())) .replace("=", ""); } catch (DecoderException e) { - logger.warn("Unable to decode hex fingerprint {} to convert it back " - + "to base64. Aborting import.", entry.getFingerprint(), e); + logger.error("Caught a decoder exception while converting hex " + + "fingerprint {} found in consensus with valid-after time {} to " + + "base64. This looks like a bug. Exiting.", + entry.getFingerprint(), validAfter, e); System.exit(1); } final String nickname = entry.getNickname(); @@ -248,8 +333,13 @@ public class ExoneraTorDatabaseImporter { } importStatusentry(validAfter, fingerprintBase64, nickname, exit, orAddresses); + importedStatusEntries++; } } + logger.debug("Parsed consensus with valid-after time {} and imported {} " + + "status entries with the Running flag into the database in {}.", + validAfter, importedStatusEntries, + Duration.between(beforeParsingConsensus, Instant.now())); } /* Import a status entry with one or more OR addresses into the @@ -304,8 +394,10 @@ public class ExoneraTorDatabaseImporter { orAddress.replaceAll("[\\[\\]]", "")); insertStatusentryStatement.setString(4, orAddress24); } else { - logger.error("Could not import status entry with IPv6 " - + "address '{}'. Exiting.", orAddress); + logger.error("Unable to parse IPv6 address {} found in status " + + "entry with base64-encoded fingerprint {} in consensus with " + + "valid-after time {}. This is likely a bug. Exiting.", + orAddress, fingerprintBase64, validAfter); System.exit(1); } } @@ -314,13 +406,20 @@ public class ExoneraTorDatabaseImporter { insertStatusentryStatement.execute(); } } catch (SQLException e) { - logger.error("Could not import status entry. Exiting.", e); + logger.error("Caught an SQL exception while importing status entry with " + + "base64-encoded fingerprint {} and valid-after time {}. Check the " + + "exception for details. Exiting.", + fingerprintBase64, validAfter, e); System.exit(1); } } /* Parse an exit list. */ private static void parseExitList(ExitList exitList) { + Instant beforeParsingExitList = Instant.now(); + LocalDateTime downloaded = LocalDateTime.ofInstant(Instant.ofEpochMilli( + exitList.getDownloadedMillis()), ZoneOffset.UTC); + int importedExitListEntries = 0; for (Entry entry : exitList.getEntries()) { for (Map.Entry<String, Long> e : entry.getExitAddresses().entrySet()) { String fingerprintBase64 = null; @@ -329,8 +428,10 @@ public class ExoneraTorDatabaseImporter { Hex.decodeHex(entry.getFingerprint().toCharArray())) .replace("=", ""); } catch (DecoderException ex) { - logger.warn("Unable to decode hex fingerprint {} to convert to " - + "base64. Aborting import.", entry.getFingerprint(), ex); + logger.error("Caught a decoder exception while converting hex " + + "fingerprint {} found in exit list downloaded (by CollecTor) " + + "at {} to base64. This looks like a bug. Exiting.", + entry.getFingerprint(), downloaded, ex); System.exit(1); } String exitAddress = e.getKey(); @@ -350,8 +451,13 @@ public class ExoneraTorDatabaseImporter { Instant.ofEpochMilli(e.getValue()), ZoneOffset.UTC); importExitlistentry(fingerprintBase64, exitAddress24, exitAddress, scanned); + importedExitListEntries++; } } + logger.debug("Parsed exit list downloaded (by CollecTor) at {} and " + + "imported {} exit list entries into the database in {}.", downloaded, + importedExitListEntries, + Duration.between(beforeParsingExitList, Instant.now())); } /* Import an exit list entry into the database. */ @@ -365,15 +471,20 @@ public class ExoneraTorDatabaseImporter { insertExitlistentryStatement.setObject(4, scanned); insertExitlistentryStatement.execute(); } catch (SQLException e) { - logger.error("Could not import exit list entry. Exiting.", e); + logger.error("Caught an SQL exception while importing exit list entry " + + "with base64-encoded fingerprint {}, exit address {}, and scan " + + "time {}. Check the exception for details. Exiting.", + fingerprintBase64, exitAddress, scanned, e); System.exit(1); } } /* Write parse history from memory to disk for the next execution. */ private static void writeImportHistoryToDisk() { - File parseHistoryFile = new File("stats/exonerator-import-history"); - parseHistoryFile.getParentFile().mkdirs(); + if (parseHistoryFile.getParentFile().mkdirs()) { + logger.debug("Created parent directory of parse history file {}.", + parseHistoryFile.getAbsoluteFile()); + } try (BufferedWriter bw = new BufferedWriter(new FileWriter( parseHistoryFile))) { for (Map.Entry<String, Long> historyEntry : @@ -382,27 +493,43 @@ public class ExoneraTorDatabaseImporter { + historyEntry.getKey() + "\n"); } } catch (IOException e) { - logger.warn("File 'stats/exonerator-import-history' could " - + "not be written. Ignoring.", e); + logger.warn("Caught an I/O exception while writing parse history file " + + "{}. The next execution might not be able to read this parse " + + "history and will parse all files in {}. Moving on, because there " + + "is nothing we can do about it.", + parseHistoryFile, importDirectory.getAbsoluteFile(), e); } + logger.debug("Wrote parse history file {}.", + parseHistoryFile.getAbsoluteFile()); } /* Close the database connection. */ private static void closeDatabaseConnection() { try { connection.close(); + logger.debug("Disconnected from database."); } catch (SQLException e) { - logger.warn("Could not close database connection. Ignoring.", e); + logger.warn("Caught an SQL exception while disconnecting from the " + + "database. Check the exception for details and ideally log into " + + "the database manually to check that everything has been imported " + + "correctly. Ignoring, because we were going to terminate anyway.", + e); } } /* Delete the exonerator-lock file to allow the next executing of this * tool. */ private static void deleteLockFile() { + Path lockFile = Paths.get("exonerator-lock"); try { - Files.delete(Paths.get("exonerator-lock")); + Files.delete(lockFile); + logger.debug("Deleted lock file {}.", lockFile); } catch (IOException e) { - logger.warn("Could not delete lock file \"exonerator-lock\"", e); + logger.warn("Caught an I/O exception while deleting lock file {}. This " + + "might prevent future executions from running until the lock file " + + "is 6 hours old and overwritten, provided that the file can be " + + "overwritten. Moving on, because we cannot do anything about it.", + lockFile, e); } } }

1 0

[exonerator/release] Set default locale US and default time zone UTC.
by karsten＠torproject.org 13 Apr '20

13 Apr '20

commit 53e94d8cd9fc889a62c0cbb39c247e98980323a4 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Apr 12 16:24:31 2020 +0200 Set default locale US and default time zone UTC. Part of #33655. --- CHANGELOG.md | 2 ++ .../torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java | 4 ++++ src/main/java/org/torproject/metrics/exonerator/QueryServlet.java | 2 -- src/main/java/org/torproject/metrics/exonerator/ServerMain.java | 5 +++++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d22bc6f..f6cc649 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ * Minor changes - Improve logging of the database importer. - Simplify logging configuration. + - Set default locale `US` and default time zone `UTC` at the + beginning of the execution. # Changes in version 4.3.0 - 2020-01-15 diff --git a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java index b4d68e5..6a12d15 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java +++ b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java @@ -39,9 +39,11 @@ import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.SortedMap; +import java.util.TimeZone; import java.util.TreeMap; /* Import Tor descriptors into the ExoneraTor database. */ @@ -52,6 +54,8 @@ public class ExoneraTorDatabaseImporter { /** Main function controlling the parsing process. */ public static void main(String[] args) { + Locale.setDefault(Locale.US); + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); logger.info("Starting ExoneraTor database importer."); readConfiguration(); openDatabaseConnection(); diff --git a/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java b/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java index dda63da..2b06ed2 100644 --- a/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java +++ b/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; -import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; import java.util.regex.Pattern; @@ -224,7 +223,6 @@ public class QueryServlet extends HttpServlet { String passedTimestampParameter) { LocalDate timestamp = null; SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); dateFormat.setLenient(false); if (passedTimestampParameter != null && passedTimestampParameter.length() > 0) { diff --git a/src/main/java/org/torproject/metrics/exonerator/ServerMain.java b/src/main/java/org/torproject/metrics/exonerator/ServerMain.java index 1819a3c..b8ff428 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ServerMain.java +++ b/src/main/java/org/torproject/metrics/exonerator/ServerMain.java @@ -9,6 +9,9 @@ import org.eclipse.jetty.xml.XmlConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Locale; +import java.util.TimeZone; + public class ServerMain { private static final Logger logger @@ -16,6 +19,8 @@ public class ServerMain { /** Starts the web server listening for incoming client connections. */ public static void main(String[] args) { + Locale.setDefault(Locale.US); + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); try { Resource jettyXml = Resource.newSystemResource("jetty.xml"); logger.info("Reading configuration from '{}'.", jettyXml);

1 0

[exonerator/release] Prepare for 4.4.0 release.
by karsten＠torproject.org 13 Apr '20

13 Apr '20

commit ca18d4cec6235bdf5caa3ee7f5a62b442916e33a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Apr 13 11:51:11 2020 +0200 Prepare for 4.4.0 release. --- CHANGELOG.md | 3 ++- build.xml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 923ab8b..afa432c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ -# Changes in version 4.?.? - 2020-0?-?? +# Changes in version 4.4.0 - 2020-04-13 * Medium changes - Default date picker to latest available date. + - Update to metrics-lib 2.11.0. * Minor changes - Improve logging of the database importer. diff --git a/build.xml b/build.xml index 419e828..4c186af 100644 --- a/build.xml +++ b/build.xml @@ -9,8 +9,8 @@ <property name="javadoc-title" value="ExoneraTor API Documentation"/> <property name="implementation-title" value="ExoneraTor" /> - <property name="release.version" value="4.3.0-dev" /> - <property name="metricslibversion" value="2.10.0" /> + <property name="release.version" value="4.4.0" /> + <property name="metricslibversion" value="2.11.0" /> <property name="jetty.version" value="-9.2.21.v20170120" /> <property name="warfile" value="exonerator-${release.version}.war"/>

1 0

[exonerator/release] Default date picker to latest available date.
by karsten＠torproject.org 13 Apr '20

13 Apr '20

commit 66d1c8189c54d9a608b0b0cdacc110a1a0c59249 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Apr 1 16:10:37 2020 +0200 Default date picker to latest available date. Implements #30260. --- CHANGELOG.md | 3 +++ .../java/org/torproject/metrics/exonerator/ExoneraTorServlet.java | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6cc649..923ab8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changes in version 4.?.? - 2020-0?-?? + * Medium changes + - Default date picker to latest available date. + * Minor changes - Improve logging of the database importer. - Simplify logging configuration. diff --git a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java index 59535ea..2649546 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java +++ b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java @@ -16,6 +16,7 @@ import java.io.PrintWriter; import java.io.StringWriter; import java.net.URL; import java.time.LocalDate; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -150,12 +151,15 @@ public class ExoneraTorServlet extends HttpServlet { ? (baseUrl + requestUri) : request.getRequestURL().toString(); /* Write form. */ + String defaultDateString = LocalDate.now(ZoneOffset.UTC) + .minusDays(2).toString(); boolean timestampOutOfRange = requestedDate.valid && (firstDate.valid && requestedDate.date.isBefore(firstDate.date) || (lastDate.valid && requestedDate.date.isAfter(lastDate.date))); this.writeForm(out, rb, relayIp, relayIpHasError || ("".equals(relayIp) && !requestedDate.empty), - requestedDate.asString, !relayIpHasError + requestedDate.valid ? requestedDate.asString : defaultDateString, + !relayIpHasError && !("".equals(relayIp) && !requestedDate.valid) && (!requestedDate.valid || timestampOutOfRange || (!"".equals(relayIp) && requestedDate.empty)), langStr);

1 0

[exonerator/master] Prepare for 4.4.0 release.
by karsten＠torproject.org 13 Apr '20

13 Apr '20

commit ca18d4cec6235bdf5caa3ee7f5a62b442916e33a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Apr 13 11:51:11 2020 +0200 Prepare for 4.4.0 release. --- CHANGELOG.md | 3 ++- build.xml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 923ab8b..afa432c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ -# Changes in version 4.?.? - 2020-0?-?? +# Changes in version 4.4.0 - 2020-04-13 * Medium changes - Default date picker to latest available date. + - Update to metrics-lib 2.11.0. * Minor changes - Improve logging of the database importer. diff --git a/build.xml b/build.xml index 419e828..4c186af 100644 --- a/build.xml +++ b/build.xml @@ -9,8 +9,8 @@ <property name="javadoc-title" value="ExoneraTor API Documentation"/> <property name="implementation-title" value="ExoneraTor" /> - <property name="release.version" value="4.3.0-dev" /> - <property name="metricslibversion" value="2.10.0" /> + <property name="release.version" value="4.4.0" /> + <property name="metricslibversion" value="2.11.0" /> <property name="jetty.version" value="-9.2.21.v20170120" /> <property name="warfile" value="exonerator-${release.version}.war"/>

1 0

[exonerator/master] Bump version to 4.4.0-dev.
by karsten＠torproject.org 13 Apr '20

13 Apr '20

commit 0ad296f7eeeea24713b8335af343ec457350c752 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Apr 13 11:57:43 2020 +0200 Bump version to 4.4.0-dev. --- CHANGELOG.md | 3 +++ build.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index afa432c..4e017b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changes in version 4.?.? - 2020-??-?? + + # Changes in version 4.4.0 - 2020-04-13 * Medium changes diff --git a/build.xml b/build.xml index 4c186af..d1ce3b4 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ <property name="javadoc-title" value="ExoneraTor API Documentation"/> <property name="implementation-title" value="ExoneraTor" /> - <property name="release.version" value="4.4.0" /> + <property name="release.version" value="4.4.0-dev" /> <property name="metricslibversion" value="2.11.0" /> <property name="jetty.version" value="-9.2.21.v20170120" /> <property name="warfile"

1 0

[metrics-lib/release] Bump version to 2.10.0-dev.
by karsten＠torproject.org 13 Apr '20

13 Apr '20

commit 4581035acf517695944442d143c5c6d931ae0022 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jan 15 22:45:16 2020 +0100 Bump version to 2.10.0-dev. --- CHANGELOG.md | 3 +++ build.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5671a86..bb8bb89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changes in version 2.1?.? - 2020-0?-?? + + # Changes in version 2.10.0 - 2020-01-15 * Medium changes diff --git a/build.xml b/build.xml index bf615f7..fcf737b 100644 --- a/build.xml +++ b/build.xml @@ -7,7 +7,7 @@ <project default="usage" name="metrics-lib" basedir="." xmlns:ivy="antlib:org.apache.ivy.ant"> - <property name="release.version" value="2.10.0" /> + <property name="release.version" value="2.10.0-dev" /> <property name="javadoc-title" value="Tor Metrics Library API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/** **/internal/** **/log/**" /> <property name="implementation-title" value="Tor Metrics Library" />

1 0