April 2020 - tor-commits - lists.torproject.org

[translation/torbutton-brandproperties_completed] https://gitweb.torproject.org/translation.git/commit/?h=torbutton-brandproperties_completed
by translation＠torproject.org 08 Apr '20

08 Apr '20

commit c95038d4e1587e7ca7c9e00232c5c1fd422766a6 Author: Translation commit bot <translation(a)torproject.org> Date: Wed Apr 8 14:51:24 2020 +0000 https://gitweb.torproject.org/translation.git/commit/?h=torbutton-brandprop… --- sv-SE/brand.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sv-SE/brand.properties b/sv-SE/brand.properties index 22f1382b26..362ab3abad 100644 --- a/sv-SE/brand.properties +++ b/sv-SE/brand.properties @@ -11,6 +11,6 @@ homePageSingleStartMain=Firefox Start, en snabb webbplats med inbyggd sökfunkti homePageImport=Importera din startsida från %S homePageMigrationPageTitle=Val av startsida -homePageMigrationDescription=Välj den startsida du vill använda: +homePageMigrationDescription=Vänligen välj den startsida du vill använda: syncBrandShortName=Synka

1 0

[translation/torbutton-brandproperties] https://gitweb.torproject.org/translation.git/commit/?h=torbutton-brandproperties
by translation＠torproject.org 08 Apr '20

08 Apr '20

commit 54db6c7f88e8b0cedb46b67b5f7a92f29da8e43c Author: Translation commit bot <translation(a)torproject.org> Date: Wed Apr 8 14:51:19 2020 +0000 https://gitweb.torproject.org/translation.git/commit/?h=torbutton-brandprop… --- sv-SE/brand.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sv-SE/brand.properties b/sv-SE/brand.properties index 22f1382b26..362ab3abad 100644 --- a/sv-SE/brand.properties +++ b/sv-SE/brand.properties @@ -11,6 +11,6 @@ homePageSingleStartMain=Firefox Start, en snabb webbplats med inbyggd sökfunkti homePageImport=Importera din startsida från %S homePageMigrationPageTitle=Val av startsida -homePageMigrationDescription=Välj den startsida du vill använda: +homePageMigrationDescription=Vänligen välj den startsida du vill använda: syncBrandShortName=Synka

1 0

[translation/bridgedb] https://gitweb.torproject.org/translation.git/commit/?h=bridgedb
by translation＠torproject.org 08 Apr '20

08 Apr '20

commit da11ff534a21ed9ad754fde5c5713c6ab649c8ec Author: Translation commit bot <translation(a)torproject.org> Date: Wed Apr 8 14:45:14 2020 +0000 https://gitweb.torproject.org/translation.git/commit/?h=bridgedb --- sv/LC_MESSAGES/bridgedb.po | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sv/LC_MESSAGES/bridgedb.po b/sv/LC_MESSAGES/bridgedb.po index d6241c2596..0ea43da53a 100644 --- a/sv/LC_MESSAGES/bridgedb.po +++ b/sv/LC_MESSAGES/bridgedb.po @@ -20,7 +20,7 @@ msgstr "" "Project-Id-Version: Tor Project\n" "Report-Msgid-Bugs-To: 'https://trac.torproject.org/projects/tor/newticket?component=BridgeDB&keywo…'\n" "POT-Creation-Date: 2020-04-07 10:14-0700\n" -"PO-Revision-Date: 2020-04-08 05:37+0000\n" +"PO-Revision-Date: 2020-04-08 14:45+0000\n" "Last-Translator: Jonatan Nyberg\n" "Language-Team: Swedish (http://www.transifex.com/otf/torproject/language/sv/)\n" "MIME-Version: 1.0\n" @@ -227,7 +227,7 @@ msgid "" "difficult for anyone watching your internet traffic to determine that you are\n" "using Tor.\n" "\n" -msgstr "BridgeDB kan tillhandahålla broar med flera %styper av Pluggable Transports%s,\nsom kan omforma din trafik till Tor Network, vilket gör det svårare för någon\nsom avlyssnar din internetuppkoppling att veta ifall du använder Tor.\n\n" +msgstr "BridgeDB kan tillhandahålla broar med flera %styper av Pluggbara transporter%s,\nsom kan omforma din trafik till Tor Network, vilket gör det svårare för någon\nsom avlyssnar din internetuppkoppling att veta ifall du använder Tor.\n\n" #. TRANSLATORS: Please DO NOT translate "Pluggable Transports". #: bridgedb/strings.py:66 @@ -235,7 +235,7 @@ msgid "" "Some bridges with IPv6 addresses are also available, though some Pluggable\n" "Transports aren't IPv6 compatible.\n" "\n" -msgstr "Det finns några broar med IPv6-adresser tillgängliga, men vissa Pluggable\nTransports är inte IPv6-kompatibla.\n" +msgstr "Det finns några broar med IPv6-adresser tillgängliga, men vissa Pluggbara\n transporter är inte IPv6-kompatibla.\n" #. TRANSLATORS: Please DO NOT translate "BridgeDB". #. TRANSLATORS: The phrase "plain-ol'-vanilla" means "plain, boring, @@ -249,7 +249,7 @@ msgid "" "Pluggable Transports %s which maybe doesn't sound as cool, but they can still\n" "help to circumvent internet censorship in many cases.\n" "\n" -msgstr "Dessutom har BridgeDB några alldeles vanliga broar %s utan Pluggable\nTransports %s vilket kanske inte låter så häftigt, men i många fall kan de \nfortfarande hjälpa dig med att kringgå internet-censur.\n\n" +msgstr "Dessutom har BridgeDB några alldeles vanliga broar %s utan Pluggbara\ntransporter %s vilket kanske inte låter så häftigt, men i många fall kan de \nfortfarande hjälpa dig med att kringgå internet-censur.\n\n" #: bridgedb/strings.py:87 bridgedb/test/test_https.py:356 msgid "What are bridges?"

1 0

[tor/release-0.4.3] Correct 'was not internal' to 'was internal' in test_external_ip()
by dgoulet＠torproject.org 08 Apr '20

08 Apr '20

commit 34faee0600e4efe2a76a1776f6c9bd3173b3e0c5 Author: Mrigyen Sawant <Mrigyen(a)users.noreply.github.com> Date: Tue Apr 7 23:32:39 2020 +0530 Correct 'was not internal' to 'was internal' in test_external_ip() --- src/test/test_addr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index af3c125a2..42232e467 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -110,7 +110,7 @@ test_addr_basic(void *arg) tt_int_op(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), OP_EQ, 1); \ t1.family = AF_INET6; \ if (tor_addr_is_internal(&t1, for_listening)) \ - TT_DIE(("%s was not internal", a)); \ + TT_DIE(("%s was internal", a)); \ STMT_END #ifndef COCCI

1 0

[tor/release-0.4.3] Merge branch 'maint-0.4.3' into release-0.4.3
by dgoulet＠torproject.org 08 Apr '20

08 Apr '20

commit a985f554c491638c56b802d93409e3bb56a24fd3 Merge: e8bbdc69d 34faee060 Author: David Goulet <dgoulet(a)torproject.org> Date: Wed Apr 8 09:42:13 2020 -0400 Merge branch 'maint-0.4.3' into release-0.4.3 src/test/test_addr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[tor/master] Merge branch 'maint-0.4.3'
by dgoulet＠torproject.org 08 Apr '20

08 Apr '20

commit c1bf819a31d729abee8cf1440cefb7b4776248cc Merge: 0b0b8a5b6 34faee060 Author: David Goulet <dgoulet(a)torproject.org> Date: Wed Apr 8 09:42:13 2020 -0400 Merge branch 'maint-0.4.3' src/test/test_addr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[tor/master] Correct 'was not internal' to 'was internal' in test_external_ip()
by dgoulet＠torproject.org 08 Apr '20

08 Apr '20

commit 34faee0600e4efe2a76a1776f6c9bd3173b3e0c5 Author: Mrigyen Sawant <Mrigyen(a)users.noreply.github.com> Date: Tue Apr 7 23:32:39 2020 +0530 Correct 'was not internal' to 'was internal' in test_external_ip() --- src/test/test_addr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index af3c125a2..42232e467 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -110,7 +110,7 @@ test_addr_basic(void *arg) tt_int_op(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), OP_EQ, 1); \ t1.family = AF_INET6; \ if (tor_addr_is_internal(&t1, for_listening)) \ - TT_DIE(("%s was not internal", a)); \ + TT_DIE(("%s was internal", a)); \ STMT_END #ifndef COCCI

1 0

[tor/maint-0.4.3] Correct 'was not internal' to 'was internal' in test_external_ip()
by dgoulet＠torproject.org 08 Apr '20

08 Apr '20

commit 34faee0600e4efe2a76a1776f6c9bd3173b3e0c5 Author: Mrigyen Sawant <Mrigyen(a)users.noreply.github.com> Date: Tue Apr 7 23:32:39 2020 +0530 Correct 'was not internal' to 'was internal' in test_external_ip() --- src/test/test_addr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/test_addr.c b/src/test/test_addr.c index af3c125a2..42232e467 100644 --- a/src/test/test_addr.c +++ b/src/test/test_addr.c @@ -110,7 +110,7 @@ test_addr_basic(void *arg) tt_int_op(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), OP_EQ, 1); \ t1.family = AF_INET6; \ if (tor_addr_is_internal(&t1, for_listening)) \ - TT_DIE(("%s was not internal", a)); \ + TT_DIE(("%s was internal", a)); \ STMT_END #ifndef COCCI

1 0

[exonerator/master] Simplify logging configuration.
by karsten＠torproject.org 08 Apr '20

08 Apr '20

commit 5a7c19700f740ff88c1a1ae05b2383851d8cf9c7 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Apr 8 08:52:45 2020 +0200 Simplify logging configuration. Implements #33549. --- CHANGELOG.md | 1 + build.xml | 2 +- src/build | 2 +- .../exonerator/ExoneraTorDatabaseImporter.java | 2 +- .../metrics/exonerator/ExoneraTorServlet.java | 4 +- .../metrics/exonerator/QueryResponse.java | 3 +- .../metrics/exonerator/QueryServlet.java | 17 +++---- .../torproject/metrics/exonerator/ServerMain.java | 7 +-- src/main/resources/logback.xml | 57 ---------------------- 9 files changed, 20 insertions(+), 75 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca10ec2..d22bc6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ * Minor changes - Improve logging of the database importer. + - Simplify logging configuration. # Changes in version 4.3.0 - 2020-01-15 diff --git a/build.xml b/build.xml index cdc919c..419e828 100644 --- a/build.xml +++ b/build.xml @@ -108,7 +108,7 @@ </archives> </restrict> <exclude name="**/Test*.class" /> - <zipfileset dir="${resources}" + <zipfileset dir="${buildresources}" prefix="" includes="logback.xml" /> <zipfileset dir="${resources}/web" diff --git a/src/build b/src/build index 264e498..b5e1a2d 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 264e498f54a20f7d299daaf2533d043f880e6a8b +Subproject commit b5e1a2d7b29e58cc0645f068a1ebf4377bf9d8b8 diff --git a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java index 53e58f7..b4d68e5 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java +++ b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java @@ -47,7 +47,7 @@ import java.util.TreeMap; /* Import Tor descriptors into the ExoneraTor database. */ public class ExoneraTorDatabaseImporter { - private static Logger logger + private static final Logger logger = LoggerFactory.getLogger(ExoneraTorDatabaseImporter.class); /** Main function controlling the parsing process. */ diff --git a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java index b3f5b59..59535ea 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java +++ b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorServlet.java @@ -36,7 +36,8 @@ public class ExoneraTorServlet extends HttpServlet { private static final long serialVersionUID = 1370088989739567509L; - private Logger logger; + private static final Logger logger + = LoggerFactory.getLogger(ExoneraTorServlet.class); private String exoneraTorHost = System.getProperty("exonerator.url", "https://exonerator.torproject.org"); @@ -49,7 +50,6 @@ public class ExoneraTorServlet extends HttpServlet { @Override public void init(ServletConfig config) throws ServletException { super.init(config); - this.logger = LoggerFactory.getLogger(ExoneraTorServlet.class); this.availableLanguageNames = new TreeMap<>(); for (String locale : this.availableLanguages) { ResourceBundle rb = ResourceBundle.getBundle("ExoneraTor", diff --git a/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java b/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java index aee8dbf..14f2fa9 100644 --- a/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java +++ b/src/main/java/org/torproject/metrics/exonerator/QueryResponse.java @@ -18,7 +18,8 @@ import java.util.SortedSet; /** Query response from the ExoneraTor database. */ public class QueryResponse { - private static Logger logger = LoggerFactory.getLogger(QueryResponse.class); + private static final Logger logger + = LoggerFactory.getLogger(QueryResponse.class); /* Actual version implemented by this class. */ private static final String VERSION = "1.0"; diff --git a/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java b/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java index ee61e28..dda63da 100644 --- a/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java +++ b/src/main/java/org/torproject/metrics/exonerator/QueryServlet.java @@ -46,7 +46,8 @@ public class QueryServlet extends HttpServlet { private static final long serialVersionUID = 7109011659099295183L; - private Logger logger; + private static final Logger logger + = LoggerFactory.getLogger(QueryServlet.class); private DataSource ds; @@ -56,15 +57,13 @@ public class QueryServlet extends HttpServlet { @Override public void init() { - this.logger = LoggerFactory.getLogger(QueryServlet.class); - /* Look up data source. */ try { Context cxt = new InitialContext(); this.ds = (DataSource) cxt.lookup("java:comp/env/jdbc/exonerator"); - this.logger.info("Successfully looked up data source."); + logger.info("Successfully looked up data source."); } catch (NamingException e) { - this.logger.warn("Could not look up data source", e); + logger.warn("Could not look up data source", e); } } @@ -341,17 +340,17 @@ public class QueryServlet extends HttpServlet { } } } catch (SQLException e) { - this.logger.warn("Result set error. Returning 'null'.", e); + logger.warn("Result set error. Returning 'null'.", e); return null; } - this.logger.info("Returned a database connection to the pool after {}.", + logger.info("Returned a database connection to the pool after {}.", Duration.between(requestedConnection, Instant.now())); } catch (SQLException e) { - this.logger.warn("Callable statement error. Returning 'null'.", e); + logger.warn("Callable statement error. Returning 'null'.", e); return null; } } catch (Throwable e) { // Catch all problems left. - this.logger.warn("Database error. Returning 'null'.", e); + logger.warn("Database error. Returning 'null'.", e); return null; } diff --git a/src/main/java/org/torproject/metrics/exonerator/ServerMain.java b/src/main/java/org/torproject/metrics/exonerator/ServerMain.java index 0663e0f..1819a3c 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ServerMain.java +++ b/src/main/java/org/torproject/metrics/exonerator/ServerMain.java @@ -11,20 +11,21 @@ import org.slf4j.LoggerFactory; public class ServerMain { - private static final Logger log = LoggerFactory.getLogger(ServerMain.class); + private static final Logger logger + = LoggerFactory.getLogger(ServerMain.class); /** Starts the web server listening for incoming client connections. */ public static void main(String[] args) { try { Resource jettyXml = Resource.newSystemResource("jetty.xml"); - log.info("Reading configuration from '{}'.", jettyXml); + logger.info("Reading configuration from '{}'.", jettyXml); XmlConfiguration configuration = new XmlConfiguration(jettyXml.getInputStream()); Server server = (Server) configuration.configure(); server.start(); server.join(); } catch (Exception ex) { - log.error("Exiting, because of: {}.", ex.getMessage(), ex); + logger.error("Exiting, because of: {}.", ex.getMessage(), ex); System.exit(1); } } diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml deleted file mode 100644 index c29c057..0000000 --- a/src/main/resources/logback.xml +++ /dev/null @@ -1,57 +0,0 @@ -<configuration debug="false"> - -  - <property name="logfile-base" value="${LOGBASE}/exonerator-" /> - -  - <property name="fileall-logname" value="${logfile-base}all" /> - <property name="fileerr-logname" value="${logfile-base}err" /> - <property name="filestatistics-logname" value="${logfile-base}statistics" /> - -  - <property name="utc-date-pattern" value="%date{ISO8601, UTC}" /> - -  - <appender name="FILEALL" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${fileall-logname}.log</file> - <encoder> - <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> - </encoder> - <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> -  - <FileNamePattern>${fileall-logname}.%d{yyyy-MM-dd}.%i.log</FileNamePattern> - <maxHistory>10</maxHistory> - <timeBasedFileNamingAndTriggeringPolicy - class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> -  - <maxFileSize>1MB</maxFileSize> - </timeBasedFileNamingAndTriggeringPolicy> - </rollingPolicy> - </appender> - - <appender name="FILEERR" class="ch.qos.logback.core.FileAppender"> - <file>${fileerr-logname}.log</file> - <encoder> - <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> - </encoder> - -  - <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> - <level>ERROR</level> - </filter> - </appender> - -  - <logger name="org.torproject" > - <appender-ref ref="FILEERR" /> - </logger> - - <logger name="org.eclipse" level="INFO" /> - <logger name="org.apache" level="INFO" /> - - <root level="INFO"> - <appender-ref ref="FILEALL" /> - </root> - -</configuration> -

1 0

[exonerator/master] Improve logging of database importer.
by karsten＠torproject.org 08 Apr '20

08 Apr '20

commit 54ad288276bd2313ff81a71406f027801e51fa55 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Mar 15 14:31:10 2020 +0100 Improve logging of database importer. This patch focuses on providing more context in log messages and unify the use of log levels. It also adds counters and timestamps as far as they are being used in log messages. And in very few cases it contains minor changes like using try-with-resource when we're touching that code anyway. But most other refactorings are left as future work. Fixes the first part of #24542. The second part will be to improve logging of the web server parts. --- CHANGELOG.md | 3 + .../exonerator/ExoneraTorDatabaseImporter.java | 239 ++++++++++++++++----- 2 files changed, 186 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c87922..ca10ec2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changes in version 4.?.? - 2020-0?-?? + * Minor changes + - Improve logging of the database importer. + # Changes in version 4.3.0 - 2020-01-15 diff --git a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java index 3dbb119..53e58f7 100644 --- a/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java +++ b/src/main/java/org/torproject/metrics/exonerator/ExoneraTorDatabaseImporter.java @@ -11,6 +11,7 @@ import org.torproject.descriptor.ExitList; import org.torproject.descriptor.ExitList.Entry; import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.UnparseableDescriptor; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Base64; @@ -25,7 +26,9 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.io.LineNumberReader; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.sql.CallableStatement; import java.sql.Connection; @@ -49,46 +52,65 @@ public class ExoneraTorDatabaseImporter { /** Main function controlling the parsing process. */ public static void main(String[] args) { + logger.info("Starting ExoneraTor database importer."); readConfiguration(); openDatabaseConnection(); prepareDatabaseStatements(); createLockFile(); + logger.info("Fetching descriptors from CollecTor."); fetchDescriptors(); + logger.info("Importing descriptors into the database."); readImportHistoryToMemory(); parseDescriptors(); writeImportHistoryToDisk(); closeDatabaseConnection(); deleteLockFile(); + logger.info("Terminating ExoneraTor database importer."); } - /* JDBC string of the ExoneraTor database. */ + /* JDBC string of the ExoneraTor database. Never include this in a log + * message, because it likely contains the database password! */ private static String jdbcString; /* Directory from which to import descriptors. */ - private static String importDirString; + private static File importDirectory; /* Learn JDBC string and directory to parse descriptors from. */ private static void readConfiguration() { File configFile = new File("config"); if (!configFile.exists()) { - logger.error("Could not find config file. Exiting."); + logger.error("Could not find configuration file {}. Make sure that this " + + "file exists. Exiting.", configFile.getAbsoluteFile()); System.exit(1); } - String line; - try { - BufferedReader br = new BufferedReader(new FileReader(configFile)); + try (BufferedReader br = new BufferedReader(new FileReader(configFile))) { + String line; while ((line = br.readLine()) != null) { if (line.startsWith("ExoneraTorDatabaseJdbc")) { jdbcString = line.split(" ")[1]; } else if (line.startsWith("ExoneraTorImportDirectory")) { - importDirString = line.split(" ")[1]; + importDirectory = new File(line.split(" ")[1]); } } - br.close(); } catch (IOException e) { - logger.error("Could not parse config file. Exiting.", e); + logger.error("Caught an I/O exception while reading configuration file " + + "{}. Make sure that this file is readable. Exiting.", + configFile.getAbsoluteFile(), e); + System.exit(1); + } catch (ArrayIndexOutOfBoundsException e) { + logger.error("Found invalid entry in configuration file {} containing " + + "fewer than 2 space-separated parts. Fix that line. Exiting.", + configFile.getAbsoluteFile()); + System.exit(1); + } + if (null == jdbcString || null == importDirectory) { + logger.error("Missing at least one mandatory line in configuration file " + + "{}. Be sure to configure ExoneraTorDatabaseJdbc and " + + "ExoneraTorImportDirectory. Exiting.", + configFile.getAbsoluteFile()); System.exit(1); } + logger.debug("Read configuration file {}.", configFile.getAbsoluteFile()); } /* Database connection. */ @@ -99,9 +121,12 @@ public class ExoneraTorDatabaseImporter { try { connection = DriverManager.getConnection(jdbcString); } catch (SQLException e) { - logger.error("Could not connect to database. Exiting.", e); + logger.error("Caught an SQL exception while connecting to the database. " + + "Make sure that the database exists and that the configured JDBC " + + "string is correct.", e); System.exit(1); } + logger.debug("Connected to the database."); } /* Callable statements to import data into the database. */ @@ -117,8 +142,10 @@ public class ExoneraTorDatabaseImporter { insertExitlistentryStatement = connection.prepareCall( "{call insert_exitlistentry_exitaddress(?, ?, ?, ?)}"); } catch (SQLException e) { - logger.warn("Could not prepare callable statements to " - + "import data into the database. Exiting.", e); + logger.error("Caught an SQL exception while preparing callable " + + "statements for importing data into the database. Make sure that " + + "the configured database user has permissions to insert data. Also " + + "make sure that the database uses the correct database schema.", e); System.exit(1); } } @@ -127,30 +154,40 @@ public class ExoneraTorDatabaseImporter { * tool to run concurrently. */ private static void createLockFile() { File lockFile = new File("exonerator-lock"); - try { - if (lockFile.exists()) { - BufferedReader br = new BufferedReader(new FileReader(lockFile)); + if (lockFile.exists()) { + try (BufferedReader br = new BufferedReader(new FileReader(lockFile))) { Instant runStarted = Instant.ofEpochMilli(Long.parseLong( br.readLine())); - br.close(); if (runStarted.plus(Duration.ofHours(6L)) .compareTo(Instant.now()) >= 0) { - logger.warn("File 'exonerator-lock' is less than 6 " - + "hours old. Exiting."); + logger.error("Lock file {} is less than 6 hours old. Either make " + + "sure that there are no other ExoneraTor database importers " + + "running and manually delete that file, or wait until the file " + + "is 6 hours old when it will be overwritten automatically. " + + "Exiting.", lockFile.getAbsoluteFile()); System.exit(1); } else { - logger.warn("File 'exonerator-lock' is at least 6 hours old." - + " Overwriting and executing anyway."); + logger.warn("Lock file {} is at least 6 hours old. Overwriting and " + + "continuing with the database import.", + lockFile.getAbsoluteFile()); } + } catch (IOException e) { + logger.error("Caught an I/O exception when reading existing lock file " + + "{}. Make sure that this file is readable. Exiting.", + lockFile.getAbsoluteFile(), e); + System.exit(1); } - BufferedWriter bw = new BufferedWriter(new FileWriter( - "exonerator-lock")); + } + try (BufferedWriter bw = new BufferedWriter(new FileWriter(lockFile))) { bw.append(String.valueOf(System.currentTimeMillis())).append("\n"); - bw.close(); } catch (IOException e) { - logger.warn("Could not create 'exonerator-lock' file. Exiting."); + logger.error("Caught an I/O exception when creating lock file {}. Make " + + "sure that the parent directory exists and that the user running " + + "the ExoneraTor database importer has permissions to create the " + + "lock file. Exiting.", lockFile.getAbsoluteFile(), e); System.exit(1); } + logger.debug("Created lock file {}.", lockFile.getAbsoluteFile()); } /* Fetch recent descriptors from CollecTor. */ @@ -159,7 +196,7 @@ public class ExoneraTorDatabaseImporter { DescriptorSourceFactory.createDescriptorCollector(); collector.collectDescriptors("https://collector.torproject.org", new String[] { "/recent/relay-descriptors/consensuses/", - "/recent/exit-lists/" }, 0L, new File(importDirString), true); + "/recent/exit-lists/" }, 0L, importDirectory, true); } /* Last and next parse histories containing paths of parsed files and @@ -168,34 +205,55 @@ public class ExoneraTorDatabaseImporter { private static SortedMap<String, Long> nextImportHistory = new TreeMap<>(); + /* Parse history file. */ + private static File parseHistoryFile = new File("stats", + "exonerator-import-history"); + /* Read stats/exonerator-import-history file from disk and remember * locally when files were last parsed. */ private static void readImportHistoryToMemory() { - File parseHistoryFile = new File("stats", - "exonerator-import-history"); if (parseHistoryFile.exists()) { - try (BufferedReader br = new BufferedReader(new FileReader( - parseHistoryFile))) { + try (LineNumberReader lnr = new LineNumberReader(new FileReader( + parseHistoryFile))) { String line; - int lineNumber = 0; - while ((line = br.readLine()) != null) { - lineNumber++; + while ((line = lnr.readLine()) != null) { + Long lastModified = null; + String filename = null; String[] parts = line.split(","); - if (parts.length != 2) { - logger.warn("File 'stats/exonerator-import-history' " - + "contains a corrupt entry in line {}. " - + "Ignoring parse history file entirely.", lineNumber); + if (parts.length == 2) { + try { + lastModified = Long.parseLong(parts[0]); + filename = parts[1]; + } catch (NumberFormatException e) { + /* Handle below. */ + } + } + if (null != lastModified && null != filename) { + lastImportHistory.put(filename, lastModified); + } else { + logger.warn("Read a corrupt entry in line {} of parse history file " + + "{}. Ignoring the parse history file entirely and moving on " + + "by parsing all descriptors in {}.", + lnr.getLineNumber(), parseHistoryFile.getAbsoluteFile(), + importDirectory.getAbsoluteFile()); lastImportHistory.clear(); return; } - long lastModified = Long.parseLong(parts[0]); - String filename = parts[1]; - lastImportHistory.put(filename, lastModified); } } catch (IOException e) { - logger.warn("Could not read import history. Ignoring.", e); + logger.warn("Caught an I/O exception while reading parse history file " + + "{}. Ignoring the parse history file entirely and moving on " + + "by parsing all descriptors in {}.", + parseHistoryFile.getAbsoluteFile(), + importDirectory.getAbsoluteFile(), e); lastImportHistory.clear(); + return; } + logger.debug("Read parse history file {} and extracted {} entries.", + parseHistoryFile.getAbsoluteFile(), lastImportHistory.size()); + } else { + logger.debug("Not reading parse history file {}, because it does not yet " + + "exist.", parseHistoryFile.getAbsoluteFile()); } } @@ -205,23 +263,48 @@ public class ExoneraTorDatabaseImporter { DescriptorSourceFactory.createDescriptorReader(); descriptorReader.setMaxDescriptorsInQueue(20); descriptorReader.setExcludedFiles(lastImportHistory); + int parsedConsensuses = 0; + int parsedExitLists = 0; + int unparseableDescriptors = 0; for (Descriptor descriptor : descriptorReader.readDescriptors( - new File(importDirString))) { + importDirectory)) { if (descriptor instanceof RelayNetworkStatusConsensus) { parseConsensus((RelayNetworkStatusConsensus) descriptor); + parsedConsensuses++; } else if (descriptor instanceof ExitList) { parseExitList((ExitList) descriptor); + parsedExitLists++; + } else if (descriptor instanceof UnparseableDescriptor) { + logger.debug("Found descriptor in {} to be unparseable. Check the " + + "descriptor parse exception and/or descriptor file for details. " + + "Skipping.", + descriptor.getDescriptorFile().getAbsoluteFile(), + ((UnparseableDescriptor) descriptor).getDescriptorParseException()); + unparseableDescriptors++; } } + if (unparseableDescriptors > 0) { + logger.warn("Found {} descriptors in {} to be unparseable and skipped " + + "them. Check the debug-level logs and/or descriptor files for " + + "details. If this happened due to a bug in the parsing code, " + + "manually delete the parse history file {} and run the database " + + "importer again. Continuing.", unparseableDescriptors, + importDirectory.getAbsoluteFile(), + parseHistoryFile.getAbsoluteFile()); + } nextImportHistory.putAll( descriptorReader.getExcludedFiles()); nextImportHistory.putAll(descriptorReader.getParsedFiles()); + logger.debug("Read {} consensuses and {} exit lists from {}.", + parsedConsensuses, parsedExitLists, importDirectory.getAbsoluteFile()); } /* Parse a consensus. */ private static void parseConsensus(RelayNetworkStatusConsensus consensus) { + Instant beforeParsingConsensus = Instant.now(); LocalDateTime validAfter = LocalDateTime.ofInstant(Instant.ofEpochMilli( consensus.getValidAfterMillis()), ZoneOffset.UTC); + int importedStatusEntries = 0; for (NetworkStatusEntry entry : consensus.getStatusEntries().values()) { if (entry.getFlags().contains("Running")) { String fingerprintBase64 = null; @@ -230,8 +313,10 @@ public class ExoneraTorDatabaseImporter { Hex.decodeHex(entry.getFingerprint().toCharArray())) .replace("=", ""); } catch (DecoderException e) { - logger.warn("Unable to decode hex fingerprint {} to convert it back " - + "to base64. Aborting import.", entry.getFingerprint(), e); + logger.error("Caught a decoder exception while converting hex " + + "fingerprint {} found in consensus with valid-after time {} to " + + "base64. This looks like a bug. Exiting.", + entry.getFingerprint(), validAfter, e); System.exit(1); } final String nickname = entry.getNickname(); @@ -248,8 +333,13 @@ public class ExoneraTorDatabaseImporter { } importStatusentry(validAfter, fingerprintBase64, nickname, exit, orAddresses); + importedStatusEntries++; } } + logger.debug("Parsed consensus with valid-after time {} and imported {} " + + "status entries with the Running flag into the database in {}.", + validAfter, importedStatusEntries, + Duration.between(beforeParsingConsensus, Instant.now())); } /* Import a status entry with one or more OR addresses into the @@ -304,8 +394,10 @@ public class ExoneraTorDatabaseImporter { orAddress.replaceAll("[\\[\\]]", "")); insertStatusentryStatement.setString(4, orAddress24); } else { - logger.error("Could not import status entry with IPv6 " - + "address '{}'. Exiting.", orAddress); + logger.error("Unable to parse IPv6 address {} found in status " + + "entry with base64-encoded fingerprint {} in consensus with " + + "valid-after time {}. This is likely a bug. Exiting.", + orAddress, fingerprintBase64, validAfter); System.exit(1); } } @@ -314,13 +406,20 @@ public class ExoneraTorDatabaseImporter { insertStatusentryStatement.execute(); } } catch (SQLException e) { - logger.error("Could not import status entry. Exiting.", e); + logger.error("Caught an SQL exception while importing status entry with " + + "base64-encoded fingerprint {} and valid-after time {}. Check the " + + "exception for details. Exiting.", + fingerprintBase64, validAfter, e); System.exit(1); } } /* Parse an exit list. */ private static void parseExitList(ExitList exitList) { + Instant beforeParsingExitList = Instant.now(); + LocalDateTime downloaded = LocalDateTime.ofInstant(Instant.ofEpochMilli( + exitList.getDownloadedMillis()), ZoneOffset.UTC); + int importedExitListEntries = 0; for (Entry entry : exitList.getEntries()) { for (Map.Entry<String, Long> e : entry.getExitAddresses().entrySet()) { String fingerprintBase64 = null; @@ -329,8 +428,10 @@ public class ExoneraTorDatabaseImporter { Hex.decodeHex(entry.getFingerprint().toCharArray())) .replace("=", ""); } catch (DecoderException ex) { - logger.warn("Unable to decode hex fingerprint {} to convert to " - + "base64. Aborting import.", entry.getFingerprint(), ex); + logger.error("Caught a decoder exception while converting hex " + + "fingerprint {} found in exit list downloaded (by CollecTor) " + + "at {} to base64. This looks like a bug. Exiting.", + entry.getFingerprint(), downloaded, ex); System.exit(1); } String exitAddress = e.getKey(); @@ -350,8 +451,13 @@ public class ExoneraTorDatabaseImporter { Instant.ofEpochMilli(e.getValue()), ZoneOffset.UTC); importExitlistentry(fingerprintBase64, exitAddress24, exitAddress, scanned); + importedExitListEntries++; } } + logger.debug("Parsed exit list downloaded (by CollecTor) at {} and " + + "imported {} exit list entries into the database in {}.", downloaded, + importedExitListEntries, + Duration.between(beforeParsingExitList, Instant.now())); } /* Import an exit list entry into the database. */ @@ -365,15 +471,20 @@ public class ExoneraTorDatabaseImporter { insertExitlistentryStatement.setObject(4, scanned); insertExitlistentryStatement.execute(); } catch (SQLException e) { - logger.error("Could not import exit list entry. Exiting.", e); + logger.error("Caught an SQL exception while importing exit list entry " + + "with base64-encoded fingerprint {}, exit address {}, and scan " + + "time {}. Check the exception for details. Exiting.", + fingerprintBase64, exitAddress, scanned, e); System.exit(1); } } /* Write parse history from memory to disk for the next execution. */ private static void writeImportHistoryToDisk() { - File parseHistoryFile = new File("stats/exonerator-import-history"); - parseHistoryFile.getParentFile().mkdirs(); + if (parseHistoryFile.getParentFile().mkdirs()) { + logger.debug("Created parent directory of parse history file {}.", + parseHistoryFile.getAbsoluteFile()); + } try (BufferedWriter bw = new BufferedWriter(new FileWriter( parseHistoryFile))) { for (Map.Entry<String, Long> historyEntry : @@ -382,27 +493,43 @@ public class ExoneraTorDatabaseImporter { + historyEntry.getKey() + "\n"); } } catch (IOException e) { - logger.warn("File 'stats/exonerator-import-history' could " - + "not be written. Ignoring.", e); + logger.warn("Caught an I/O exception while writing parse history file " + + "{}. The next execution might not be able to read this parse " + + "history and will parse all files in {}. Moving on, because there " + + "is nothing we can do about it.", + parseHistoryFile, importDirectory.getAbsoluteFile(), e); } + logger.debug("Wrote parse history file {}.", + parseHistoryFile.getAbsoluteFile()); } /* Close the database connection. */ private static void closeDatabaseConnection() { try { connection.close(); + logger.debug("Disconnected from database."); } catch (SQLException e) { - logger.warn("Could not close database connection. Ignoring.", e); + logger.warn("Caught an SQL exception while disconnecting from the " + + "database. Check the exception for details and ideally log into " + + "the database manually to check that everything has been imported " + + "correctly. Ignoring, because we were going to terminate anyway.", + e); } } /* Delete the exonerator-lock file to allow the next executing of this * tool. */ private static void deleteLockFile() { + Path lockFile = Paths.get("exonerator-lock"); try { - Files.delete(Paths.get("exonerator-lock")); + Files.delete(lockFile); + logger.debug("Deleted lock file {}.", lockFile); } catch (IOException e) { - logger.warn("Could not delete lock file \"exonerator-lock\"", e); + logger.warn("Caught an I/O exception while deleting lock file {}. This " + + "might prevent future executions from running until the lock file " + + "is 6 hours old and overwritten, provided that the file can be " + + "overwritten. Moving on, because we cannot do anything about it.", + lockFile, e); } } }

1 0