[tor-commits] [metrics-web/master] Simplify logging configuration.

commit 334550d6d5c50e064c49f6ee713cff561e137fc6 Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue Mar 31 10:30:46 2020 +0200

Simplify logging configuration.

Implements #33549. --- CHANGELOG.md | 1 + src/build | 2 +- .../torproject/metrics/stats/bridgedb/Main.java | 16 +++--- .../org/torproject/metrics/stats/bwhist/Main.java | 14 +++--- .../bwhist/RelayDescriptorDatabaseImporter.java | 43 ++++++++-------- .../org/torproject/metrics/stats/clients/Main.java | 24 ++++----- .../metrics/stats/connbidirect/Main.java | 44 ++++++++-------- .../metrics/stats/hidserv/Aggregator.java | 7 +-- .../stats/hidserv/ComputedNetworkFractions.java | 12 ++--- .../metrics/stats/hidserv/DocumentStore.java | 17 ++++--- .../stats/hidserv/ExtrapolatedHidServStats.java | 6 +-- .../metrics/stats/hidserv/Extrapolator.java | 5 +- .../org/torproject/metrics/stats/hidserv/Main.java | 16 +++--- .../torproject/metrics/stats/hidserv/Parser.java | 19 +++---- .../stats/hidserv/ReportedHidServStats.java | 4 +- .../torproject/metrics/stats/hidserv/Simulate.java | 12 ++--- .../org/torproject/metrics/stats/main/Main.java | 30 +++++------ .../torproject/metrics/stats/onionperf/Main.java | 22 ++++---- .../org/torproject/metrics/stats/servers/Main.java | 24 ++++----- .../org/torproject/metrics/stats/totalcw/Main.java | 24 ++++----- .../torproject/metrics/stats/webstats/Main.java | 32 ++++++------ .../org/torproject/metrics/web/ServerMain.java | 7 +-- .../org/torproject/metrics/web/UpdateNews.java | 8 +-- src/main/resources/logback.xml | 58 ---------------------- src/submods/metrics-lib | 2 +- 25 files changed, 201 insertions(+), 248 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md index b1571c6..323d0e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ links into https:// links. - Set default locale `US` at the beginning of the execution. - Set default time zone `UTC` at the beginning of the execution. + - Simplify logging configuration.

# Changes in version 1.3.0 - 2019-11-09 diff --git a/src/build b/src/build index 264e498..fd85646 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 264e498f54a20f7d299daaf2533d043f880e6a8b +Subproject commit fd856466bcb260f53ef69a24c102d0e49d171cc3 diff --git a/src/main/java/org/torproject/metrics/stats/bridgedb/Main.java b/src/main/java/org/torproject/metrics/stats/bridgedb/Main.java index 989c695..f7e9fb4 100644 --- a/src/main/java/org/torproject/metrics/stats/bridgedb/Main.java +++ b/src/main/java/org/torproject/metrics/stats/bridgedb/Main.java @@ -24,7 +24,7 @@ import java.util.TreeMap;

public class Main {

- private static Logger log = LoggerFactory.getLogger(Main.class); + private static final Logger logger = LoggerFactory.getLogger(Main.class);

private static final Path bridgedbStatsCsvFile = org.torproject.metrics.stats.main.Main.modulesDir.toPath() @@ -55,7 +55,7 @@ public class Main { } String[] lineParts = line.split(","); if (lineParts.length != 4) { - log.warn("Skipping unrecognized line '{}' in {}.", line, + logger.warn("Skipping unrecognized line '{}' in {}.", line, bridgedbStatsCsvFile.toAbsolutePath()); continue; } @@ -64,8 +64,8 @@ public class Main { long value = Long.parseLong(lineParts[3]); readStatistics.put(key, value); } - log.debug("Read {} containing {} non-header lines.", bridgedbStatsCsvFile, - readStatistics.size()); + logger.debug("Read {} containing {} non-header lines.", + bridgedbStatsCsvFile, readStatistics.size()); } return readStatistics; } @@ -82,7 +82,7 @@ public class Main { } BridgedbMetrics bridgedbMetrics = (BridgedbMetrics) descriptor; if (!"1".equals(bridgedbMetrics.bridgedbMetricsVersion())) { - log.warn("Unable to process BridgeDB metrics version {} != 1.", + logger.warn("Unable to process BridgeDB metrics version {} != 1.", bridgedbMetrics.bridgedbMetricsVersion()); continue; } @@ -100,7 +100,7 @@ public class Main { continue; } if (bridgedbMetricCount.getValue() < 10) { - log.warn("Skipping too small BridgeDB metric count {} < 10 in {}.", + logger.warn("Skipping too small BridgeDB metric count {} < 10 in {}.", bridgedbMetricCount.getValue(), descriptor.getDescriptorFile().getAbsolutePath()); continue; @@ -141,8 +141,8 @@ public class Main { statistic.getValue())); } Files.write(bridgedbStatsCsvFile, lines, StandardOpenOption.CREATE); - log.debug("Wrote {} containing {} non-header lines.", bridgedbStatsCsvFile, - lines.size() - 1); + logger.debug("Wrote {} containing {} non-header lines.", + bridgedbStatsCsvFile, lines.size() - 1); } }

diff --git a/src/main/java/org/torproject/metrics/stats/bwhist/Main.java b/src/main/java/org/torproject/metrics/stats/bwhist/Main.java index 4e92f1c..e3befc0 100644 --- a/src/main/java/org/torproject/metrics/stats/bwhist/Main.java +++ b/src/main/java/org/torproject/metrics/stats/bwhist/Main.java @@ -15,7 +15,7 @@ import java.util.Arrays; */ public class Main {

- private static Logger log = LoggerFactory.getLogger(Main.class); + private static final Logger logger = LoggerFactory.getLogger(Main.class);

private static String[] paths = { "recent/relay-descriptors/consensuses", @@ -34,9 +34,9 @@ public class Main { /** Executes this data-processing module. */ public static void main(String[] args) throws Exception {

- log.info("Starting bwhist module."); + logger.info("Starting bwhist module.");

- log.info("Reading descriptors and inserting relevant parts into the " + logger.info("Reading descriptors and inserting relevant parts into the " + "database."); File[] descriptorDirectories = Arrays.stream(paths).map((String path) -> new File(org.torproject.metrics.stats.main.Main.descriptorsDir, @@ -47,17 +47,17 @@ public class Main { historyFile, jdbcString); database.importRelayDescriptors();

- log.info("Aggregating database entries."); + logger.info("Aggregating database entries."); database.aggregate();

- log.info("Querying aggregated statistics from the database."); + logger.info("Querying aggregated statistics from the database."); new Writer().write(new File(baseDir, "stats/bandwidth.csv").toPath(), database.queryBandwidth());

- log.info("Closing database connection."); + logger.info("Closing database connection."); database.closeConnection();

- log.info("Terminating bwhist module."); + logger.info("Terminating bwhist module."); } }

diff --git a/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java b/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java index 7b08f77..2afbecf 100644 --- a/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java +++ b/src/main/java/org/torproject/metrics/stats/bwhist/RelayDescriptorDatabaseImporter.java @@ -85,7 +85,7 @@ public final class RelayDescriptorDatabaseImporter { */ private CallableStatement csH;

- private static Logger log + private static final Logger logger = LoggerFactory.getLogger(RelayDescriptorDatabaseImporter.class);

/** @@ -141,7 +141,7 @@ public final class RelayDescriptorDatabaseImporter { + "(date) VALUES (?)"); this.scheduledUpdates = new HashSet<>(); } catch (SQLException e) { - log.warn("Could not connect to database or prepare statements.", e); + logger.warn("Could not connect to database or prepare statements.", e); } }

@@ -160,7 +160,7 @@ public final class RelayDescriptorDatabaseImporter { this.dateTimeFormat.format(timestamp).substring(0, 10) + " 00:00:00").getTime(); } catch (ParseException e) { - log.warn("Internal parsing error.", e); + logger.warn("Internal parsing error.", e); return; } if (!this.scheduledUpdates.contains(dateMillis)) { @@ -206,8 +206,8 @@ public final class RelayDescriptorDatabaseImporter { insertedStatusEntries.add(fingerprint); } } catch (SQLException e) { - log.warn("Could not add network status consensus entry. We won't make " - + "any further SQL requests in this execution.", e); + logger.warn("Could not add network status consensus entry. We won't " + + "make any further SQL requests in this execution.", e); this.importIntoDatabase = false; } } @@ -304,7 +304,7 @@ public final class RelayDescriptorDatabaseImporter { for (String bandwidthHistoryString : bandwidthHistoryStrings) { String[] parts = bandwidthHistoryString.split(" "); if (parts.length != 6) { - log.debug("Bandwidth history line does not have expected " + logger.debug("Bandwidth history line does not have expected " + "number of elements. Ignoring this line."); continue; } @@ -312,13 +312,13 @@ public final class RelayDescriptorDatabaseImporter { try { intervalLength = Long.parseLong(parts[3].substring(1)); } catch (NumberFormatException e) { - log.debug("Bandwidth history line does not have valid interval length " - + "'{} {}'. Ignoring this line.", parts[3], parts[4]); + logger.debug("Bandwidth history line does not have valid interval " + + "length '{} {}'. Ignoring this line.", parts[3], parts[4]); continue; } String[] values = parts[5].split(","); if (intervalLength % 900L != 0L) { - log.debug("Bandwidth history line does not contain " + logger.debug("Bandwidth history line does not contain " + "multiples of 15-minute intervals. Ignoring this line."); continue; } else if (intervalLength != 900L) { @@ -336,7 +336,7 @@ public final class RelayDescriptorDatabaseImporter { values = newValues; intervalLength = 900L; } catch (NumberFormatException e) { - log.debug("Number format exception while parsing " + logger.debug("Number format exception while parsing " + "bandwidth history line. Ignoring this line."); continue; } @@ -350,15 +350,16 @@ public final class RelayDescriptorDatabaseImporter { dateStart = dateTimeFormat.parse(parts[1] + " 00:00:00") .getTime(); } catch (ParseException e) { - log.debug("Parse exception while parsing timestamp in " + logger.debug("Parse exception while parsing timestamp in " + "bandwidth history line. Ignoring this line."); continue; } if (Math.abs(published - intervalEnd) > 7L * 24L * 60L * 60L * 1000L) { - log.debug("Extra-info descriptor publication time {} and last interval " - + "time {} in {} line differ by more than 7 days! Not adding this " - + "line!", dateTimeFormat.format(published), intervalEndTime, type); + logger.debug("Extra-info descriptor publication time {} and last " + + "interval time {} in {} line differ by more than 7 days! Not " + + "adding this line!", dateTimeFormat.format(published), + intervalEndTime, type); continue; } long currentIntervalEnd = intervalEnd; @@ -384,7 +385,7 @@ public final class RelayDescriptorDatabaseImporter { currentIntervalEnd -= intervalLength * 1000L; } } catch (NumberFormatException e) { - log.debug("Number format exception while parsing " + logger.debug("Number format exception while parsing " + "bandwidth history line. Ignoring this line."); continue; } @@ -432,7 +433,7 @@ public final class RelayDescriptorDatabaseImporter { this.csH.executeBatch(); } } catch (SQLException | ParseException e) { - log.warn("Could not insert bandwidth " + logger.warn("Could not insert bandwidth " + "history line into database. We won't make any " + "further SQL requests in this execution.", e); this.importIntoDatabase = false; @@ -539,8 +540,8 @@ public final class RelayDescriptorDatabaseImporter { public void commit() {

/* Log stats about imported descriptors. */ - log.info("Finished importing relay descriptors: {} network status entries " - + "and {} bandwidth history elements", rrsCount, rhsCount); + logger.info("Finished importing relay descriptors: {} network status " + + "entries and {} bandwidth history elements", rrsCount, rhsCount);

/* Insert scheduled updates a second time, just in case the refresh * run has started since inserting them the first time in which case @@ -555,7 +556,7 @@ public final class RelayDescriptorDatabaseImporter { this.psU.execute(); } } catch (SQLException e) { - log.warn("Could not add scheduled dates " + logger.warn("Could not add scheduled dates " + "for the next refresh run.", e); } } @@ -567,7 +568,7 @@ public final class RelayDescriptorDatabaseImporter {

this.conn.commit(); } catch (SQLException e) { - log.warn("Could not commit final records to database", e); + logger.warn("Could not commit final records to database", e); } } } @@ -637,7 +638,7 @@ public final class RelayDescriptorDatabaseImporter { try { this.conn.close(); } catch (SQLException e) { - log.warn("Could not close database connection.", e); + logger.warn("Could not close database connection.", e); } } } diff --git a/src/main/java/org/torproject/metrics/stats/clients/Main.java b/src/main/java/org/torproject/metrics/stats/clients/Main.java index d89a82a..bfa9214 100644 --- a/src/main/java/org/torproject/metrics/stats/clients/Main.java +++ b/src/main/java/org/torproject/metrics/stats/clients/Main.java @@ -23,7 +23,7 @@ import java.util.TreeMap;

public class Main {

- private static Logger log = LoggerFactory.getLogger(Main.class); + private static final Logger logger = LoggerFactory.getLogger(Main.class);

private static Database database;

@@ -33,36 +33,36 @@ public class Main { /** Executes this data-processing module. */ public static void main(String[] args) throws Exception {

- log.info("Starting clients module."); + logger.info("Starting clients module.");

- log.info("Connecting to database."); + logger.info("Connecting to database."); database = new Database();

- log.info("Reading relay descriptors and importing relevant parts into the " - + "database."); + logger.info("Reading relay descriptors and importing relevant parts into " + + "the database."); parseRelayDescriptors();

- log.info("Reading bridge descriptors and importing relevant parts into the " - + "database."); + logger.info("Reading bridge descriptors and importing relevant parts into " + + "the database."); parseBridgeDescriptors();

- log.info("Processing newly imported data."); + logger.info("Processing newly imported data."); database.processImported(); database.commit();

- log.info("Querying aggregated statistics from the database."); + logger.info("Querying aggregated statistics from the database."); new Writer().write(new File(baseDir, "stats/userstats.csv").toPath(), database.queryEstimated()); new Writer().write(new File(baseDir, "stats/userstats-combined.csv") .toPath(), database.queryCombined());

- log.info("Disconnecting from database."); + logger.info("Disconnecting from database."); database.close();

- log.info("Running detector."); + logger.info("Running detector."); new Detector().detect();

- log.info("Terminating clients module."); + logger.info("Terminating clients module."); }

private static final long ONE_HOUR_MILLIS = 60L * 60L * 1000L; diff --git a/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java b/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java index 5e71534..2abf202 100644 --- a/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java +++ b/src/main/java/org/torproject/metrics/stats/connbidirect/Main.java @@ -34,7 +34,7 @@ import java.util.TreeSet;

public class Main {

- private static Logger log = LoggerFactory.getLogger(Main.class); + private static final Logger logger = LoggerFactory.getLogger(Main.class);

static class RawStat implements Comparable<RawStat> {

@@ -79,12 +79,12 @@ public class Main { return new RawStat(dateDays, fingerprint, fractionRead, fractionWrite, fractionBoth); } else { - log.warn("Could not deserialize raw statistic from string '{}'.", + logger.warn("Could not deserialize raw statistic from string '{}'.", string); return null; } } catch (NumberFormatException e) { - log.warn("Could not deserialize raw statistic from string '{}'.", + logger.warn("Could not deserialize raw statistic from string '{}'.", string, e); return null; } @@ -144,13 +144,13 @@ public class Main { SortedMap<String, Long> parseHistory = parseParseHistory( readStringFromFile(parseHistoryFile)); if (parseHistory == null) { - log.warn("Could not parse {}. Proceeding without parse history.", + logger.warn("Could not parse {}. Proceeding without parse history.", parseHistoryFile.getAbsolutePath()); } SortedMap<String, Short> aggregateStats = parseAggregateStats( readStringFromFile(aggregateStatsFile)); if (aggregateStats == null) { - log.warn("Could not parse previously aggregated " + logger.warn("Could not parse previously aggregated " + "statistics. Not proceeding, because we would otherwise " + "lose previously aggregated values for which we don't have " + "raw statistics anymore."); @@ -160,7 +160,7 @@ public class Main { parseHistory = addRawStatsFromDescriptors(newRawStats, descriptorsDirectories, parseHistory); if (parseHistory == null) { - log.warn("Could not parse raw statistics from " + logger.warn("Could not parse raw statistics from " + "descriptors. Not proceeding, because we would otherwise " + "leave out those descriptors in future runs."); return; @@ -169,7 +169,7 @@ public class Main { SortedSet<RawStat> rawStats = parseRawStats( readStringFromFile(rawStatsFile)); if (rawStats == null) { - log.warn("Could not parse previously parsed raw " + logger.warn("Could not parse previously parsed raw " + "statistics. Not proceeding, because we might otherwise " + "leave out previously parsed statistics in the aggregates."); return; @@ -189,7 +189,7 @@ public class Main { sb.append("\n ") .append(dateFormat.format(conflictingDate * ONE_DAY_IN_MILLIS)); } - log.warn(sb.toString()); + logger.warn(sb.toString()); return; } updateAggregateStats(aggregateStats, rawStats); @@ -248,19 +248,19 @@ public class Main { while ((line = lnr.readLine()) != null) { String[] parts = line.split(","); if (parts.length < 2) { - log.warn("Invalid line {} in parse history: '{}'.", + logger.warn("Invalid line {} in parse history: '{}'.", lnr.getLineNumber(), line); return null; } parsedParseHistory.put(parts[0], Long.parseLong(parts[1])); } } catch (IOException e) { - log.warn("Unexpected I/O exception while reading line {} from parse " + logger.warn("Unexpected I/O exception while reading line {} from parse " + "history.", lnr.getLineNumber(), e); return null; } catch (NumberFormatException e) { - log.warn("Invalid line {} in parse history: '{}'.", lnr.getLineNumber(), - line, e); + logger.warn("Invalid line {} in parse history: '{}'.", + lnr.getLineNumber(), line, e); return null; } return parsedParseHistory; @@ -295,14 +295,14 @@ public class Main { String line = ""; try { if (!AGGREGATE_STATS_HEADER.equals(lnr.readLine())) { - log.warn("First line of aggregate statistics does not " + logger.warn("First line of aggregate statistics does not " + "contain the header line. Is this the correct file?"); return null; } while ((line = lnr.readLine()) != null) { String[] parts = line.split(","); if (parts.length != 4) { - log.warn("Invalid line {} in aggregate statistics: '{}'.", + logger.warn("Invalid line {} in aggregate statistics: '{}'.", lnr.getLineNumber(), line); return null; } @@ -310,11 +310,11 @@ public class Main { + parts[2], Short.parseShort(parts[3])); } } catch (IOException e) { - log.warn("Unexpected I/O exception while reading line {} from aggregate " - + "statistics.", lnr.getLineNumber(), e); + logger.warn("Unexpected I/O exception while reading line {} from " + + "aggregate statistics.", lnr.getLineNumber(), e); return null; } catch (NumberFormatException e) { - log.warn("Invalid line {} in aggregate statistics: '{}'.", + logger.warn("Invalid line {} in aggregate statistics: '{}'.", lnr.getLineNumber(), line, e); return null; } @@ -341,19 +341,19 @@ public class Main { while ((line = lnr.readLine()) != null) { RawStat rawStat = RawStat.fromString(line); if (rawStat == null) { - log.warn("Invalid line {} in raw statistics: '{}'.", + logger.warn("Invalid line {} in raw statistics: '{}'.", lnr.getLineNumber(), line); return null; } parsedRawStats.add(rawStat); } } catch (IOException e) { - log.warn("Unexpected I/O exception while reading line {} from raw " + logger.warn("Unexpected I/O exception while reading line {} from raw " + "statistics.", lnr.getLineNumber(), e); return null; } catch (NumberFormatException e) { - log.warn("Invalid line {} in raw statistics: '{}'.", lnr.getLineNumber(), - line, e); + logger.warn("Invalid line {} in raw statistics: '{}'.", + lnr.getLineNumber(), line, e); return null; } return parsedRawStats; @@ -392,7 +392,7 @@ public class Main { int write = extraInfo.getConnBiDirectWrite(); int both = extraInfo.getConnBiDirectBoth(); if (below < 0 || read < 0 || write < 0 || both < 0) { - log.debug("Could not parse incomplete conn-bi-direct statistics. " + logger.debug("Could not parse incomplete conn-bi-direct statistics. " + "Skipping descriptor."); return null; } diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java b/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java index cb52598..8ca00a0 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/Aggregator.java @@ -24,7 +24,8 @@ import java.util.TreeMap; * statistics and the total network fraction of reporting relays. */ public class Aggregator {

- private static Logger log = LoggerFactory.getLogger(Aggregator.class); + private static final Logger logger + = LoggerFactory.getLogger(Aggregator.class);

/** Document file containing extrapolated hidden-service statistics. */ private File extrapolatedHidServStatsFile; @@ -62,7 +63,7 @@ public class Aggregator { this.extrapolatedHidServStatsStore.retrieve( this.extrapolatedHidServStatsFile); if (extrapolatedStats == null) { - log.warn("Unable to retrieve extrapolated hidden-service " + logger.warn("Unable to retrieve extrapolated hidden-service " + "statistics from file {}. Skipping aggregation step.", this.extrapolatedHidServStatsFile.getAbsolutePath()); return; @@ -188,7 +189,7 @@ public class Aggregator { this.hidservStatsCsvFile))) { bw.write(sb.toString()); } catch (IOException e) { - log.warn("Unable to write results to {}. Ignoring.", + logger.warn("Unable to write results to {}. Ignoring.", this.extrapolatedHidServStatsFile.getAbsolutePath()); } } diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java b/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java index 3f3f12d..d110cbd 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/ComputedNetworkFractions.java @@ -15,7 +15,7 @@ import java.util.Map; * status entries and bandwidth weights in a network status consensus. */ public class ComputedNetworkFractions implements Document {

- private static Logger log + private static final Logger logger = LoggerFactory.getLogger(ComputedNetworkFractions.class);

/** Relay fingerprint consisting of 40 upper-case hex characters. */ @@ -137,18 +137,18 @@ public class ComputedNetworkFractions implements Document { @Override public boolean parse(String[] formattedStrings) { if (formattedStrings.length != 2) { - log.warn("Invalid number of formatted strings. Skipping."); + logger.warn("Invalid number of formatted strings. Skipping."); return false; } String[] firstParts = formattedStrings[0].split(",", 2); if (firstParts.length != 2) { - log.warn("Invalid number of comma-separated values. Skipping."); + logger.warn("Invalid number of comma-separated values. Skipping."); return false; } String fingerprint = firstParts[0]; String[] secondParts = formattedStrings[1].split(",", 3); if (secondParts.length != 3) { - log.warn("Invalid number of comma-separated values. Skipping."); + logger.warn("Invalid number of comma-separated values. Skipping."); return false; } String validAfterDate = firstParts[1]; @@ -166,7 +166,7 @@ public class ComputedNetworkFractions implements Document { if (validAfterDateMillis == DateTimeHelper.NO_TIME_AVAILABLE || validAfterTimeMillis < 0L || validAfterTimeMillis >= DateTimeHelper.ONE_DAY) { - log.warn("Invalid date/hour format. Skipping."); + logger.warn("Invalid date/hour format. Skipping."); return false; } long validAfterMillis = validAfterDateMillis + validAfterTimeMillis; @@ -179,7 +179,7 @@ public class ComputedNetworkFractions implements Document { ? 0.0 : Double.parseDouble(secondParts[2]); return true; } catch (NumberFormatException e) { - log.warn("Invalid number format. Skipping."); + logger.warn("Invalid number format. Skipping."); return false; } } diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java b/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java index 1cfcf08..fe223c7 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/DocumentStore.java @@ -26,7 +26,8 @@ import java.util.TreeSet; * interface to a file and later to retrieve them. */ public class DocumentStore<T extends Document> {

- private static Logger log = LoggerFactory.getLogger(DocumentStore.class); + private static final Logger logger + = LoggerFactory.getLogger(DocumentStore.class);

/** Document class, needed to create new instances when retrieving * documents. */ @@ -47,7 +48,7 @@ public class DocumentStore<T extends Document> { /* Retrieve existing documents. */ Set<T> retrievedDocuments = this.retrieve(documentFile); if (retrievedDocuments == null) { - log.warn("Unable to read and update {}. Not storing documents.", + logger.warn("Unable to read and update {}. Not storing documents.", documentFile.getAbsoluteFile()); return false; } @@ -68,7 +69,7 @@ public class DocumentStore<T extends Document> { File documentTempFile = new File(documentFile.getAbsoluteFile() + ".tmp"); if (documentTempFile.exists()) { - log.warn("Temporary document file {} still exists, " + logger.warn("Temporary document file {} still exists, " + "indicating that a previous execution did not terminate " + "cleanly. Not storing documents.", documentTempFile.getAbsoluteFile()); @@ -90,7 +91,7 @@ public class DocumentStore<T extends Document> { documentFile.delete(); documentTempFile.renameTo(documentFile); } catch (IOException e) { - log.warn("Unable to write {}. Not storing documents.", + logger.warn("Unable to write {}. Not storing documents.", documentFile.getAbsolutePath(), e); return false; } @@ -125,7 +126,7 @@ public class DocumentStore<T extends Document> { if (!line.startsWith(" ")) { formattedString0 = line; } else if (formattedString0 == null) { - log.warn("First line in {} must not start with a space. Not " + logger.warn("First line in {} must not start with a space. Not " + "retrieving any previously stored documents.", documentFile.getAbsolutePath()); return null; @@ -140,7 +141,7 @@ public class DocumentStore<T extends Document> { T document = this.clazz.getDeclaredConstructor().newInstance(); if (!document.parse(new String[] { formattedString0, line.substring(1) })) { - log.warn("Unable to read line {} from {}. Not retrieving any " + logger.warn("Unable to read line {} from {}. Not retrieving any " + "previously stored documents.", lnr.getLineNumber(), documentFile.getAbsolutePath()); return null; @@ -149,12 +150,12 @@ public class DocumentStore<T extends Document> { } } } catch (IOException e) { - log.warn("Unable to read {}. Not retrieving any previously stored " + logger.warn("Unable to read {}. Not retrieving any previously stored " + "documents.", documentFile.getAbsolutePath(), e); return null; } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { - log.warn("Unable to read {}. Cannot instantiate document object.", + logger.warn("Unable to read {}. Cannot instantiate document object.", documentFile.getAbsolutePath(), e); return null; } diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java b/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java index 71048f3..c46ee2f 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/ExtrapolatedHidServStats.java @@ -11,7 +11,7 @@ import org.slf4j.LoggerFactory; * computed network fractions in the statistics interval. */ public class ExtrapolatedHidServStats implements Document {

- private static Logger log + private static final Logger logger = LoggerFactory.getLogger(ExtrapolatedHidServStats.class);

/** Date of statistics interval end in milliseconds. */ @@ -136,7 +136,7 @@ public class ExtrapolatedHidServStats implements Document { @Override public boolean parse(String[] formattedStrings) { if (formattedStrings.length != 2) { - log.warn("Invalid number of formatted strings: {}. Skipping.", + logger.warn("Invalid number of formatted strings: {}. Skipping.", formattedStrings.length); return false; } @@ -144,7 +144,7 @@ public class ExtrapolatedHidServStats implements Document { DateTimeHelper.ISO_DATE_FORMAT); String[] secondParts = formattedStrings[1].split(",", 5); if (secondParts.length != 5) { - log.warn("Invalid number of comma-separated values: {}. Skipping.", + logger.warn("Invalid number of comma-separated values: {}. Skipping.", secondParts.length); return false; } diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/Extrapolator.java b/src/main/java/org/torproject/metrics/stats/hidserv/Extrapolator.java index 6bb47b8..a1f5028 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/Extrapolator.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/Extrapolator.java @@ -20,7 +20,8 @@ import java.util.TreeSet; * observed by the relay. */ public class Extrapolator {

- private static Logger log = LoggerFactory.getLogger(Extrapolator.class); + private static final Logger logger + = LoggerFactory.getLogger(Extrapolator.class);

/** Document file containing previously parsed reported hidden-service * statistics. */ @@ -89,7 +90,7 @@ public class Extrapolator {

/* Make sure that all documents could be retrieved correctly. */ if (extrapolatedStats == null || reportedStats == null) { - log.warn("Could not read previously parsed or extrapolated " + logger.warn("Could not read previously parsed or extrapolated " + "hidserv-stats. Skipping."); return false; } diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/Main.java b/src/main/java/org/torproject/metrics/stats/hidserv/Main.java index 1711dbb..ba65f8e 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/Main.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/Main.java @@ -14,7 +14,7 @@ import java.io.File; * do not overlap. */ public class Main {

- private static Logger log = LoggerFactory.getLogger(Main.class); + private static final Logger logger = LoggerFactory.getLogger(Main.class);

private static final File baseDir = new File( org.torproject.metrics.stats.main.Main.modulesDir, "hidserv"); @@ -34,7 +34,7 @@ public class Main {

/* Initialize parser and read parse history to avoid parsing * descriptor files that haven't changed since the last execution. */ - log.info("Initializing parser and reading parse history..."); + logger.info("Initializing parser and reading parse history..."); DocumentStore<ReportedHidServStats> reportedHidServStatsStore = new DocumentStore<>(ReportedHidServStats.class); DocumentStore<ComputedNetworkFractions> @@ -46,28 +46,28 @@ public class Main {

/* Parse new descriptors and store their contents using the document * stores. */ - log.info("Parsing descriptors..."); + logger.info("Parsing descriptors..."); parser.parseDescriptors();

/* Write the parse history to avoid parsing descriptor files again * next time. It's okay to do this now and not at the end of the * execution, because even if something breaks apart below, it's safe * not to parse descriptor files again. */ - log.info("Writing parse history..."); + logger.info("Writing parse history..."); parser.writeParseHistory();

/* Extrapolate reported statistics using computed network fractions * and write the result to disk using a document store. The result is * a single file with extrapolated network totals based on reports by * single relays. */ - log.info("Extrapolating statistics..."); + logger.info("Extrapolating statistics..."); DocumentStore<ExtrapolatedHidServStats> extrapolatedHidServStatsStore = new DocumentStore<>(ExtrapolatedHidServStats.class); Extrapolator extrapolator = new Extrapolator(statusDirectory, reportedHidServStatsStore, computedNetworkFractionsStore, extrapolatedHidServStatsStore); if (!extrapolator.extrapolateHidServStats()) { - log.warn("Could not extrapolate statistics. Terminating."); + logger.warn("Could not extrapolate statistics. Terminating."); return; }

@@ -75,7 +75,7 @@ public class Main { * This includes calculating daily weighted interquartile means, among * other statistics. Write the result to a .csv file that can be * processed by other tools. */ - log.info("Aggregating statistics..."); + logger.info("Aggregating statistics..."); File hidservStatsExtrapolatedCsvFile = new File(baseDir, "stats/hidserv.csv"); Aggregator aggregator = new Aggregator(statusDirectory, @@ -83,7 +83,7 @@ public class Main { aggregator.aggregateHidServStats();

/* End this execution. */ - log.info("Terminating."); + logger.info("Terminating."); } }

diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/Parser.java b/src/main/java/org/torproject/metrics/stats/hidserv/Parser.java index 46a6607..d1d2328 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/Parser.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/Parser.java @@ -35,7 +35,7 @@ import java.util.TreeSet; * document files for later use. */ public class Parser {

- private static Logger log = LoggerFactory.getLogger(Parser.class); + private static final Logger logger = LoggerFactory.getLogger(Parser.class);

/** File containing tuples of last-modified times and file names of * descriptor files parsed in the previous execution. */ @@ -111,12 +111,12 @@ public class Parser { String[] parts = line.split(" ", 2); excludedFiles.put(parts[1], Long.parseLong(parts[0])); } catch (NumberFormatException e) { - log.warn("Illegal line '{}' in parse history. Skipping line.", line, - e); + logger.warn("Illegal line '{}' in parse history. Skipping line.", + line, e); } } } catch (IOException e) { - log.warn("Could not read history file '{}'. Not " + logger.warn("Could not read history file '{}'. Not " + "excluding descriptors in this execution.", this.parseHistoryFile.getAbsolutePath(), e); } @@ -151,8 +151,9 @@ public class Parser { + "\n"); } } catch (IOException e) { - log.warn("Could not write history file '{}'. Not excluding descriptors " - + "in next execution.", this.parseHistoryFile.getAbsolutePath(), e); + logger.warn("Could not write history file '{}'. Not excluding " + + "descriptors in next execution.", + this.parseHistoryFile.getAbsolutePath(), e); } }

@@ -231,7 +232,7 @@ public class Parser { } else if (extraInfoDescriptor.getHidservStatsEndMillis() >= 0L || extraInfoDescriptor.getHidservRendRelayedCells() != null || extraInfoDescriptor.getHidservDirOnionsSeen() != null) { - log.warn("Relay {} published incomplete hidserv-stats. Ignoring.", + logger.warn("Relay {} published incomplete hidserv-stats. Ignoring.", fingerprint); } } @@ -252,7 +253,7 @@ public class Parser { SortedMap<String, Integer> bandwidthWeights = consensus.getBandwidthWeights(); if (bandwidthWeights == null) { - log.warn("Consensus with valid-after time {} doesn't contain any Wxx " + logger.warn("Consensus with valid-after time {} doesn't contain any Wxx " + "weights. Skipping.", DateTimeHelper.format(consensus.getValidAfterMillis())); return; @@ -264,7 +265,7 @@ public class Parser { new TreeSet<>(Arrays.asList("Wmg,Wmm,Wme,Wmd".split(","))); expectedWeightKeys.removeAll(bandwidthWeights.keySet()); if (!expectedWeightKeys.isEmpty()) { - log.warn("Consensus with valid-after time {} doesn't contain expected " + logger.warn("Consensus with valid-after time {} doesn't contain expected " + "Wmx weights. Skipping.", DateTimeHelper.format(consensus.getValidAfterMillis())); return; diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/ReportedHidServStats.java b/src/main/java/org/torproject/metrics/stats/hidserv/ReportedHidServStats.java index 5b79a65..95942af 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/ReportedHidServStats.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/ReportedHidServStats.java @@ -11,7 +11,7 @@ import org.slf4j.LoggerFactory; * by the relay in the "hidserv-" lines of its extra-info descriptor. */ public class ReportedHidServStats implements Document {

- private static Logger log + private static final Logger logger = LoggerFactory.getLogger(ReportedHidServStats.class);

/* Relay fingerprint consisting of 40 upper-case hex characters. */ @@ -115,7 +115,7 @@ public class ReportedHidServStats implements Document { @Override public boolean parse(String[] formattedStrings) { if (formattedStrings.length != 2) { - log.warn("Invalid number of formatted strings: {} Skipping.", + logger.warn("Invalid number of formatted strings: {} Skipping.", formattedStrings.length); return false; } diff --git a/src/main/java/org/torproject/metrics/stats/hidserv/Simulate.java b/src/main/java/org/torproject/metrics/stats/hidserv/Simulate.java index 696fc1d..21f20a9 100644 --- a/src/main/java/org/torproject/metrics/stats/hidserv/Simulate.java +++ b/src/main/java/org/torproject/metrics/stats/hidserv/Simulate.java @@ -24,7 +24,7 @@ import java.util.TreeSet; * contains its own main method.) */ public class Simulate {

- private static Logger log = LoggerFactory.getLogger(Simulate.class); + private static final Logger logger = LoggerFactory.getLogger(Simulate.class);

private static File simCellsCsvFile = new File("out/csv/sim-cells.csv"); @@ -34,11 +34,11 @@ public class Simulate {

/** Runs two simulations to evaluate this data-processing module. */ public static void main(String[] args) throws Exception { - log.info("Simulating extrapolation of rendezvous cells"); + logger.info("Simulating extrapolation of rendezvous cells"); simulateManyCells(); - log.info("Simulating extrapolation of .onions"); + logger.info("Simulating extrapolation of .onions"); simulateManyOnions(); - log.info("Terminating."); + logger.info("Terminating."); }

private static Random rnd = new Random(); @@ -51,7 +51,7 @@ public class Simulate { final int numberOfExtrapolations = 1000; for (int i = 0; i < numberOfExtrapolations; i++) { bw.write(simulateCells(i)); - log.info("."); + logger.info("."); } bw.close(); } @@ -64,7 +64,7 @@ public class Simulate { final int numberOfExtrapolations = 1000; for (int i = 0; i < numberOfExtrapolations; i++) { bw.write(simulateOnions(i)); - log.info("."); + logger.info("."); } bw.close(); } diff --git a/src/main/java/org/torproject/metrics/stats/main/Main.java b/src/main/java/org/torproject/metrics/stats/main/Main.java index 6badd96..41cba3a 100644 --- a/src/main/java/org/torproject/metrics/stats/main/Main.java +++ b/src/main/java/org/torproject/metrics/stats/main/Main.java @@ -15,7 +15,7 @@ import java.util.TimeZone;

public class Main {

- private static final Logger log = LoggerFactory.getLogger(Main.class); + private static final Logger logger = LoggerFactory.getLogger(Main.class);

private static final String baseDir = System.getProperty("metrics.basedir", "/srv/metrics.torproject.org/metrics"); @@ -29,7 +29,7 @@ public class Main { /** Start the metrics update run. */ public static void main(String[] args) {

- log.info("Starting metrics update run."); + logger.info("Starting metrics update run.");

Locale.setDefault(Locale.US); TimeZone.setDefault(TimeZone.getTimeZone("UTC")); @@ -40,11 +40,11 @@ public class Main { continue; } if (outputDir.mkdirs()) { - log.info("Successfully created module base directory {} and any " + logger.info("Successfully created module base directory {} and any " + "nonexistent parent directories.", outputDir.getAbsolutePath()); } else { - log.error("Unable to create module base directory {} and any " + logger.error("Unable to create module base directory {} and any " + "nonexistent parent directories. Exiting.", outputDir.getAbsolutePath()); return; @@ -67,19 +67,19 @@ public class Main {

for (Class<?> module : modules) { try { - log.info("Starting {} module.", module.getName()); + logger.info("Starting {} module.", module.getName()); module.getDeclaredMethod("main", String[].class) .invoke(null, (Object) args); - log.info("Completed {} module.", module.getName()); + logger.info("Completed {} module.", module.getName()); } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { - log.warn("Caught an exception when invoking the main method of the {} " - + "module. Moving on to the next module, if available.", + logger.warn("Caught an exception when invoking the main method of the " + + "{} module. Moving on to the next module, if available.", module.getName(), e); } }

- log.info("Making module data available."); + logger.info("Making module data available."); File[] moduleStatsDirs = new File[] { new File(modulesDir, "connbidirect/stats"), new File(modulesDir, "onionperf/stats"), @@ -96,13 +96,15 @@ public class Main { List<String> copiedFiles = new ArrayList<>(); for (File moduleStatsDir : moduleStatsDirs) { if (!moduleStatsDir.exists()) { - log.warn("Skipping nonexistent module stats dir {}.", moduleStatsDir); + logger.warn("Skipping nonexistent module stats dir {}.", + moduleStatsDir); continue; } File[] moduleStatsFiles = moduleStatsDir.isDirectory() ? moduleStatsDir.listFiles() : new File[] { moduleStatsDir }; if (null == moduleStatsFiles) { - log.warn("Skipping nonexistent module stats dir {}.", moduleStatsDir); + logger.warn("Skipping nonexistent module stats dir {}.", + moduleStatsDir); continue; } for (File statsFile : moduleStatsFiles) { @@ -115,16 +117,16 @@ public class Main { StandardCopyOption.REPLACE_EXISTING); copiedFiles.add(statsFile.getName()); } catch (IOException e) { - log.warn("Unable to copy module stats file {} to stats output " + logger.warn("Unable to copy module stats file {} to stats output " + "directory {}. Skipping.", statsFile, statsDir, e); } } } if (!copiedFiles.isEmpty()) { - log.info("Successfully copied {} files to stats output directory: {}", + logger.info("Successfully copied {} files to stats output directory: {}", copiedFiles.size(), copiedFiles); }

- log.info("Completed metrics update run."); + logger.info("Completed metrics update run."); } } diff --git a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java index 4325c01..e1b063c 100644 --- a/src/main/java/org/torproject/metrics/stats/onionperf/Main.java +++ b/src/main/java/org/torproject/metrics/stats/onionperf/Main.java @@ -31,7 +31,7 @@ import java.util.Set; public class Main {

/** Logger for this class. */ - private static Logger log = LoggerFactory.getLogger(Main.class); + private static final Logger logger = LoggerFactory.getLogger(Main.class);

private static final String jdbcString = String.format( "jdbc:postgresql://localhost/onionperf?user=%s&password=%s", @@ -43,7 +43,7 @@ public class Main {

/** Executes this data-processing module. */ public static void main(String[] args) throws Exception { - log.info("Starting onionperf module."); + logger.info("Starting onionperf module."); Connection connection = connectToDatabase(); importOnionPerfFiles(connection); writeStatistics(new File(baseDir, "stats/torperf-1.1.csv").toPath(), @@ -56,15 +56,15 @@ public class Main { new File(baseDir, "stats/onionperf-throughput.csv").toPath(), queryThroughput(connection)); disconnectFromDatabase(connection); - log.info("Terminated onionperf module."); + logger.info("Terminated onionperf module."); }

private static Connection connectToDatabase() throws SQLException { - log.info("Connecting to database."); + logger.info("Connecting to database."); Connection connection = DriverManager.getConnection(jdbcString); connection.setAutoCommit(false); - log.info("Successfully connected to database."); + logger.info("Successfully connected to database."); return connection; }

@@ -240,7 +240,7 @@ public class Main {

static List<String> queryOnionPerf(Connection connection) throws SQLException { - log.info("Querying statistics from database."); + logger.info("Querying statistics from database."); List<String> statistics = new ArrayList<>(); statistics .add("date,filesize,source,server,q1,md,q3,timeouts,failures,requests"); @@ -268,7 +268,7 @@ public class Main {

static List<String> queryBuildTimes(Connection connection) throws SQLException { - log.info("Querying buildtime statistics from database."); + logger.info("Querying buildtime statistics from database."); List<String> statistics = new ArrayList<>(); statistics.add("date,source,position,q1,md,q3"); Statement st = connection.createStatement(); @@ -291,7 +291,7 @@ public class Main {

static List<String> queryLatencies(Connection connection) throws SQLException { - log.info("Querying latency statistics from database."); + logger.info("Querying latency statistics from database."); List<String> statistics = new ArrayList<>(); statistics.add("date,source,server,low,q1,md,q3,high"); Statement st = connection.createStatement(); @@ -316,7 +316,7 @@ public class Main {

static List<String> queryThroughput(Connection connection) throws SQLException { - log.info("Querying throughput statistics from database."); + logger.info("Querying throughput statistics from database."); List<String> statistics = new ArrayList<>(); statistics.add("date,source,server,low,q1,md,q3,high"); Statement st = connection.createStatement(); @@ -361,14 +361,14 @@ public class Main { static void writeStatistics(Path webstatsPath, List<String> statistics) throws IOException { webstatsPath.toFile().getParentFile().mkdirs(); - log.info("Writing {} lines to {}.", statistics.size(), + logger.info("Writing {} lines to {}.", statistics.size(), webstatsPath.toFile().getAbsolutePath()); Files.write(webstatsPath, statistics, StandardCharsets.UTF_8); }

private static void disconnectFromDatabase(Connection connection) throws SQLException { - log.info("Disconnecting from database."); + logger.info("Disconnecting from database."); connection.close(); } } diff --git a/src/main/java/org/torproject/metrics/stats/servers/Main.java b/src/main/java/org/torproject/metrics/stats/servers/Main.java index 1fc853f..3258189 100644 --- a/src/main/java/org/torproject/metrics/stats/servers/Main.java +++ b/src/main/java/org/torproject/metrics/stats/servers/Main.java @@ -22,7 +22,7 @@ import java.util.Arrays; * statistics to CSV files. */ public class Main {

- private static Logger log = LoggerFactory.getLogger(Main.class); + private static final Logger logger = LoggerFactory.getLogger(Main.class);

private static final File baseDir = new File( org.torproject.metrics.stats.main.Main.modulesDir, "servers"); @@ -40,9 +40,9 @@ public class Main { /** Run the module. */ public static void main(String[] args) throws Exception {

- log.info("Starting servers module."); + logger.info("Starting servers module.");

- log.info("Reading descriptors and inserting relevant parts into the " + logger.info("Reading descriptors and inserting relevant parts into the " + "database."); DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader(); File historyFile = new File(baseDir, "status/read-descriptors"); @@ -64,30 +64,30 @@ public class Main { database.insertStatus(parser.parseBridgeNetworkStatus( (BridgeNetworkStatus) descriptor)); } else if (null != descriptor.getRawDescriptorBytes()) { - log.debug("Skipping unknown descriptor of type {} starting with " + logger.debug("Skipping unknown descriptor of type {} starting with " + "'{}'.", descriptor.getClass(), new String(descriptor.getRawDescriptorBytes(), 0, Math.min(descriptor.getRawDescriptorLength(), 100))); } else { - log.debug("Skipping unknown, empty descriptor of type {}.", + logger.debug("Skipping unknown, empty descriptor of type {}.", descriptor.getClass()); } }

- log.info("Aggregating database entries."); + logger.info("Aggregating database entries."); database.aggregate();

- log.info("Committing all updated parts in the database."); + logger.info("Committing all updated parts in the database."); database.commit(); } catch (SQLException sqle) { - log.error("Cannot recover from SQL exception while inserting or " + logger.error("Cannot recover from SQL exception while inserting or " + "aggregating data. Rolling back and exiting.", sqle); database.rollback(); return; } reader.saveHistoryFile(historyFile);

- log.info("Querying aggregated statistics from the database."); + logger.info("Querying aggregated statistics from the database."); File outputDir = new File(baseDir, "stats"); new Writer().write(new File(outputDir, "ipv6servers.csv").toPath(), database.queryServersIpv6()); @@ -102,10 +102,10 @@ public class Main { new Writer().write(new File(outputDir, "platforms.csv").toPath(), database.queryPlatforms());

- log.info("Terminating servers module."); + logger.info("Terminating servers module."); } catch (SQLException sqle) { - log.error("Cannot recover from SQL exception while querying. Not writing " - + "output file.", sqle); + logger.error("Cannot recover from SQL exception while querying. Not " + + "writing output file.", sqle); } } } diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Main.java b/src/main/java/org/torproject/metrics/stats/totalcw/Main.java index 3be41f9..c19defd 100644 --- a/src/main/java/org/torproject/metrics/stats/totalcw/Main.java +++ b/src/main/java/org/torproject/metrics/stats/totalcw/Main.java @@ -21,7 +21,7 @@ import java.util.Arrays; * CSV file. */ public class Main {

- private static Logger log = LoggerFactory.getLogger(Main.class); + private static final Logger logger = LoggerFactory.getLogger(Main.class);

private static final File baseDir = new File( org.torproject.metrics.stats.main.Main.modulesDir, "totalcw"); @@ -35,10 +35,10 @@ public class Main { /** Run the module. */ public static void main(String[] args) throws Exception {

- log.info("Starting totalcw module."); + logger.info("Starting totalcw module.");

- log.info("Reading consensuses and votes and inserting relevant parts into " - + "the database."); + logger.info("Reading consensuses and votes and inserting relevant parts " + + "into the database."); DescriptorReader reader = DescriptorSourceFactory.createDescriptorReader(); File historyFile = new File(baseDir, "status/read-descriptors"); reader.setHistoryFile(historyFile); @@ -56,33 +56,33 @@ public class Main { database.insertVote(parser.parseRelayNetworkStatusVote( (RelayNetworkStatusVote) descriptor)); } else { - log.debug("Skipping unknown descriptor of type {}.", + logger.debug("Skipping unknown descriptor of type {}.", descriptor.getClass()); } }

- log.info("Committing all updated parts in the database."); + logger.info("Committing all updated parts in the database."); database.commit(); } catch (SQLException sqle) { - log.error("Cannot recover from SQL exception while inserting data. " + logger.error("Cannot recover from SQL exception while inserting data. " + "Rolling back and exiting.", sqle); database.rollback(); return; } reader.saveHistoryFile(historyFile);

- log.info("Querying aggregated statistics from the database."); + logger.info("Querying aggregated statistics from the database."); Iterable<OutputLine> output = database.queryTotalcw(); File outputFile = new File(baseDir, "stats/totalcw.csv"); - log.info("Writing aggregated statistics to {}.", outputFile); + logger.info("Writing aggregated statistics to {}.", outputFile); if (null != output) { new Writer().write(outputFile.toPath(), output); }

- log.info("Terminating totalcw module."); + logger.info("Terminating totalcw module."); } catch (SQLException sqle) { - log.error("Cannot recover from SQL exception while querying. Not writing " - + "output file.", sqle); + logger.error("Cannot recover from SQL exception while querying. Not " + + "writing output file.", sqle); } } } diff --git a/src/main/java/org/torproject/metrics/stats/webstats/Main.java b/src/main/java/org/torproject/metrics/stats/webstats/Main.java index bca86c5..7ce099e 100644 --- a/src/main/java/org/torproject/metrics/stats/webstats/Main.java +++ b/src/main/java/org/torproject/metrics/stats/webstats/Main.java @@ -42,7 +42,7 @@ import java.util.TreeSet; public class Main {

/** Logger for this class. */ - private static Logger log = LoggerFactory.getLogger(Main.class); + private static final Logger logger = LoggerFactory.getLogger(Main.class);

private static final String jdbcString = String.format( "jdbc:postgresql://localhost/webstats?user=%s&password=%s", @@ -72,7 +72,7 @@ public class Main {

/** Executes this data-processing module. */ public static void main(String[] args) throws Exception { - log.info("Starting webstats module."); + logger.info("Starting webstats module."); Connection connection = connectToDatabase(); SortedSet<String> skipFiles = queryImportedFileNames(connection); importLogFiles(connection, skipFiles, @@ -84,21 +84,21 @@ public class Main { writeStatistics(new File(baseDir, "stats/webstats.csv").toPath(), statistics); disconnectFromDatabase(connection); - log.info("Terminated webstats module."); + logger.info("Terminated webstats module."); }

private static Connection connectToDatabase() throws SQLException { - log.info("Connecting to database."); + logger.info("Connecting to database."); Connection connection = DriverManager.getConnection(jdbcString); connection.setAutoCommit(false); - log.info("Successfully connected to database."); + logger.info("Successfully connected to database."); return connection; }

static SortedSet<String> queryImportedFileNames(Connection connection) throws SQLException { - log.info("Querying previously imported log files."); + logger.info("Querying previously imported log files."); SortedSet<String> importedLogFileUrls = new TreeSet<>(); Statement st = connection.createStatement(); String queryString = "SELECT server, site, log_date FROM files"; @@ -110,7 +110,7 @@ public class Main { rs.getDate(3).toLocalDate().format(dateFormat))); } } - log.info("Found {} previously imported log files.", + logger.info("Found {} previously imported log files.", importedLogFileUrls.size()); return importedLogFileUrls; } @@ -142,11 +142,11 @@ public class Main { logFile.getPhysicalHost(), logFile.getVirtualHost(), logFile.getLogDate(), parsedLogLines); } catch (DescriptorParseException exc) { - log.warn("Cannot parse log file with file name {}. Retrying in the " + logger.warn("Cannot parse log file with file name {}. Retrying in the " + "next run.", logFile.getDescriptorFile().getName(), exc); } catch (SQLException exc) { - log.warn("Cannot import log file with file name {} into the database. " - + "Rolling back and retrying in the next run.", + logger.warn("Cannot import log file with file name {} into the " + + "database. Rolling back and retrying in the next run.", logFile.getDescriptorFile().getName(), exc); try { connection.rollback(); @@ -173,7 +173,7 @@ public class Main { + COUNT + ") VALUES (?, CAST(? AS method), ?, ?, ?)"); int fileId = insertFile(psFiles, urlString, server, site, logDate); if (fileId < 0) { - log.debug("Skipping previously imported log file {}.", urlString); + logger.debug("Skipping previously imported log file {}.", urlString); return; } for (Map.Entry<String, Long> requests : parsedLogLines.entrySet()) { @@ -185,7 +185,7 @@ public class Main { int resourceId = insertResource(psResourcesSelect, psResourcesInsert, resource); if (resourceId < 0) { - log.error("Could not retrieve auto-generated key for new resources " + logger.error("Could not retrieve auto-generated key for new resources " + "entry."); connection.rollback(); return; @@ -194,7 +194,7 @@ public class Main { count); } connection.commit(); - log.debug("Finished importing log file with file name {} into database.", + logger.debug("Finished importing log file with file name {} into database.", urlString); }

@@ -265,7 +265,7 @@ public class Main {

static SortedSet<String> queryWebstats(Connection connection) throws SQLException { - log.info("Querying statistics from database."); + logger.info("Querying statistics from database."); SortedSet<String> statistics = new TreeSet<>(); Statement st = connection.createStatement(); String queryString = "SELECT " + ALL_COLUMNS + " FROM webstats"; @@ -295,14 +295,14 @@ public class Main { List<String> lines = new ArrayList<>(); lines.add(ALL_COLUMNS); lines.addAll(statistics); - log.info("Writing {} lines to {}.", lines.size(), + logger.info("Writing {} lines to {}.", lines.size(), webstatsPath.toFile().getAbsolutePath()); Files.write(webstatsPath, lines, StandardCharsets.UTF_8); }

private static void disconnectFromDatabase(Connection connection) throws SQLException { - log.info("Disconnecting from database."); + logger.info("Disconnecting from database."); connection.close(); } } diff --git a/src/main/java/org/torproject/metrics/web/ServerMain.java b/src/main/java/org/torproject/metrics/web/ServerMain.java index bb03086..21f8529 100644 --- a/src/main/java/org/torproject/metrics/web/ServerMain.java +++ b/src/main/java/org/torproject/metrics/web/ServerMain.java @@ -13,21 +13,22 @@ import java.util.Locale;

public class ServerMain {

- private static final Logger log = LoggerFactory.getLogger(ServerMain.class); + private static final Logger logger + = LoggerFactory.getLogger(ServerMain.class);

/** Starts the web server listening for incoming client connections. */ public static void main(String[] args) { Locale.setDefault(Locale.US); try { Resource jettyXml = Resource.newSystemResource("jetty.xml"); - log.info("Reading configuration from '{}'.", jettyXml); + logger.info("Reading configuration from '{}'.", jettyXml); XmlConfiguration configuration = new XmlConfiguration(jettyXml.getInputStream()); Server server = (Server) configuration.configure(); server.start(); server.join(); } catch (Exception ex) { - log.error("Exiting, because of: {}.", ex.getMessage(), ex); + logger.error("Exiting, because of: {}.", ex.getMessage(), ex); System.exit(1); } } diff --git a/src/main/java/org/torproject/metrics/web/UpdateNews.java b/src/main/java/org/torproject/metrics/web/UpdateNews.java index 8f4440e..07b1d75 100644 --- a/src/main/java/org/torproject/metrics/web/UpdateNews.java +++ b/src/main/java/org/torproject/metrics/web/UpdateNews.java @@ -18,7 +18,8 @@ import java.util.Locale;

public class UpdateNews {

- private static Logger log = LoggerFactory.getLogger(UpdateNews.class); + private static final Logger logger + = LoggerFactory.getLogger(UpdateNews.class);

/** Update news. */ public static void main(String[] args) throws Exception { @@ -79,7 +80,7 @@ public class UpdateNews { int space = desc.indexOf(" ", open); int close = desc.indexOf("]", open); if (open < 0 || space < 0 || close < 0) { - log.warn("Cannot convert link in line {}. Exiting.", line); + logger.warn("Cannot convert link in line {}. Exiting.", line); System.exit(1); } desc = desc.substring(0, open) + "<a href="" @@ -91,7 +92,8 @@ public class UpdateNews { int open = desc.indexOf("`"); int close = desc.indexOf("`", open + 1); if (open < 0 || close < 0) { - log.warn("Cannot convert code fragment in line {}. Exiting.", line); + logger.warn("Cannot convert code fragment in line {}. Exiting.", + line); System.exit(1); } desc = desc.substring(0, open) + "<code>" diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml deleted file mode 100644 index 7789feb..0000000 --- a/src/main/resources/logback.xml +++ /dev/null @@ -1,58 +0,0 @@ -<configuration debug="false"> - <statusListener class="ch.qos.logback.core.status.NopStatusListener" /> - - <!-- a path and a prefix --> - <property name="logfile-base" value="${LOGBASE}/metrics-web-" /> - - <!-- log file names --> - <property name="fileall-logname" value="${logfile-base}all" /> - <property name="fileerr-logname" value="${logfile-base}err" /> - <property name="filestatistics-logname" value="${logfile-base}statistics" /> - - <!-- date pattern --> - <property name="utc-date-pattern" value="%date{ISO8601, UTC}" /> - - <!-- appender section --> - <appender name="FILEALL" class="ch.qos.logback.core.rolling.RollingFileAppender"> - <file>${fileall-logname}.log</file> - <encoder> - <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> - </encoder> - <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> - <!-- rollover daily --> - <FileNamePattern>${fileall-logname}.%d{yyyy-MM-dd}.%i.log</FileNamePattern> - <maxHistory>10</maxHistory> - <timeBasedFileNamingAndTriggeringPolicy - class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> - <!-- or whenever the file size reaches 1MB --> - <maxFileSize>1MB</maxFileSize> - </timeBasedFileNamingAndTriggeringPolicy> - </rollingPolicy> - </appender> - - <appender name="FILEERR" class="ch.qos.logback.core.FileAppender"> - <file>${fileerr-logname}.log</file> - <encoder> - <pattern>${utc-date-pattern} %level %logger{20}:%line %msg%n</pattern> - </encoder> - - <!-- ERROR or worse --> - <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> - <level>ERROR</level> - </filter> - </appender> - - <!-- logger section --> - <logger name="org.torproject" > - <appender-ref ref="FILEERR" /> - </logger> - - <logger name="org.eclipse" level="INFO" /> - <logger name="org.apache" level="INFO" /> - - <root level="ALL"> - <appender-ref ref="FILEALL" /> - </root> - -</configuration> - diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 81570c4..d7d5303 160000 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 81570c4dbc097089f367c104c7ef5a77bee29763 +Subproject commit d7d5303e76a69f5fd0fe2b3b9f6be9f25f1fd824