January 2020 - tor-commits - lists.torproject.org

[collector/release] Add some real tests for the webstats module.
by karsten＠torproject.org 15 Jan '20

15 Jan '20

commit 3002d6bc6b6bf84953cf842cbf6b3b18dc944879 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Dec 11 12:16:05 2019 +0100 Add some real tests for the webstats module. --- src/build | 2 +- .../collector/webstats/SanitizeWeblogsTest.java | 290 ++++++++++++++++++++- 2 files changed, 287 insertions(+), 5 deletions(-) diff --git a/src/build b/src/build index eb16cb3..264e498 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit eb16cb359db41722e6089bafb1e26808df4338df +Subproject commit 264e498f54a20f7d299daaf2533d043f880e6a8b diff --git a/src/test/java/org/torproject/metrics/collector/webstats/SanitizeWeblogsTest.java b/src/test/java/org/torproject/metrics/collector/webstats/SanitizeWeblogsTest.java index a550c41..21617b5 100644 --- a/src/test/java/org/torproject/metrics/collector/webstats/SanitizeWeblogsTest.java +++ b/src/test/java/org/torproject/metrics/collector/webstats/SanitizeWeblogsTest.java @@ -4,17 +4,299 @@ package org.torproject.metrics.collector.webstats; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.WebServerAccessLog; +import org.torproject.metrics.collector.Main; +import org.torproject.metrics.collector.conf.Configuration; +import org.torproject.metrics.collector.conf.Key; + +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.SortedMap; +import java.util.TreeMap; public class SanitizeWeblogsTest { + /** Sample original web server access logs as input for tests. */ + private static final String[][] inputLogs = new String[][] { + { "metrics.torproject.org-access.log-20191120.gz", + "0.0.0.0 - - [19/Nov/2019:00:00:00 +0000] " + + "\"GET /networksize.html HTTP/1.1\" 200 3269 \"-\" \"-\" -\n" + + "0.0.0.0 - - [19/Nov/2019:00:00:00 +0000] " + + "\"GET /networksize.png?start=2019-08-21&end=2019-11-19 HTTP/1.1\" " + + "200 39383 \"-\" \"-\" -\n" + + "0.0.0.0 - - [19/Nov/2019:00:00:00 +0000] " + + "\"GET /userstats-relay-country.html HTTP/1.1\" 200 7350 " + + "\"-\" \"-\" -\n" + + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] " + + "\"GET /collector/recent/relay-descriptors/ HTTP/1.1\" 200 10227 " + + "\"-\" \"-\" -\n" }, + { "metrics.torproject.org-access.log-20191121.gz", + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] " + + "\"HEAD /collector/recent/relay-descriptors/microdescs/ " + + "HTTP/1.1\" 200 - \"-\" \"-\" -\n" + + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] " + + "\"HEAD /collector/recent/exit-lists/ HTTP/1.1\" 200 " + + "- \"-\" \"-\" -\n" + + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] " + + "\"GET /collector/archive/bridge-descriptors/extra-infos/ " + + "HTTP/1.1\" 200 48013 \"-\" \"-\" -\n" + + "0.0.0.0 - - [21/Nov/2019:00:00:00 +0000] " + + "\"GET /images/cc/sk.png HTTP/1.1\" 200 395 \"-\" \"-\" -\n" }, + { "metrics.torproject.org-access.log-20191122.gz", + "0.0.0.0 - - [21/Nov/2019:00:00:00 +0000] " + + "\"GET /images/favicon.ico HTTP/1.1\" 200 1150 \"-\" \"-\" -\n" + + "0.0.0.0 - - [21/Nov/2019:00:00:00 +0000] " + + "\"GET /images/flags/authority.png HTTP/1.1\" 200 325 " + + "\"https://metrics.torproject.org/rs.html\" \"-\" -\n" + + "0.0.0.0 - - [21/Nov/2019:00:00:00 +0000] " + + "\"GET /news.atom HTTP/1.1\" 200 36362 \"-\" \"-\" -\n" + + "0.0.0.0 - - [22/Nov/2019:00:00:00 +0000] " + + "\"GET /onionperf-buildtimes.csv HTTP/1.1\" 200 270336 " + + "\"-\" \"-\" -\n" }, + { "metrics.torproject.org-access.log-20191123.gz", + "0.0.0.0 - - [22/Nov/2019:00:00:00 +0000] " + + "\"GET /userstats-relay-country.html?" + + "start=2010-01-01&end=2019-11-22&country=vn&events=off HTTP/1.1\" " + + "200 35517 \"-\" \"-\" -\n" + + "0.0.0.0 - - [22/Nov/2019:00:00:00 +0000] " + + "\"GET /userstats-relay-country.png?" + + "start=2010-01-01&end=2019-11-22&country=vn&events=off HTTP/1.1\" " + + "200 28041 \"-\" \"-\" -\n" + + "0.0.0.0 - - [22/Nov/2019:00:00:00 +0000] " + + "\"GET /userstats-relay-country.png?" + + "start=2010-01-01&end=2019-11-22&country=vn&events=off HTTP/1.1\" " + + "200 28041 \"-\" \"-\" -\n" + + "0.0.0.0 - - [23/Nov/2019:00:00:00 +0000] \"GET / HTTP/1.1\" " + + "200 3336 \"-\" \"-\" -\n" } + }; + + /** Sanitized web server access logs as output of tests. */ + private static final String[][] outputLogs = new String[][] { + { "metrics.torproject.org_meronense.torproject.org_" + + "access.log_20191120.xz", + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] " + + "\"GET /collector/archive/bridge-descriptors/extra-infos/ " + + "HTTP/1.1\" 200 48013\n" + + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] " + + "\"GET /collector/recent/relay-descriptors/ HTTP/1.1\" 200 10227\n" + + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] " + + "\"HEAD /collector/recent/exit-lists/ HTTP/1.1\" 200 -\n" + + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] " + + "\"HEAD /collector/recent/relay-descriptors/microdescs/ " + + "HTTP/1.1\" 200 -\n" }, + { "metrics.torproject.org_meronense.torproject.org_" + + "access.log_20191121.xz", + "0.0.0.0 - - [21/Nov/2019:00:00:00 +0000] " + + "\"GET /images/cc/sk.png HTTP/1.1\" 200 395\n" + + "0.0.0.0 - - [21/Nov/2019:00:00:00 +0000] " + + "\"GET /images/favicon.ico HTTP/1.1\" 200 1150\n" + + "0.0.0.0 - - [21/Nov/2019:00:00:00 +0000] " + + "\"GET /images/flags/authority.png HTTP/1.1\" 200 325\n" + + "0.0.0.0 - - [21/Nov/2019:00:00:00 +0000] " + + "\"GET /news.atom HTTP/1.1\" 200 36362\n" } + }; + + /** Temporary folder containing all files for this test. */ + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + /** Directory containing web server logs to sanitize. */ + private Path inputDirectory; + + /** Directory storing all intermediate state that needs to be preserved + * between processing runs. */ + private Path statsDirectory; + + /** Directory holding sanitized bridge descriptor files for tarballs. */ + private Path outDirectory; + + /** Directory holding recent sanitized web server logs. */ + private Path recentDirectory; + + /** CollecTor configuration for this test. */ + private Configuration configuration; + + /** Prepares the temporary folder and the various builders for this + * test. */ + @Before + public void createTemporaryFolderAndBuilders() + throws IOException { + this.inputDirectory = this.temporaryFolder.newFolder("in", + "webstats", "meronense.torproject.org").toPath(); + this.statsDirectory = this.temporaryFolder.newFolder("stats").toPath(); + this.outDirectory = this.temporaryFolder.newFolder("out").toPath(); + this.recentDirectory = this.temporaryFolder.newFolder("indexed", "recent") + .toPath(); + this.initializeTestConfiguration(); + } + + /** Initializes a configuration for the bridge descriptor sanitizer. */ + private void initializeTestConfiguration() throws IOException { + this.configuration = new Configuration(); + this.configuration.load(getClass().getClassLoader().getResourceAsStream( + Main.CONF_FILE)); + this.configuration.setProperty(Key.WebstatsActivated.name(), "true"); + this.configuration.setProperty(Key.WebstatsLocalOrigins.name(), + this.inputDirectory.toString()); + this.configuration.setProperty(Key.StatsPath.name(), + this.statsDirectory.toString()); + this.configuration.setProperty(Key.RecentPath.name(), + this.recentDirectory.toString()); + this.configuration.setProperty(Key.OutputPath.name(), + this.outDirectory.toString()); + } + + private void writeInputFiles(String[] ... inputLogs) throws IOException { + for (String[] inputLog : inputLogs) { + Path inputLogFile = this.inputDirectory.resolve(inputLog[0]); + if (!Files.exists(inputLogFile.getParent())) { + Files.createDirectories(inputLogFile.getParent()); + } + try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter( + new GzipCompressorOutputStream( + Files.newOutputStream(inputLogFile))))) { + bw.write(inputLog[1]); + } + } + } + + private void deleteInputFiles(String[] ... deleteLogs) throws IOException { + for (String[] deleteLog : deleteLogs) { + Path deleteLogFile = this.inputDirectory.resolve(deleteLog[0]); + Files.delete(deleteLogFile); + } + } + + private void sanitizeWeblogs() { + SanitizeWeblogs sw = new SanitizeWeblogs(this.configuration); + sw.startProcessing(); + } + + private void compareResults(String[] ... outputLogs) + throws DescriptorParseException { + SortedMap<String, WebServerAccessLog> parsedLogs = new TreeMap<>(); + for (Descriptor descriptor + : DescriptorSourceFactory.createDescriptorReader() + .readDescriptors(this.recentDirectory.toFile())) { + if (!(descriptor instanceof WebServerAccessLog)) { + fail("Parsed descriptor of unknown type."); + } else { + WebServerAccessLog wsal = (WebServerAccessLog) descriptor; + parsedLogs.put(wsal.getDescriptorFile().getName(), wsal); + } + } + assertEquals(outputLogs.length, parsedLogs.size()); + for (String[] outputLog : outputLogs) { + String expectedLogFilename = outputLog[0]; + List<String> expectedLogLines = Arrays.asList(outputLog[1].split("\n")); + assertTrue(parsedLogs.containsKey(expectedLogFilename)); + List<String> actualLogLines = new ArrayList<>(); + parsedLogs.get(expectedLogFilename).logLines() + .forEach((line) -> actualLogLines.add(line.toString())); + assertEquals(expectedLogLines, actualLogLines); + } + } + + @Test + public void testSingleRun() throws Exception { + this.writeInputFiles(inputLogs); + this.sanitizeWeblogs(); + this.compareResults(outputLogs); + } + @Test - public void bytesForTest() { - String lines = "line\nline\nline\nline\nline\n" - + "line\nline\nline\nline\nline\n"; - assertEquals(lines, new String(SanitizeWeblogs.bytesFor("line", 10))); + public void testSubsequentRuns() throws Exception { + for (String[] inputLog : inputLogs) { + this.writeInputFiles(inputLog); + this.sanitizeWeblogs(); + } + this.compareResults(outputLogs); } + @Test + public void testSubsequentRunsReverseOrder() throws Exception { + for (int i = inputLogs.length - 1; i >= 0; i--) { + this.writeInputFiles(inputLogs[i]); + this.sanitizeWeblogs(); + } + this.compareResults(outputLogs); + } + + @Test + public void testSlidingWindow() throws Exception { + this.writeInputFiles(inputLogs[0], inputLogs[1], inputLogs[2]); + this.sanitizeWeblogs(); + this.compareResults(outputLogs[0]); + this.deleteInputFiles(inputLogs[0]); + this.writeInputFiles(inputLogs[3]); + this.sanitizeWeblogs(); + this.compareResults(outputLogs); + } + + @Test + public void testSingleDayNoLimit() throws Exception { + this.configuration.setProperty(Key.WebstatsLimits.name(), "false"); + this.writeInputFiles(new String[][] { + { "metrics.torproject.org-access.log-20191120.gz", + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] " + + "\"GET /collector/recent/relay-descriptors/ " + + "HTTP/1.1\" 200 10227 \"-\" \"-\" -\n" + + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] " + + "\"HEAD /collector/recent/relay-descriptors/microdescs/ " + + "HTTP/1.1\" 200 - \"-\" \"-\" -\n" + + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] " + + "\"HEAD /collector/recent/exit-lists/ " + + "HTTP/1.1\" 200 - \"-\" \"-\" -\n" + + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] " + + "\"GET /collector/archive/bridge-descriptors/extra-infos/ " + + "HTTP/1.1\" 200 48013 \"-\" \"-\" -\n" } }); + this.sanitizeWeblogs(); + this.compareResults(outputLogs[0]); + } + + @Test + public void testErrorLog() throws Exception { + this.configuration.setProperty(Key.WebstatsLimits.name(), "false"); + this.writeInputFiles(new String[][] { + { "metrics.torproject.org-error.log-20191121.gz", + "[Thu Nov 21 15:13:15.211234 2019] [authz_core:error] " + + "[pid 12920:tid 139635582793920] [client 127.0.0.1:59912]\n" } }); + this.sanitizeWeblogs(); + this.compareResults(); + } + + @Test + public void testNonMatchingLines() throws Exception { + this.configuration.setProperty(Key.WebstatsLimits.name(), "false"); + this.writeInputFiles(new String[][] { + { "metrics.torproject.org-access.log-20191121.gz", + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] \"GET /favicon.ico " + + "HTTP/1.1\" 404 8903 \"-\" \"-\" -\n" + + "0.0.0.0 - - [20/Nov/2019:00:00:00 +0000] \"POST /con.php " + + "HTTP/1.1\" 301 320 \"http://metrics.torproject.org/con.php\" " + + "\"-\" -\n" + + "[Thu Nov 21 15:13:15.211234 2019] [authz_core:error] " + + "[pid 12920:tid 139635582793920] [client 127.0.0.1:59912]\n" } }); + this.sanitizeWeblogs(); + this.compareResults(); + } }

1 0

[collector/release] Remember processed files between module runs.
by karsten＠torproject.org 15 Jan '20

15 Jan '20

commit 741401a0daffda52fd1de81b29b276ed9e939ba5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jan 7 13:30:28 2020 +0100 Remember processed files between module runs. The three recently added modules to archive Snowflake statistics, bridge pool assignments, and BridgeDB metrics have in common that they process any input files regardless of whether they already processed them before. The problem is that the input files processed by these modules are either never removed (Snowflake statistics) or only removed manually by the operator (bridge pool assignments and BridgeDB statistics). The effect is that non-recent BridgeDB metrics and bridge pool assignments are being placed in the indexed/recent/ directory in the next execution after they are deleted for being older than 72 hours. The same would happen with Snowflake statistics after the operator removes them from the out/ directory. The fix is to use a state file containing file names of previously processed files and only process a file not found in there. This is the same approach as taken for bridge descriptor tarballs. --- CHANGELOG.md | 3 ++ .../bridgedb/BridgedbMetricsProcessor.java | 18 +++++++++ .../BridgePoolAssignmentsProcessor.java | 19 +++++++++ .../metrics/collector/cron/CollecTorMain.java | 47 ++++++++++++++++++++++ .../snowflake/SnowflakeStatsDownloader.java | 13 ++++++ 5 files changed, 100 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe7937c..fb295d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ - Give up on periodically checking the configuration file for updates and reloading it in case of changes. - Avoid reprocessing webstats files. + - Remember processed files between module runs for archived + Snowflake statistics, bridge pool assignments, and BridgeDB + metrics. * Minor changes - Remove dependency on metrics-lib's internal package. diff --git a/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java index c6b939b..0073ee3 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java @@ -24,7 +24,9 @@ import java.nio.file.Paths; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Arrays; +import java.util.SortedSet; import java.util.Stack; +import java.util.TreeSet; public class BridgedbMetricsProcessor extends CollecTorMain { @@ -40,6 +42,11 @@ public class BridgedbMetricsProcessor extends CollecTorMain { private File inputDirectory; /** + * File containing file names of previously parsed BridgeDB metrics files. + */ + private Path parsedBridgedbMetricsFile; + + /** * Directory for writing BridgeDB statistics files to be archived in tarballs. */ private String outputPathName; @@ -88,11 +95,19 @@ public class BridgedbMetricsProcessor extends CollecTorMain { protected void startProcessing() throws ConfigurationException { logger.info("Starting BridgeDB statistics module of CollecTor."); this.initializeConfiguration(); + SortedSet<Path> previouslyProcessedFiles = this.readProcessedFiles( + this.parsedBridgedbMetricsFile); + SortedSet<Path> processedFiles = new TreeSet<>(); logger.info("Reading BridgeDB statistics files in {}.", this.inputDirectory); for (Descriptor descriptor : DescriptorSourceFactory.createDescriptorReader() .readDescriptors(this.inputDirectory)) { + processedFiles.add(descriptor.getDescriptorFile().toPath()); + if (previouslyProcessedFiles.contains( + descriptor.getDescriptorFile().toPath())) { + continue; + } if (descriptor instanceof BridgedbMetrics) { BridgedbMetrics bridgedbMetrics = (BridgedbMetrics) descriptor; BridgedbMetricsPersistence persistence @@ -114,6 +129,7 @@ public class BridgedbMetricsProcessor extends CollecTorMain { } logger.info("Cleaning up directory {} containing recent files.", this.recentPathName); + this.writeProcessedFiles(this.parsedBridgedbMetricsFile, processedFiles); this.cleanUpRsyncDirectory(); logger.info("Finished processing BridgeDB statistics file(s)."); } @@ -123,6 +139,8 @@ public class BridgedbMetricsProcessor extends CollecTorMain { * storing them in instance attributes. */ private void initializeConfiguration() throws ConfigurationException { + this.parsedBridgedbMetricsFile = this.config.getPath(Key.StatsPath) + .resolve("processed-bridgedb-metrics"); this.outputPathName = config.getPath(Key.OutputPath).toString(); this.recentPathName = config.getPath(Key.RecentPath).toString(); this.inputDirectory = diff --git a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java index 6034e1d..ffae262 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java +++ b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java @@ -25,6 +25,7 @@ import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.time.DateTimeException; import java.time.Instant; @@ -36,8 +37,10 @@ import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.SortedMap; +import java.util.SortedSet; import java.util.Stack; import java.util.TreeMap; +import java.util.TreeSet; public class BridgePoolAssignmentsProcessor extends CollecTorMain { @@ -54,6 +57,11 @@ public class BridgePoolAssignmentsProcessor extends CollecTorMain { private File assignmentsDirectory; /** + * File containing file names of previously parsed assignments files. + */ + private Path parsedBridgePoolAssignmentsFile; + + /** * Directory containing sanitized bridge pool assignments for tarballs. */ private String outputPathName; @@ -117,9 +125,16 @@ public class BridgePoolAssignmentsProcessor extends CollecTorMain { protected void startProcessing() throws ConfigurationException { logger.info("Starting bridge-pool-assignments module of CollecTor."); this.initializeConfiguration(); + SortedSet<Path> previouslyProcessedFiles = this.readProcessedFiles( + this.parsedBridgePoolAssignmentsFile); + SortedSet<Path> processedFiles = new TreeSet<>(); List<File> assignmentFiles = this.listAssignmentFiles(); LocalDateTime latestPublished = null; for (File assignmentFile : assignmentFiles) { + processedFiles.add(assignmentFile.toPath()); + if (previouslyProcessedFiles.contains(assignmentFile.toPath())) { + continue; + } logger.info("Processing bridge pool assignment file '{}'...", assignmentFile.getAbsolutePath()); SortedMap<LocalDateTime, SortedMap<String, String>> @@ -161,6 +176,8 @@ public class BridgePoolAssignmentsProcessor extends CollecTorMain { + "published at {}, which is more than 5:30 hours in the past.", latestPublished); } + this.writeProcessedFiles(this.parsedBridgePoolAssignmentsFile, + processedFiles); this.cleanUpRsyncDirectory(); logger.info("Finished processing bridge pool assignment file(s)."); } @@ -170,6 +187,8 @@ public class BridgePoolAssignmentsProcessor extends CollecTorMain { * storing them in instance attributes. */ private void initializeConfiguration() throws ConfigurationException { + this.parsedBridgePoolAssignmentsFile = this.config.getPath(Key.StatsPath) + .resolve("parsed-bridge-pool-assignments"); this.outputPathName = Paths.get(config.getPath(Key.OutputPath).toString(), "bridge-pool-assignments").toString(); this.recentPathName = Paths.get(config.getPath(Key.RecentPath).toString(), diff --git a/src/main/java/org/torproject/metrics/collector/cron/CollecTorMain.java b/src/main/java/org/torproject/metrics/collector/cron/CollecTorMain.java index 49174ca..979c3ca 100644 --- a/src/main/java/org/torproject/metrics/collector/cron/CollecTorMain.java +++ b/src/main/java/org/torproject/metrics/collector/cron/CollecTorMain.java @@ -16,9 +16,14 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; import java.util.concurrent.Callable; public abstract class CollecTorMain extends SyncManager @@ -128,5 +133,47 @@ public abstract class CollecTorMain extends SyncManager + ioe.getMessage(), ioe); } } + + /** + * Read file names of processed files from the given state file. + * + * @param stateFile State file to read file names from. + * @return File names of processed files. + */ + public SortedSet<Path> readProcessedFiles(Path stateFile) { + SortedSet<Path> processedFiles = new TreeSet<>(); + if (Files.exists(stateFile)) { + try { + for (String line : Files.readAllLines(stateFile)) { + processedFiles.add(Paths.get(line)); + } + } catch (IOException e) { + logger.warn("I/O error while reading processed files.", e); + } + } + return processedFiles; + } + + /** + * Write file names of processed files to the state file. + * + * @param stateFile State file to write file names to. + * @param processedFiles File names of processed files. + */ + public void writeProcessedFiles(Path stateFile, + SortedSet<Path> processedFiles) { + List<String> lines = new ArrayList<>(); + for (Path processedFile : processedFiles) { + lines.add(processedFile.toString()); + } + try { + if (!Files.exists(stateFile)) { + Files.createDirectories(stateFile.getParent()); + } + Files.write(stateFile, lines); + } catch (IOException e) { + logger.warn("I/O error while writing processed files.", e); + } + } } diff --git a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java index f40c311..cbca74a 100644 --- a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java @@ -23,6 +23,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.time.LocalDateTime; import java.util.Arrays; import java.util.SortedSet; @@ -71,6 +73,11 @@ public class SnowflakeStatsDownloader extends CollecTorMain { } logger.debug("Finished downloading {}.", url); + Path parsedSnowflakeStatsFile = this.config.getPath(Key.StatsPath) + .resolve("processed-snowflake-stats"); + SortedSet<Path> previouslyProcessedFiles = this.readProcessedFiles( + parsedSnowflakeStatsFile); + SortedSet<Path> processedFiles = new TreeSet<>(); DescriptorParser descriptorParser = DescriptorSourceFactory.createDescriptorParser(); SortedSet<LocalDateTime> snowflakeStatsEnds = new TreeSet<>(); @@ -85,6 +92,11 @@ public class SnowflakeStatsDownloader extends CollecTorMain { = new SnowflakeStatsPersistence(snowflakeStats); File tarballFile = new File(outputPathName + "/" + persistence.getStoragePath()); + Path relativeFileName = Paths.get(tarballFile.getName()); + processedFiles.add(relativeFileName); + if (previouslyProcessedFiles.contains(relativeFileName)) { + continue; + } if (tarballFile.exists()) { continue; } @@ -106,6 +118,7 @@ public class SnowflakeStatsDownloader extends CollecTorMain { snowflakeStatsEnds.last()); } + this.writeProcessedFiles(parsedSnowflakeStatsFile, processedFiles); this.cleanUpRsyncDirectory(); }

1 0

[collector/release] Remove dependency on metrics-lib's log package (4/4).
by karsten＠torproject.org 15 Jan '20

15 Jan '20

commit 8263cc7bdbb0a632f12a84fb2051dd9a25c28142 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Nov 23 18:07:41 2019 +0100 Remove dependency on metrics-lib's log package (4/4). - Remove package-internal abstract class. --- .../collector/webstats/LogDescriptorImpl.java | 117 --------------------- .../collector/webstats/SanitizeWeblogs.java | 2 +- .../collector/webstats/WebServerAccessLogImpl.java | 82 +++++++++++++-- 3 files changed, 73 insertions(+), 128 deletions(-) diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java deleted file mode 100644 index e20be27..0000000 --- a/src/main/java/org/torproject/metrics/collector/webstats/LogDescriptorImpl.java +++ /dev/null @@ -1,117 +0,0 @@ -/* Copyright 2017--2018 The Tor Project - * See LICENSE for licensing information */ - -package org.torproject.metrics.collector.webstats; - -import org.torproject.descriptor.DescriptorParseException; -import org.torproject.descriptor.LogDescriptor; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * Base class for log descriptors. - * - * @since 2.2.0 - */ -public abstract class LogDescriptorImpl implements LogDescriptor { - - /** Logfile name parts separator. */ - public static final String SEP = "_"; - - /** The log's file name should contain this string. */ - public static final String MARKER = ".log"; - - private static Pattern filenamePattern = Pattern.compile( - "(?:\\S*)" + MARKER + SEP + "(?:[0-9a-zA-Z]*)(?:\\.?)([a-zA-Z2]*)"); - - private final File descriptorFile; - - /** Byte array for plain, i.e. uncompressed, log data. */ - private byte[] logBytes; - - private FileType fileType; - - private List<String> unrecognizedLines = new ArrayList<>(); - - /** - * This constructor performs basic operations on the given bytes. - * - * An unknown compression type (see {@link #getCompressionType}) - * is interpreted as missing compression. In this case the bytes - * will be compressed to the given compression type. - * - * @since 2.2.0 - */ - protected LogDescriptorImpl(byte[] logBytes, File descriptorFile, - String logName) throws DescriptorParseException { - this.logBytes = logBytes; - this.descriptorFile = descriptorFile; - try { - Matcher mat = filenamePattern.matcher(logName); - if (!mat.find()) { - throw new DescriptorParseException( - "Log file name doesn't comply to standard: " + logName); - } - this.fileType = FileType.findType(mat.group(1).toUpperCase()); - if (FileType.PLAIN == this.fileType) { - this.fileType = FileType.XZ; - this.logBytes = this.fileType.compress(this.logBytes); - } - } catch (Exception ex) { - throw new DescriptorParseException("Cannot parse file " - + logName + " from file " + descriptorFile.getName(), ex); - } - } - - @Override - public InputStream decompressedByteStream() throws DescriptorParseException { - try { - return this.fileType.decompress(new ByteArrayInputStream(this.logBytes)); - } catch (Exception ex) { - throw new DescriptorParseException("Cannot provide deflated stream of " - + this.descriptorFile + ".", ex); - } - } - - public String getCompressionType() { - return this.fileType.name().toLowerCase(); - } - - @Override - public byte[] getRawDescriptorBytes() { - return this.logBytes; - } - - public void setRawDescriptorBytes(byte[] bytes) { - this.logBytes = bytes; - } - - @Override - public int getRawDescriptorLength() { - return this.logBytes.length; - } - - @Override - public List<String> getAnnotations() { - return Collections.emptyList(); - } - - @Override - public List<String> getUnrecognizedLines() { - return this.unrecognizedLines; - } - - @Override - public File getDescriptorFile() { - return descriptorFile; - } - -} - diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java index 51318f8..fc7c64f 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java @@ -135,7 +135,7 @@ public class SanitizeWeblogs extends CollecTorMain { private void storeSortedAndForget(String virtualHost, String physicalHost, LocalDate date, Map<String, Long> lineCounts) { - String name = new StringJoiner(LogDescriptorImpl.SEP) + String name = new StringJoiner(WebServerAccessLogImpl.SEP) .add(virtualHost).add(physicalHost) .add(WebServerAccessLogImpl.MARKER) .add(date.format(DateTimeFormatter.BASIC_ISO_DATE)) diff --git a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java index 4ec9e39..af77c94 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java @@ -7,13 +7,15 @@ import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.WebServerAccessLog; import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.File; +import java.io.InputStream; import java.io.InputStreamReader; -import java.nio.file.Paths; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -26,8 +28,10 @@ import java.util.stream.Stream; * * @since 2.2.0 */ -public class WebServerAccessLogImpl extends LogDescriptorImpl - implements WebServerAccessLog { +public class WebServerAccessLogImpl implements WebServerAccessLog { + + /** Logfile name parts separator. */ + public static final String SEP = "_"; /** The log's name should include this string. */ public static final String MARKER = "access.log"; @@ -37,6 +41,15 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl = Pattern.compile("(\\S*)" + SEP + "(\\S*)" + SEP + "" + MARKER + SEP + "(\\d*)(?:\\.?)([a-zA-Z]*)"); + private final File descriptorFile; + + /** Byte array for plain, i.e. uncompressed, log data. */ + private byte[] logBytes; + + private FileType fileType; + + private List<String> unrecognizedLines = new ArrayList<>(); + private final String physicalHost; private final String virtualHost; @@ -58,25 +71,31 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl * The immediate parent name is taken to be the physical host collecting the * logs. */ - protected WebServerAccessLogImpl(byte[] logBytes, File file, String logName) - throws DescriptorParseException { - super(logBytes, file, logName); + protected WebServerAccessLogImpl(byte[] logBytes, File descriptorFile, + String logName) throws DescriptorParseException { + this.logBytes = logBytes; + this.descriptorFile = descriptorFile; try { - String fn = Paths.get(logName).getFileName().toString(); - Matcher mat = filenamePattern.matcher(fn); + Matcher mat = filenamePattern.matcher(logName); if (!mat.find()) { throw new DescriptorParseException( - "WebServerAccessLog file name doesn't comply to standard: " + fn); + "Log file name doesn't comply to standard: " + logName); } this.virtualHost = mat.group(1); this.physicalHost = mat.group(2); if (null == this.virtualHost || null == this.physicalHost || this.virtualHost.isEmpty() || this.physicalHost.isEmpty()) { throw new DescriptorParseException( - "WebServerAccessLog file name doesn't comply to standard: " + fn); + "WebServerAccessLog file name doesn't comply to standard: " + + logName); } String ymd = mat.group(3); this.logDate = LocalDate.parse(ymd, DateTimeFormatter.BASIC_ISO_DATE); + this.fileType = FileType.findType(mat.group(4).toUpperCase()); + if (FileType.PLAIN == this.fileType) { + this.fileType = FileType.XZ; + this.logBytes = this.fileType.compress(this.logBytes); + } } catch (DescriptorParseException dpe) { throw dpe; // escalate } catch (Exception pe) { @@ -86,6 +105,49 @@ public class WebServerAccessLogImpl extends LogDescriptorImpl } @Override + public InputStream decompressedByteStream() throws DescriptorParseException { + try { + return this.fileType.decompress(new ByteArrayInputStream(this.logBytes)); + } catch (Exception ex) { + throw new DescriptorParseException("Cannot provide deflated stream of " + + this.descriptorFile + ".", ex); + } + } + + public String getCompressionType() { + return this.fileType.name().toLowerCase(); + } + + @Override + public byte[] getRawDescriptorBytes() { + return this.logBytes; + } + + public void setRawDescriptorBytes(byte[] bytes) { + this.logBytes = bytes; + } + + @Override + public int getRawDescriptorLength() { + return this.logBytes.length; + } + + @Override + public List<String> getAnnotations() { + return Collections.emptyList(); + } + + @Override + public List<String> getUnrecognizedLines() { + return this.unrecognizedLines; + } + + @Override + public File getDescriptorFile() { + return descriptorFile; + } + + @Override public String getPhysicalHost() { return this.physicalHost; }

1 0

[collector/release] Update copyright to 2020.
by karsten＠torproject.org 15 Jan '20

15 Jan '20

commit d2a74b676a0d1c8563638ca3607a866b95877949 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jan 15 21:36:34 2020 +0100 Update copyright to 2020. --- LICENSE | 2 +- src/main/java/org/torproject/metrics/collector/Main.java | 2 +- .../torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java | 2 +- .../metrics/collector/bridgedescs/BridgeDescriptorParser.java | 2 +- .../torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java | 2 +- .../org/torproject/metrics/collector/bridgedescs/DescriptorBuilder.java | 2 +- .../metrics/collector/bridgedescs/SanitizedBridgesWriter.java | 2 +- .../metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java | 2 +- src/main/java/org/torproject/metrics/collector/conf/Annotation.java | 2 +- src/main/java/org/torproject/metrics/collector/conf/Configuration.java | 2 +- .../org/torproject/metrics/collector/conf/ConfigurationException.java | 2 +- src/main/java/org/torproject/metrics/collector/conf/Key.java | 2 +- src/main/java/org/torproject/metrics/collector/conf/SourceType.java | 2 +- src/main/java/org/torproject/metrics/collector/cron/CollecTorMain.java | 2 +- src/main/java/org/torproject/metrics/collector/cron/Scheduler.java | 2 +- src/main/java/org/torproject/metrics/collector/cron/ShutdownHook.java | 2 +- .../java/org/torproject/metrics/collector/downloader/Downloader.java | 2 +- .../org/torproject/metrics/collector/exitlists/ExitListDownloader.java | 2 +- .../java/org/torproject/metrics/collector/indexer/CreateIndexJson.java | 2 +- .../java/org/torproject/metrics/collector/indexer/DirectoryNode.java | 2 +- src/main/java/org/torproject/metrics/collector/indexer/FileNode.java | 2 +- src/main/java/org/torproject/metrics/collector/indexer/IndexNode.java | 2 +- src/main/java/org/torproject/metrics/collector/indexer/IndexerTask.java | 2 +- .../org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java | 2 +- .../torproject/metrics/collector/persist/BandwidthFilePersistence.java | 2 +- .../metrics/collector/persist/BridgeExtraInfoPersistence.java | 2 +- .../metrics/collector/persist/BridgePoolAssignmentPersistence.java | 2 +- .../metrics/collector/persist/BridgeServerDescriptorPersistence.java | 2 +- .../metrics/collector/persist/BridgedbMetricsPersistence.java | 2 +- .../org/torproject/metrics/collector/persist/ConsensusPersistence.java | 2 +- .../org/torproject/metrics/collector/persist/DescriptorPersistence.java | 2 +- .../org/torproject/metrics/collector/persist/ExitlistPersistence.java | 2 +- .../org/torproject/metrics/collector/persist/ExtraInfoPersistence.java | 2 +- .../torproject/metrics/collector/persist/MicroConsensusPersistence.java | 2 +- .../org/torproject/metrics/collector/persist/OnionPerfPersistence.java | 2 +- .../java/org/torproject/metrics/collector/persist/PersistenceUtils.java | 2 +- .../metrics/collector/persist/ServerDescriptorPersistence.java | 2 +- .../torproject/metrics/collector/persist/SnowflakeStatsPersistence.java | 2 +- .../org/torproject/metrics/collector/persist/StatusPersistence.java | 2 +- .../java/org/torproject/metrics/collector/persist/VotePersistence.java | 2 +- .../metrics/collector/persist/WebServerAccessLogPersistence.java | 2 +- .../java/org/torproject/metrics/collector/persist/package-info.java | 2 +- .../java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java | 2 +- .../java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java | 2 +- .../metrics/collector/relaydescs/CachedRelayDescriptorReader.java | 2 +- .../org/torproject/metrics/collector/relaydescs/ReferenceChecker.java | 2 +- .../metrics/collector/relaydescs/RelayDescriptorDownloader.java | 2 +- .../torproject/metrics/collector/relaydescs/RelayDescriptorParser.java | 2 +- .../metrics/collector/snowflake/SnowflakeStatsDownloader.java | 2 +- src/main/java/org/torproject/metrics/collector/sync/Criterium.java | 2 +- .../java/org/torproject/metrics/collector/sync/ProcessCriterium.java | 2 +- src/main/java/org/torproject/metrics/collector/sync/SyncManager.java | 2 +- .../java/org/torproject/metrics/collector/sync/SyncPersistence.java | 2 +- src/main/java/org/torproject/metrics/collector/sync/package-info.java | 2 +- src/main/java/org/torproject/metrics/collector/webstats/FileType.java | 2 +- src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java | 2 +- .../java/org/torproject/metrics/collector/webstats/LogMetadata.java | 2 +- .../java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java | 2 +- .../torproject/metrics/collector/webstats/WebServerAccessLogImpl.java | 2 +- .../torproject/metrics/collector/webstats/WebServerAccessLogLine.java | 2 +- src/main/resources/bootstrap-development.sh | 2 +- src/main/resources/collector.properties | 2 +- src/main/resources/create-tarballs.sh | 2 +- src/test/java/org/torproject/metrics/collector/MainTest.java | 2 +- .../metrics/collector/bridgedescs/BridgeDescriptorParserTest.java | 2 +- .../metrics/collector/bridgedescs/ExtraInfoTestDescriptorBuilder.java | 2 +- .../collector/bridgedescs/NetworkStatusTestDescriptorBuilder.java | 2 +- .../metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java | 2 +- .../metrics/collector/bridgedescs/ServerTestDescriptorBuilder.java | 2 +- .../torproject/metrics/collector/bridgedescs/TarballTestBuilder.java | 2 +- .../torproject/metrics/collector/bridgedescs/TestDescriptorBuilder.java | 2 +- .../java/org/torproject/metrics/collector/conf/ConfigurationTest.java | 2 +- src/test/java/org/torproject/metrics/collector/cron/Broken.java | 2 +- .../java/org/torproject/metrics/collector/cron/CollecTorMainTest.java | 2 +- src/test/java/org/torproject/metrics/collector/cron/SchedulerTest.java | 2 +- .../org/torproject/metrics/collector/downloader/DownloaderTest.java | 2 +- .../org/torproject/metrics/collector/indexer/CreateIndexJsonTest.java | 2 +- .../java/org/torproject/metrics/collector/indexer/IndexerTaskTest.java | 2 +- .../java/org/torproject/metrics/collector/persist/PersistUtilsTest.java | 2 +- .../torproject/metrics/collector/relaydescs/ReferenceCheckerTest.java | 2 +- src/test/java/org/torproject/metrics/collector/sync/FileCollector.java | 2 +- .../java/org/torproject/metrics/collector/sync/SyncPersistenceTest.java | 2 +- .../java/org/torproject/metrics/collector/webstats/LogFileMapTest.java | 2 +- .../java/org/torproject/metrics/collector/webstats/LogMetadataTest.java | 2 +- .../org/torproject/metrics/collector/webstats/SanitizeWeblogsTest.java | 2 +- 85 files changed, 85 insertions(+), 85 deletions(-) diff --git a/LICENSE b/LICENSE index eb96349..9aaf268 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2010--2017 The Tor Project +Copyright 2010--2020 The Tor Project Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/src/main/java/org/torproject/metrics/collector/Main.java b/src/main/java/org/torproject/metrics/collector/Main.java index 861567f..77bb2ca 100644 --- a/src/main/java/org/torproject/metrics/collector/Main.java +++ b/src/main/java/org/torproject/metrics/collector/Main.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector; diff --git a/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java index 9bb4f36..c6b939b 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java @@ -1,4 +1,4 @@ -/* Copyright 2019 The Tor Project +/* Copyright 2019--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.bridgedb; diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java index 778d187..b5e30bc 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParser.java @@ -1,4 +1,4 @@ -/* Copyright 2010--2018 The Tor Project +/* Copyright 2010--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.bridgedescs; diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java index 041807e..de9cd4b 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/BridgeSnapshotReader.java @@ -1,4 +1,4 @@ -/* Copyright 2010--2018 The Tor Project +/* Copyright 2010--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.bridgedescs; diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/DescriptorBuilder.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/DescriptorBuilder.java index 8fc25b4..b4b63e7 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/DescriptorBuilder.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/DescriptorBuilder.java @@ -1,4 +1,4 @@ -/* Copyright 2018 The Tor Project +/* Copyright 2018--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.bridgedescs; diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java index 45f2a1a..6aee057 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java +++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java @@ -1,4 +1,4 @@ -/* Copyright 2010--2018 The Tor Project +/* Copyright 2010--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.bridgedescs; diff --git a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java index 875273d..6034e1d 100644 --- a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java +++ b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java @@ -1,4 +1,4 @@ -/* Copyright 2011--2019 The Tor Project +/* Copyright 2011--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.bridgepools; diff --git a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java index ff5119e..0d3530e 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java +++ b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.conf; diff --git a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java index cfdd9c8..52e1d05 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java +++ b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.conf; diff --git a/src/main/java/org/torproject/metrics/collector/conf/ConfigurationException.java b/src/main/java/org/torproject/metrics/collector/conf/ConfigurationException.java index 5257434..fb45ee8 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/ConfigurationException.java +++ b/src/main/java/org/torproject/metrics/collector/conf/ConfigurationException.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.conf; diff --git a/src/main/java/org/torproject/metrics/collector/conf/Key.java b/src/main/java/org/torproject/metrics/collector/conf/Key.java index d0b8a5a..866cbad 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/Key.java +++ b/src/main/java/org/torproject/metrics/collector/conf/Key.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.conf; diff --git a/src/main/java/org/torproject/metrics/collector/conf/SourceType.java b/src/main/java/org/torproject/metrics/collector/conf/SourceType.java index ed5df4d..6d2ea84 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/SourceType.java +++ b/src/main/java/org/torproject/metrics/collector/conf/SourceType.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.conf; diff --git a/src/main/java/org/torproject/metrics/collector/cron/CollecTorMain.java b/src/main/java/org/torproject/metrics/collector/cron/CollecTorMain.java index 09796e3..49174ca 100644 --- a/src/main/java/org/torproject/metrics/collector/cron/CollecTorMain.java +++ b/src/main/java/org/torproject/metrics/collector/cron/CollecTorMain.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.cron; diff --git a/src/main/java/org/torproject/metrics/collector/cron/Scheduler.java b/src/main/java/org/torproject/metrics/collector/cron/Scheduler.java index 054f03f..03292f4 100644 --- a/src/main/java/org/torproject/metrics/collector/cron/Scheduler.java +++ b/src/main/java/org/torproject/metrics/collector/cron/Scheduler.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.cron; diff --git a/src/main/java/org/torproject/metrics/collector/cron/ShutdownHook.java b/src/main/java/org/torproject/metrics/collector/cron/ShutdownHook.java index 70699f6..c8f0807 100644 --- a/src/main/java/org/torproject/metrics/collector/cron/ShutdownHook.java +++ b/src/main/java/org/torproject/metrics/collector/cron/ShutdownHook.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.cron; diff --git a/src/main/java/org/torproject/metrics/collector/downloader/Downloader.java b/src/main/java/org/torproject/metrics/collector/downloader/Downloader.java index a1f3852..e82737e 100644 --- a/src/main/java/org/torproject/metrics/collector/downloader/Downloader.java +++ b/src/main/java/org/torproject/metrics/collector/downloader/Downloader.java @@ -1,4 +1,4 @@ -/* Copyright 2019 The Tor Project +/* Copyright 2019--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.downloader; diff --git a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java index b4bee15..49e176b 100644 --- a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java @@ -1,4 +1,4 @@ -/* Copyright 2010--2018 The Tor Project +/* Copyright 2010--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.exitlists; diff --git a/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java b/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java index 15aa31d..6613e9f 100644 --- a/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java +++ b/src/main/java/org/torproject/metrics/collector/indexer/CreateIndexJson.java @@ -1,4 +1,4 @@ -/* Copyright 2015--2019 The Tor Project +/* Copyright 2015--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.indexer; diff --git a/src/main/java/org/torproject/metrics/collector/indexer/DirectoryNode.java b/src/main/java/org/torproject/metrics/collector/indexer/DirectoryNode.java index a369d08..c537709 100644 --- a/src/main/java/org/torproject/metrics/collector/indexer/DirectoryNode.java +++ b/src/main/java/org/torproject/metrics/collector/indexer/DirectoryNode.java @@ -1,4 +1,4 @@ -/* Copyright 2019 The Tor Project +/* Copyright 2019--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.indexer; diff --git a/src/main/java/org/torproject/metrics/collector/indexer/FileNode.java b/src/main/java/org/torproject/metrics/collector/indexer/FileNode.java index c007196..c3031a4 100644 --- a/src/main/java/org/torproject/metrics/collector/indexer/FileNode.java +++ b/src/main/java/org/torproject/metrics/collector/indexer/FileNode.java @@ -1,4 +1,4 @@ -/* Copyright 2019 The Tor Project +/* Copyright 2019--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.indexer; diff --git a/src/main/java/org/torproject/metrics/collector/indexer/IndexNode.java b/src/main/java/org/torproject/metrics/collector/indexer/IndexNode.java index 8b7a46b..7d7569c 100644 --- a/src/main/java/org/torproject/metrics/collector/indexer/IndexNode.java +++ b/src/main/java/org/torproject/metrics/collector/indexer/IndexNode.java @@ -1,4 +1,4 @@ -/* Copyright 2019 The Tor Project +/* Copyright 2019--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.indexer; diff --git a/src/main/java/org/torproject/metrics/collector/indexer/IndexerTask.java b/src/main/java/org/torproject/metrics/collector/indexer/IndexerTask.java index 03c750b..dee2c30 100644 --- a/src/main/java/org/torproject/metrics/collector/indexer/IndexerTask.java +++ b/src/main/java/org/torproject/metrics/collector/indexer/IndexerTask.java @@ -1,4 +1,4 @@ -/* Copyright 2019 The Tor Project +/* Copyright 2019--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.indexer; diff --git a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java index b106be7..dc1a675 100644 --- a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java @@ -1,4 +1,4 @@ -/* Copyright 2012--2018 The Tor Project +/* Copyright 2012--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.onionperf; diff --git a/src/main/java/org/torproject/metrics/collector/persist/BandwidthFilePersistence.java b/src/main/java/org/torproject/metrics/collector/persist/BandwidthFilePersistence.java index aeaf0ee..bbbfca5 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/BandwidthFilePersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/BandwidthFilePersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/BridgeExtraInfoPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/BridgeExtraInfoPersistence.java index 23245c3..04cb473 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/BridgeExtraInfoPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/BridgeExtraInfoPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/BridgePoolAssignmentPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/BridgePoolAssignmentPersistence.java index 5613060..e729dab 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/BridgePoolAssignmentPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/BridgePoolAssignmentPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/BridgeServerDescriptorPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/BridgeServerDescriptorPersistence.java index 7694f14..27c37d0 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/BridgeServerDescriptorPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/BridgeServerDescriptorPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/BridgedbMetricsPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/BridgedbMetricsPersistence.java index a72ffe2..a6ca106 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/BridgedbMetricsPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/BridgedbMetricsPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2019 The Tor Project +/* Copyright 2019--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/ConsensusPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/ConsensusPersistence.java index b48077f..2ae9357 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/ConsensusPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/ConsensusPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java index 20cd570..7c648ef 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/ExitlistPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/ExitlistPersistence.java index 307f3fa..3f85600 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/ExitlistPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/ExitlistPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/ExtraInfoPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/ExtraInfoPersistence.java index 4ac8cba..33b4c3c 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/ExtraInfoPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/ExtraInfoPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/MicroConsensusPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/MicroConsensusPersistence.java index a3e189a..5514c28 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/MicroConsensusPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/MicroConsensusPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java index bc654a4..7ed16a2 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/OnionPerfPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2017--2018 The Tor Project +/* Copyright 2017--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java index 78eac72..72ad73a 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java +++ b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/ServerDescriptorPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/ServerDescriptorPersistence.java index bb0ae64..71a2493 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/ServerDescriptorPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/ServerDescriptorPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/SnowflakeStatsPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/SnowflakeStatsPersistence.java index ee6e029..d9ae41c 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/SnowflakeStatsPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/SnowflakeStatsPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2019 The Tor Project +/* Copyright 2019--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/StatusPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/StatusPersistence.java index c3cbf9e..c379410 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/StatusPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/StatusPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/VotePersistence.java b/src/main/java/org/torproject/metrics/collector/persist/VotePersistence.java index 0c1f80a..461ca40 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/VotePersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/VotePersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java index 0001957..1c4d519 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/WebServerAccessLogPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/main/java/org/torproject/metrics/collector/persist/package-info.java b/src/main/java/org/torproject/metrics/collector/persist/package-info.java index 75ec95e..2b32235 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/package-info.java +++ b/src/main/java/org/torproject/metrics/collector/persist/package-info.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ /** This package contains decorating classes for descriptors that diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java index 3326833..5a80058 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java @@ -1,4 +1,4 @@ -/* Copyright 2010--2018 The Tor Project +/* Copyright 2010--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.relaydescs; diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java index 911ac2a..e0e1623 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java @@ -1,4 +1,4 @@ -/* Copyright 2010--2018 The Tor Project +/* Copyright 2010--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.relaydescs; diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java index 0a8012a..21a3041 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java @@ -1,4 +1,4 @@ -/* Copyright 2010--2018 The Tor Project +/* Copyright 2010--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.relaydescs; diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ReferenceChecker.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ReferenceChecker.java index f3a21ba..5ff77e7 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/ReferenceChecker.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ReferenceChecker.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.relaydescs; diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java index 3337633..b0fcebb 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java @@ -1,4 +1,4 @@ -/* Copyright 2010--2018 The Tor Project +/* Copyright 2010--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.relaydescs; diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java index abbd777..3905458 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java @@ -1,4 +1,4 @@ -/* Copyright 2010--2018 The Tor Project +/* Copyright 2010--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.relaydescs; diff --git a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java index cb5f0cc..f40c311 100644 --- a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java @@ -1,4 +1,4 @@ -/* Copyright 2019 The Tor Project +/* Copyright 2019--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.snowflake; diff --git a/src/main/java/org/torproject/metrics/collector/sync/Criterium.java b/src/main/java/org/torproject/metrics/collector/sync/Criterium.java index a90de8e..67cebdc 100644 --- a/src/main/java/org/torproject/metrics/collector/sync/Criterium.java +++ b/src/main/java/org/torproject/metrics/collector/sync/Criterium.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.sync; diff --git a/src/main/java/org/torproject/metrics/collector/sync/ProcessCriterium.java b/src/main/java/org/torproject/metrics/collector/sync/ProcessCriterium.java index d4ec149..ed442fa 100644 --- a/src/main/java/org/torproject/metrics/collector/sync/ProcessCriterium.java +++ b/src/main/java/org/torproject/metrics/collector/sync/ProcessCriterium.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.sync; diff --git a/src/main/java/org/torproject/metrics/collector/sync/SyncManager.java b/src/main/java/org/torproject/metrics/collector/sync/SyncManager.java index 5a1a142..e42ae61 100644 --- a/src/main/java/org/torproject/metrics/collector/sync/SyncManager.java +++ b/src/main/java/org/torproject/metrics/collector/sync/SyncManager.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.sync; diff --git a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java index f6678fe..f81e164 100644 --- a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.sync; diff --git a/src/main/java/org/torproject/metrics/collector/sync/package-info.java b/src/main/java/org/torproject/metrics/collector/sync/package-info.java index 53e64b7..0761fd5 100644 --- a/src/main/java/org/torproject/metrics/collector/sync/package-info.java +++ b/src/main/java/org/torproject/metrics/collector/sync/package-info.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ /** This package coordinates syncing and merging the fetched data. diff --git a/src/main/java/org/torproject/metrics/collector/webstats/FileType.java b/src/main/java/org/torproject/metrics/collector/webstats/FileType.java index 15b1e00..489bba4 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/FileType.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/FileType.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2019 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.webstats; diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java index d3e7dcd..5be6b50 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/LogFileMap.java @@ -1,4 +1,4 @@ -/* Copyright 2017--2018 The Tor Project +/* Copyright 2017--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.webstats; diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogMetadata.java b/src/main/java/org/torproject/metrics/collector/webstats/LogMetadata.java index b30c13a..d3bf8fb 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/LogMetadata.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/LogMetadata.java @@ -1,4 +1,4 @@ -/* Copyright 2017--2018 The Tor Project +/* Copyright 2017--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.webstats; diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java index b3fee06..6c8a495 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java @@ -1,4 +1,4 @@ -/* Copyright 2017--2018 The Tor Project +/* Copyright 2017--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.webstats; diff --git a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java index f091aa1..0e51c05 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java @@ -1,4 +1,4 @@ -/* Copyright 2017--2018 The Tor Project +/* Copyright 2017--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.webstats; diff --git a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogLine.java b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogLine.java index bc03c0a..816064a 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogLine.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogLine.java @@ -1,4 +1,4 @@ -/* Copyright 2018 The Tor Project +/* Copyright 2018--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.webstats; diff --git a/src/main/resources/bootstrap-development.sh b/src/main/resources/bootstrap-development.sh index d301e25..ebdc1de 100755 --- a/src/main/resources/bootstrap-development.sh +++ b/src/main/resources/bootstrap-development.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright 2016--2018 The Tor Project +# Copyright 2016--2020 The Tor Project # See LICENSE for licensing information # # Use for submodule initialization and checkout. diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index 6422120..61baed5 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -223,4 +223,4 @@ BridgedbMetricsLocalOrigins = in/bridgedb-stats ## Retrieve files from the following instances. ## List of URLs separated by comma. BridgedbMetricsSyncOrigins = https://collector.torproject.org -# \ No newline at end of file +# diff --git a/src/main/resources/create-tarballs.sh b/src/main/resources/create-tarballs.sh index 3927960..07952c7 100755 --- a/src/main/resources/create-tarballs.sh +++ b/src/main/resources/create-tarballs.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2016--2018 The Tor Project +# Copyright 2016--2020 The Tor Project # See LICENSE for licensing information. # # Script for creating descriptor tarballs on a CollecTor instance, diff --git a/src/test/java/org/torproject/metrics/collector/MainTest.java b/src/test/java/org/torproject/metrics/collector/MainTest.java index f3f2500..02c16df 100644 --- a/src/test/java/org/torproject/metrics/collector/MainTest.java +++ b/src/test/java/org/torproject/metrics/collector/MainTest.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector; diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParserTest.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParserTest.java index 4ea2e97..de82ab7 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParserTest.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/BridgeDescriptorParserTest.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.bridgedescs; diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/ExtraInfoTestDescriptorBuilder.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/ExtraInfoTestDescriptorBuilder.java index 45d6925..f7e6acc 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/ExtraInfoTestDescriptorBuilder.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/ExtraInfoTestDescriptorBuilder.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.bridgedescs; diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/NetworkStatusTestDescriptorBuilder.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/NetworkStatusTestDescriptorBuilder.java index d04fc3c..60540e4 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/NetworkStatusTestDescriptorBuilder.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/NetworkStatusTestDescriptorBuilder.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.bridgedescs; diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java index 1267d4a..67e9738 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriterTest.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.bridgedescs; diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/ServerTestDescriptorBuilder.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/ServerTestDescriptorBuilder.java index c0a6bf5..ec0807d 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/ServerTestDescriptorBuilder.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/ServerTestDescriptorBuilder.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.bridgedescs; diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/TarballTestBuilder.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/TarballTestBuilder.java index 08dca84..0009463 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/TarballTestBuilder.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/TarballTestBuilder.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.bridgedescs; diff --git a/src/test/java/org/torproject/metrics/collector/bridgedescs/TestDescriptorBuilder.java b/src/test/java/org/torproject/metrics/collector/bridgedescs/TestDescriptorBuilder.java index 81df984..03c7267 100644 --- a/src/test/java/org/torproject/metrics/collector/bridgedescs/TestDescriptorBuilder.java +++ b/src/test/java/org/torproject/metrics/collector/bridgedescs/TestDescriptorBuilder.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.bridgedescs; diff --git a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java index 6f5d16c..c45cd4f 100644 --- a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.conf; diff --git a/src/test/java/org/torproject/metrics/collector/cron/Broken.java b/src/test/java/org/torproject/metrics/collector/cron/Broken.java index 44aa88b..a4a624a 100644 --- a/src/test/java/org/torproject/metrics/collector/cron/Broken.java +++ b/src/test/java/org/torproject/metrics/collector/cron/Broken.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.cron; diff --git a/src/test/java/org/torproject/metrics/collector/cron/CollecTorMainTest.java b/src/test/java/org/torproject/metrics/collector/cron/CollecTorMainTest.java index cc124a4..2818127 100644 --- a/src/test/java/org/torproject/metrics/collector/cron/CollecTorMainTest.java +++ b/src/test/java/org/torproject/metrics/collector/cron/CollecTorMainTest.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.cron; diff --git a/src/test/java/org/torproject/metrics/collector/cron/SchedulerTest.java b/src/test/java/org/torproject/metrics/collector/cron/SchedulerTest.java index 3f20646..c41f52d 100644 --- a/src/test/java/org/torproject/metrics/collector/cron/SchedulerTest.java +++ b/src/test/java/org/torproject/metrics/collector/cron/SchedulerTest.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.cron; diff --git a/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java index aef8c0c..fb21498 100644 --- a/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java +++ b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java @@ -1,4 +1,4 @@ -/* Copyright 2019 The Tor Project +/* Copyright 2019--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.downloader; diff --git a/src/test/java/org/torproject/metrics/collector/indexer/CreateIndexJsonTest.java b/src/test/java/org/torproject/metrics/collector/indexer/CreateIndexJsonTest.java index db00032..9c01293 100644 --- a/src/test/java/org/torproject/metrics/collector/indexer/CreateIndexJsonTest.java +++ b/src/test/java/org/torproject/metrics/collector/indexer/CreateIndexJsonTest.java @@ -1,4 +1,4 @@ -/* Copyright 2019 The Tor Project +/* Copyright 2019--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.indexer; diff --git a/src/test/java/org/torproject/metrics/collector/indexer/IndexerTaskTest.java b/src/test/java/org/torproject/metrics/collector/indexer/IndexerTaskTest.java index 8e5e6f4..6c67214 100644 --- a/src/test/java/org/torproject/metrics/collector/indexer/IndexerTaskTest.java +++ b/src/test/java/org/torproject/metrics/collector/indexer/IndexerTaskTest.java @@ -1,4 +1,4 @@ -/* Copyright 2019 The Tor Project +/* Copyright 2019--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.indexer; diff --git a/src/test/java/org/torproject/metrics/collector/persist/PersistUtilsTest.java b/src/test/java/org/torproject/metrics/collector/persist/PersistUtilsTest.java index 6774923..a33b94f 100644 --- a/src/test/java/org/torproject/metrics/collector/persist/PersistUtilsTest.java +++ b/src/test/java/org/torproject/metrics/collector/persist/PersistUtilsTest.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.persist; diff --git a/src/test/java/org/torproject/metrics/collector/relaydescs/ReferenceCheckerTest.java b/src/test/java/org/torproject/metrics/collector/relaydescs/ReferenceCheckerTest.java index 9489315..a68cd32 100644 --- a/src/test/java/org/torproject/metrics/collector/relaydescs/ReferenceCheckerTest.java +++ b/src/test/java/org/torproject/metrics/collector/relaydescs/ReferenceCheckerTest.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.relaydescs; diff --git a/src/test/java/org/torproject/metrics/collector/sync/FileCollector.java b/src/test/java/org/torproject/metrics/collector/sync/FileCollector.java index e84a62f..01710bf 100644 --- a/src/test/java/org/torproject/metrics/collector/sync/FileCollector.java +++ b/src/test/java/org/torproject/metrics/collector/sync/FileCollector.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.sync; diff --git a/src/test/java/org/torproject/metrics/collector/sync/SyncPersistenceTest.java b/src/test/java/org/torproject/metrics/collector/sync/SyncPersistenceTest.java index a8f7095..acd326b 100644 --- a/src/test/java/org/torproject/metrics/collector/sync/SyncPersistenceTest.java +++ b/src/test/java/org/torproject/metrics/collector/sync/SyncPersistenceTest.java @@ -1,4 +1,4 @@ -/* Copyright 2016--2018 The Tor Project +/* Copyright 2016--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.sync; diff --git a/src/test/java/org/torproject/metrics/collector/webstats/LogFileMapTest.java b/src/test/java/org/torproject/metrics/collector/webstats/LogFileMapTest.java index 89cfc10..d765cbd 100644 --- a/src/test/java/org/torproject/metrics/collector/webstats/LogFileMapTest.java +++ b/src/test/java/org/torproject/metrics/collector/webstats/LogFileMapTest.java @@ -1,4 +1,4 @@ -/* Copyright 2017--2018 The Tor Project +/* Copyright 2017--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.webstats; diff --git a/src/test/java/org/torproject/metrics/collector/webstats/LogMetadataTest.java b/src/test/java/org/torproject/metrics/collector/webstats/LogMetadataTest.java index 24629ab..20d7656 100644 --- a/src/test/java/org/torproject/metrics/collector/webstats/LogMetadataTest.java +++ b/src/test/java/org/torproject/metrics/collector/webstats/LogMetadataTest.java @@ -1,4 +1,4 @@ -/* Copyright 2017--2018 The Tor Project +/* Copyright 2017--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.webstats; diff --git a/src/test/java/org/torproject/metrics/collector/webstats/SanitizeWeblogsTest.java b/src/test/java/org/torproject/metrics/collector/webstats/SanitizeWeblogsTest.java index 21617b5..803975e 100644 --- a/src/test/java/org/torproject/metrics/collector/webstats/SanitizeWeblogsTest.java +++ b/src/test/java/org/torproject/metrics/collector/webstats/SanitizeWeblogsTest.java @@ -1,4 +1,4 @@ -/* Copyright 2017--2018 The Tor Project +/* Copyright 2017--2020 The Tor Project * See LICENSE for licensing information */ package org.torproject.metrics.collector.webstats;

1 0

[collector/release] Avoid reprocessing webstats files.
by karsten＠torproject.org 15 Jan '20

15 Jan '20

commit d48163379c2604626a62da775aafe68b5be62186 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Dec 11 12:22:40 2019 +0100 Avoid reprocessing webstats files. Web servers typically provide us with the last 14 days of request logs. We shouldn't process the whole 14 days over and over. Instead we should only process new logs files and any other log files containing log lines from newly written dates. In some cases web servers stop serving a given virtual host or stop acting as web server at all. However, in these cases we're left with 14 days of logs per virtual host. Ideally, these logs would get cleaned up, but until that's the case, we should at least not reprocess these files over and over. In order to avoid reprocessing webstats files, we need a new state file with log dates contained in given input files. We use that state file to determine which of the previously processed webstats files to re-process, so that we can write complete daily logs. --- CHANGELOG.md | 1 + .../metrics/collector/webstats/LogMetadata.java | 17 +++ .../collector/webstats/SanitizeWeblogs.java | 155 ++++++++++++++++++--- .../collector/webstats/WebServerAccessLogImpl.java | 19 +++ 4 files changed, 170 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73abdea..fe7937c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ * Medium changes - Give up on periodically checking the configuration file for updates and reloading it in case of changes. + - Avoid reprocessing webstats files. * Minor changes - Remove dependency on metrics-lib's internal package. diff --git a/src/main/java/org/torproject/metrics/collector/webstats/LogMetadata.java b/src/main/java/org/torproject/metrics/collector/webstats/LogMetadata.java index 879e8d7..b30c13a 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/LogMetadata.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/LogMetadata.java @@ -81,5 +81,22 @@ public class LogMetadata { } return Optional.ofNullable(metadata); } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (other == null || getClass() != other.getClass()) { + return false; + } + LogMetadata that = (LogMetadata) other; + return path.toString().equals(that.path.toString()); + } + + @Override + public int hashCode() { + return path.hashCode(); + } } diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java index fc7c64f..b3fee06 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java @@ -25,14 +25,18 @@ import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -56,8 +60,10 @@ public class SanitizeWeblogs extends CollecTorMain { private static final int LIMIT = 2; private static final String WEBSTATS = "webstats"; - private String outputPathName; - private String recentPathName; + private Path outputDirectory; + private Path recentDirectory; + private Path processedWebstatsFile; + private boolean limits; /** @@ -84,14 +90,22 @@ public class SanitizeWeblogs extends CollecTorMain { try { Files.createDirectories(this.config.getPath(Key.OutputPath)); Files.createDirectories(this.config.getPath(Key.RecentPath)); - this.outputPathName = this.config.getPath(Key.OutputPath).toString(); - this.recentPathName = this.config.getPath(Key.RecentPath).toString(); + Files.createDirectories(this.config.getPath(Key.StatsPath)); + this.outputDirectory = this.config.getPath(Key.OutputPath); + this.recentDirectory = this.config.getPath(Key.RecentPath); + this.processedWebstatsFile = this.config.getPath(Key.StatsPath) + .resolve("processed-webstats"); this.limits = this.config.getBool(Key.WebstatsLimits); Set<SourceType> sources = this.config.getSourceTypeSet( Key.WebstatsSources); if (sources.contains(SourceType.Local)) { log.info("Processing logs using batch value {}.", BATCH); - findCleanWrite(this.config.getPath(Key.WebstatsLocalOrigins)); + Map<LogMetadata, Set<LocalDate>> previouslyProcessedWebstats + = this.readProcessedWebstats(); + Map<LogMetadata, Set<LocalDate>> newlyProcessedWebstats + = this.findCleanWrite(this.config.getPath(Key.WebstatsLocalOrigins), + previouslyProcessedWebstats); + this.writeProcessedWebstats(newlyProcessedWebstats); long cutOffMillis = System.currentTimeMillis() - 3L * 24L * 60L * 60L * 1000L; PersistenceUtils.cleanDirectory(this.config.getPath(Key.RecentPath), @@ -103,7 +117,32 @@ public class SanitizeWeblogs extends CollecTorMain { } } - private void findCleanWrite(Path dir) { + private Map<LogMetadata, Set<LocalDate>> readProcessedWebstats() { + Map<LogMetadata, Set<LocalDate>> processedWebstats = new HashMap<>(); + if (Files.exists(this.processedWebstatsFile)) { + try { + for (String line : Files.readAllLines(this.processedWebstatsFile)) { + String[] lineParts = line.split(",", 2); + Optional<LogMetadata> logMetadata + = LogMetadata.create(Paths.get(lineParts[1])); + if (logMetadata.isPresent()) { + processedWebstats.putIfAbsent(logMetadata.get(), new HashSet<>()); + LocalDate containedLogDate = LocalDate.parse(lineParts[0]); + processedWebstats.get(logMetadata.get()).add(containedLogDate); + } + } + } catch (IOException e) { + log.error("Cannot read state file {}.", this.processedWebstatsFile, e); + } + log.debug("Read state file containing {} log files.", + processedWebstats.size()); + } + return processedWebstats; + } + + private Map<LogMetadata, Set<LocalDate>> findCleanWrite(Path dir, + Map<LogMetadata, Set<LocalDate>> previouslyProcessedWebstats) { + Map<LogMetadata, Set<LocalDate>> newlyProcessedWebstats = new HashMap<>(); LogFileMap fileMapIn = new LogFileMap(dir); log.info("Found log files for {} virtual hosts.", fileMapIn.size()); for (Map.Entry<String,TreeMap<String,TreeMap<LocalDate,LogMetadata>>> @@ -113,24 +152,76 @@ public class SanitizeWeblogs extends CollecTorMain { : virtualEntry.getValue().entrySet()) { String physicalHost = physicalEntry.getKey(); log.info("Processing logs for {} on {}.", virtualHost, physicalHost); - Map<LocalDate, Map<String, Long>> linesByDate - = physicalEntry.getValue().values().stream().parallel() - .flatMap(metadata -> sanitzedLineStream(metadata).entrySet() - .stream()) - .collect(groupingBy(Map.Entry::getKey, - reducing(Collections.emptyMap(), Map.Entry::getValue, - (e1, e2) -> Stream.concat(e1.entrySet().stream(), e2.entrySet() - .stream()) - .collect(groupingByConcurrent(Map.Entry::getKey, - summingLong(Map.Entry::getValue)))))); - LocalDate[] interval = determineInterval(linesByDate.keySet()); - linesByDate.entrySet().stream() - .filter((entry) -> entry.getKey().isAfter(interval[0]) - && entry.getKey().isBefore(interval[1])).parallel() + /* Go through current input log files for given virtual and physical + * host, and either look up contained log dates from the last execution, + * or parse files to memory now. */ + Map<LocalDate, Map<String, Long>> sanitizedLinesByDate + = new HashMap<>(); + Set<LogMetadata> previouslyReadFiles = new HashSet<>(); + for (LogMetadata logMetadata : physicalEntry.getValue().values()) { + Set<LocalDate> containedLogDates; + if (previouslyProcessedWebstats.containsKey(logMetadata)) { + containedLogDates = previouslyProcessedWebstats.get(logMetadata); + for (LocalDate date : containedLogDates) { + sanitizedLinesByDate.putIfAbsent(date, new TreeMap<>()); + } + previouslyReadFiles.add(logMetadata); + } else { + containedLogDates = sanitizeWebstatsLog(sanitizedLinesByDate, + logMetadata); + } + newlyProcessedWebstats.put(logMetadata, containedLogDates); + } + /* Determine log dates that are safe to be written to disk now and that + * we didn't write to disk before. */ + Set<LocalDate> storeDates = new HashSet<>(); + LocalDate[] interval = determineInterval(sanitizedLinesByDate.keySet()); + for (LocalDate newDate : sanitizedLinesByDate.keySet()) { + if (newDate.isAfter(interval[0]) && newDate.isBefore(interval[1])) { + WebServerAccessLogPersistence walp + = new WebServerAccessLogPersistence( + new WebServerAccessLogImpl(virtualHost, physicalHost, newDate)); + Path outputPath = this.outputDirectory + .resolve(walp.getStoragePath()); + if (!Files.exists(outputPath)) { + storeDates.add(newDate); + } + } + } + /* Reprocess previously read files containing log dates that we're going + * to write to disk below. */ + for (LogMetadata previouslyReadFile : previouslyReadFiles) { + if (!Collections.disjoint(storeDates, + newlyProcessedWebstats.get(previouslyReadFile))) { + sanitizeWebstatsLog(sanitizedLinesByDate, previouslyReadFile); + } + } + /* Write sanitized log files to disk. */ + sanitizedLinesByDate.entrySet().stream() + .filter((entry) -> storeDates.contains(entry.getKey())).parallel() .forEach((entry) -> storeSortedAndForget(virtualHost, physicalHost, entry.getKey(), entry.getValue())); } } + return newlyProcessedWebstats; + } + + private Set<LocalDate> sanitizeWebstatsLog( + Map<LocalDate, Map<String, Long>> sanitizedLinesByDate, + LogMetadata logFile) { + Map<LocalDate, Map<String, Long>> newlySanitizedLinesByDate + = sanitzedLineStream(logFile); + for (Map.Entry<LocalDate, Map<String, Long>> e + : newlySanitizedLinesByDate.entrySet()) { + sanitizedLinesByDate.putIfAbsent(e.getKey(), new TreeMap<>()); + Map<String, Long> newlySanitizedLines + = sanitizedLinesByDate.get(e.getKey()); + for (Map.Entry<String, Long> e1 : e.getValue().entrySet()) { + newlySanitizedLines.put(e1.getKey(), + newlySanitizedLines.getOrDefault(e1.getKey(), 0L) + e1.getValue()); + } + } + return newlySanitizedLinesByDate.keySet(); } private void storeSortedAndForget(String virtualHost, String physicalHost, @@ -149,8 +240,8 @@ public class SanitizeWeblogs extends CollecTorMain { new WebServerAccessLogImpl(toCompressedBytes(retainedLines), new File(name), name)); log.debug("Storing {}.", name); - walp.storeOut(this.outputPathName); - walp.storeRecent(this.recentPathName); + walp.storeOut(this.outputDirectory.toString()); + walp.storeRecent(this.recentDirectory.toString()); } catch (DescriptorParseException dpe) { log.error("Cannot store log desriptor {}.", name, dpe); } catch (Throwable th) { // catch all else @@ -279,5 +370,25 @@ public class SanitizeWeblogs extends CollecTorMain { return Collections.emptyMap(); } + private void writeProcessedWebstats( + Map<LogMetadata, Set<LocalDate>> newlyProcessedWebstats) { + try { + if (!Files.exists(this.processedWebstatsFile.getParent())) { + Files.createDirectories(this.processedWebstatsFile.getParent()); + } + List<String> lines = new ArrayList<>(); + for (Map.Entry<LogMetadata, Set<LocalDate>> e + : newlyProcessedWebstats.entrySet()) { + for (LocalDate logLineDate : e.getValue()) { + lines.add(String.format("%s,%s", logLineDate, e.getKey().path)); + } + } + Files.write(this.processedWebstatsFile, lines); + } catch (IOException e) { + log.error("Cannot write state file {}.", this.processedWebstatsFile, e); + } + log.debug("Wrote state file containing {} log files.", + newlyProcessedWebstats.size()); + } } diff --git a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java index af77c94..f091aa1 100644 --- a/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java +++ b/src/main/java/org/torproject/metrics/collector/webstats/WebServerAccessLogImpl.java @@ -104,6 +104,25 @@ public class WebServerAccessLogImpl implements WebServerAccessLog { } } + /** + * Creates an empty WebServerAccessLog from the given filename parts. + * + * This instance is not intended to be written to disk, as it doesn't have + * any content. The main intention of this instance is to compute storage + * paths. + * + * @param virtualHost Virtual host name. + * @param physicalHost Physical host name. + * @param logDate Log date. + */ + protected WebServerAccessLogImpl(String virtualHost, String physicalHost, + LocalDate logDate) { + this.descriptorFile = null; + this.virtualHost = virtualHost; + this.physicalHost = physicalHost; + this.logDate = logDate; + } + @Override public InputStream decompressedByteStream() throws DescriptorParseException { try {

1 0

[collector/release] Update to metrics-lib 2.10.0.
by karsten＠torproject.org 15 Jan '20

15 Jan '20

commit 27e41ea7393c24bc72429915dd74c309c2583fa2 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jan 15 22:58:17 2020 +0100 Update to metrics-lib 2.10.0. --- CHANGELOG.md | 1 + build.xml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb295d6..028755f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ * Minor changes - Remove dependency on metrics-lib's internal package. + - Update to metrics-lib 2.10.0. # Changes in version 1.13.1 - 2019-11-11 diff --git a/build.xml b/build.xml index c75b69e..312244f 100644 --- a/build.xml +++ b/build.xml @@ -12,7 +12,7 @@ <property name="release.version" value="1.13.1-dev" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> - <property name="metricslibversion" value="2.9.1" /> + <property name="metricslibversion" value="2.10.0" /> <property name="jarincludes" value="collector.properties logback.xml" /> <patternset id="runtime" >

1 0

[collector/release] Prepare for 1.14.0 release.
by karsten＠torproject.org 15 Jan '20

15 Jan '20

commit 3a9f05e01f1abe8315d25c780b3aab72376412f5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jan 15 23:07:02 2020 +0100 Prepare for 1.14.0 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 028755f..898338c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.1?.? - 2019-1?-?? +# Changes in version 1.14.0 - 2020-01-15 * Medium changes - Give up on periodically checking the configuration file for diff --git a/build.xml b/build.xml index 312244f..5293d41 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.13.1-dev" /> + <property name="release.version" value="1.14.0" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.10.0" />

1 0

[translation/support-portal] https://gitweb.torproject.org/translation.git/commit/?h=support-portal
by translation＠torproject.org 15 Jan '20

15 Jan '20

commit c55f9b762f22c8524b42a0112913ba93a5476fb5 Author: Translation commit bot <translation(a)torproject.org> Date: Wed Jan 15 22:23:56 2020 +0000 https://gitweb.torproject.org/translation.git/commit/?h=support-portal --- contents+ru.po | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contents+ru.po b/contents+ru.po index f2a6396707..94ed840d22 100644 --- a/contents+ru.po +++ b/contents+ru.po @@ -4255,6 +4255,8 @@ msgid "" "If you are unable to connect to an onion service, please see [I cannot reach" " X.onion!](../../onionservices/onionservices-3)." msgstr "" +"Если у вас проблемы с доступом к onion-ресурсу, пожалуйста, см. страницу [Я " +"не могу открыть ресурс .onion](../../onionservices/onionservices-3)." #: https//support.torproject.org/censorship/censorship-7/ #: (content/censorship/censorship-7/contents+en.lrquestion.title)

1 0

[collector/master] Prepare for 1.14.0 release.
by karsten＠torproject.org 15 Jan '20

15 Jan '20

commit 3a9f05e01f1abe8315d25c780b3aab72376412f5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jan 15 23:07:02 2020 +0100 Prepare for 1.14.0 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 028755f..898338c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.1?.? - 2019-1?-?? +# Changes in version 1.14.0 - 2020-01-15 * Medium changes - Give up on periodically checking the configuration file for diff --git a/build.xml b/build.xml index 312244f..5293d41 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.13.1-dev" /> + <property name="release.version" value="1.14.0" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.10.0" />

1 0

[collector/master] Bump version to 1.14.0-dev.
by karsten＠torproject.org 15 Jan '20

15 Jan '20

commit c75f0c781ec45b6b56aedea5738692d0404bdc8a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jan 15 23:20:01 2020 +0100 Bump version to 1.14.0-dev. --- CHANGELOG.md | 3 +++ build.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 898338c..d3088c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changes in version 1.1?.? - 2020-0?-?? + + # Changes in version 1.14.0 - 2020-01-15 * Medium changes diff --git a/build.xml b/build.xml index 5293d41..5d90b46 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.14.0" /> + <property name="release.version" value="1.14.0-dev" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.10.0" />

1 0