October 2019 - tor-commits - lists.torproject.org

[collector/release] Create utility class for downloading from HTTP servers
by karsten＠torproject.org 18 Oct '19

18 Oct '19

commit dc3a2e4aea03ca44be033dafc3dcc3d7e0d5c07f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Sep 26 11:48:18 2019 +0200 Create utility class for downloading from HTTP servers Also skip two tests in ConfigurationTest that together take 53 seconds which has the effect that we don't run tests very often. Implements #31599. --- CHANGELOG.md | 6 + src/build | 2 +- .../metrics/collector/downloader/Downloader.java | 62 ++++++++++ .../collector/exitlists/ExitListDownloader.java | 33 ++--- .../collector/onionperf/OnionPerfDownloader.java | 23 +++- .../relaydescs/RelayDescriptorDownloader.java | 32 +---- .../snowflake/SnowflakeStatsDownloader.java | 56 ++------- .../metrics/collector/conf/ConfigurationTest.java | 3 + .../collector/downloader/DownloaderTest.java | 134 +++++++++++++++++++++ 9 files changed, 250 insertions(+), 101 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e30b086..6e72f02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# Changes in version 1.12.0 - 2019-??-?? + + * Medium changes + - Require Mockito 1.10.19 as dependency for running tests. + + # Changes in version 1.11.1 - 2019-09-19 * Minor changes diff --git a/src/build b/src/build index 07c2a00..62a964a 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4 +Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47 diff --git a/src/main/java/org/torproject/metrics/collector/downloader/Downloader.java b/src/main/java/org/torproject/metrics/collector/downloader/Downloader.java new file mode 100644 index 0000000..a1f3852 --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/downloader/Downloader.java @@ -0,0 +1,62 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.downloader; + +import java.io.BufferedInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.zip.InflaterInputStream; + +/** + * Utility class for downloading resources from HTTP servers. + */ +public class Downloader { + + /** + * Download the given URL from an HTTP server and return downloaded bytes. + * + * @param url URL to download. + * @return Downloaded bytes, or {@code null} if the resource was not found. + * @throws IOException Thrown if anything goes wrong while downloading. + */ + public static byte[] downloadFromHttpServer(URL url) throws IOException { + return Downloader.downloadFromHttpServer(url, false); + } + + /** + * Download the given URL from an HTTP server, possibly inflate the response, + * and return downloaded bytes. + * + * @param url URL to download. + * @param isDeflated Whether the response is deflated. + * @return Downloaded bytes, or {@code null} if the resource was not found. + * @throws IOException Thrown if anything goes wrong while downloading. + */ + public static byte[] downloadFromHttpServer(URL url, boolean isDeflated) + throws IOException { + ByteArrayOutputStream downloadedBytes = new ByteArrayOutputStream(); + HttpURLConnection huc = (HttpURLConnection) url.openConnection(); + huc.setRequestMethod("GET"); + huc.setReadTimeout(5000); + huc.connect(); + int response = huc.getResponseCode(); + if (response != 200) { + return null; + } + try (BufferedInputStream in = isDeflated + ? new BufferedInputStream(new InflaterInputStream( + huc.getInputStream())) + : new BufferedInputStream(huc.getInputStream())) { + int len; + byte[] data = new byte[1024]; + while ((len = in.read(data, 0, 1024)) >= 0) { + downloadedBytes.write(data, 0, len); + } + } + return downloadedBytes.toByteArray(); + } +} + diff --git a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java index 66fc1a7..b4bee15 100644 --- a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java @@ -12,16 +12,15 @@ import org.torproject.metrics.collector.conf.Configuration; import org.torproject.metrics.collector.conf.ConfigurationException; import org.torproject.metrics.collector.conf.Key; import org.torproject.metrics.collector.cron.CollecTorMain; +import org.torproject.metrics.collector.downloader.Downloader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedInputStream; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.net.HttpURLConnection; import java.net.URL; import java.nio.file.Paths; import java.text.SimpleDateFormat; @@ -77,32 +76,18 @@ public class ExitListDownloader extends CollecTorMain { sb.append("Downloaded ").append(dateTimeFormat.format(downloadedDate)) .append("\n"); URL url = config.getUrl(Key.ExitlistUrl); - try { - HttpURLConnection huc = (HttpURLConnection) url.openConnection(); - huc.setRequestMethod("GET"); - huc.setReadTimeout(5000); - huc.connect(); - int response = huc.getResponseCode(); - if (response != 200) { - logger.warn("Could not download exit list. Response code {}", - response); - return; - } - try (BufferedInputStream in = new BufferedInputStream( - huc.getInputStream())) { - int len; - byte[] data = new byte[1024]; - while ((len = in.read(data, 0, 1024)) >= 0) { - sb.append(new String(data, 0, len)); - } - } - downloadedExitList = sb.toString(); - logger.debug("Finished downloading exit list."); + byte[] downloadedBytes; + try { + downloadedBytes = Downloader.downloadFromHttpServer(url); } catch (IOException e) { logger.warn("Failed downloading exit list", e); return; } - if (downloadedExitList == null) { + if (null != downloadedBytes) { + sb.append(new String(downloadedBytes)); + downloadedExitList = sb.toString(); + logger.debug("Finished downloading exit list."); + } else { logger.warn("Failed downloading exit list."); return; } diff --git a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java index ca307a5..b106be7 100644 --- a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java @@ -11,6 +11,7 @@ import org.torproject.metrics.collector.conf.Configuration; import org.torproject.metrics.collector.conf.ConfigurationException; import org.torproject.metrics.collector.conf.Key; import org.torproject.metrics.collector.cron.CollecTorMain; +import org.torproject.metrics.collector.downloader.Downloader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,7 +22,6 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URL; @@ -200,12 +200,25 @@ public class OnionPerfDownloader extends CollecTorMain { /* Download file contents to temporary file. */ File tempFile = new File(this.recentDirectory, "." + tpfFileName); + byte[] downloadedBytes; + try { + downloadedBytes = Downloader.downloadFromHttpServer( + new URL(baseUrl + tpfFileName)); + } catch (IOException e) { + logger.warn("Unable to download '{}{}'. Skipping.", baseUrl, tpfFileName, + e); + return; + } + if (null == downloadedBytes) { + logger.warn("Unable to download '{}{}'. Skipping.", baseUrl, tpfFileName); + return; + } tempFile.getParentFile().mkdirs(); - try (InputStream is = new URL(baseUrl + tpfFileName).openStream()) { - Files.copy(is, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + try { + Files.write(tempFile.toPath(), downloadedBytes); } catch (IOException e) { - logger.warn("Unable to download '{}{}' to temporary file '{}'. " - + "Skipping.", baseUrl, tpfFileName, tempFile, e); + logger.warn("Unable to write previously downloaded '{}{}' to temporary " + + "file '{}'. Skipping.", baseUrl, tpfFileName, tempFile, e); return; } diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java index f1179d8..3337633 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java @@ -3,20 +3,19 @@ package org.torproject.metrics.collector.relaydescs; +import org.torproject.metrics.collector.downloader.Downloader; + import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.digest.DigestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.BufferedWriter; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; -import java.net.HttpURLConnection; import java.net.URL; import java.nio.charset.StandardCharsets; import java.text.ParseException; @@ -33,7 +32,6 @@ import java.util.SortedSet; import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; -import java.util.zip.InflaterInputStream; /** * Downloads relay descriptors from the directory authorities via HTTP. @@ -875,29 +873,11 @@ public class RelayDescriptorDownloader { String fullUrl = "http://" + authority + resource + (isCompressed ? ".z" : ""); URL url = new URL(fullUrl); - HttpURLConnection huc = (HttpURLConnection) url.openConnection(); - huc.setRequestMethod("GET"); - huc.setReadTimeout(5000); - huc.connect(); - int response = huc.getResponseCode(); - if (response == 200) { - try (BufferedInputStream in - = isCompressed ? new BufferedInputStream(new InflaterInputStream( - huc.getInputStream())) - : new BufferedInputStream(huc.getInputStream())) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] data = new byte[1024]; - while ((len = in.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - allData = baos.toByteArray(); - } - } - logger.debug("Downloaded {} -> {} ({} bytes)", fullUrl, response, - allData == null ? 0 : allData.length); + allData = Downloader.downloadFromHttpServer(url, isCompressed); int receivedDescriptors = 0; - if (allData != null) { + logger.debug("Downloaded {} -> ({} bytes)", fullUrl, + allData == null ? 0 : allData.length); + if (null != allData) { if (resource.startsWith("/tor/status-vote/")) { this.rdp.parse(allData, null); receivedDescriptors = 1; diff --git a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java index 4f7994e..cb5f0cc 100644 --- a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java @@ -12,18 +12,16 @@ import org.torproject.metrics.collector.conf.Configuration; import org.torproject.metrics.collector.conf.ConfigurationException; import org.torproject.metrics.collector.conf.Key; import org.torproject.metrics.collector.cron.CollecTorMain; +import org.torproject.metrics.collector.downloader.Downloader; import org.torproject.metrics.collector.persist.SnowflakeStatsPersistence; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedInputStream; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.net.HttpURLConnection; import java.net.URL; import java.time.LocalDateTime; import java.util.Arrays; @@ -60,9 +58,15 @@ public class SnowflakeStatsDownloader extends CollecTorMain { this.recentPathName = config.getPath(Key.RecentPath).toString(); logger.debug("Downloading snowflake stats..."); URL url = config.getUrl(Key.SnowflakeStatsUrl); - ByteArrayOutputStream downloadedSnowflakeStats - = this.downloadFromHttpServer(url); - if (null == downloadedSnowflakeStats) { + byte[] downloadedBytes; + try { + downloadedBytes = Downloader.downloadFromHttpServer(url); + } catch (IOException e) { + logger.warn("Failed downloading {}.", url, e); + return; + } + if (null == downloadedBytes) { + logger.warn("Could not download {}.", url); return; } logger.debug("Finished downloading {}.", url); @@ -72,7 +76,7 @@ public class SnowflakeStatsDownloader extends CollecTorMain { SortedSet<LocalDateTime> snowflakeStatsEnds = new TreeSet<>(); String outputPathName = config.getPath(Key.OutputPath).toString(); for (Descriptor descriptor : descriptorParser.parseDescriptors( - downloadedSnowflakeStats.toByteArray(), null, null)) { + downloadedBytes, null, null)) { if (descriptor instanceof SnowflakeStats) { SnowflakeStats snowflakeStats = (SnowflakeStats) descriptor; LocalDateTime snowflakeStatsEnd = snowflakeStats.snowflakeStatsEnd(); @@ -106,44 +110,6 @@ public class SnowflakeStatsDownloader extends CollecTorMain { } /** - * Download the given URL from an HTTP server and return a stream with - * downloaded bytes. - * - * <p>If anything goes wrong while downloading, log a warning and return - * {@code null}.</p> - * - * @param url URL to download. - * @return Stream with downloaded bytes, or {@code null} if an error has - * occurred. - */ - private ByteArrayOutputStream downloadFromHttpServer(URL url) { - ByteArrayOutputStream downloadedBytes = new ByteArrayOutputStream(); - try { - HttpURLConnection huc = (HttpURLConnection) url.openConnection(); - huc.setRequestMethod("GET"); - huc.setReadTimeout(5000); - huc.connect(); - int response = huc.getResponseCode(); - if (response != 200) { - logger.warn("Could not download {}. Response code {}", url, response); - return null; - } - try (BufferedInputStream in = new BufferedInputStream( - huc.getInputStream())) { - int len; - byte[] data = new byte[1024]; - while ((len = in.read(data, 0, 1024)) >= 0) { - downloadedBytes.write(data, 0, len); - } - } - } catch (IOException e) { - logger.warn("Failed downloading {}.", url, e); - return null; - } - return downloadedBytes; - } - - /** * Write the given byte array(s) to the given file. * * <p>If the file already exists, it is overwritten. If the parent directory diff --git a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java index 201d541..67a9082 100644 --- a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java @@ -12,6 +12,7 @@ import org.torproject.metrics.collector.MainTest; import org.torproject.metrics.collector.cron.CollecTorMain; import org.torproject.metrics.collector.cron.Dummy; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -127,6 +128,7 @@ public class ConfigurationTest { } } + @Ignore("This test takes 40 seconds, which is too long.") @Test() public void testUrlArrayValues() throws Exception { URL[] array = new URL[randomSource.nextInt(30) + 1]; @@ -180,6 +182,7 @@ public class ConfigurationTest { conf.setWatchableSourceAndLoad(Paths.get("/tmp/phantom.path")); } + @Ignore("This test takes 13 seconds, which is too long.") @Test() public void testConfigChange() throws Exception { Configuration conf = new Configuration(); diff --git a/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java new file mode 100644 index 0000000..19db782 --- /dev/null +++ b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java @@ -0,0 +1,134 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.downloader; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.SocketTimeoutException; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; +import java.net.URLStreamHandlerFactory; +import java.util.HashMap; +import java.util.Map; + +/** + * Test class for {@link Downloader}. + * + * <p>This test class is heavily based on the blog post + * <a href="https://claritysoftware.co.uk/mocking-javas-url-with-mockito/">"How + * to mock the Java URL class with Mockito"</a> by nathan from March 10, + * 2017.</p> + */ +public class DownloaderTest { + + /** + * Custom {@link URLStreamHandler} that allows us to control the + * {@link URLConnection URLConnections} that are returned by {@link URL URLs} + * in the code under test. + */ + private static class HttpUrlStreamHandler extends URLStreamHandler { + + private Map<URL, URLConnection> connections = new HashMap(); + + @Override + protected URLConnection openConnection(URL url) throws IOException { + return connections.get(url); + } + + private void resetConnections() { + connections = new HashMap(); + } + + private HttpUrlStreamHandler addConnection(URL url, + URLConnection urlConnection) { + connections.put(url, urlConnection); + return this; + } + } + + private static HttpUrlStreamHandler httpUrlStreamHandler; + + /** + * Set up our own stream handler for all tests in this class. + */ + @BeforeClass + public static void setupUrlStreamHandlerFactory() { + URLStreamHandlerFactory urlStreamHandlerFactory + = mock(URLStreamHandlerFactory.class); + URL.setURLStreamHandlerFactory(urlStreamHandlerFactory); + httpUrlStreamHandler = new HttpUrlStreamHandler(); + given(urlStreamHandlerFactory.createURLStreamHandler("http")) + .willReturn(httpUrlStreamHandler); + } + + /** + * Clear any connections from previously run tests. + */ + @Before + public void reset() { + httpUrlStreamHandler.resetConnections(); + } + + @Test + public void testExistingResource() throws Exception { + URL requestedUrl = new URL("http://example.org/exists"); + byte[] expectedDownloadedBytes = "content".getBytes(); + HttpURLConnection urlConnection = mock(HttpURLConnection.class); + httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); + given(urlConnection.getResponseCode()).willReturn(200); + given(urlConnection.getInputStream()).willReturn( + new ByteArrayInputStream(expectedDownloadedBytes)); + byte[] downloadedBytes = Downloader.downloadFromHttpServer(requestedUrl); + assertArrayEquals(expectedDownloadedBytes, downloadedBytes); + } + + @Test + public void testNonExistingResource() throws Exception { + URL requestedUrl = new URL("http://example.org/notfound"); + HttpURLConnection urlConnection = mock(HttpURLConnection.class); + httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); + given(urlConnection.getResponseCode()).willReturn(404); + byte[] downloadedBytes = Downloader.downloadFromHttpServer(requestedUrl); + assertNull(downloadedBytes); + } + + @Test + public void testEmptyResource() throws Exception { + URL requestedUrl = new URL("http://example.org/empty"); + byte[] expectedDownloadedBytes = new byte[0]; + HttpURLConnection urlConnection = mock(HttpURLConnection.class); + httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); + given(urlConnection.getResponseCode()).willReturn(200); + given(urlConnection.getInputStream()).willReturn( + new ByteArrayInputStream(expectedDownloadedBytes)); + byte[] downloadedBytes = Downloader.downloadFromHttpServer(requestedUrl); + assertEquals(0, downloadedBytes.length); + } + + @Test(expected = SocketTimeoutException.class) + public void testTimeout() throws Exception { + URL requestedUrl = new URL("http://example.org/timeout"); + SocketTimeoutException expectedException = new SocketTimeoutException(); + HttpURLConnection urlConnection = mock(HttpURLConnection.class); + httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); + given(urlConnection.getResponseCode()).willReturn(200); + given(urlConnection.getInputStream()).willThrow(expectedException); + Downloader.downloadFromHttpServer(requestedUrl); + fail("Should have thrown a SocketTimeoutException."); + } +} +

1 0

[collector/release] Add new BridgedbMetrics module.
by karsten＠torproject.org 18 Oct '19

18 Oct '19

commit f2abf679c890f3f050c839cf32910900ae73ee76 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Sep 18 15:57:52 2019 +0200 Add new BridgedbMetrics module. Implements part of #19332. --- CHANGELOG.md | 2 + build.xml | 2 +- .../org/torproject/metrics/collector/Main.java | 3 + .../bridgedb/BridgedbMetricsProcessor.java | 190 +++++++++++++++++++++ .../metrics/collector/conf/Annotation.java | 1 + .../metrics/collector/conf/Configuration.java | 1 + .../org/torproject/metrics/collector/conf/Key.java | 8 +- .../persist/BridgedbMetricsPersistence.java | 37 ++++ .../collector/persist/DescriptorPersistence.java | 1 + .../metrics/collector/sync/SyncPersistence.java | 5 + src/main/resources/collector.properties | 17 ++ .../metrics/collector/conf/ConfigurationTest.java | 2 +- .../metrics/collector/cron/CollecTorMainTest.java | 1 + 13 files changed, 267 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e72f02..ca4d7b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ * Medium changes - Require Mockito 1.10.19 as dependency for running tests. + - Archive BridgeDB statistics. + - Update to metrics-lib 2.8.0. # Changes in version 1.11.1 - 2019-09-19 diff --git a/build.xml b/build.xml index 6b097a3..892566b 100644 --- a/build.xml +++ b/build.xml @@ -12,7 +12,7 @@ <property name="release.version" value="1.11.1-dev" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> - <property name="metricslibversion" value="2.7.0" /> + <property name="metricslibversion" value="2.8.0" /> <property name="jarincludes" value="collector.properties logback.xml" /> <patternset id="runtime" > diff --git a/src/main/java/org/torproject/metrics/collector/Main.java b/src/main/java/org/torproject/metrics/collector/Main.java index 3150ffc..3438bda 100644 --- a/src/main/java/org/torproject/metrics/collector/Main.java +++ b/src/main/java/org/torproject/metrics/collector/Main.java @@ -3,6 +3,7 @@ package org.torproject.metrics.collector; +import org.torproject.metrics.collector.bridgedb.BridgedbMetricsProcessor; import org.torproject.metrics.collector.bridgedescs.SanitizedBridgesWriter; import org.torproject.metrics.collector.bridgepools.BridgePoolAssignmentsProcessor; import org.torproject.metrics.collector.conf.Configuration; @@ -59,6 +60,8 @@ public class Main { collecTorMains.put(Key.WebstatsActivated, SanitizeWeblogs.class); collecTorMains.put(Key.SnowflakeStatsActivated, SnowflakeStatsDownloader.class); + collecTorMains.put(Key.BridgedbMetricsActivated, + BridgedbMetricsProcessor.class); } private static Configuration conf = new Configuration(); diff --git a/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java new file mode 100644 index 0000000..7ae4502 --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java @@ -0,0 +1,190 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.bridgedb; + +import org.torproject.descriptor.BridgedbMetrics; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.UnparseableDescriptor; +import org.torproject.metrics.collector.conf.Configuration; +import org.torproject.metrics.collector.conf.ConfigurationException; +import org.torproject.metrics.collector.conf.Key; +import org.torproject.metrics.collector.cron.CollecTorMain; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.Stack; + +public class BridgedbMetricsProcessor extends CollecTorMain { + + /** + * Class logger. + */ + private static final Logger logger = LoggerFactory.getLogger( + BridgedbMetricsProcessor.class); + + /** + * Directory for reading BridgeDB statistics files. + */ + private File inputDirectory; + + /** + * Directory for writing BridgeDB statistics files to be archived in tarballs. + */ + private String outputPathName; + + /** + * Directory for writing recently processed BridgeDB statistics files. + */ + private String recentPathName; + + /** + * File name format. + */ + private DateTimeFormatter filenameFormat = DateTimeFormatter.ofPattern( + "uuuu/MM/dd/uuuu-MM-dd-HH-mm-ss"); + + /** + * Initialize this class with the given configuration. + */ + public BridgedbMetricsProcessor(Configuration config) { + super(config); + } + + /** + * Return the module identifier. + * + * @return Module identifier. + */ + @Override + public String module() { + return "BridgedbMetrics"; + } + + /** + * Return the synchronization marker. + * + * @return Synchronization marker. + */ + @Override + protected String syncMarker() { + return "BridgedbMetrics"; + } + + /** + * Start processing files, which includes reading BridgeDB statistics files + * from disk, possibly decompressing them and splitting them by date, and + * writing them back to disk. + * + * @throws ConfigurationException Thrown if configuration values cannot be + * obtained. + */ + @Override + protected void startProcessing() throws ConfigurationException { + logger.info("Starting BridgeDB statistics module of CollecTor."); + this.initializeConfiguration(); + logger.info("Reading BridgeDB statistics files in {}.", + this.inputDirectory); + for (Descriptor descriptor + : DescriptorSourceFactory.createDescriptorReader() + .readDescriptors(this.inputDirectory)) { + if (descriptor instanceof BridgedbMetrics) { + BridgedbMetrics bridgedbMetrics = (BridgedbMetrics) descriptor; + Path tarballPath = Paths.get(this.outputPathName, + bridgedbMetrics.bridgedbMetricsEnd().format(this.filenameFormat)); + Path rsyncPath = Paths.get(this.recentPathName, + bridgedbMetrics.bridgedbMetricsEnd().format(this.filenameFormat)); + this.writeDescriptor(bridgedbMetrics.getRawDescriptorBytes(), + tarballPath, rsyncPath); + } else if (descriptor instanceof UnparseableDescriptor) { + logger.warn("Skipping unparseable descriptor in file {}.", + descriptor.getDescriptorFile(), + ((UnparseableDescriptor) descriptor).getDescriptorParseException()); + } else { + logger.warn("Skipping unexpected descriptor of type {} in file {}.", + descriptor.getClass(), descriptor.getDescriptorFile()); + } + } + logger.info("Cleaning up directory {} containing recent files.", + this.recentPathName); + this.cleanUpRsyncDirectory(); + logger.info("Finished processing BridgeDB statistics file(s)."); + } + + /** + * Initialize configuration by obtaining current configuration values and + * storing them in instance attributes. + */ + private void initializeConfiguration() throws ConfigurationException { + this.outputPathName = Paths.get(config.getPath(Key.OutputPath).toString(), + "bridgedb-metrics").toString(); + this.recentPathName = Paths.get(config.getPath(Key.RecentPath).toString(), + "bridgedb-metrics").toString(); + this.inputDirectory = + config.getPath(Key.BridgedbMetricsLocalOrigins).toFile(); + } + + /** + * Write the given raw descriptor bytes to the given files, and stop at the + * first file that already exists. + * + * @param rawDescriptorBytes Raw descriptor bytes to write. + * @param outputPaths One or more paths to write to. + */ + private void writeDescriptor(byte[] rawDescriptorBytes, + Path ... outputPaths) { + for (Path outputPath : outputPaths) { + try { + File outputFile = outputPath.toFile(); + if (outputFile.exists()) { + continue; + } + if (!outputFile.getParentFile().exists() + && !outputFile.getParentFile().mkdirs()) { + logger.warn("Could not create parent directories of {}.", outputFile); + return; + } + Files.write(outputPath, rawDescriptorBytes); + } catch (IOException e) { + logger.warn("Unable to write descriptor to file {}.", outputPath, e); + } + } + } + + /** + * Delete all files from the rsync directory that have not been modified in + * the last three days. + */ + public void cleanUpRsyncDirectory() { + Instant cutOff = Instant.now().minus(3L, ChronoUnit.DAYS); + Stack<File> allFiles = new Stack<>(); + allFiles.add(new File(this.recentPathName)); + while (!allFiles.isEmpty()) { + File file = allFiles.pop(); + if (file.isDirectory()) { + File[] filesInDirectory = file.listFiles(); + if (null != filesInDirectory) { + allFiles.addAll(Arrays.asList(filesInDirectory)); + } + } else if (Instant.ofEpochMilli(file.lastModified()).isBefore(cutOff)) { + try { + Files.deleteIfExists(file.toPath()); + } catch (IOException e) { + logger.warn("Unable to delete file {} that is apparently older than " + + "three days.", file, e); + } + } + } + } +} diff --git a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java index 7d2bbe9..ff5119e 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java +++ b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java @@ -7,6 +7,7 @@ package org.torproject.metrics.collector.conf; public enum Annotation { BandwidthFile("@type bandwidth-file 1.0\n"), + BridgedbMetrics("@type bridgedb-metrics 1.0\n"), BridgeExtraInfo("@type bridge-extra-info 1.3\n"), BridgePoolAssignment("@type bridge-pool-assignment 1.0\n"), BridgeServer("@type bridge-server-descriptor 1.2\n"), diff --git a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java index 59229e3..56be34c 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java +++ b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java @@ -89,6 +89,7 @@ public class Configuration extends Observable implements Cloneable { if (!(this.getBool(Key.RelaydescsActivated) || this.getBool(Key.BridgedescsActivated) || this.getBool(Key.BridgePoolAssignmentsActivated) + || this.getBool(Key.BridgedbMetricsActivated) || this.getBool(Key.ExitlistsActivated) || this.getBool(Key.UpdateindexActivated) || this.getBool(Key.OnionPerfActivated) diff --git a/src/main/java/org/torproject/metrics/collector/conf/Key.java b/src/main/java/org/torproject/metrics/collector/conf/Key.java index dfef673..390feed 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/Key.java +++ b/src/main/java/org/torproject/metrics/collector/conf/Key.java @@ -78,7 +78,13 @@ public enum Key { SnowflakeStatsPeriodMinutes(Integer.class), SnowflakeStatsUrl(URL.class), SnowflakeStatsSources(SourceType[].class), - SnowflakeStatsSyncOrigins(URL[].class); + SnowflakeStatsSyncOrigins(URL[].class), + BridgedbMetricsActivated(Boolean.class), + BridgedbMetricsOffsetMinutes(Integer.class), + BridgedbMetricsPeriodMinutes(Integer.class), + BridgedbMetricsSources(SourceType[].class), + BridgedbMetricsLocalOrigins(Path.class), + BridgedbMetricsSyncOrigins(URL[].class); private Class clazz; private static Set<String> keys; diff --git a/src/main/java/org/torproject/metrics/collector/persist/BridgedbMetricsPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/BridgedbMetricsPersistence.java new file mode 100644 index 0000000..a72ffe2 --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/persist/BridgedbMetricsPersistence.java @@ -0,0 +1,37 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.persist; + +import org.torproject.descriptor.BridgedbMetrics; +import org.torproject.metrics.collector.conf.Annotation; + +import java.nio.file.Paths; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; + +public class BridgedbMetricsPersistence + extends DescriptorPersistence<BridgedbMetrics> { + + private static final String BRIDGEDB_STATS = "bridgedb-metrics"; + + public BridgedbMetricsPersistence(BridgedbMetrics desc) { + super(desc, Annotation.BridgedbMetrics.bytes()); + calculatePaths(); + } + + private void calculatePaths() { + DateTimeFormatter directoriesFormatter = DateTimeFormatter + .ofPattern("uuuu/MM/dd").withZone(ZoneOffset.UTC); + String[] directories = this.desc.bridgedbMetricsEnd() + .format(directoriesFormatter).split("/"); + DateTimeFormatter fileFormatter = DateTimeFormatter + .ofPattern("uuuu-MM-dd-HH-mm-ss").withZone(ZoneOffset.UTC); + String fileOut = this.desc.bridgedbMetricsEnd().format(fileFormatter) + + "-bridgedb-metrics"; + this.recentPath = Paths.get(BRIDGEDB_STATS, fileOut).toString(); + this.storagePath = Paths.get(BRIDGEDB_STATS, directories[0], directories[1], + directories[2], fileOut).toString(); + } +} + diff --git a/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java index 3e7a06b..20cd570 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java @@ -20,6 +20,7 @@ public abstract class DescriptorPersistence<T extends Descriptor> { protected static final String BRIDGEDESCS = "bridge-descriptors"; protected static final String BRIDGEPOOLASSIGNMENTS = "bridge-pool-assignments"; + protected static final String BRIDGEDBSTATS = "bridgedb-stats"; protected static final String DASH = "-"; protected static final String DOT = "."; protected static final String MICRODESC = "microdesc"; diff --git a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java index cfc3dbe..f6678fe 100644 --- a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java @@ -8,6 +8,7 @@ import org.torproject.descriptor.BridgeExtraInfoDescriptor; import org.torproject.descriptor.BridgeNetworkStatus; import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.BridgeServerDescriptor; +import org.torproject.descriptor.BridgedbMetrics; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.ExitList; import org.torproject.descriptor.RelayExtraInfoDescriptor; @@ -24,6 +25,7 @@ import org.torproject.metrics.collector.persist.BandwidthFilePersistence; import org.torproject.metrics.collector.persist.BridgeExtraInfoPersistence; import org.torproject.metrics.collector.persist.BridgePoolAssignmentPersistence; import org.torproject.metrics.collector.persist.BridgeServerDescriptorPersistence; +import org.torproject.metrics.collector.persist.BridgedbMetricsPersistence; import org.torproject.metrics.collector.persist.ConsensusPersistence; import org.torproject.metrics.collector.persist.DescriptorPersistence; import org.torproject.metrics.collector.persist.ExitlistPersistence; @@ -154,6 +156,9 @@ public class SyncPersistence { case "SnowflakeStats": descPersist = new SnowflakeStatsPersistence((SnowflakeStats) desc); break; + case "BridgedbStats": + descPersist = new BridgedbMetricsPersistence((BridgedbMetrics) desc); + break; default: log.trace("Invalid descriptor type {} for sync-merge.", clazz.getName()); diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index b180a3e..e7cadf7 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -59,6 +59,12 @@ SnowflakeStatsActivated = false SnowflakeStatsPeriodMinutes = 480 # offset in minutes since the epoch and SnowflakeStatsOffsetMinutes = 100 +# the following defines, if this module is activated +BridgedbMetricsActivated = false +# period in minutes +BridgedbMetricsPeriodMinutes = 480 +# offset in minutes since the epoch and +BridgedbMetricsOffsetMinutes = 340 ########################################## ## All below can be changed at runtime. @@ -216,3 +222,14 @@ SnowflakeStatsSyncOrigins = https://collector.torproject.org ## Where to download snowflake statistics from. SnowflakeStatsUrl = https://snowflake-broker.torproject.net/metrics # +######## BridgeDB statistics ######## +# +## Define descriptor sources +# possible values: Local, Sync +BridgedbMetricsSources = Local +## Relative path to directory to import BridgeDB metrics from. +BridgedbMetricsLocalOrigins = in/bridgedb-stats +## Retrieve files from the following instances. +## List of URLs separated by comma. +BridgedbMetricsSyncOrigins = https://collector.torproject.org +# \ No newline at end of file diff --git a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java index 7845909..7e9ea28 100644 --- a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java @@ -39,7 +39,7 @@ public class ConfigurationTest { public void testKeyCount() { assertEquals("The number of properties keys in enum Key changed." + "\n This test class should be adapted.", - 65, Key.values().length); + 71, Key.values().length); } @Test() diff --git a/src/test/java/org/torproject/metrics/collector/cron/CollecTorMainTest.java b/src/test/java/org/torproject/metrics/collector/cron/CollecTorMainTest.java index 99f1f48..cc124a4 100644 --- a/src/test/java/org/torproject/metrics/collector/cron/CollecTorMainTest.java +++ b/src/test/java/org/torproject/metrics/collector/cron/CollecTorMainTest.java @@ -70,6 +70,7 @@ public class CollecTorMainTest { case "Relay": case "Bridge": case "BridgePoolAssignments": + case "BridgedbMetrics": case "Exitlist": case "OnionPerf": case "Webstats":

1 0

[collector/release] Update to latest metrics-base.
by karsten＠torproject.org 18 Oct '19

18 Oct '19

commit 5c87bf6b63c054ddaeb0c627259a9d387903f5ac Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Oct 18 10:59:03 2019 +0200 Update to latest metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 62a964a..d82fff9 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47 +Subproject commit d82fff984634fe006ac7b0b102e7f48a52ca20d9

1 0

[collector/release] Prepare for 1.12.0 release.
by karsten＠torproject.org 18 Oct '19

18 Oct '19

commit fcdaeb86814c33cfa9b119d5bc396dbf69dda8e6 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Oct 18 11:05:42 2019 +0200 Prepare for 1.12.0 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca4d7b6..f30ea3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.12.0 - 2019-??-?? +# Changes in version 1.12.0 - 2019-10-18 * Medium changes - Require Mockito 1.10.19 as dependency for running tests. diff --git a/build.xml b/build.xml index 892566b..c78f5c6 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.11.1-dev" /> + <property name="release.version" value="1.12.0" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.8.0" />

1 0

[collector/master] Add new BridgedbMetrics module.
by karsten＠torproject.org 18 Oct '19

18 Oct '19

commit f2abf679c890f3f050c839cf32910900ae73ee76 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Sep 18 15:57:52 2019 +0200 Add new BridgedbMetrics module. Implements part of #19332. --- CHANGELOG.md | 2 + build.xml | 2 +- .../org/torproject/metrics/collector/Main.java | 3 + .../bridgedb/BridgedbMetricsProcessor.java | 190 +++++++++++++++++++++ .../metrics/collector/conf/Annotation.java | 1 + .../metrics/collector/conf/Configuration.java | 1 + .../org/torproject/metrics/collector/conf/Key.java | 8 +- .../persist/BridgedbMetricsPersistence.java | 37 ++++ .../collector/persist/DescriptorPersistence.java | 1 + .../metrics/collector/sync/SyncPersistence.java | 5 + src/main/resources/collector.properties | 17 ++ .../metrics/collector/conf/ConfigurationTest.java | 2 +- .../metrics/collector/cron/CollecTorMainTest.java | 1 + 13 files changed, 267 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e72f02..ca4d7b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ * Medium changes - Require Mockito 1.10.19 as dependency for running tests. + - Archive BridgeDB statistics. + - Update to metrics-lib 2.8.0. # Changes in version 1.11.1 - 2019-09-19 diff --git a/build.xml b/build.xml index 6b097a3..892566b 100644 --- a/build.xml +++ b/build.xml @@ -12,7 +12,7 @@ <property name="release.version" value="1.11.1-dev" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> - <property name="metricslibversion" value="2.7.0" /> + <property name="metricslibversion" value="2.8.0" /> <property name="jarincludes" value="collector.properties logback.xml" /> <patternset id="runtime" > diff --git a/src/main/java/org/torproject/metrics/collector/Main.java b/src/main/java/org/torproject/metrics/collector/Main.java index 3150ffc..3438bda 100644 --- a/src/main/java/org/torproject/metrics/collector/Main.java +++ b/src/main/java/org/torproject/metrics/collector/Main.java @@ -3,6 +3,7 @@ package org.torproject.metrics.collector; +import org.torproject.metrics.collector.bridgedb.BridgedbMetricsProcessor; import org.torproject.metrics.collector.bridgedescs.SanitizedBridgesWriter; import org.torproject.metrics.collector.bridgepools.BridgePoolAssignmentsProcessor; import org.torproject.metrics.collector.conf.Configuration; @@ -59,6 +60,8 @@ public class Main { collecTorMains.put(Key.WebstatsActivated, SanitizeWeblogs.class); collecTorMains.put(Key.SnowflakeStatsActivated, SnowflakeStatsDownloader.class); + collecTorMains.put(Key.BridgedbMetricsActivated, + BridgedbMetricsProcessor.class); } private static Configuration conf = new Configuration(); diff --git a/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java new file mode 100644 index 0000000..7ae4502 --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java @@ -0,0 +1,190 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.bridgedb; + +import org.torproject.descriptor.BridgedbMetrics; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.UnparseableDescriptor; +import org.torproject.metrics.collector.conf.Configuration; +import org.torproject.metrics.collector.conf.ConfigurationException; +import org.torproject.metrics.collector.conf.Key; +import org.torproject.metrics.collector.cron.CollecTorMain; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.Arrays; +import java.util.Stack; + +public class BridgedbMetricsProcessor extends CollecTorMain { + + /** + * Class logger. + */ + private static final Logger logger = LoggerFactory.getLogger( + BridgedbMetricsProcessor.class); + + /** + * Directory for reading BridgeDB statistics files. + */ + private File inputDirectory; + + /** + * Directory for writing BridgeDB statistics files to be archived in tarballs. + */ + private String outputPathName; + + /** + * Directory for writing recently processed BridgeDB statistics files. + */ + private String recentPathName; + + /** + * File name format. + */ + private DateTimeFormatter filenameFormat = DateTimeFormatter.ofPattern( + "uuuu/MM/dd/uuuu-MM-dd-HH-mm-ss"); + + /** + * Initialize this class with the given configuration. + */ + public BridgedbMetricsProcessor(Configuration config) { + super(config); + } + + /** + * Return the module identifier. + * + * @return Module identifier. + */ + @Override + public String module() { + return "BridgedbMetrics"; + } + + /** + * Return the synchronization marker. + * + * @return Synchronization marker. + */ + @Override + protected String syncMarker() { + return "BridgedbMetrics"; + } + + /** + * Start processing files, which includes reading BridgeDB statistics files + * from disk, possibly decompressing them and splitting them by date, and + * writing them back to disk. + * + * @throws ConfigurationException Thrown if configuration values cannot be + * obtained. + */ + @Override + protected void startProcessing() throws ConfigurationException { + logger.info("Starting BridgeDB statistics module of CollecTor."); + this.initializeConfiguration(); + logger.info("Reading BridgeDB statistics files in {}.", + this.inputDirectory); + for (Descriptor descriptor + : DescriptorSourceFactory.createDescriptorReader() + .readDescriptors(this.inputDirectory)) { + if (descriptor instanceof BridgedbMetrics) { + BridgedbMetrics bridgedbMetrics = (BridgedbMetrics) descriptor; + Path tarballPath = Paths.get(this.outputPathName, + bridgedbMetrics.bridgedbMetricsEnd().format(this.filenameFormat)); + Path rsyncPath = Paths.get(this.recentPathName, + bridgedbMetrics.bridgedbMetricsEnd().format(this.filenameFormat)); + this.writeDescriptor(bridgedbMetrics.getRawDescriptorBytes(), + tarballPath, rsyncPath); + } else if (descriptor instanceof UnparseableDescriptor) { + logger.warn("Skipping unparseable descriptor in file {}.", + descriptor.getDescriptorFile(), + ((UnparseableDescriptor) descriptor).getDescriptorParseException()); + } else { + logger.warn("Skipping unexpected descriptor of type {} in file {}.", + descriptor.getClass(), descriptor.getDescriptorFile()); + } + } + logger.info("Cleaning up directory {} containing recent files.", + this.recentPathName); + this.cleanUpRsyncDirectory(); + logger.info("Finished processing BridgeDB statistics file(s)."); + } + + /** + * Initialize configuration by obtaining current configuration values and + * storing them in instance attributes. + */ + private void initializeConfiguration() throws ConfigurationException { + this.outputPathName = Paths.get(config.getPath(Key.OutputPath).toString(), + "bridgedb-metrics").toString(); + this.recentPathName = Paths.get(config.getPath(Key.RecentPath).toString(), + "bridgedb-metrics").toString(); + this.inputDirectory = + config.getPath(Key.BridgedbMetricsLocalOrigins).toFile(); + } + + /** + * Write the given raw descriptor bytes to the given files, and stop at the + * first file that already exists. + * + * @param rawDescriptorBytes Raw descriptor bytes to write. + * @param outputPaths One or more paths to write to. + */ + private void writeDescriptor(byte[] rawDescriptorBytes, + Path ... outputPaths) { + for (Path outputPath : outputPaths) { + try { + File outputFile = outputPath.toFile(); + if (outputFile.exists()) { + continue; + } + if (!outputFile.getParentFile().exists() + && !outputFile.getParentFile().mkdirs()) { + logger.warn("Could not create parent directories of {}.", outputFile); + return; + } + Files.write(outputPath, rawDescriptorBytes); + } catch (IOException e) { + logger.warn("Unable to write descriptor to file {}.", outputPath, e); + } + } + } + + /** + * Delete all files from the rsync directory that have not been modified in + * the last three days. + */ + public void cleanUpRsyncDirectory() { + Instant cutOff = Instant.now().minus(3L, ChronoUnit.DAYS); + Stack<File> allFiles = new Stack<>(); + allFiles.add(new File(this.recentPathName)); + while (!allFiles.isEmpty()) { + File file = allFiles.pop(); + if (file.isDirectory()) { + File[] filesInDirectory = file.listFiles(); + if (null != filesInDirectory) { + allFiles.addAll(Arrays.asList(filesInDirectory)); + } + } else if (Instant.ofEpochMilli(file.lastModified()).isBefore(cutOff)) { + try { + Files.deleteIfExists(file.toPath()); + } catch (IOException e) { + logger.warn("Unable to delete file {} that is apparently older than " + + "three days.", file, e); + } + } + } + } +} diff --git a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java index 7d2bbe9..ff5119e 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java +++ b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java @@ -7,6 +7,7 @@ package org.torproject.metrics.collector.conf; public enum Annotation { BandwidthFile("@type bandwidth-file 1.0\n"), + BridgedbMetrics("@type bridgedb-metrics 1.0\n"), BridgeExtraInfo("@type bridge-extra-info 1.3\n"), BridgePoolAssignment("@type bridge-pool-assignment 1.0\n"), BridgeServer("@type bridge-server-descriptor 1.2\n"), diff --git a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java index 59229e3..56be34c 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/Configuration.java +++ b/src/main/java/org/torproject/metrics/collector/conf/Configuration.java @@ -89,6 +89,7 @@ public class Configuration extends Observable implements Cloneable { if (!(this.getBool(Key.RelaydescsActivated) || this.getBool(Key.BridgedescsActivated) || this.getBool(Key.BridgePoolAssignmentsActivated) + || this.getBool(Key.BridgedbMetricsActivated) || this.getBool(Key.ExitlistsActivated) || this.getBool(Key.UpdateindexActivated) || this.getBool(Key.OnionPerfActivated) diff --git a/src/main/java/org/torproject/metrics/collector/conf/Key.java b/src/main/java/org/torproject/metrics/collector/conf/Key.java index dfef673..390feed 100644 --- a/src/main/java/org/torproject/metrics/collector/conf/Key.java +++ b/src/main/java/org/torproject/metrics/collector/conf/Key.java @@ -78,7 +78,13 @@ public enum Key { SnowflakeStatsPeriodMinutes(Integer.class), SnowflakeStatsUrl(URL.class), SnowflakeStatsSources(SourceType[].class), - SnowflakeStatsSyncOrigins(URL[].class); + SnowflakeStatsSyncOrigins(URL[].class), + BridgedbMetricsActivated(Boolean.class), + BridgedbMetricsOffsetMinutes(Integer.class), + BridgedbMetricsPeriodMinutes(Integer.class), + BridgedbMetricsSources(SourceType[].class), + BridgedbMetricsLocalOrigins(Path.class), + BridgedbMetricsSyncOrigins(URL[].class); private Class clazz; private static Set<String> keys; diff --git a/src/main/java/org/torproject/metrics/collector/persist/BridgedbMetricsPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/BridgedbMetricsPersistence.java new file mode 100644 index 0000000..a72ffe2 --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/persist/BridgedbMetricsPersistence.java @@ -0,0 +1,37 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.persist; + +import org.torproject.descriptor.BridgedbMetrics; +import org.torproject.metrics.collector.conf.Annotation; + +import java.nio.file.Paths; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; + +public class BridgedbMetricsPersistence + extends DescriptorPersistence<BridgedbMetrics> { + + private static final String BRIDGEDB_STATS = "bridgedb-metrics"; + + public BridgedbMetricsPersistence(BridgedbMetrics desc) { + super(desc, Annotation.BridgedbMetrics.bytes()); + calculatePaths(); + } + + private void calculatePaths() { + DateTimeFormatter directoriesFormatter = DateTimeFormatter + .ofPattern("uuuu/MM/dd").withZone(ZoneOffset.UTC); + String[] directories = this.desc.bridgedbMetricsEnd() + .format(directoriesFormatter).split("/"); + DateTimeFormatter fileFormatter = DateTimeFormatter + .ofPattern("uuuu-MM-dd-HH-mm-ss").withZone(ZoneOffset.UTC); + String fileOut = this.desc.bridgedbMetricsEnd().format(fileFormatter) + + "-bridgedb-metrics"; + this.recentPath = Paths.get(BRIDGEDB_STATS, fileOut).toString(); + this.storagePath = Paths.get(BRIDGEDB_STATS, directories[0], directories[1], + directories[2], fileOut).toString(); + } +} + diff --git a/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java index 3e7a06b..20cd570 100644 --- a/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/persist/DescriptorPersistence.java @@ -20,6 +20,7 @@ public abstract class DescriptorPersistence<T extends Descriptor> { protected static final String BRIDGEDESCS = "bridge-descriptors"; protected static final String BRIDGEPOOLASSIGNMENTS = "bridge-pool-assignments"; + protected static final String BRIDGEDBSTATS = "bridgedb-stats"; protected static final String DASH = "-"; protected static final String DOT = "."; protected static final String MICRODESC = "microdesc"; diff --git a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java index cfc3dbe..f6678fe 100644 --- a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java +++ b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java @@ -8,6 +8,7 @@ import org.torproject.descriptor.BridgeExtraInfoDescriptor; import org.torproject.descriptor.BridgeNetworkStatus; import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.BridgeServerDescriptor; +import org.torproject.descriptor.BridgedbMetrics; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.ExitList; import org.torproject.descriptor.RelayExtraInfoDescriptor; @@ -24,6 +25,7 @@ import org.torproject.metrics.collector.persist.BandwidthFilePersistence; import org.torproject.metrics.collector.persist.BridgeExtraInfoPersistence; import org.torproject.metrics.collector.persist.BridgePoolAssignmentPersistence; import org.torproject.metrics.collector.persist.BridgeServerDescriptorPersistence; +import org.torproject.metrics.collector.persist.BridgedbMetricsPersistence; import org.torproject.metrics.collector.persist.ConsensusPersistence; import org.torproject.metrics.collector.persist.DescriptorPersistence; import org.torproject.metrics.collector.persist.ExitlistPersistence; @@ -154,6 +156,9 @@ public class SyncPersistence { case "SnowflakeStats": descPersist = new SnowflakeStatsPersistence((SnowflakeStats) desc); break; + case "BridgedbStats": + descPersist = new BridgedbMetricsPersistence((BridgedbMetrics) desc); + break; default: log.trace("Invalid descriptor type {} for sync-merge.", clazz.getName()); diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties index b180a3e..e7cadf7 100644 --- a/src/main/resources/collector.properties +++ b/src/main/resources/collector.properties @@ -59,6 +59,12 @@ SnowflakeStatsActivated = false SnowflakeStatsPeriodMinutes = 480 # offset in minutes since the epoch and SnowflakeStatsOffsetMinutes = 100 +# the following defines, if this module is activated +BridgedbMetricsActivated = false +# period in minutes +BridgedbMetricsPeriodMinutes = 480 +# offset in minutes since the epoch and +BridgedbMetricsOffsetMinutes = 340 ########################################## ## All below can be changed at runtime. @@ -216,3 +222,14 @@ SnowflakeStatsSyncOrigins = https://collector.torproject.org ## Where to download snowflake statistics from. SnowflakeStatsUrl = https://snowflake-broker.torproject.net/metrics # +######## BridgeDB statistics ######## +# +## Define descriptor sources +# possible values: Local, Sync +BridgedbMetricsSources = Local +## Relative path to directory to import BridgeDB metrics from. +BridgedbMetricsLocalOrigins = in/bridgedb-stats +## Retrieve files from the following instances. +## List of URLs separated by comma. +BridgedbMetricsSyncOrigins = https://collector.torproject.org +# \ No newline at end of file diff --git a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java index 7845909..7e9ea28 100644 --- a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java @@ -39,7 +39,7 @@ public class ConfigurationTest { public void testKeyCount() { assertEquals("The number of properties keys in enum Key changed." + "\n This test class should be adapted.", - 65, Key.values().length); + 71, Key.values().length); } @Test() diff --git a/src/test/java/org/torproject/metrics/collector/cron/CollecTorMainTest.java b/src/test/java/org/torproject/metrics/collector/cron/CollecTorMainTest.java index 99f1f48..cc124a4 100644 --- a/src/test/java/org/torproject/metrics/collector/cron/CollecTorMainTest.java +++ b/src/test/java/org/torproject/metrics/collector/cron/CollecTorMainTest.java @@ -70,6 +70,7 @@ public class CollecTorMainTest { case "Relay": case "Bridge": case "BridgePoolAssignments": + case "BridgedbMetrics": case "Exitlist": case "OnionPerf": case "Webstats":

1 0

[collector/master] Prepare for 1.12.0 release.
by karsten＠torproject.org 18 Oct '19

18 Oct '19

commit fcdaeb86814c33cfa9b119d5bc396dbf69dda8e6 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Oct 18 11:05:42 2019 +0200 Prepare for 1.12.0 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca4d7b6..f30ea3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.12.0 - 2019-??-?? +# Changes in version 1.12.0 - 2019-10-18 * Medium changes - Require Mockito 1.10.19 as dependency for running tests. diff --git a/build.xml b/build.xml index 892566b..c78f5c6 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.11.1-dev" /> + <property name="release.version" value="1.12.0" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.8.0" />

1 0

[collector/master] Update to latest metrics-base.
by karsten＠torproject.org 18 Oct '19

18 Oct '19

commit 5c87bf6b63c054ddaeb0c627259a9d387903f5ac Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Oct 18 10:59:03 2019 +0200 Update to latest metrics-base. --- src/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build b/src/build index 62a964a..d82fff9 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47 +Subproject commit d82fff984634fe006ac7b0b102e7f48a52ca20d9

1 0

[community/master] Bringing onion services section in line with other sections
by pili＠torproject.org 18 Oct '19

18 Oct '19

commit f160b86710d2af6346650c2670939177c9f04299 Author: Pili Guerra <pili(a)piliguerra.com> Date: Fri Oct 11 11:20:23 2019 +0200 Bringing onion services section in line with other sections --- content/onion-services/ssl-tls/contents.lr | 21 ------------- templates/onion-services.html | 49 ++++++++++++++---------------- 2 files changed, 22 insertions(+), 48 deletions(-) diff --git a/content/onion-services/ssl-tls/contents.lr b/content/onion-services/ssl-tls/contents.lr deleted file mode 100644 index e60a121..0000000 --- a/content/onion-services/ssl-tls/contents.lr +++ /dev/null @@ -1,21 +0,0 @@ -section: onion services ---- -section_id: onion-services ---- -color: primary ---- -image: lock ---- -_template: layout.html ---- -title: Adding SSL/TLS to a .onion site ---- -subtitle: Add more security and authenticity for your .onion. ---- -key: 2 ---- -html: two-columns-page.html ---- -body: - -Coming soon diff --git a/templates/onion-services.html b/templates/onion-services.html index 94e834a..a4a7ed6 100644 --- a/templates/onion-services.html +++ b/templates/onion-services.html @@ -7,43 +7,30 @@ <img src="/static/images/home/png/{{ this.section_id }}.png" class="img-fluid" alt="{{ _(this.section) }}" /> </div> </div> - <a href="{{ this.path|url }}overview/"><h2 class="text-primary"><u>{{ _('Learn more about onion services here') }} - <i class="fas fa-arrow-right"></i></u> - </h2></a> - <div class="row py-5"> + <div class="row"> {% for child in this.children|sort(attribute='key') %} - {% if child.key != '0' %} - <div class="col-sm-6 col-md-6 col-sm-12 col-xl-6 py-4"> - <div class="card h-100 border-0"> + <div class="col-sm-6 col-md-6 col-sm-12 col-xl-6 py-3"> + <div class="card h-100"> <div class="card-body"> - <h2 class="text-primary display-2"><i class="fas fa-{{ child.image }}"></i></h2> - <h4 class="card-title text-primary">{{ child.title }}</h4> - <p class="card-text">{{ child.subtitle }}</p> - <a href="{{ child.path|url }}">{{ child.cta }} <i class="fas fa-arrow-right"></i></a> + <div class="card-body"> + <h4 class="card-title text-primary"> + <a href="{{ child.path|url(alt=this.alt) }}">{{ _(child.title) }}</a> + </h4> + <p class="card-text">{{ _(child.subtitle) }}</p> + </div> </div> </div> </div> - {% endif %} {% endfor %} </div> <div class="row py-5 text-center mx-auto"> <h2 class="display-4 text-primary text-center mx-auto">{{ _('Onionize any website') }}</h2> <p class="text-center"> - {{ _('There\'s a toolkit that lets you take any existing website and host it as a .onion site. You would wanna do this because .onion sites are more secure than just regular sites. We show you how to use this toolkit and onionize a site.') }} + {{ _('There\'s a toolkit that lets you take any existing website and host it as a .onion site. You would want do this because .onion sites are more secure than just regular sites. We show you how to use this toolkit and onionize a site.') }} </p> - </div> - <div class="row mx-auto"> - <div class="col-6"> - <a href="https://github.com/alecmuffett/eotk#i-want-to-create-my-own-project"> - <h3 class="text-primary text-center mx-auto"><u>{{ _('Learn more') }} - <i class="fas fa-arrow-right"></i></u> - </h3></a> - </div> - <div class="col-6"> - <a href="https://github.com/alecmuffett/eotk"><h3 class="text-primary text-center mx-auto"><u>{{ _('Check out the code') }} - <i class="fas fa-arrow-right"></i></u> - </h3></a> - </div> + <a class="btn btn-lg btn-outline-primary mx-auto my-3" href="https://github.com/alecmuffett/eotk"> + {{ _('Check out the code') }}<i class="ml-3 pt-2 fas fa-arrow-right"></i> + </a> </div> <div class="row py-5 text-center mx-auto"> <h2 class="display-4 text-primary text-center mx-auto">{{ _('Featured .onion sites') }}</h2> @@ -63,6 +50,14 @@ </div> </div> {% endfor %} - + </div> + <div class="row py-5 text-center mx-auto"> + <h2 class="display-4 text-primary text-center mx-auto my-3">{{ _('Learn More') }}</h2> + <p class="text-center"> + {{ _('Are you interested in learning more about onion services? Join our tor-onions mailing list to speak with other onion service operators.')}} + </p> + <a class="btn btn-lg btn-outline-primary mx-auto my-3" href="https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-onions"> + {{ _('Tor Onions mailing list') }}<i class="ml-3 pt-2 fas fa-arrow-right"></i> + </a> </div> </div>

1 0

[community/master] Changed buttons to ordered list
by pili＠torproject.org 18 Oct '19

18 Oct '19

commit 7c7daf97db0a30f224a2d6af6cbeaefd0412b25e Author: LilyLME <38939048+LilyLME(a)users.noreply.github.com> Date: Sat Oct 5 17:27:41 2019 +0100 Changed buttons to ordered list The steps followed to sign up on Transifex to become a Tor translator are supposed to be an ordered list, so that the reader follows them chronologically. --- .../localization/becoming-tor-translator/contents.lr | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/content/localization/becoming-tor-translator/contents.lr b/content/localization/becoming-tor-translator/contents.lr index 229f70d..1e48e58 100644 --- a/content/localization/becoming-tor-translator/contents.lr +++ b/content/localization/becoming-tor-translator/contents.lr @@ -29,19 +29,19 @@ In order to begin contributing you will have to sign up with Transifex. Here's a # Signing Up On Transifex -- Head over to the [Transifex signup page](https://transifex.com/signup/). +1. Head over to the [Transifex signup page](https://transifex.com/signup/). Enter your information into the fields and click the 'Sign Up' button: ![Sign up to Transifex](/static/images/localization/tr1.png) -- Fill out the next page with your name and select "Localization" and "Translator" from the drop-down menus: +1. Fill out the next page with your name and select "Localization" and "Translator" from the drop-down menus: ![Fill out details](/static/images/localization/tr2.png) -- On the next page, select 'Join an existing project' and continue. -- On the next page, select the languages you speak from the drop-down menu and continue. -- You are now signed up! Go to the [Tor Transifex page](https://www.transifex.com/otf/torproject/). -- Click the blue 'Join Team' button on the far right: +1. On the next page, select 'Join an existing project' and continue. +1. On the next page, select the languages you speak from the drop-down menu and continue. +1. You are now signed up! Go to the [Tor Transifex page](https://www.transifex.com/otf/torproject/). +1. Click the blue 'Join Team' button on the far right: ![Join Team](/static/images/localization/tr3.png) -- Select the language you would like to translate from the dropdown menu: +1. Select the language you would like to translate from the dropdown menu: ![Choose Language](/static/images/localization/tr4.png) -- A notification will now show up on the top of the page like so: +1. A notification will now show up on the top of the page like so: ![Request Submitted](/static/images/localization/tr5.png) -After your membership is approved, you're ready to begin. \ No newline at end of file +After your membership is approved, you're ready to begin.

1 0

[community/develop] Merge commit '0a622f691b17b36394b3cf5e340060c99226f935' into develop
by pili＠torproject.org 18 Oct '19

18 Oct '19

commit 084aa9873382a5b0573fc018352a6bb29ac39a20 Merge: b37bee8 0a622f6 Author: Pili Guerra <pili(a)piliguerra.com> Date: Fri Oct 18 10:09:43 2019 +0200 Merge commit '0a622f691b17b36394b3cf5e340060c99226f935' into develop .../localization/becoming-tor-translator/contents.lr | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-)

1 0