October 2019 - tor-commits - lists.torproject.org

[translation/snowflakeaddon-messages.json] https://gitweb.torproject.org/translation.git/commit/?h=snowflakeaddon-messages.json
by translation＠torproject.org 18 Oct '19

18 Oct '19

commit b1aa454c1cce1cceb51c717b283ea9b03561141c Author: Translation commit bot <translation(a)torproject.org> Date: Fri Oct 18 09:47:14 2019 +0000 https://gitweb.torproject.org/translation.git/commit/?h=snowflakeaddon-mess… --- zh_CN/messages.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zh_CN/messages.json b/zh_CN/messages.json index ad6a8a3e8..ac6bd9ee2 100644 --- a/zh_CN/messages.json +++ b/zh_CN/messages.json @@ -33,7 +33,7 @@ "message": "Snowflake 是一个用来对抗互联网审查的系统。被审查的人们可以使用 Snowflake 来访问互联网。他们的连接将通过由志愿者们所运行的 Snowflake 代理来访问网络。了解更多关于 Snowflake 如何运行的资讯，请参阅<a href=\"https://trac.torproject.org/projects/tor/wiki/doc/Snowflake/\" data-msgid=\"__MSG_docWiki__\">文档 wiki</a>。" }, "docWiki": { - "message": "documentation wiki" + "message": "文档 wiki" }, "browser": { "message": "浏览器" @@ -45,13 +45,13 @@ "message": "拓展" }, "installExtension": { - "message": "如果您的互联网访问并不受到审查，你可以考虑安装 Snowflake 拓展来帮助那些身陷审查网络中的用户。无需担心人们通过您的代理访问哪些网站。他们可见的 IP 地址是他们的 Tor 出口节点的 IP 地址，而不是您的。" + "message": "如果您的互联网访问并未受到审查，你可以考虑安装 Snowflake 拓展来帮助那些身陷审查网络中的用户。无需担心人们通过您的代理访问哪些网站。他们可见的 IP 地址是他们的 Tor 出口节点的 IP 地址，而不是您的。" }, "installFirefox": { - "message": "Install in Firefox" + "message": "安装到 Firefox 浏览器" }, "installChrome": { - "message": "Install in Chrome" + "message": "安装到 Google Chrome 浏览器" }, "reportingBugs": { "message": "报告缺陷" @@ -63,13 +63,13 @@ "message": "<a href=\"https://trac.torproject.org/projects/tor/register\">创建一个账户</a>或者使用共享的 cypherpunks 账号 writecode 密码<a href=\"https://trac.torproject.org/projects/tor/login\">登录</a>，" }, "bugTracker": { - "message": "<a href=\"https://trac.torproject.org/projects/tor/newticket?component=Circumvention%…">填写工单</a> 并使用我们的漏洞追踪器。" + "message": "使用我们的漏洞追踪器<a href=\"https://trac.torproject.org/projects/tor/newticket?component=Circumvention%…">填写工单</a> 。" }, "descriptive": { - "message": "请尽量描述您的情况，如果可能，请同时报告您的日志文件，这将帮助我们重现漏洞。考虑添加关键词 snowflake-webextension 或者 snowflake-client 来让我们知道您在 Snowflake 系统的哪个部分中遇到了问题。" + "message": "请尽量客观地描述您的情况，如果可能，请同时报告您的日志文件，这将帮助我们重现漏洞。考虑添加关键词 snowflake-webextension 或者 snowflake-client 来让我们知道您在 Snowflake 系统的哪个部分中遇到了问题。" }, "embed": { - "message": "Embed" + "message": "嵌入式插件" }, "possible": { "message": "现在可以在任何网站上嵌入 Snowflake 徽章："

1 0

[tor-browser-build/master] Bump tor-browser branch for nightly
by gk＠torproject.org 18 Oct '19

18 Oct '19

commit d85ebd30305a598b2ed5fc8578cf004095056a2f Author: Georg Koppen <gk(a)torproject.org> Date: Fri Oct 18 09:32:46 2019 +0000 Bump tor-browser branch for nightly --- projects/firefox/config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/firefox/config b/projects/firefox/config index 4370f01..fe1b5d4 100644 --- a/projects/firefox/config +++ b/projects/firefox/config @@ -8,9 +8,9 @@ git_submodule: 1 gpg_keyring: torbutton.gpg var: - firefox_platform_version: 68.1.0 + firefox_platform_version: 68.2.0 firefox_version: '[% c("var/firefox_platform_version") %]esr' - torbrowser_branch: 9.0 + torbrowser_branch: 9.5 torbrowser_update_channel: alpha branding_directory: '[% IF c("var/android") %]mobile/android[% ELSE %]browser[% END %]/branding/alpha' copyright_year: '[% exec("git show -s --format=%ci").remove("-.*") %]' @@ -51,7 +51,7 @@ targets: branding_directory: '[% IF c("var/android") %]mobile/android[% ELSE %]browser[% END %]/branding/official' nightly: - git_hash: 'tor-browser-[% c("var/firefox_version") %]-[% c("var/torbrowser_branch") %]-3' + git_hash: 'tor-browser-[% c("var/firefox_version") %]-[% c("var/torbrowser_branch") %]-1' tag_gpg_id: 0 var: torbrowser_update_channel: default

1 0

[tor-browser-build/master] Fix typo in Changelog
by gk＠torproject.org 18 Oct '19

18 Oct '19

commit 5460ef952ed966834344158257a600f230ab60b0 Author: Georg Koppen <gk(a)torproject.org> Date: Fri Oct 18 09:32:18 2019 +0000 Fix typo in Changelog --- projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt b/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt index 56a7725..2930090 100644 --- a/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt +++ b/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt @@ -364,7 +364,7 @@ Tor Browser 9.0a1 -- May 21 2019 * Update Tor Launcher to 0.2.19 * Bug 28044: Integrate Tor Launcher into tor-browser * Bug 29627: Moat: add support for obfsproxy's meek_lite - * Bug 30139: Remove FTE bits + * Bug 30319: Remove FTE bits * Translations update * Bug 28044: Integrate Tor Launcher into tor-browser * Bug 30372: Backport letterboxing (bug 1538130)

1 0

[translation/tor-launcher-properties_completed] https://gitweb.torproject.org/translation.git/commit/?h=tor-launcher-properties_completed
by translation＠torproject.org 18 Oct '19

18 Oct '19

commit e69a1007a5353912b0940c181c5d77539e11dd76 Author: Translation commit bot <translation(a)torproject.org> Date: Fri Oct 18 09:22:42 2019 +0000 https://gitweb.torproject.org/translation.git/commit/?h=tor-launcher-proper… --- mk/torlauncher.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/torlauncher.properties b/mk/torlauncher.properties index 806ac8d57..4055e2407 100644 --- a/mk/torlauncher.properties +++ b/mk/torlauncher.properties @@ -52,7 +52,7 @@ torlauncher.forAssistance2=За помош, посетете ја %S torlauncher.copiedNLogMessages=Копирањето е завршено. %S Tor лог пораките се подготвени да бидат залепени во уредувач на текст или во порака за е-пошта. -torlauncher.bootstrapStatus.starting=Започнувањето +torlauncher.bootstrapStatus.starting=Започнување torlauncher.bootstrapStatus.conn_pt=Поврзување со мост torlauncher.bootstrapStatus.conn_done_pt=Поврзан со мост torlauncher.bootstrapStatus.conn_proxy=Поврзување со прокси

1 0

[translation/tor-launcher-properties] https://gitweb.torproject.org/translation.git/commit/?h=tor-launcher-properties
by translation＠torproject.org 18 Oct '19

18 Oct '19

commit 269254258791793d7f9a4fa5d719e80ab703546b Author: Translation commit bot <translation(a)torproject.org> Date: Fri Oct 18 09:22:36 2019 +0000 https://gitweb.torproject.org/translation.git/commit/?h=tor-launcher-proper… --- mk/torlauncher.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mk/torlauncher.properties b/mk/torlauncher.properties index 806ac8d57..4055e2407 100644 --- a/mk/torlauncher.properties +++ b/mk/torlauncher.properties @@ -52,7 +52,7 @@ torlauncher.forAssistance2=За помош, посетете ја %S torlauncher.copiedNLogMessages=Копирањето е завршено. %S Tor лог пораките се подготвени да бидат залепени во уредувач на текст или во порака за е-пошта. -torlauncher.bootstrapStatus.starting=Започнувањето +torlauncher.bootstrapStatus.starting=Започнување torlauncher.bootstrapStatus.conn_pt=Поврзување со мост torlauncher.bootstrapStatus.conn_done_pt=Поврзан со мост torlauncher.bootstrapStatus.conn_proxy=Поврзување со прокси

1 0

[collector/master] Bump version to 1.12.0-dev.
by karsten＠torproject.org 18 Oct '19

18 Oct '19

commit 0e57dae0bfd6f3f7e0c5b5c78773818343ba9550 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Oct 18 11:11:38 2019 +0200 Bump version to 1.12.0-dev. --- CHANGELOG.md | 3 +++ build.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f30ea3e..bb328ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +# Changes in version 1.??.? - 2019-??-?? + + # Changes in version 1.12.0 - 2019-10-18 * Medium changes diff --git a/build.xml b/build.xml index c78f5c6..8276e63 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.12.0" /> + <property name="release.version" value="1.12.0-dev" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.8.0" />

1 0

[collector/release] Use metrics-base ant task to fetch metrics-lib.
by karsten＠torproject.org 18 Oct '19

18 Oct '19

commit de03530b98bd66eee404e4483d930baf62a8f3e1 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Sep 27 10:05:35 2019 +0200 Use metrics-base ant task to fetch metrics-lib. Implements part of #31649. --- .gitlab-ci.yml | 7 +------ src/build | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5bd5623..897c9da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,5 @@ variables: GIT_STRATEGY: clone - METRICS_LIB_VERSION: "2.7.0" JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" stages: @@ -15,11 +14,7 @@ test: - git submodule init - git submodule update - mkdir lib - - mkdir tmp - - pushd tmp - - curl https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$M… | tar xzf - - - popd - - mv tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar lib + - ant fetch-metrics-lib - ant -lib /usr/share/java resolve - ant test - ant checks diff --git a/src/build b/src/build index 0674958..07c2a00 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af +Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4

1 0

[collector/release] Tweak URL-based unit tests.
by karsten＠torproject.org 18 Oct '19

18 Oct '19

commit 834ef7c5e76778e31f9ca8569e69423b32700acf Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Sep 27 12:23:19 2019 +0200 Tweak URL-based unit tests. Turns out that one of the tests that we disabled earlier was only slow, because creating a URL instance to https://something.dummy.org/ triggered an actual DNS resolve. There's absolutely no need to do this, so we can as well use localhost in all our test URLs. --- .../org/torproject/metrics/collector/conf/ConfigurationTest.java | 5 ++--- .../torproject/metrics/collector/downloader/DownloaderTest.java | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java index 67a9082..7845909 100644 --- a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java @@ -128,13 +128,12 @@ public class ConfigurationTest { } } - @Ignore("This test takes 40 seconds, which is too long.") @Test() public void testUrlArrayValues() throws Exception { URL[] array = new URL[randomSource.nextInt(30) + 1]; for (int i = 0; i < array.length; i++) { - array[i] = new URL("https://" - + Integer.toBinaryString(randomSource.nextInt(100)) + ".dummy.org"); + array[i] = new URL("http://localhost/" + + Integer.toBinaryString(randomSource.nextInt(100))); } String input = Arrays.toString(array).replace("[", "").replace("]", "") diff --git a/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java index 19db782..0617940 100644 --- a/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java +++ b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java @@ -85,7 +85,7 @@ public class DownloaderTest { @Test public void testExistingResource() throws Exception { - URL requestedUrl = new URL("http://example.org/exists"); + URL requestedUrl = new URL("http://localhost/exists"); byte[] expectedDownloadedBytes = "content".getBytes(); HttpURLConnection urlConnection = mock(HttpURLConnection.class); httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); @@ -98,7 +98,7 @@ public class DownloaderTest { @Test public void testNonExistingResource() throws Exception { - URL requestedUrl = new URL("http://example.org/notfound"); + URL requestedUrl = new URL("http://localhost/notfound"); HttpURLConnection urlConnection = mock(HttpURLConnection.class); httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); given(urlConnection.getResponseCode()).willReturn(404); @@ -108,7 +108,7 @@ public class DownloaderTest { @Test public void testEmptyResource() throws Exception { - URL requestedUrl = new URL("http://example.org/empty"); + URL requestedUrl = new URL("http://localhost/empty"); byte[] expectedDownloadedBytes = new byte[0]; HttpURLConnection urlConnection = mock(HttpURLConnection.class); httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); @@ -121,7 +121,7 @@ public class DownloaderTest { @Test(expected = SocketTimeoutException.class) public void testTimeout() throws Exception { - URL requestedUrl = new URL("http://example.org/timeout"); + URL requestedUrl = new URL("http://localhost/timeout"); SocketTimeoutException expectedException = new SocketTimeoutException(); HttpURLConnection urlConnection = mock(HttpURLConnection.class); httpUrlStreamHandler.addConnection(requestedUrl, urlConnection);

1 0

[collector/release] Bump version to 1.11.1-dev.
by karsten＠torproject.org 18 Oct '19

18 Oct '19

commit 18877fdb4d270a6ef02664eb07271c064fab785f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Sep 19 15:11:37 2019 +0200 Bump version to 1.11.1-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 9f2a0a6..6b097a3 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,7 @@ <property name="javadoc-title" value="CollecTor API Documentation"/> <property name="implementation-title" value="CollecTor" /> - <property name="release.version" value="1.11.1" /> + <property name="release.version" value="1.11.1-dev" /> <property name="project-main-class" value="org.torproject.metrics.collector.Main" /> <property name="name" value="collector"/> <property name="metricslibversion" value="2.7.0" />

1 0

[collector/release] Create utility class for downloading from HTTP servers
by karsten＠torproject.org 18 Oct '19

18 Oct '19

commit dc3a2e4aea03ca44be033dafc3dcc3d7e0d5c07f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Sep 26 11:48:18 2019 +0200 Create utility class for downloading from HTTP servers Also skip two tests in ConfigurationTest that together take 53 seconds which has the effect that we don't run tests very often. Implements #31599. --- CHANGELOG.md | 6 + src/build | 2 +- .../metrics/collector/downloader/Downloader.java | 62 ++++++++++ .../collector/exitlists/ExitListDownloader.java | 33 ++--- .../collector/onionperf/OnionPerfDownloader.java | 23 +++- .../relaydescs/RelayDescriptorDownloader.java | 32 +---- .../snowflake/SnowflakeStatsDownloader.java | 56 ++------- .../metrics/collector/conf/ConfigurationTest.java | 3 + .../collector/downloader/DownloaderTest.java | 134 +++++++++++++++++++++ 9 files changed, 250 insertions(+), 101 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e30b086..6e72f02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# Changes in version 1.12.0 - 2019-??-?? + + * Medium changes + - Require Mockito 1.10.19 as dependency for running tests. + + # Changes in version 1.11.1 - 2019-09-19 * Minor changes diff --git a/src/build b/src/build index 07c2a00..62a964a 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4 +Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47 diff --git a/src/main/java/org/torproject/metrics/collector/downloader/Downloader.java b/src/main/java/org/torproject/metrics/collector/downloader/Downloader.java new file mode 100644 index 0000000..a1f3852 --- /dev/null +++ b/src/main/java/org/torproject/metrics/collector/downloader/Downloader.java @@ -0,0 +1,62 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.downloader; + +import java.io.BufferedInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.zip.InflaterInputStream; + +/** + * Utility class for downloading resources from HTTP servers. + */ +public class Downloader { + + /** + * Download the given URL from an HTTP server and return downloaded bytes. + * + * @param url URL to download. + * @return Downloaded bytes, or {@code null} if the resource was not found. + * @throws IOException Thrown if anything goes wrong while downloading. + */ + public static byte[] downloadFromHttpServer(URL url) throws IOException { + return Downloader.downloadFromHttpServer(url, false); + } + + /** + * Download the given URL from an HTTP server, possibly inflate the response, + * and return downloaded bytes. + * + * @param url URL to download. + * @param isDeflated Whether the response is deflated. + * @return Downloaded bytes, or {@code null} if the resource was not found. + * @throws IOException Thrown if anything goes wrong while downloading. + */ + public static byte[] downloadFromHttpServer(URL url, boolean isDeflated) + throws IOException { + ByteArrayOutputStream downloadedBytes = new ByteArrayOutputStream(); + HttpURLConnection huc = (HttpURLConnection) url.openConnection(); + huc.setRequestMethod("GET"); + huc.setReadTimeout(5000); + huc.connect(); + int response = huc.getResponseCode(); + if (response != 200) { + return null; + } + try (BufferedInputStream in = isDeflated + ? new BufferedInputStream(new InflaterInputStream( + huc.getInputStream())) + : new BufferedInputStream(huc.getInputStream())) { + int len; + byte[] data = new byte[1024]; + while ((len = in.read(data, 0, 1024)) >= 0) { + downloadedBytes.write(data, 0, len); + } + } + return downloadedBytes.toByteArray(); + } +} + diff --git a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java index 66fc1a7..b4bee15 100644 --- a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java @@ -12,16 +12,15 @@ import org.torproject.metrics.collector.conf.Configuration; import org.torproject.metrics.collector.conf.ConfigurationException; import org.torproject.metrics.collector.conf.Key; import org.torproject.metrics.collector.cron.CollecTorMain; +import org.torproject.metrics.collector.downloader.Downloader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedInputStream; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.net.HttpURLConnection; import java.net.URL; import java.nio.file.Paths; import java.text.SimpleDateFormat; @@ -77,32 +76,18 @@ public class ExitListDownloader extends CollecTorMain { sb.append("Downloaded ").append(dateTimeFormat.format(downloadedDate)) .append("\n"); URL url = config.getUrl(Key.ExitlistUrl); - try { - HttpURLConnection huc = (HttpURLConnection) url.openConnection(); - huc.setRequestMethod("GET"); - huc.setReadTimeout(5000); - huc.connect(); - int response = huc.getResponseCode(); - if (response != 200) { - logger.warn("Could not download exit list. Response code {}", - response); - return; - } - try (BufferedInputStream in = new BufferedInputStream( - huc.getInputStream())) { - int len; - byte[] data = new byte[1024]; - while ((len = in.read(data, 0, 1024)) >= 0) { - sb.append(new String(data, 0, len)); - } - } - downloadedExitList = sb.toString(); - logger.debug("Finished downloading exit list."); + byte[] downloadedBytes; + try { + downloadedBytes = Downloader.downloadFromHttpServer(url); } catch (IOException e) { logger.warn("Failed downloading exit list", e); return; } - if (downloadedExitList == null) { + if (null != downloadedBytes) { + sb.append(new String(downloadedBytes)); + downloadedExitList = sb.toString(); + logger.debug("Finished downloading exit list."); + } else { logger.warn("Failed downloading exit list."); return; } diff --git a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java index ca307a5..b106be7 100644 --- a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java @@ -11,6 +11,7 @@ import org.torproject.metrics.collector.conf.Configuration; import org.torproject.metrics.collector.conf.ConfigurationException; import org.torproject.metrics.collector.conf.Key; import org.torproject.metrics.collector.cron.CollecTorMain; +import org.torproject.metrics.collector.downloader.Downloader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,7 +22,6 @@ import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; -import java.io.InputStream; import java.io.InputStreamReader; import java.net.MalformedURLException; import java.net.URL; @@ -200,12 +200,25 @@ public class OnionPerfDownloader extends CollecTorMain { /* Download file contents to temporary file. */ File tempFile = new File(this.recentDirectory, "." + tpfFileName); + byte[] downloadedBytes; + try { + downloadedBytes = Downloader.downloadFromHttpServer( + new URL(baseUrl + tpfFileName)); + } catch (IOException e) { + logger.warn("Unable to download '{}{}'. Skipping.", baseUrl, tpfFileName, + e); + return; + } + if (null == downloadedBytes) { + logger.warn("Unable to download '{}{}'. Skipping.", baseUrl, tpfFileName); + return; + } tempFile.getParentFile().mkdirs(); - try (InputStream is = new URL(baseUrl + tpfFileName).openStream()) { - Files.copy(is, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + try { + Files.write(tempFile.toPath(), downloadedBytes); } catch (IOException e) { - logger.warn("Unable to download '{}{}' to temporary file '{}'. " - + "Skipping.", baseUrl, tpfFileName, tempFile, e); + logger.warn("Unable to write previously downloaded '{}{}' to temporary " + + "file '{}'. Skipping.", baseUrl, tpfFileName, tempFile, e); return; } diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java index f1179d8..3337633 100644 --- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java @@ -3,20 +3,19 @@ package org.torproject.metrics.collector.relaydescs; +import org.torproject.metrics.collector.downloader.Downloader; + import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.digest.DigestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedInputStream; import java.io.BufferedReader; import java.io.BufferedWriter; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; -import java.net.HttpURLConnection; import java.net.URL; import java.nio.charset.StandardCharsets; import java.text.ParseException; @@ -33,7 +32,6 @@ import java.util.SortedSet; import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; -import java.util.zip.InflaterInputStream; /** * Downloads relay descriptors from the directory authorities via HTTP. @@ -875,29 +873,11 @@ public class RelayDescriptorDownloader { String fullUrl = "http://" + authority + resource + (isCompressed ? ".z" : ""); URL url = new URL(fullUrl); - HttpURLConnection huc = (HttpURLConnection) url.openConnection(); - huc.setRequestMethod("GET"); - huc.setReadTimeout(5000); - huc.connect(); - int response = huc.getResponseCode(); - if (response == 200) { - try (BufferedInputStream in - = isCompressed ? new BufferedInputStream(new InflaterInputStream( - huc.getInputStream())) - : new BufferedInputStream(huc.getInputStream())) { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] data = new byte[1024]; - while ((len = in.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - allData = baos.toByteArray(); - } - } - logger.debug("Downloaded {} -> {} ({} bytes)", fullUrl, response, - allData == null ? 0 : allData.length); + allData = Downloader.downloadFromHttpServer(url, isCompressed); int receivedDescriptors = 0; - if (allData != null) { + logger.debug("Downloaded {} -> ({} bytes)", fullUrl, + allData == null ? 0 : allData.length); + if (null != allData) { if (resource.startsWith("/tor/status-vote/")) { this.rdp.parse(allData, null); receivedDescriptors = 1; diff --git a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java index 4f7994e..cb5f0cc 100644 --- a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java +++ b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java @@ -12,18 +12,16 @@ import org.torproject.metrics.collector.conf.Configuration; import org.torproject.metrics.collector.conf.ConfigurationException; import org.torproject.metrics.collector.conf.Key; import org.torproject.metrics.collector.cron.CollecTorMain; +import org.torproject.metrics.collector.downloader.Downloader; import org.torproject.metrics.collector.persist.SnowflakeStatsPersistence; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedInputStream; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.net.HttpURLConnection; import java.net.URL; import java.time.LocalDateTime; import java.util.Arrays; @@ -60,9 +58,15 @@ public class SnowflakeStatsDownloader extends CollecTorMain { this.recentPathName = config.getPath(Key.RecentPath).toString(); logger.debug("Downloading snowflake stats..."); URL url = config.getUrl(Key.SnowflakeStatsUrl); - ByteArrayOutputStream downloadedSnowflakeStats - = this.downloadFromHttpServer(url); - if (null == downloadedSnowflakeStats) { + byte[] downloadedBytes; + try { + downloadedBytes = Downloader.downloadFromHttpServer(url); + } catch (IOException e) { + logger.warn("Failed downloading {}.", url, e); + return; + } + if (null == downloadedBytes) { + logger.warn("Could not download {}.", url); return; } logger.debug("Finished downloading {}.", url); @@ -72,7 +76,7 @@ public class SnowflakeStatsDownloader extends CollecTorMain { SortedSet<LocalDateTime> snowflakeStatsEnds = new TreeSet<>(); String outputPathName = config.getPath(Key.OutputPath).toString(); for (Descriptor descriptor : descriptorParser.parseDescriptors( - downloadedSnowflakeStats.toByteArray(), null, null)) { + downloadedBytes, null, null)) { if (descriptor instanceof SnowflakeStats) { SnowflakeStats snowflakeStats = (SnowflakeStats) descriptor; LocalDateTime snowflakeStatsEnd = snowflakeStats.snowflakeStatsEnd(); @@ -106,44 +110,6 @@ public class SnowflakeStatsDownloader extends CollecTorMain { } /** - * Download the given URL from an HTTP server and return a stream with - * downloaded bytes. - * - * If anything goes wrong while downloading, log a warning and return - * {@code null}. - * - * @param url URL to download. - * @return Stream with downloaded bytes, or {@code null} if an error has - * occurred. - */ - private ByteArrayOutputStream downloadFromHttpServer(URL url) { - ByteArrayOutputStream downloadedBytes = new ByteArrayOutputStream(); - try { - HttpURLConnection huc = (HttpURLConnection) url.openConnection(); - huc.setRequestMethod("GET"); - huc.setReadTimeout(5000); - huc.connect(); - int response = huc.getResponseCode(); - if (response != 200) { - logger.warn("Could not download {}. Response code {}", url, response); - return null; - } - try (BufferedInputStream in = new BufferedInputStream( - huc.getInputStream())) { - int len; - byte[] data = new byte[1024]; - while ((len = in.read(data, 0, 1024)) >= 0) { - downloadedBytes.write(data, 0, len); - } - } - } catch (IOException e) { - logger.warn("Failed downloading {}.", url, e); - return null; - } - return downloadedBytes; - } - - /** * Write the given byte array(s) to the given file. * * If the file already exists, it is overwritten. If the parent directory diff --git a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java index 201d541..67a9082 100644 --- a/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java +++ b/src/test/java/org/torproject/metrics/collector/conf/ConfigurationTest.java @@ -12,6 +12,7 @@ import org.torproject.metrics.collector.MainTest; import org.torproject.metrics.collector.cron.CollecTorMain; import org.torproject.metrics.collector.cron.Dummy; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -127,6 +128,7 @@ public class ConfigurationTest { } } + @Ignore("This test takes 40 seconds, which is too long.") @Test() public void testUrlArrayValues() throws Exception { URL[] array = new URL[randomSource.nextInt(30) + 1]; @@ -180,6 +182,7 @@ public class ConfigurationTest { conf.setWatchableSourceAndLoad(Paths.get("/tmp/phantom.path")); } + @Ignore("This test takes 13 seconds, which is too long.") @Test() public void testConfigChange() throws Exception { Configuration conf = new Configuration(); diff --git a/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java new file mode 100644 index 0000000..19db782 --- /dev/null +++ b/src/test/java/org/torproject/metrics/collector/downloader/DownloaderTest.java @@ -0,0 +1,134 @@ +/* Copyright 2019 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.collector.downloader; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; +import static org.mockito.BDDMockito.given; +import static org.mockito.Mockito.mock; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.SocketTimeoutException; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLStreamHandler; +import java.net.URLStreamHandlerFactory; +import java.util.HashMap; +import java.util.Map; + +/** + * Test class for {@link Downloader}. + * + * This test class is heavily based on the blog post + * <a href="https://claritysoftware.co.uk/mocking-javas-url-with-mockito/">"How + * to mock the Java URL class with Mockito"</a> by nathan from March 10, + * 2017. + */ +public class DownloaderTest { + + /** + * Custom {@link URLStreamHandler} that allows us to control the + * {@link URLConnection URLConnections} that are returned by {@link URL URLs} + * in the code under test. + */ + private static class HttpUrlStreamHandler extends URLStreamHandler { + + private Map<URL, URLConnection> connections = new HashMap(); + + @Override + protected URLConnection openConnection(URL url) throws IOException { + return connections.get(url); + } + + private void resetConnections() { + connections = new HashMap(); + } + + private HttpUrlStreamHandler addConnection(URL url, + URLConnection urlConnection) { + connections.put(url, urlConnection); + return this; + } + } + + private static HttpUrlStreamHandler httpUrlStreamHandler; + + /** + * Set up our own stream handler for all tests in this class. + */ + @BeforeClass + public static void setupUrlStreamHandlerFactory() { + URLStreamHandlerFactory urlStreamHandlerFactory + = mock(URLStreamHandlerFactory.class); + URL.setURLStreamHandlerFactory(urlStreamHandlerFactory); + httpUrlStreamHandler = new HttpUrlStreamHandler(); + given(urlStreamHandlerFactory.createURLStreamHandler("http")) + .willReturn(httpUrlStreamHandler); + } + + /** + * Clear any connections from previously run tests. + */ + @Before + public void reset() { + httpUrlStreamHandler.resetConnections(); + } + + @Test + public void testExistingResource() throws Exception { + URL requestedUrl = new URL("http://example.org/exists"); + byte[] expectedDownloadedBytes = "content".getBytes(); + HttpURLConnection urlConnection = mock(HttpURLConnection.class); + httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); + given(urlConnection.getResponseCode()).willReturn(200); + given(urlConnection.getInputStream()).willReturn( + new ByteArrayInputStream(expectedDownloadedBytes)); + byte[] downloadedBytes = Downloader.downloadFromHttpServer(requestedUrl); + assertArrayEquals(expectedDownloadedBytes, downloadedBytes); + } + + @Test + public void testNonExistingResource() throws Exception { + URL requestedUrl = new URL("http://example.org/notfound"); + HttpURLConnection urlConnection = mock(HttpURLConnection.class); + httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); + given(urlConnection.getResponseCode()).willReturn(404); + byte[] downloadedBytes = Downloader.downloadFromHttpServer(requestedUrl); + assertNull(downloadedBytes); + } + + @Test + public void testEmptyResource() throws Exception { + URL requestedUrl = new URL("http://example.org/empty"); + byte[] expectedDownloadedBytes = new byte[0]; + HttpURLConnection urlConnection = mock(HttpURLConnection.class); + httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); + given(urlConnection.getResponseCode()).willReturn(200); + given(urlConnection.getInputStream()).willReturn( + new ByteArrayInputStream(expectedDownloadedBytes)); + byte[] downloadedBytes = Downloader.downloadFromHttpServer(requestedUrl); + assertEquals(0, downloadedBytes.length); + } + + @Test(expected = SocketTimeoutException.class) + public void testTimeout() throws Exception { + URL requestedUrl = new URL("http://example.org/timeout"); + SocketTimeoutException expectedException = new SocketTimeoutException(); + HttpURLConnection urlConnection = mock(HttpURLConnection.class); + httpUrlStreamHandler.addConnection(requestedUrl, urlConnection); + given(urlConnection.getResponseCode()).willReturn(200); + given(urlConnection.getInputStream()).willThrow(expectedException); + Downloader.downloadFromHttpServer(requestedUrl); + fail("Should have thrown a SocketTimeoutException."); + } +} +

1 0