November 2019 - tor-commits - lists.torproject.org

[metrics-web/release] Adds GitLab CI for tests and checks
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit d4086dde65cb51ca1a39340b1b7a4e9d54981914 Author: Iain R. Learmonth <irl(a)fsfe.org> Date: Thu Sep 5 13:44:34 2019 +0100 Adds GitLab CI for tests and checks Configuration is held in the .gitlab-ci.yml file. Fixes: #31402 --- .gitlab-ci.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..d10fcb8 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,28 @@ +variables: + GIT_STRATEGY: clone + METRICS_LIB_VERSION: "2.6.2" + EXONERATOR_VERSION: "4.1.0" + JAVA_TOOL_OPTIONS: "-Dfile.encoding=UTF-8" + +stages: + - test + +test: + stage: test + image: debian:buster + script: + - apt update + - apt -y install default-jdk ant ivy git curl + - git submodule init + - git submodule update + - mkdir lib + - mkdir tmp + - pushd tmp + - curl https://dist.torproject.org/metrics-lib/$METRICS_LIB_VERSION/metrics-lib-$M… | tar xzf - + - curl https://dist.torproject.org/exonerator/$EXONERATOR_VERSION/exonerator-$EXON… | tar xzf - + - popd + - mv tmp/metrics-lib-$METRICS_LIB_VERSION/generated/dist/metrics-lib-$METRICS_LIB_VERSION-thin.jar lib + - mv tmp/exonerator-$EXONERATOR_VERSION/generated/dist/exonerator-$EXONERATOR_VERSION-thin.jar lib + - ant -lib /usr/share/java resolve + - ant test + - ant checks

1 0

[metrics-web/release] Fix generated metrics-lib JavaDocs.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit bba56f6a536c34e858b1e78256e3209a5d27cd7e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Aug 7 21:33:34 2019 +0200 Fix generated metrics-lib JavaDocs. --- build.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/build.xml b/build.xml index b4a27e5..d8c0c07 100644 --- a/build.xml +++ b/build.xml @@ -210,6 +210,7 @@ dir="${basedir}/src/submods/metrics-lib" failonerror="true" > <arg value="clean" /> + <arg value="resolve" /> <arg value="docs" /> </exec> </target>

1 0

[metrics-web/release] Update to latest metrics-base and metrics-lib.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 84105ea10be7932b9cc11807991f0bda39ae2d75 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jul 29 10:26:43 2019 +0200 Update to latest metrics-base and metrics-lib. --- src/build | 2 +- src/submods/metrics-lib | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build b/src/build index e639c69..ed48192 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit e639c697e9e94c6dbb26e946e5247c20a62c0661 +Subproject commit ed4819215aa60cc8935f955d9404f30e83dd892b diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index e723c06..7032e73 160000 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit e723c065b764ecfbb3bb96d4c491e67398b7f21b +Subproject commit 7032e73a6471f3a4d7782ef65fc3d4cf00bb0ac1

1 0

[metrics-web/release] Upgrade to latest metrics-base and -lib.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit e2940260a1879d3a4354e826108366746b10a243 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 2 14:35:54 2019 +0200 Upgrade to latest metrics-base and -lib. --- src/build | 2 +- src/submods/metrics-lib | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build b/src/build index 07c2a00..62a964a 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 07c2a00c27f0d536223f8b5a61fc91e60eb524d4 +Subproject commit 62a964a7702086ca6ba0e2307809d4f983163b47 diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index c1581dd..8e2f671 160000 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit c1581dd8ca26798dc90ea2189ca41636b50e1e0d +Subproject commit 8e2f67107f1e6add0db3fa209cb89c3b69f7bf06

1 0

[metrics-web/release] Adds guidelines for new metrics page
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 66704ba064c13f6349722edd8d63f93b5d2244ec Author: Iain R. Learmonth <irl(a)fsfe.org> Date: Wed Jul 24 14:54:19 2019 +0100 Adds guidelines for new metrics page Fixes: #29315 --- .../metrics/web/MetricsGuidelinesServlet.java | 25 +++ src/main/resources/web.xml | 11 + src/main/resources/web/jsps/metrics-guidelines.jsp | 223 +++++++++++++++++++++ src/main/resources/web/jsps/sources.jsp | 6 + 4 files changed, 265 insertions(+) diff --git a/src/main/java/org/torproject/metrics/web/MetricsGuidelinesServlet.java b/src/main/java/org/torproject/metrics/web/MetricsGuidelinesServlet.java new file mode 100644 index 0000000..41dae13 --- /dev/null +++ b/src/main/java/org/torproject/metrics/web/MetricsGuidelinesServlet.java @@ -0,0 +1,25 @@ +/* Copyright 2018 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.metrics.web; + +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +public class MetricsGuidelinesServlet extends AnyServlet { + + private static final long serialVersionUID = 6099009779662419291L; + + @Override + public void doGet(HttpServletRequest request, + HttpServletResponse response) throws IOException, ServletException { + + request.setAttribute("categories", this.categories); + request.getRequestDispatcher("WEB-INF/metrics-guidelines.jsp") + .forward(request, response); + } +} + diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml index 045dd2e..f92813a 100644 --- a/src/main/resources/web.xml +++ b/src/main/resources/web.xml @@ -218,6 +218,17 @@ </servlet-mapping> <servlet> + <servlet-name>MetricsGuidelinesServlet</servlet-name> + <servlet-class> + org.torproject.metrics.web.MetricsGuidelinesServlet + </servlet-class> + </servlet> + <servlet-mapping> + <servlet-name>MetricsGuidelinesServlet</servlet-name> + <url-pattern>/metrics-guidelines.html</url-pattern> + </servlet-mapping> + + <servlet> <servlet-name>NewsServlet</servlet-name> <servlet-class> org.torproject.metrics.web.NewsServlet diff --git a/src/main/resources/web/jsps/metrics-guidelines.jsp b/src/main/resources/web/jsps/metrics-guidelines.jsp new file mode 100644 index 0000000..0f06fe5 --- /dev/null +++ b/src/main/resources/web/jsps/metrics-guidelines.jsp @@ -0,0 +1,223 @@ +<%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core" %> +<%@ taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions" %> +<jsp:include page="top.jsp"> + <jsp:param name="pageTitle" value="Sources – Tor Metrics"/> + <jsp:param name="navActive" value="Sources"/> +</jsp:include> + +<div class="container"> +<ul class="breadcrumb"> +<li><a href="/">Home</a></li> +<li><a href="sources.html">Sources</a></li> +<li class="active">Guidelines for getting your data into Tor Metrics</li> +</ul> +</div> + +<div class="container"> + +<h1>Guidelines for getting your data into Tor Metrics +<a href="#metrics-guidelines" id="metrics-guidelines" class="anchor">#</a></h1> + +<h2>Scope and preliminaries</h2> + +<p>This document provides guidelines to authors and operators of tools that + collect data about the publicly deployed Tor network that would like to + contribute data, or allow data to be contributed easily by others using + the tool, to Tor Metrics.</p> + +<p>This document does not discuss how to ensure measurements are safe, for this + refer to the Research Safety Board Guidelines [<a href="#ref-0">0</a>] and Guidelines for + Performing Safe Measurement on the Internet [<a href="#ref-1">1</a>].</p> + +<h2>What data belongs on Tor Metrics?</h2> + +<ul> +<li>If it happens in the public deployed Tor network it likely belongs on Tor + Metrics.</li> +<li>If it happens for a short term only, like for a research project, it's + unlikely worth the effort to have Tor Metrics archive, publish, aggregate, + and visualize it. In this case you should collect the data yourself (keeping + in mind research ethics!), and we can later talk about linking to it or even + using it as external data.</li> +<li>If your data is a combination of existing data on Tor Metrics plus maybe + external data, we shouldn't add it, either. In such a case we should rather + talk about extending our services towards what your service does, if that + makes sense.</li> +</ul> + +<h2>What data do you want to see on Tor Metrics?</h2> + +<p>This section aims to help you organize your thoughts before making a request + to the Metrics team. It might be that there are good reasons that something is + not done in one of the preferred ways, but ideally data collection tools can + be written with this guidelines in mind.</p> + +<ul> +<li>What is your data about? Is it about servers or users or both? Is it + passively gathered or actively measured or both?</li> +<li>This will help us to decide how we might present the data on Tor Metrics + and perhaps which other datasets we have that might benefit from being + combined with the new dataset.</li> +<li>Is there a way for you to aggregate the data before you hand it over to us? + Of course this requires more thinking upfront, but it's a great way to ensure + not to give out too sensitive data to us or anyone else. It's not always + possible or even useful to aggregate data and discard the original data, + though. Two examples:<ol> + <li>Relays count how many clients download the consensus from them and from + which country they connect. When 24 hours have passed, they include the + count by country in their next extra-info descriptor. This is aggregated + data. The obviously more sensitive, non-aggregated variant would be for + relays to provide a log of clients downloading consensuses.</li> + <li>The torproject.org webservers keep highly sanitized logs of web clients + making requests to them that we sanitize even more before we archive them. + This is non-aggregated data. The possibly less sensitive aggregated variant + would be for webservers to count requests by requested URL or similar.</li> + </ol></li> +<li>Is the data you're planning to give us too sensitive? If so, can you sanitize + it yourself before giving it to us (we can help you with that), or does the + sanitizing need to happen on our side (we should still involve you in this + case)? +<p>There are currently cases where Tor Metrics performs the sanitization of + data before archiving, but the preferred system would sanitize the data as + close to the source as possible to minimize the possibility that sensitive + data could be leaked.</li> +<li>How will you expect that Tor Metrics will fetch your data? For most data + currently, CollecTor fetches from a web server secured with TLS. This is + the easiest and quickest method to implement and so there should be a good + reason to not use this method.</li> +<li>When is your data available and for how long? Ideally, we'd survive reboots + or downtimes on our side for up to 72 hours without losing any of your data. + Typically, you'd implement this using a cache. If that is hard or impossible + to do on your side, we'll have to think about adding redundancy on our side. + That's all possible and we did it before, it'll just make the process take + longer.</li> +<li>Do you expect any difficulties on our side to write code that processes your + data? If we only need to fetch and store your data, probably not. But if we + have to inflate, parse, verify, combine, sanitize, split, and deflate your + data, maybe. And if we need to include fancy crypto libraries in order to + process your data, then for sure. Any intuitions you have about possible + difficulties would be good to know, even if things turn out to be easier in + the end. +<p>As far as possible, use simple formats for providing data. The Tor Directory + Protocol meta-format [<a href="#ref-2">2</a>, §1.2] is a simple format for which we already have + parsers. Without good reason, do not serialize to formats such as YAML, TOML, + etc. as this would require adding a new parsing library into Tor Metrics just + to parse the new data.</p></li> +<li>How much data do you think you'll give us over the next five years? A + ballpark figure is fine, like the number of bytes as a power of ten.</li> + </ul> + +<h2>What belongs into the data format for the data to be archived?</h2> + +<dl> +<dt>Timestamp</dt><dd>We're using the timestamp to place the data item into the right + archive file, among other things. Exception: microdescriptors do not contain + a timestamp, which makes them a pain to archive.</dd> +<dt>Source identifier</dt><dd>Ideally, we'd expect a cryptographic identifier of the + source, but if that is not available, any identifier will do. Exception: exit + lists do not contain a source identifier, because there happened to be just + one exit list scanner in the network; you can see how this doesn't scale so + well.</dd> +<dt>Generator identifier</dt><dd>The name of software and its version (either release + or a commit reference) that produced the result. If a bug is discovered in the + software then this allows us to see which data may have been affected by it.</dd> +<dt>Network location</dt><dd>If performing active measurement, the network location of + the vantage point (e.g. IP address, ASN, and/or country) can help to provide + context when comparing between different vantage points.</dd> +<dt>Signature</dt><dd>The signature is the proof that the source produced the data item, + not us. And even if we don't verify all signatures, others might want to do + that. If you are using the Tor Directory Protocol meta-format to serialize + your metrics then signing metrics using RSA or Ed25519 signatures can be + done easily. Signatures should not be detached to keep fetching, archiving + and validating simple. Exception: hello, exit list, you again!</dd> +</dl> + +<h2>First steps</h2> + +<p>You're still reading, so it seems that we caught your interest! How should we + start?</p> + +<ul> +<li>Is the data already publicly available somewhere and all you want is discuss + a way to include it in Tor Metrics? That's easy then. Just share with us what + you have and we can talk.</li> +<li>If the data is not public yet, do you maybe have a data format that we can + discuss? Bonus points if it comes with samples, but only if you're absolutely + certain that the data is safe to be published.</li> +<li>If you have none of the above, can you share logs with us, so that we can + help you derive a possible data format? It doesn't need to be recent logs + (even though time might not magically make your data safe to be published). + You could edit the logs and take out any parts you think are too sensitive. + And you should encrypt the data before sending it to us.</li> +<li>If you have nothing at all yet, let's talk anyway. Describe to us what you think would be good to include in Tor Metrics, and we'll figure something + out.</li> +</ul> + +<h2>How will Tor Metrics include the data?</h2> + + <p>It's a process to get your data on Tor Metrics, and not a short one. Let's go + through the necessary steps for doing it. After each step we should together + decide whether we're ready to move forward, need to take a step back, or maybe + even stop the project, because we found out that it's not what we wanted.</p> + +<ul> + <li>If you can, give us a few months as heads-up. Ideally, it won't take us that + long to do this project, but we'd prefer to make room for it in our next + six-month roadmap. Otherwise we might not be able to do it right away.</li> +<li> We discuss your data format with you and other Tor developers on the public + tor-dev@ mailing list. Maybe you or we need to write a Tor proposal for this.</li> +<li>We write a documentation page for the data format plus any necessary + sanitizing steps. See the Tor Metrics website and the tor-spec Git repository + for a couple of examples.</li> +<li>We write code for metrics-lib and/or Stem to parse your data and verify the + data format. At this point we'll find out if there are any misunderstandings + regarding data types or data structure that we haven't seen before.</li> +<li>We write code for CollecTor to fetch and archive your data, but without + publishing just yet. As part of this we also agree on file names and URLs + where your data will later be available.</li> +<li>We make a one-time visualization using your data, mostly as a sanity check. + You'd be surprised how many issues are hiding well enough that we would + otherwise not find them.</li> +<li>At this point we can think about adding your data to our services like + Onionoo, Relay Search, and ExoneraTor and our visualizations on Tor Metrics. + Typically, we'd do that as a separate project, though.</li> +<li>Finally we make your data available for download on CollecTor and put the + documentation on the Tor Metrics website. We announce that your data is now + on Tor Metrics.</li> +</ul> + +<h2>Maintenance</h2> + +<p>Congratulations, your data is now on Tor Metrics. But that's not the end of the + story! Here's what we need you to do as long as we have your data:</p> + +<ul><li>Make sure that we always get the data by whatever means we came up with + together. Avoid longer downtimes and fix any related issues in a timely + fashion. We do care about this, because people will come to us and complain + that "our" data is not up-to-date, when it may in fact be your fault.</li> +<li>If you're planning to make any changes that affect the data format or the way + how the data comes to us, talk to us beforehand with enough time to make such + changes. Several weeks in advance would be good, because we may have to + inform our users about upcoming changes and give them some time to update + their tools.</li> + <li>Let's be honest: we had to remove data from Tor Metrics in the past, because + the services providing them have become unreliable or unmaintained. In such a + case we'd talk to you and try to improve the situation. But if that doesn't + work, we'd remove your data from Tor Metrics with enough heads up time for + you and others to prepare. We'd very likely archive your data and keep it + around in such a case. Sorry, and thanks for understanding!</li> +</ul> + +<h2>References</h2> + +[<a id="ref-0">0</a>] Tor Project. Research Safety Board Guidelines. + <a href="https://research.torproject.org/safetyboard/#guidelines">https://research.torproject.org/safetyboard/#guidelines</a><br> +[<a id="ref-1">1</a>] I. Learmonth. Guidelines for Performing Safe Measurement on the Internet. + (Work-in-progress). <a href="https://datatracker.ietf.org/doc/draft-learmonth-pearg-safe-internet-measur…">https://datatracker.ietf.org/doc/draft-learmonth-pearg-safe-internet-measur…</a><br> +[<a id="ref-2">2</a>] Tor Project. Tor Directory Protocol, version 3. + <a href="https://spec.torproject.org/dir-spec">https://spec.torproject.org/dir-spec</a> + +</div> + +<jsp:include page="bottom.jsp"/> + diff --git a/src/main/resources/web/jsps/sources.jsp b/src/main/resources/web/jsps/sources.jsp index a43d681..1f2930c 100644 --- a/src/main/resources/web/jsps/sources.jsp +++ b/src/main/resources/web/jsps/sources.jsp @@ -41,6 +41,12 @@ </ul> </div> + <div class="container"> + <h2>New sources <a href="#new" name="new" class="anchor">#</a></h2> + <p>We have written <a href="/metrics-guidelines.html">guidelines for Tor developers</a> that would like metrics collected by their service to be integrated + with Tor Metrics.</p> + </div> + <div class="container"> <h2>Specifications <a href="#specifications" name="specifications" class="anchor">#</a></h2> <p>The following specification documents are available for Tor network data.</p>

1 0

[metrics-web/release] Remove Cobertura from the build process.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit a881d0f480fe8c4289c47a9e14fdcd016a2ade9e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Aug 14 11:47:41 2019 +0200 Remove Cobertura from the build process. --- CHANGELOG.md | 1 + src/build | 2 +- src/submods/metrics-lib | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c0f935..41bb5df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ copied to the `lib/` directory manually. Current dependency versions resolved by Ivy are the same as in Debian stretch with few exceptions. + - Remove Cobertura from the build process. # Changes in version 1.2.0 - 2018-08-25 diff --git a/src/build b/src/build index 256e619..75cda35 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 256e6192ad0066e48c6abb0ee9ac71714c7a3a0f +Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index e172870..53b16d1 160000 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit e1728709b5ed30dace2dd5b7e47a3e3c5e9e5736 +Subproject commit 53b16d192da4a7fa5c1987a89a60257061602882

1 0

[metrics-web/release] Update CollecTor page to include snowflake stats.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 412ac16376bb93e9be80b776a64435940f16681d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Aug 14 10:58:38 2019 +0200 Update CollecTor page to include snowflake stats. Implements #29461. --- src/main/resources/web/jsps/collector.jsp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index a05b7e0..42280a9 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -196,6 +196,15 @@ <td><a href="/collector/recent/relay-descriptors/bandwidths/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/relay-descriptors/bandwidths/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> +<tr class="tableHeadline"> + <td colspan="3"><b><a href="#snowflake-stats">Snowflake Statistics</a></b></td> +</tr> +<tr> + <td><a href="#type-snowflake-stats">Snowflake Statistics</a></td> + <td><code>@type snowflake-stats 1.0</code></td> + <td><a href="/collector/recent/snowflakes/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> + <a href="/collector/archive/snowflakes/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> +</tr> </tbody> </table> @@ -792,6 +801,21 @@ These are available from a DirPort's </p> +<br> +<h2 id="snowflake-stats" class="hover">Snowflake Statistics +<a href="#snowflake-stats" class="anchor">#</a> +</h2> + +<h3 id="type-snowflake-stats" class="hover">Snowflake Statistics +<small><code>@type snowflake-stats 1.0</code></small> +<a href="/collector/recent/snowflakes/" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> +<a href="/collector/archive/snowflakes/" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a> +<a href="#type-snowflake-stats" class="anchor">#</a> +</h3> + +<p> +Snowflake statistics containing aggregated information about snowflake proxies and snowflake clients as generated by the snowflake broker. +</p> </div> <br>

1 0

[metrics-web/release] Update to latest metrics-base and metrics-lib.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 3a663e8ff6b5b0c0757fe787cf92eaac91734ad5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Sep 17 10:05:27 2019 +0200 Update to latest metrics-base and metrics-lib. This update includes an update of PostgreSQL JDBC driver version to 42.2.5. --- CHANGELOG.md | 1 + build.xml | 2 +- src/build | 2 +- src/submods/metrics-lib | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41bb5df..2147ed7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ versions resolved by Ivy are the same as in Debian stretch with few exceptions. - Remove Cobertura from the build process. + - Update PostgreSQL JDBC driver version to 42.2.5. # Changes in version 1.2.0 - 2018-08-25 diff --git a/build.xml b/build.xml index 038a82f..3e8048f 100644 --- a/build.xml +++ b/build.xml @@ -57,7 +57,7 @@ <include name="metrics-lib-${metricslibversion}-thin.jar"/> <include name="commons-compress-1.13.jar"/> <include name="commons-math3-3.6.1.jar"/> - <include name="postgresql-9.4.1212.jar"/> + <include name="postgresql-42.2.5.jar"/> <include name="javax.servlet-api-3.1.0.jar"/> <include name="xz-1.6.jar"/> </patternset> diff --git a/src/build b/src/build index 75cda35..0674958 160000 --- a/src/build +++ b/src/build @@ -1 +1 @@ -Subproject commit 75cda35ba7375ab0f5b12fa214620c3a29917097 +Subproject commit 0674958512c705255d4ec4631d3280068ce0a7af diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 7e396f0..c324d11 160000 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 7e396f01ef6de4cb3cd7e2b7fc977b4af7b97e7f +Subproject commit c324d11035059bd9677af120dc6471aebebf980a

1 0

[metrics-web/release] Add "Tor Browser updates by release channel" graph.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit cba1711544fbfea9e88ba4161f94b26231b28d5f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 2 12:24:38 2019 +0200 Add "Tor Browser updates by release channel" graph. Implements #31755. --- src/main/R/rserver/rserve-init.R | 46 ++++++++++++++++++++++ src/main/resources/web.xml | 4 ++ src/main/resources/web/json/categories.json | 1 + src/main/resources/web/json/metrics.json | 11 ++++++ .../resources/web/jsps/reproducible-metrics.jsp | 1 + src/main/resources/web/jsps/stats.jsp | 22 +++++++++++ 6 files changed, 85 insertions(+) diff --git a/src/main/R/rserver/rserve-init.R b/src/main/R/rserver/rserve-init.R index af74ef7..5eac4d7 100644 --- a/src/main/R/rserver/rserve-init.R +++ b/src/main/R/rserver/rserve-init.R @@ -1399,6 +1399,52 @@ plot_webstats_tb_locale <- function(start_p, end_p, path_p) { ggsave(filename = path_p, width = 8, height = 5, dpi = 150) } +prepare_webstats_tb_channel <- function(start_p = NULL, end_p = NULL) { + read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), + col_types = cols( + log_date = col_date(format = ""), + request_type = col_factor(levels = NULL), + platform = col_skip(), + channel = col_factor(levels = NULL), + locale = col_skip(), + incremental = col_skip(), + count = col_double())) %>% + filter(if (!is.null(start_p)) log_date >= as.Date(start_p) else TRUE) %>% + filter(if (!is.null(end_p)) log_date <= as.Date(end_p) else TRUE) %>% + filter(request_type %in% c("tbup", "tbur")) %>% + filter(channel %in% c("a", "r")) %>% + group_by(log_date, channel, request_type) %>% + summarize(count = sum(count)) %>% + dcast(log_date + channel ~ request_type, value.var = "count") %>% + rename(date = log_date, update_pings = tbup, update_requests = tbur) +} + +plot_webstats_tb_channel <- function(start_p, end_p, path_p) { + prepare_webstats_tb_channel(start_p, end_p) %>% + gather(request_type, count, -c(date, channel)) %>% + unite("request_type_channel", request_type, channel) %>% + mutate(request_type_channel = factor(request_type_channel, + levels = c("update_pings_r", "update_pings_a", + "update_requests_r", "update_requests_a"), + labels = c("Update pings (stable)", "Update pings (alpha)", + "Update requests (stable)", "Update requests (alpha)"))) %>% + ungroup() %>% + complete(date = full_seq(date, period = 1), + nesting(request_type_channel)) %>% + ggplot(aes(x = date, y = count)) + + geom_point() + + geom_line() + + scale_x_date(name = "", breaks = custom_breaks, + labels = custom_labels, minor_breaks = custom_minor_breaks) + + scale_y_continuous(name = "", labels = formatter, limits = c(0, NA)) + + facet_grid(request_type_channel ~ ., scales = "free_y") + + theme(strip.text.y = element_text(angle = 0, hjust = 0, size = rel(1.5)), + strip.background = element_rect(fill = NA)) + + ggtitle("Tor Browser updates by release channel") + + labs(caption = copyright_notice) + ggsave(filename = path_p, width = 8, height = 5, dpi = 150) +} + prepare_webstats_tm <- function(start_p = NULL, end_p = NULL) { read_csv(file = paste(stats_dir, "webstats.csv", sep = ""), col_types = cols( diff --git a/src/main/resources/web.xml b/src/main/resources/web.xml index 2c68fd7..10d12ee 100644 --- a/src/main/resources/web.xml +++ b/src/main/resources/web.xml @@ -51,6 +51,7 @@ <url-pattern>/webstats-tb.html</url-pattern> <url-pattern>/webstats-tb-platform.html</url-pattern> <url-pattern>/webstats-tb-locale.html</url-pattern> + <url-pattern>/webstats-tb-channel.html</url-pattern> <url-pattern>/webstats-tm.html</url-pattern> <url-pattern>/relays-ipv6.html</url-pattern> <url-pattern>/bridges-ipv6.html</url-pattern> @@ -182,6 +183,9 @@ <url-pattern>/webstats-tb-locale.png</url-pattern> <url-pattern>/webstats-tb-locale.pdf</url-pattern> <url-pattern>/webstats-tb-locale.csv</url-pattern> + <url-pattern>/webstats-tb-channel.png</url-pattern> + <url-pattern>/webstats-tb-channel.pdf</url-pattern> + <url-pattern>/webstats-tb-channel.csv</url-pattern> <url-pattern>/webstats-tm.png</url-pattern> <url-pattern>/webstats-tm.pdf</url-pattern> <url-pattern>/webstats-tm.csv</url-pattern> diff --git a/src/main/resources/web/json/categories.json b/src/main/resources/web/json/categories.json index 89742bc..3771631 100644 --- a/src/main/resources/web/json/categories.json +++ b/src/main/resources/web/json/categories.json @@ -88,6 +88,7 @@ "webstats-tb", "webstats-tb-platform", "webstats-tb-locale", + "webstats-tb-channel", "webstats-tm" ] } diff --git a/src/main/resources/web/json/metrics.json b/src/main/resources/web/json/metrics.json index e6bab04..54ce78b 100644 --- a/src/main/resources/web/json/metrics.json +++ b/src/main/resources/web/json/metrics.json @@ -416,6 +416,17 @@ ] }, { + "id": "webstats-tb-channel", + "title": "Tor Browser updates by release channel", + "type": "Graph", + "description": "<p>This graph shows absolute numbers of requests to Tor's web servers made by Tor Browser to check whether a newer version is available or to download a newer version, broken down by release channel.</p>", + "function": "webstats_tb_channel", + "parameters": [ + "start", + "end" + ] + }, + { "id": "webstats-tm", "title": "Tor Messenger downloads and updates", "type": "Graph", diff --git a/src/main/resources/web/jsps/reproducible-metrics.jsp b/src/main/resources/web/jsps/reproducible-metrics.jsp index d6e7d0e..9b21aa7 100644 --- a/src/main/resources/web/jsps/reproducible-metrics.jsp +++ b/src/main/resources/web/jsps/reproducible-metrics.jsp @@ -803,6 +803,7 @@ Applications <a href="#applications" name="applications" class="anchor">#</a></h <li>Tor Browser downloads and updates <a href="/webstats-tb.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Tor Browser downloads and updates by platform <a href="/webstats-tb-platform.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Tor Browser downloads and updates by locale <a href="/webstats-tb-locale.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> +<li>Tor Browser updates by release channel <a href="/webstats-tb-channel.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> <li>Tor Messenger downloads and updates <a href="/webstats-tm.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a></li> </ul> diff --git a/src/main/resources/web/jsps/stats.jsp b/src/main/resources/web/jsps/stats.jsp index eac75e3..3ce295d 100644 --- a/src/main/resources/web/jsps/stats.jsp +++ b/src/main/resources/web/jsps/stats.jsp @@ -53,6 +53,7 @@ https://metrics.torproject.org/identifier.csv <li><b>May 29, 2019:</b> Extended <a href="#onionperf-latencies">Circuit round-trip latencies</a> graph to contain high/low values.</li> <li><b>June 2, 2019:</b> Added <a href="#onionperf-throughput">Throughput</a> graph.</li> <li><b>August 5, 2019:</b> Re-added the <a href="#bandwidth">Total relay bandwidth</a> graph due to popular demand.</li> +<li><b>October 2, 2019:</b> Added <a href="#webstats-tb-channel">Tor Browser updates by release channel</a> graph.</li> </ul> </div> @@ -735,6 +736,27 @@ Applications <a href="#applications" name="applications" class="anchor">#</a></h <li><b>update_pings:</b> Number of Tor Browser update pings.</li> </ul> +<h3>Tor Browser updates by release channel +<a href="/webstats-tb-channel.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> +<a href="/webstats-tb-channel.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a> +<a href="#webstats-tb-channel" name="webstats-tb-channel" class="anchor">#</a></h3> + +<h4>Parameters</h4> + +<ul> +<li><b>start:</b> First UTC date (YYYY-MM-DD) to include in the file.</li> +<li><b>end:</b> Last UTC date (YYYY-MM-DD) to include in the file.</li> +</ul> + +<h4>Columns</h4> + +<ul> +<li><b>date:</b> UTC date (YYYY-MM-DD) when requests to <code>torproject.org</code> web servers have been logged.</li> +<li><b>channel:</b> Release channel, like "r" for stable releases and "a" for alpha releases.</li> +<li><b>update_pings:</b> Number of Tor Browser update pings.</li> +<li><b>update_requests:</b> Number of Tor Browser update requests.</li> +</ul> + <h3>Tor Messenger downloads and updates <a href="/webstats-tm.html" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> graph</a> <a href="/webstats-tm.csv" class="btn btn-primary btn-xs"><i class="fa fa-chevron-right" aria-hidden="true"></i> data</a>

1 0

[metrics-web/release] Update to metrics-lib 2.9.1 and ExoneraTor 4.2.0.
by karsten＠torproject.org 09 Nov '19

09 Nov '19

commit 9d34311e53389493e68d5b885da9928454807044 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Nov 9 20:38:51 2019 +0100 Update to metrics-lib 2.9.1 and ExoneraTor 4.2.0. --- CHANGELOG.md | 1 + build.xml | 4 ++-- src/submods/metrics-lib | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2147ed7..b8a4ed0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ few exceptions. - Remove Cobertura from the build process. - Update PostgreSQL JDBC driver version to 42.2.5. + - Update to metrics-lib 2.9.1 and ExoneraTor 4.2.0. # Changes in version 1.2.0 - 2018-08-25 diff --git a/build.xml b/build.xml index 959d069..3efe4bc 100644 --- a/build.xml +++ b/build.xml @@ -10,8 +10,8 @@ <property name="javadoc-title" value="MetricsWeb API Documentation"/> <property name="implementation-title" value="metrics-web" /> <property name="release.version" value="1.2.0-dev" /> - <property name="metricslibversion" value="2.6.2" /> - <property name="exoneratorversion" value="4.1.0" /> + <property name="metricslibversion" value="2.9.1" /> + <property name="exoneratorversion" value="4.2.0" /> <property name="jetty.version" value="-9.2.21.v20170120" /> <property name="warfile" value="metrics-web-${release.version}.war"/> diff --git a/src/submods/metrics-lib b/src/submods/metrics-lib index 8763af4..87d998c 160000 --- a/src/submods/metrics-lib +++ b/src/submods/metrics-lib @@ -1 +1 @@ -Subproject commit 8763af44465445b2257f9740d4ce341f8334eead +Subproject commit 87d998c542688c7e465486de24ef07c1cbece0b0

1 0