commit 157174ff31a8dcbbbcab1763a4b151706e671a1b Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Apr 2 12:28:41 2020 +0200
Add first/last published months to CollecTor page.
Implements #24941. --- .../torproject/metrics/web/CollecTorServlet.java | 16 ++--- .../metrics/web/CollectorDirectoryProvider.java | 6 +- .../torproject/metrics/web/DirectoryListing.java | 72 ++++++++++++++++++++-- src/main/resources/web/jsps/collector.jsp | 46 ++++++++++---- 4 files changed, 113 insertions(+), 27 deletions(-)
diff --git a/src/main/java/org/torproject/metrics/web/CollecTorServlet.java b/src/main/java/org/torproject/metrics/web/CollecTorServlet.java index b9661b9..b985d32 100644 --- a/src/main/java/org/torproject/metrics/web/CollecTorServlet.java +++ b/src/main/java/org/torproject/metrics/web/CollecTorServlet.java @@ -4,8 +4,7 @@ package org.torproject.metrics.web;
import java.io.IOException; -import java.util.List; -import java.util.Map; +import java.util.HashMap;
import javax.servlet.ServletConfig; import javax.servlet.ServletException; @@ -45,21 +44,24 @@ public class CollecTorServlet extends AnyServlet { requestedPath = requestedPath.substring(requestedPath.indexOf( "/collector")); } - Map<String, List<String[]>> index; + DirectoryListing index = this.collectorDirectory.getIndex(); if ("/collector.html".equals(requestedPath)) { request.setAttribute("categories", this.categories); + request.setAttribute("published", null == index + ? new HashMap<String, String>() : index.getFormattedPublishedTimes()); request.getRequestDispatcher("WEB-INF/collector.jsp").forward(request, response); - } else if (null == (index = this.collectorDirectory.getIndex())) { + } else if (null == index) { response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, "Index of CollecTor files unavailable."); } else if (!requestedPath.endsWith("/") - && index.containsKey(requestedPath + "/")) { + && index.getFormattedTableEntries().containsKey(requestedPath + "/")) { response.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); response.setHeader("Location", requestedPath + "/"); - } else if (index.containsKey(requestedPath)) { + } else if (index.getFormattedTableEntries().containsKey(requestedPath)) { request.setAttribute("categories", this.categories); - request.setAttribute("files", index.get(requestedPath)); + request.setAttribute("files", + index.getFormattedTableEntries().get(requestedPath)); request.getRequestDispatcher("/WEB-INF/collector-files.jsp").forward( request, response); } else { diff --git a/src/main/java/org/torproject/metrics/web/CollectorDirectoryProvider.java b/src/main/java/org/torproject/metrics/web/CollectorDirectoryProvider.java index 8d3a018..54281bd 100644 --- a/src/main/java/org/torproject/metrics/web/CollectorDirectoryProvider.java +++ b/src/main/java/org/torproject/metrics/web/CollectorDirectoryProvider.java @@ -3,8 +3,6 @@
package org.torproject.metrics.web;
-import java.util.List; -import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -23,7 +21,7 @@ public class CollectorDirectoryProvider implements Runnable { Executors.newScheduledThreadPool(1);
/** Last known directory listings. */ - private final AtomicReference<Map<String, List<String[]>>> index + private final AtomicReference<DirectoryListing> index = new AtomicReference<>(null);
CollectorDirectoryProvider(String host) { @@ -33,7 +31,7 @@ public class CollectorDirectoryProvider implements Runnable {
/** Returns the index object in a thread-safe way, blocking the invoking * thread at most 10 seconds if no index object is available. */ - Map<String, List<String[]>> getIndex() { + DirectoryListing getIndex() { if (null == this.index.get()) { long waitingSinceMillis = System.currentTimeMillis(); do { diff --git a/src/main/java/org/torproject/metrics/web/DirectoryListing.java b/src/main/java/org/torproject/metrics/web/DirectoryListing.java index aceec59..8664ca8 100644 --- a/src/main/java/org/torproject/metrics/web/DirectoryListing.java +++ b/src/main/java/org/torproject/metrics/web/DirectoryListing.java @@ -15,6 +15,8 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.URLConnection; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -24,16 +26,27 @@ import java.util.TreeSet;
/** Map with all directory listings for all directories and subdirectories * contained in an index.json file. */ -public class DirectoryListing extends HashMap<String, List<String[]>> - implements Map<String, List<String[]>> { +public class DirectoryListing {
private IndexNode index;
+ private Map<String, List<String[]>> formattedTableEntries = new HashMap<>(); + + private Map<String, String> formattedPublishedTimes = new HashMap<>(); + DirectoryListing(IndexNode index) { this.index = index; extractDirectoryListings(); }
+ public Map<String, List<String[]>> getFormattedTableEntries() { + return this.formattedTableEntries; + } + + public Map<String, String> getFormattedPublishedTimes() { + return this.formattedPublishedTimes; + } + /** * Parsed {@code index.json} file, which can be the root node ("index node"), * an inner node ("directory node"), or a leaf node ("file node"). @@ -72,6 +85,18 @@ public class DirectoryListing extends HashMap<String, List<String[]>> String lastModified;
/** + * Earliest publication timestamp of contained descriptors using pattern + * {@code "YYYY-MM-DD HH:MM"} in the UTC timezone. + */ + String firstPublished; + + /** + * Latest publication timestamp of contained descriptors using pattern + * {@code "YYYY-MM-DD HH:MM"} in the UTC timezone. + */ + String lastPublished; + + /** * Compare two index nodes by their (relative) path in alphabetic order. * * @param other The other index node to compare to. @@ -144,11 +169,16 @@ public class DirectoryListing extends HashMap<String, List<String[]>> /** Extracts directory listing from an index node by visiting all nodes. */ private void extractDirectoryListings() { Map<IndexNode, String> directoryNodes = new HashMap<>(); - this.put("/collector/", + this.formattedTableEntries.put("/collector/", formatTableEntries("", "/", this.index.directories, this.index.files)); for (IndexNode directory : this.index.directories) { directoryNodes.put(directory, "/"); } + LocalDate today = LocalDate.now(); + String todayString = today + .format(DateTimeFormatter.ISO_DATE).substring(0, 7); + String lastWeekString = today.minusWeeks(1L) + .format(DateTimeFormatter.ISO_DATE).substring(0, 7); while (!directoryNodes.isEmpty()) { IndexNode currentDirectoryNode = directoryNodes.keySet().iterator().next(); @@ -160,10 +190,14 @@ public class DirectoryListing extends HashMap<String, List<String[]>> parentPath, currentDirectoryNode.path)); } } - this.put(String - .format("/collector%s%s/", parentPath, currentDirectoryNode.path), + String currentDirectoryPath = String + .format("/collector%s%s/", parentPath, currentDirectoryNode.path); + this.formattedTableEntries.put(currentDirectoryPath, formatTableEntries(parentPath, currentDirectoryNode.path + "/", currentDirectoryNode.directories, currentDirectoryNode.files)); + this.formattedPublishedTimes.put(currentDirectoryPath, + formatPublishedTimes(todayString, lastWeekString, + currentDirectoryNode.files)); } }
@@ -202,5 +236,33 @@ public class DirectoryListing extends HashMap<String, List<String[]>> return String .format("%.1f %siB", bytes / Math.pow(1024, exp), pre); } + + /** Formats first and last published times for a given directory. */ + private String formatPublishedTimes(String todayString, String lastWeekString, + SortedSet<IndexNode> files) { + if (null != files) { + SortedSet<String> publishedYearMonths = new TreeSet<>(); + for (IndexNode file : files) { + if (null != file.firstPublished && file.firstPublished.length() > 6) { + publishedYearMonths.add(file.firstPublished.substring(0, 7)); + } + if (null != file.lastPublished && file.lastPublished.length() > 6) { + publishedYearMonths.add(file.lastPublished.substring(0, 7)); + } + } + if (publishedYearMonths.isEmpty()) { + return ""; + } else if (publishedYearMonths.last().equals(todayString) + || publishedYearMonths.last().equals(lastWeekString)) { + return String.format("%s to present", publishedYearMonths.first()); + } else if (publishedYearMonths.size() == 1) { + return publishedYearMonths.first(); + } else { + return String.format("%s to %s", publishedYearMonths.first(), + publishedYearMonths.last()); + } + } + return ""; + } }
diff --git a/src/main/resources/web/jsps/collector.jsp b/src/main/resources/web/jsps/collector.jsp index d56bdfe..0e5bf06 100644 --- a/src/main/resources/web/jsps/collector.jsp +++ b/src/main/resources/web/jsps/collector.jsp @@ -48,63 +48,72 @@ <h1 id="available-descriptors" class="hover">Available Descriptors <a href="#available-descriptors" class="anchor">#</a></h1>
-<p>Descriptors are available in two different file formats: recent descriptors that were published in the last 72 hours are available as plain text, and archived descriptors covering over 10 years of Tor network history are available as compressed tarballs.</p> +<p>Descriptors are available in two different file formats: recent descriptors that were published in the last 72 hours are available as plain text, and archived descriptors covering many years of Tor network history are available as compressed tarballs.</p>
<table class="table"> <thead> <tr> <th>Descriptor Type</th> <th>Type Annotation</th> +<th>Published</th> <th class="thDescriptors">Descriptors</th> </tr> </thead> <tbody> <tr class="tableHeadline"> - <td colspan="3"><b><a href="#relay-descriptors">Tor Relay Descriptors</a></b></td> + <td colspan="4"><b><a href="#relay-descriptors">Tor Relay Descriptors</a></b></td> </tr> <tr> <td><a href="#type-server-descriptor">Relay Server Descriptors</a></td> <td><code>@type server-descriptor 1.0</code></td> + <td>${published["/collector/archive/relay-descriptors/server-descriptors/"]}</td> <td><a href="/collector/recent/relay-descriptors/server-descriptors/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/relay-descriptors/server-descriptors/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr> <td><a href="#type-extra-info">Relay Extra-info Descriptors</a></td> <td><code>@type extra-info 1.0</code></td> + <td>${published["/collector/archive/relay-descriptors/extra-infos/"]}</td> <td><a href="/collector/recent/relay-descriptors/extra-infos/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/relay-descriptors/extra-infos/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr> <td><a href="#type-network-status-consensus-3">Network Status Consensuses</a></td> <td><code>@type network-status-consensus-3 1.0</code></td> + <td>${published["/collector/archive/relay-descriptors/consensuses/"]}</td> <td><a href="/collector/recent/relay-descriptors/consensuses/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/relay-descriptors/consensuses/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr> <td><a href="#type-network-status-vote-3">Network Status Votes</a></td> <td><code>@type network-status-vote-3 1.0</code></td> + <td>${published["/collector/archive/relay-descriptors/votes/"]}</td> <td><a href="/collector/recent/relay-descriptors/votes/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/relay-descriptors/votes/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr> <td><a href="#type-dir-key-certificate-3">Directory Key Certificates</a></td> <td><code>@type dir-key-certificate-3 1.0</code></td> + <td></td> <td><a href="/collector/archive/relay-descriptors/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr> <td><a href="#type-detached-signature-3">Detached Signatures</a></td> <td><code>@type detached-signature-3 1.0</code></td> <td></td> + <td></td> </tr> <tr> <td><a href="#type-network-status-microdesc-consensus-3">Microdescriptor Consensuses</a></td> <td><code>@type network-status-microdesc-consensus-3 1.0</code></td> + <td>${published["/collector/archive/relay-descriptors/microdescs/"]}</td> <td><a href="/collector/recent/relay-descriptors/microdescs/consensus-microdesc/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/relay-descriptors/microdescs/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr> <td><a href="#type-microdescriptor">Microdescriptors</a></td> <td><code>@type microdescriptor 1.0</code></td> + <td>${published["/collector/archive/relay-descriptors/microdescs/"]}</td> <td><a href="/collector/recent/relay-descriptors/microdescs/micro/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/relay-descriptors/microdescs/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> @@ -112,111 +121,126 @@ <td><a href="#type-network-status-entry-3">Network Status Entries</a></td> <td><code>@type network-status-entry-3 1.0</code></td> <td></td> + <td></td> </tr> <tr> <td><a href="#type-network-status-2">Version 2 Network Statuses</a></td> <td><code>@type network-status-2 1.0</code></td> + <td>${published["/collector/archive/relay-descriptors/statuses/"]}</td> <td><a href="/collector/archive/relay-descriptors/statuses/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr> <td><a href="#type-directory">Version 1 Directories</a></td> <td><code>@type directory 1.0</code></td> + <td>${published["/collector/archive/relay-descriptors/tor/"]}</td> <td><a href="/collector/archive/relay-descriptors/tor/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr class="tableHeadline"> - <td colspan="3"><b><a href="#bridge-descriptors">Tor Bridge Descriptors</a></b></td> + <td colspan="4"><b><a href="#bridge-descriptors">Tor Bridge Descriptors</a></b></td> </tr> <tr> <td><a href="#type-bridge-network-status">Bridge Network Statuses</a></td> <td><code>@type bridge-network-status 1.2</code></td> + <td>${published["/collector/archive/bridge-descriptors/statuses/"]}</td> <td><a href="/collector/recent/bridge-descriptors/statuses/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/bridge-descriptors/statuses/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr> <td><a href="#type-bridge-server-descriptor">Bridge Server Descriptors</a></td> <td><code>@type bridge-server-descriptor 1.2</code></td> + <td>${published["/collector/archive/bridge-descriptors/server-descriptors/"]}</td> <td><a href="/collector/recent/bridge-descriptors/server-descriptors/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/bridge-descriptors/server-descriptors/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr> <td><a href="#type-bridge-extra-info">Bridge Extra-info Descriptors</a></td> <td><code>@type bridge-extra-info 1.3</code></td> + <td>${published["/collector/archive/bridge-descriptors/extra-infos/"]}</td> <td><a href="/collector/recent/bridge-descriptors/extra-infos/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/bridge-descriptors/extra-infos/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr class="tableHeadline"> - <td colspan="3"><b><a href="#tor-hidden-service-descriptors">Tor Hidden Service Descriptors</a></b></td> + <td colspan="4"><b><a href="#tor-hidden-service-descriptors">Tor Hidden Service Descriptors</a></b></td> </tr> <tr> <td><a href="#type-hidden-service-descriptor">Hidden Service Descriptors</a></td> <td><code>@type hidden-service-descriptor 1.0</code></td> <td></td> + <td></td> </tr> <tr> <td><a href="#type-hidden-service-descriptor-3">Hidden Service Descriptors v3</a></td> <td><code>@type hidden-service-descriptor-3 1.0</code></td> <td></td> + <td></td> </tr> <tr class="tableHeadline"> - <td colspan="3"><b><a href="#bridge-pool-assignments">BridgeDB's Bridge Pool Assignments</a></b></td> + <td colspan="4"><b><a href="#bridge-pool-assignments">BridgeDB's Bridge Pool Assignments</a></b></td> </tr> <tr> <td><a href="#type-bridge-pool-assignment">Bridge Pool Assignments</a></td> <td><code>@type bridge-pool-assignment 1.0</code></td> + <td>${published["/collector/archive/bridge-pool-assignments/"]}</td> <td><a href="/collector/recent/bridge-pool-assignments/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/bridge-pool-assignments/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr class="tableHeadline"> - <td colspan="3"><b><a href="#exit-lists">TorDNSEL's Exit Lists</a></b></td> + <td colspan="4"><b><a href="#exit-lists">TorDNSEL's Exit Lists</a></b></td> </tr> <tr> <td><a href="#type-tordnsel">Exit Lists</a></td> <td><code>@type tordnsel 1.0</code></td> + <td>${published["/collector/archive/exit-lists/"]}</td> <td><a href="/collector/recent/exit-lists/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/exit-lists/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr class="tableHeadline"> - <td colspan="3"><b><a href="#torperf">Torperf's and OnionPerf's Performance Data</a></b></td> + <td colspan="4"><b><a href="#torperf">Torperf's and OnionPerf's Performance Data</a></b></td> </tr> <tr> <td><a href="#type-torperf">Torperf Measurement Results</a></td> <td><code>@type torperf 1.1</code></td> + <td>${published["/collector/archive/torperf/"]}</td> <td><a href="/collector/recent/torperf/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/torperf/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr class="tableHeadline"> - <td colspan="3"><b><a href="#webstats">Tor web server logs</a></b></td> + <td colspan="4"><b><a href="#webstats">Tor web server logs</a></b></td> </tr> <tr> <td><a href="#type-webstats">Tor web server logs</a></td> <td></td> + <td>${published["/collector/archive/webstats/"]}</td> <td><a href="/collector/recent/webstats/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/webstats/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr class="tableHeadline"> - <td colspan="3"><b><a href="#bandwidth-files">Bandwidth Files</a></b></td> + <td colspan="4"><b><a href="#bandwidth-files">Bandwidth Files</a></b></td> </tr> <tr> <td><a href="#type-bandwidth-file">Bandwidth Files</a></td> <td><code>@type bandwidth-file 1.0</code></td> + <td>${published["/collector/archive/relay-descriptors/bandwidths/"]}</td> <td><a href="/collector/recent/relay-descriptors/bandwidths/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/relay-descriptors/bandwidths/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr class="tableHeadline"> - <td colspan="3"><b><a href="#snowflake-stats">Snowflake Statistics</a></b></td> + <td colspan="4"><b><a href="#snowflake-stats">Snowflake Statistics</a></b></td> </tr> <tr> <td><a href="#type-snowflake-stats">Snowflake Statistics</a></td> <td><code>@type snowflake-stats 1.0</code></td> + <td>${published["/collector/archive/snowflakes/"]}</td> <td><a href="/collector/recent/snowflakes/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/snowflakes/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr> <tr class="tableHeadline"> - <td colspan="3"><b><a href="#bridgedb-metrics">BridgeDB metrics</a></b></td> + <td colspan="4"><b><a href="#bridgedb-metrics">BridgeDB metrics</a></b></td> </tr> <tr> <td><a href="#type-bridgedb-metrics">BridgeDB metrics</a></td> <td><code>@type bridgedb-metrics 1.0</code></td> + <td>${published["/collector/archive/bridgedb-metrics/"]}</td> <td><a href="/collector/recent/bridgedb-metrics/" class="btn btn-primary btn-xs pull-left"><i class="fa fa-chevron-right" aria-hidden="true"></i> recent</a> <a href="/collector/archive/bridgedb-metrics/" class="btn btn-primary btn-xs pull-right"><i class="fa fa-chevron-right" aria-hidden="true"></i> archive</a></td> </tr>