[tor-commits] [onionoo/master] Add first_seen_days and last_seen_days parameters.
karsten at torproject.org
karsten at torproject.org
Wed Apr 10 07:45:47 UTC 2013
commit 63f9e228dc3b02bf0831476f7dbd36721ddfb531
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Tue Apr 9 15:19:59 2013 +0200
Add first_seen_days and last_seen_days parameters.
Implements step 3 of #6509.
---
src/org/torproject/onionoo/ResourceServlet.java | 127 ++++++++++++++++++++++-
web/index.html | 23 ++++
2 files changed, 147 insertions(+), 3 deletions(-)
diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java
index 8c88781..03e129b 100644
--- a/src/org/torproject/onionoo/ResourceServlet.java
+++ b/src/org/torproject/onionoo/ResourceServlet.java
@@ -16,8 +16,10 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TimeZone;
+import java.util.TreeMap;
import java.util.TreeSet;
import java.util.regex.Pattern;
@@ -56,6 +58,9 @@ public class ResourceServlet extends HttpServlet {
bridgeFingerprintSummaryLines = null;
private Map<String, Set<String>> relaysByCountryCode = null,
relaysByASNumber = null, relaysByFlag = null;
+ private SortedMap<Integer, Set<String>> relaysByFirstSeenDays = null,
+ bridgesByFirstSeenDays = null, relaysByLastSeenDays = null,
+ bridgesByLastSeenDays = null;
private void readSummaryFile() {
if (!summaryFile.exists()) {
readSummaryFile = false;
@@ -70,6 +75,11 @@ public class ResourceServlet extends HttpServlet {
relaysByCountryCode = new HashMap<String, Set<String>>(),
relaysByASNumber = new HashMap<String, Set<String>>(),
relaysByFlag = new HashMap<String, Set<String>>();
+ SortedMap<Integer, Set<String>>
+ relaysByFirstSeenDays = new TreeMap<Integer, Set<String>>(),
+ bridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(),
+ relaysByLastSeenDays = new TreeMap<Integer, Set<String>>(),
+ bridgesByLastSeenDays = new TreeMap<Integer, Set<String>>();
CurrentNodes cn = new CurrentNodes();
cn.readRelaySearchDataFile(this.summaryFile);
cn.setRelayRunningBits();
@@ -82,6 +92,7 @@ public class ResourceServlet extends HttpServlet {
this.bridgesPublishedString = dateTimeFormat.format(
cn.getLastPublishedMillis());
List<String> orderRelaysByConsensusWeight = new ArrayList<String>();
+ long now = System.currentTimeMillis();
for (Node entry : cn.getCurrentRelays().values()) {
String fingerprint = entry.getFingerprint().toUpperCase();
String hashedFingerprint = entry.getHashedFingerprint().
@@ -118,6 +129,24 @@ public class ResourceServlet extends HttpServlet {
relaysByFlag.get(flagLowerCase).add(fingerprint);
relaysByFlag.get(flagLowerCase).add(hashedFingerprint);
}
+ int daysSinceFirstSeen = (int) ((now - entry.getFirstSeenMillis())
+ / 86400000L);
+ if (!relaysByFirstSeenDays.containsKey(daysSinceFirstSeen)) {
+ relaysByFirstSeenDays.put(daysSinceFirstSeen,
+ new HashSet<String>());
+ }
+ relaysByFirstSeenDays.get(daysSinceFirstSeen).add(fingerprint);
+ relaysByFirstSeenDays.get(daysSinceFirstSeen).add(
+ hashedFingerprint);
+ int daysSinceLastSeen = (int) ((now - entry.getLastSeenMillis())
+ / 86400000L);
+ if (!relaysByLastSeenDays.containsKey(daysSinceLastSeen)) {
+ relaysByLastSeenDays.put(daysSinceLastSeen,
+ new HashSet<String>());
+ }
+ relaysByLastSeenDays.get(daysSinceLastSeen).add(fingerprint);
+ relaysByLastSeenDays.get(daysSinceLastSeen).add(
+ hashedFingerprint);
}
Collections.sort(orderRelaysByConsensusWeight);
relaysByConsensusWeight = new ArrayList<String>();
@@ -131,6 +160,26 @@ public class ResourceServlet extends HttpServlet {
String line = this.formatBridgeSummaryLine(entry);
bridgeFingerprintSummaryLines.put(hashedFingerprint, line);
bridgeFingerprintSummaryLines.put(hashedHashedFingerprint, line);
+ int daysSinceFirstSeen = (int) ((now - entry.getFirstSeenMillis())
+ / 86400000L);
+ if (!bridgesByFirstSeenDays.containsKey(daysSinceFirstSeen)) {
+ bridgesByFirstSeenDays.put(daysSinceFirstSeen,
+ new HashSet<String>());
+ }
+ bridgesByFirstSeenDays.get(daysSinceFirstSeen).add(
+ hashedFingerprint);
+ bridgesByFirstSeenDays.get(daysSinceFirstSeen).add(
+ hashedHashedFingerprint);
+ int daysSinceLastSeen = (int) ((now - entry.getLastSeenMillis())
+ / 86400000L);
+ if (!bridgesByLastSeenDays.containsKey(daysSinceLastSeen)) {
+ bridgesByLastSeenDays.put(daysSinceLastSeen,
+ new HashSet<String>());
+ }
+ bridgesByLastSeenDays.get(daysSinceLastSeen).add(
+ hashedFingerprint);
+ bridgesByLastSeenDays.get(daysSinceLastSeen).add(
+ hashedHashedFingerprint);
}
this.relaysByConsensusWeight = relaysByConsensusWeight;
this.relayFingerprintSummaryLines = relayFingerprintSummaryLines;
@@ -138,6 +187,10 @@ public class ResourceServlet extends HttpServlet {
this.relaysByCountryCode = relaysByCountryCode;
this.relaysByASNumber = relaysByASNumber;
this.relaysByFlag = relaysByFlag;
+ this.relaysByFirstSeenDays = relaysByFirstSeenDays;
+ this.relaysByLastSeenDays = relaysByLastSeenDays;
+ this.bridgesByFirstSeenDays = bridgesByFirstSeenDays;
+ this.bridgesByLastSeenDays = bridgesByLastSeenDays;
}
this.summaryFileLastModified = summaryFile.lastModified();
this.readSummaryFile = true;
@@ -225,9 +278,9 @@ public class ResourceServlet extends HttpServlet {
/* Make sure that the request doesn't contain any unknown
* parameters. */
- Set<String> knownParameters = new HashSet<String>(Arrays.asList(
- "type,running,search,lookup,country,as,flag,order,limit,offset".
- split(",")));
+ Set<String> knownParameters = new HashSet<String>(Arrays.asList((
+ "type,running,search,lookup,country,as,flag,first_seen_days,"
+ + "last_seen_days,order,limit,offset").split(",")));
for (String parameterKey : parameterMap.keySet()) {
if (!knownParameters.contains(parameterKey)) {
response.sendError(HttpServletResponse.SC_BAD_REQUEST);
@@ -313,6 +366,30 @@ public class ResourceServlet extends HttpServlet {
}
this.filterByFlag(filteredRelays, filteredBridges, flagParameter);
}
+ if (parameterMap.containsKey("first_seen_days")) {
+ int[] days = this.parseDaysParameter(
+ parameterMap.get("first_seen_days"));
+ if (days == null) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ this.filterNodesByDays(filteredRelays, this.relaysByFirstSeenDays,
+ days);
+ this.filterNodesByDays(filteredBridges, this.bridgesByFirstSeenDays,
+ days);
+ }
+ if (parameterMap.containsKey("last_seen_days")) {
+ int[] days = this.parseDaysParameter(
+ parameterMap.get("last_seen_days"));
+ if (days == null) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ this.filterNodesByDays(filteredRelays, this.relaysByLastSeenDays,
+ days);
+ this.filterNodesByDays(filteredBridges, this.bridgesByLastSeenDays,
+ days);
+ }
/* Re-order and limit results. */
List<String> orderedRelays = new ArrayList<String>();
@@ -468,6 +545,33 @@ public class ResourceServlet extends HttpServlet {
return parameter;
}
+ private static Pattern daysPattern = Pattern.compile("^[0-9-]{1,10}$");
+ private int[] parseDaysParameter(String parameter) {
+ if (!daysPattern.matcher(parameter).matches()) {
+ return null;
+ }
+ int x = 0, y = Integer.MAX_VALUE;
+ try {
+ if (!parameter.contains("-")) {
+ x = y = Integer.parseInt(parameter);
+ } else {
+ String[] parts = parameter.split("-", 2);
+ if (parts[0].length() > 0) {
+ x = Integer.parseInt(parts[0]);
+ }
+ if (parts.length > 1 && parts[1].length() > 0) {
+ y = Integer.parseInt(parts[1]);
+ }
+ }
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ if (x > y) {
+ return null;
+ }
+ return new int[] { x, y };
+ }
+
private void filterByType(Map<String, String> filteredRelays,
Map<String, String> filteredBridges, boolean relaysRequested) {
if (relaysRequested) {
@@ -664,6 +768,23 @@ public class ResourceServlet extends HttpServlet {
filteredBridges.clear();
}
+ private void filterNodesByDays(Map<String, String> filteredNodes,
+ SortedMap<Integer, Set<String>> nodesByDays, int[] days) {
+ Set<String> removeNodes = new HashSet<String>();
+ for (Set<String> nodes : nodesByDays.headMap(days[0]).values()) {
+ removeNodes.addAll(nodes);
+ }
+ if (days[1] < Integer.MAX_VALUE) {
+ for (Set<String> nodes :
+ nodesByDays.tailMap(days[1] + 1).values()) {
+ removeNodes.addAll(nodes);
+ }
+ }
+ for (String fingerprint : removeNodes) {
+ filteredNodes.remove(fingerprint);
+ }
+ }
+
private void writeRelays(List<String> relays, PrintWriter pw,
String resourceType) {
pw.write("{\"relays_published\":\"" + this.relaysPublishedString
diff --git a/web/index.html b/web/index.html
index cfd857d..9427978 100755
--- a/web/index.html
+++ b/web/index.html
@@ -675,6 +675,29 @@ first parameter value will be considered.
Filtering by flag is case-insensitive.
<font color="blue">Added on April 9.</font>
</td></tr>
+<!--
+<tr><td><b><font color="blue">first_seen_days</font></b></td>
+<td>Return only relays or bridges which
+have first been seen during the given range of days ago.
+A parameter value "x-y" with x >= y returns relays or bridges that have
+first been seen at least x and at most y days ago.
+Accepted short forms are "x", "x-", and "-y" which are interpreted as
+"x-x", "x-infinity", and "0-y".
+<font color="blue">Added on April 9.</font>
+</td></tr>
+<tr><td><b><font color="blue">last_seen_days</font></b></td>
+<td>Return only relays or bridges which
+have last been seen during the given range of days ago.
+A parameter value "x-y" with x >= y returns relays or bridges that have
+last been seen at least x and at most y days ago.
+Accepted short forms are "x", "x-", and "-y" which are interpreted as
+"x-x", "x-infinity", and "0-y".
+Note that relays and bridges that haven't been running in the past week
+are never included in results, so that setting x to 8 or higher will
+always lead to an empty result set.
+<font color="blue">Added on April 9.</font>
+</td></tr>
+-->
</table>
<p>Relay and/or bridge documents in the response can be ordered and
limited by providing further parameters.
More information about the tor-commits
mailing list