commit 03c4a3d3a08a9426b76583383e087a60e947dfed Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Jul 3 23:00:27 2014 +0200
Add new family parameter.
The family parameter accepts a fingerprint and returns documents from all relays in the family of this relay. This parameter can be useful for websites showing aggregate data from all relays run by the same person or organization.
Implements #12521. --- src/org/torproject/onionoo/DocumentStore.java | 4 +- .../onionoo/NodeDetailsStatusUpdater.java | 24 ++++++--- src/org/torproject/onionoo/NodeIndexer.java | 28 +++++++++- src/org/torproject/onionoo/NodeStatus.java | 30 +++++++++-- src/org/torproject/onionoo/RequestHandler.java | 23 +++++++++ src/org/torproject/onionoo/ResourceServlet.java | 12 ++++- .../torproject/onionoo/ResourceServletTest.java | 54 +++++++++++++++++--- web/protocol.html | 17 ++++++ 8 files changed, 171 insertions(+), 21 deletions(-)
diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index d80999e..e8f8878 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -283,7 +283,7 @@ public class DocumentStore { aSNumber = null, contact = null; SortedSet<String> orAddressesAndPorts = new TreeSet<String>(), exitAddresses = new TreeSet<String>(), - relayFlags = new TreeSet<String>(); + relayFlags = new TreeSet<String>(), family = null; long lastSeenMillis = -1L, consensusWeight = -1L, lastRdnsLookup = -1L, firstSeenMillis = -1L, lastChangedAddresses = -1L; @@ -342,7 +342,7 @@ public class DocumentStore { lastSeenMillis, orPort, dirPort, relayFlags, consensusWeight, countryCode, hostName, lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, lastChangedAddresses, aSNumber, contact, - recommendedVersion); + recommendedVersion, family); nodeStatus.setRunning(running); return nodeStatus; } catch (Exception e) { diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java index 72b98cd..2859063 100644 --- a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java +++ b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java @@ -102,9 +102,8 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, Logger.printStatusTime("Started reverse domain name lookups"); this.lookUpCitiesAndASes(); Logger.printStatusTime("Looked up cities and ASes"); - this.setRunningBitsContactsAndExitAddresses(); - Logger.printStatusTime("Set running bits, contacts, and exit " - + "addresses"); + this.setDescriptorPartsOfNodeStatus(); + Logger.printStatusTime("Set descriptor parts of node statuses."); this.calculatePathSelectionProbabilities(); Logger.printStatusTime("Calculated path selection probabilities"); this.finishReverseDomainNameLookups(); @@ -154,7 +153,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, fingerprint, address, orAddressesAndPorts, null, validAfterMillis, orPort, dirPort, relayFlags, consensusWeight, null, null, -1L, defaultPolicy, portList, validAfterMillis, - validAfterMillis, null, null, recommendedVersion); + validAfterMillis, null, null, recommendedVersion, null); if (this.knownNodes.containsKey(fingerprint)) { this.knownNodes.get(fingerprint).update(newNodeStatus); } else { @@ -184,7 +183,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, NodeStatus newNodeStatus = new NodeStatus(false, nickname, fingerprint, address, orAddressesAndPorts, null, publishedMillis, orPort, dirPort, relayFlags, -1L, "??", null, - -1L, null, null, publishedMillis, -1L, null, null, null); + -1L, null, null, publishedMillis, -1L, null, null, null, null); if (this.knownNodes.containsKey(fingerprint)) { this.knownNodes.get(fingerprint).update(newNodeStatus); } else { @@ -215,7 +214,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } }
- private void setRunningBitsContactsAndExitAddresses() { + private void setDescriptorPartsOfNodeStatus() { for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { String fingerprint = e.getKey(); NodeStatus node = e.getValue(); @@ -236,6 +235,19 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } } } + if (detailsStatus.getFamily() != null && + !detailsStatus.getFamily().isEmpty()) { + SortedSet<String> familyFingerprints = new TreeSet<String>(); + for (String familyMember : detailsStatus.getFamily()) { + if (familyMember.startsWith("$") && + familyMember.length() == 41) { + familyFingerprints.add(familyMember.substring(1)); + } + } + if (!familyFingerprints.isEmpty()) { + node.setFamilyFingerprints(familyFingerprints); + } + } } } if (!node.isRelay() && node.getRelayFlags().contains("Running") && diff --git a/src/org/torproject/onionoo/NodeIndexer.java b/src/org/torproject/onionoo/NodeIndexer.java index b1935e1..8a04afb 100644 --- a/src/org/torproject/onionoo/NodeIndexer.java +++ b/src/org/torproject/onionoo/NodeIndexer.java @@ -105,6 +105,14 @@ class NodeIndex { return relaysByContact; }
+ private Map<String, Set<String>> relaysByFamily = null; + public void setRelaysByFamily(Map<String, Set<String>> relaysByFamily) { + this.relaysByFamily = relaysByFamily; + } + public Map<String, Set<String>> getRelaysByFamily() { + return this.relaysByFamily; + } + private SortedMap<Integer, Set<String>> relaysByFirstSeenDays; public void setRelaysByFirstSeenDays( SortedMap<Integer, Set<String>> relaysByFirstSeenDays) { @@ -241,7 +249,8 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByASNumber = new HashMap<String, Set<String>>(), newRelaysByFlag = new HashMap<String, Set<String>>(), newBridgesByFlag = new HashMap<String, Set<String>>(), - newRelaysByContact = new HashMap<String, Set<String>>(); + newRelaysByContact = new HashMap<String, Set<String>>(), + newRelaysByFamily = new HashMap<String, Set<String>>(); SortedMap<Integer, Set<String>> newRelaysByFirstSeenDays = new TreeMap<Integer, Set<String>>(), newBridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(), @@ -305,6 +314,9 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByFlag.get(flagLowerCase).add(fingerprint); newRelaysByFlag.get(flagLowerCase).add(hashedFingerprint); } + if (entry.getFamilyFingerprints() != null) { + newRelaysByFamily.put(fingerprint, entry.getFamilyFingerprints()); + } int daysSinceFirstSeen = (int) ((time.currentTimeMillis() - entry.getFirstSeenMillis()) / DateTimeHelper.ONE_DAY); if (!newRelaysByFirstSeenDays.containsKey(daysSinceFirstSeen)) { @@ -335,6 +347,19 @@ public class NodeIndexer implements ServletContextListener, Runnable { for (String relay : orderRelaysByConsensusWeight) { newRelaysByConsensusWeight.add(relay.split(" ")[1]); } + for (Map.Entry<String, Set<String>> e : + newRelaysByFamily.entrySet()) { + String fingerprint = e.getKey(); + Set<String> inMutualFamilyRelation = new HashSet<String>(); + for (String otherFingerprint : e.getValue()) { + if (newRelaysByFamily.containsKey(otherFingerprint) && + newRelaysByFamily.get(otherFingerprint).contains( + fingerprint)) { + inMutualFamilyRelation.add(otherFingerprint); + } + } + e.getValue().retainAll(inMutualFamilyRelation); + } for (NodeStatus entry : currentBridges) { String hashedFingerprint = entry.getFingerprint().toUpperCase(); String hashedHashedFingerprint = entry.getHashedFingerprint(). @@ -385,6 +410,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { newNodeIndex.setRelaysByFlag(newRelaysByFlag); newNodeIndex.setBridgesByFlag(newBridgesByFlag); newNodeIndex.setRelaysByContact(newRelaysByContact); + newNodeIndex.setRelaysByFamily(newRelaysByFamily); newNodeIndex.setRelaysByFirstSeenDays(newRelaysByFirstSeenDays); newNodeIndex.setRelaysByLastSeenDays(newRelaysByLastSeenDays); newNodeIndex.setBridgesByFirstSeenDays(newBridgesByFirstSeenDays); diff --git a/src/org/torproject/onionoo/NodeStatus.java b/src/org/torproject/onionoo/NodeStatus.java index c829f7f..1bc74f9 100644 --- a/src/org/torproject/onionoo/NodeStatus.java +++ b/src/org/torproject/onionoo/NodeStatus.java @@ -294,6 +294,15 @@ public class NodeStatus extends Document { return this.recommendedVersion; }
+ private SortedSet<String> familyFingerprints; + public void setFamilyFingerprints( + SortedSet<String> familyFingerprints) { + this.familyFingerprints = familyFingerprints; + } + public SortedSet<String> getFamilyFingerprints() { + return this.familyFingerprints; + } + public NodeStatus(boolean isRelay, String nickname, String fingerprint, String address, SortedSet<String> orAddressesAndPorts, SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, @@ -301,7 +310,7 @@ public class NodeStatus extends Document { String countryCode, String hostName, long lastRdnsLookup, String defaultPolicy, String portList, long firstSeenMillis, long lastChangedAddresses, String aSNumber, String contact, - Boolean recommendedVersion) { + Boolean recommendedVersion, SortedSet<String> familyFingerprints) { this.isRelay = isRelay; this.nickname = nickname; this.fingerprint = fingerprint; @@ -348,6 +357,7 @@ public class NodeStatus extends Document { this.aSNumber = aSNumber; this.contact = contact; this.recommendedVersion = recommendedVersion; + this.familyFingerprints = familyFingerprints; }
public static NodeStatus fromString(String documentString) { @@ -356,7 +366,7 @@ public class NodeStatus extends Document { countryCode = null, hostName = null, defaultPolicy = null, portList = null, aSNumber = null, contact = null; SortedSet<String> orAddressesAndPorts = null, exitAddresses = null, - relayFlags = null; + relayFlags = null, familyFingerprints = null; long lastSeenMillis = -1L, consensusWeight = -1L, lastRdnsLookup = -1L, firstSeenMillis = -1L, lastChangedAddresses = -1L; @@ -455,6 +465,10 @@ public class NodeStatus extends Document { recommendedVersion = parts[21].equals("null") ? null : parts[21].equals("true"); } + if (parts.length > 22 && !parts[22].equals("null")) { + familyFingerprints = new TreeSet<String>(Arrays.asList( + parts[22].split(";"))); + } } catch (NumberFormatException e) { System.err.println("Number format exception while parsing node " + "status line '" + documentString + "': " + e.getMessage() @@ -473,7 +487,7 @@ public class NodeStatus extends Document { lastSeenMillis, orPort, dirPort, relayFlags, consensusWeight, countryCode, hostName, lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, lastChangedAddresses, aSNumber, contact, - recommendedVersion); + recommendedVersion, familyFingerprints); return newNodeStatus; }
@@ -551,6 +565,16 @@ public class NodeStatus extends Document { sb.append("\t" + (this.contact != null ? this.contact : "")); sb.append("\t" + (this.recommendedVersion == null ? "null" : this.recommendedVersion ? "true" : "false")); + if (this.familyFingerprints == null || + this.familyFingerprints.isEmpty()) { + sb.append("\tnull"); + } else { + sb.append("\t"); + written = 0; + for (String familyFingerprint : this.familyFingerprints) { + sb.append((written++ > 0 ? ";" : "") + familyFingerprint); + } + } return sb.toString(); } } diff --git a/src/org/torproject/onionoo/RequestHandler.java b/src/org/torproject/onionoo/RequestHandler.java index 67be625..66559e6 100644 --- a/src/org/torproject/onionoo/RequestHandler.java +++ b/src/org/torproject/onionoo/RequestHandler.java @@ -104,6 +104,11 @@ public class RequestHandler { lastSeenDays.length); }
+ private String family; + public void setFamily(String family) { + this.family = family; + } + private Map<String, NodeStatus> filteredRelays = new HashMap<String, NodeStatus>();
@@ -127,6 +132,7 @@ public class RequestHandler { this.filterNodesByFirstSeenDays(); this.filterNodesByLastSeenDays(); this.filterByContact(); + this.filterByFamily(); this.order(); this.offset(); this.limit(); @@ -442,6 +448,23 @@ public class RequestHandler { this.filteredBridges.clear(); }
+ private void filterByFamily() { + if (this.family == null) { + return; + } + Set<String> removeRelays = new HashSet<String>( + this.filteredRelays.keySet()); + removeRelays.remove(this.family); + if (this.nodeIndex.getRelaysByFamily().containsKey(this.family)) { + removeRelays.removeAll(this.nodeIndex.getRelaysByFamily(). + get(this.family)); + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + this.filteredBridges.clear(); + } + private void order() { if (this.order != null && this.order.length == 1) { List<String> orderBy = new ArrayList<String>( diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java index 1817504..dad6043 100644 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ b/src/org/torproject/onionoo/ResourceServlet.java @@ -148,7 +148,7 @@ public class ResourceServlet extends HttpServlet { Set<String> knownParameters = new HashSet<String>(Arrays.asList(( "type,running,search,lookup,fingerprint,country,as,flag," + "first_seen_days,last_seen_days,contact,order,limit,offset," - + "fields").split(","))); + + "fields,family").split(","))); for (String parameterKey : parameterMap.keySet()) { if (!knownParameters.contains(parameterKey)) { response.sendError(HttpServletResponse.SC_BAD_REQUEST); @@ -303,6 +303,16 @@ public class ResourceServlet extends HttpServlet { } rh.setLimit(limitParameter); } + if (parameterMap.containsKey("family")) { + String familyParameter = this.parseFingerprintParameter( + parameterMap.get("family")); + if (familyParameter == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + String family = familyParameter.toUpperCase(); + rh.setFamily(family); + } rh.handleRequest();
ResponseBuilder rb = new ResponseBuilder(); diff --git a/test/org/torproject/onionoo/ResourceServletTest.java b/test/org/torproject/onionoo/ResourceServletTest.java index 3b047c6..ff3ba35 100644 --- a/test/org/torproject/onionoo/ResourceServletTest.java +++ b/test/org/torproject/onionoo/ResourceServletTest.java @@ -106,7 +106,9 @@ public class ResourceServletTest { + "2013-04-19\t05:00:00\t9001\t0\tRunning,Valid\t20\tde\tnull\t" + "-1\treject\t1-65535\t2013-04-18\t05:00:00\t" + "2013-04-19\t05:00:00\tAS8767\ttorkaz <klaus dot zufall at " - + "gmx dot de> fb-token:np5_g_83jmf="); + + "gmx dot de> fb-token:np5_g_83jmf=\tnull\t" + + "001C13B3A55A71B977CA65EC85539D79C653A3FC;" + + "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B"); this.relays.put("001C13B3A55A71B977CA65EC85539D79C653A3FC", "r\tFerrari458\t001C13B3A55A71B977CA65EC85539D79C653A3FC\t" + "68.38.171.200;[2001:4f8:3:2e::51]:9001;\t" @@ -114,30 +116,32 @@ public class ResourceServletTest { + "Fast,Named,Running,V2Dir,Valid\t1140\tus\t" + "c-68-38-171-200.hsd1.pa.comcast.net\t1366805763009\treject\t" + "1-65535\t2013-02-12\t16:00:00\t2013-02-26\t18:00:00\t" - + "AS7922\t"); + + "AS7922\tnull\tnull\t000C5F55BD4814B917CC474BD537F1A3B33CCE2A"); this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", "r\tTimMayTribute\t0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B\t" + "89.69.68.246;;\t2013-04-22\t20:00:00\t9001\t9030\t" + "Fast,Running,Unnamed,V2Dir,Valid\t63\ta1\tnull\t-1\treject\t" + "1-65535\t2013-04-16\t18:00:00\t2013-04-16\t18:00:00\t" + "AS6830\t1024D/51E2A1C7 steven j. murdoch " - + "tor+steven.murdoch@cl.cam.ac.uk fb-token:5sr_k_zs2wm="); + + "tor+steven.murdoch@cl.cam.ac.uk fb-token:5sr_k_zs2wm=\t" + + "null\tnull"); this.bridges = new TreeMap<String, String>(); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", "b\tec2bridgercc7f31fe\t" + "0000831B236DFF73D409AD17B40E2A728A53994F\t10.199.7.176;;\t" + "2013-04-21\t18:07:03\t443\t0\tValid\t-1\t??\tnull\t-1\t" - + "null\tnull\t2013-04-20\t15:37:04\tnull\tnull\tnull\tnull"); + + "null\tnull\t2013-04-20\t15:37:04\tnull\tnull\tnull\tnull\t" + + "null\tnull"); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", "b\tUnnamed\t0002D9BDBBC230BD9C78FF502A16E0033EF87E0C\t" + "10.0.52.84;;\t2013-04-20\t17:37:04\t443\t0\tValid\t-1\t??\t" + "null\t-1\tnull\tnull\t2013-04-14\t07:07:05\tnull\tnull\tnull\t" - + "null"); + + "null\tnull\tnull"); this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", "b\tgummy\t1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756\t" + "10.63.169.98;;\t2013-04-24\t01:07:04\t9001\t0\tRunning,Valid\t" + "-1\t??\tnull\t-1\tnull\tnull\t2013-01-16\t21:07:04\tnull\t" - + "null\tnull\tnull"); + + "null\tnull\tnull\tnull\tnull"); }
private void runTest(String requestURI, @@ -287,7 +291,7 @@ public class ResourceServletTest { "r TorkaZ 000C5F55BD4814B917CC474BD537F1A3B33CCE2A " + "62.216.201.221;; 2013-04-19 05:00:00 9001 0 Running,Valid 20 " + "de null -1 reject 1-65535 2013-04-18 05:00:00 2013-04-19 " - + "05:00:00 AS8767"); + + "05:00:00 AS8767 null null null"); this.runTest("/summary", null); assertEquals("2013-04-19 05:00:00", this.summaryDocument.relays_published); @@ -306,7 +310,7 @@ public class ResourceServletTest { this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", "b ec2bridgercc7f31fe 0000831B236DFF73D409AD17B40E2A728A53994F " + "10.199.7.176;; 2013-04-21 18:07:03 443 0 Valid -1 ?? null -1 " - + "null null 2013-04-20 15:37:04 null null null"); + + "null null 2013-04-20 15:37:04 null null null null null null"); this.runTest("/summary", null); assertEquals("2013-04-21 18:07:03", this.summaryDocument.bridges_published); @@ -1211,5 +1215,39 @@ public class ResourceServletTest { this.assertErrorStatusCode( "/summary?limit=one", 400); } + + @Test() + public void testFamilyTorkaZ() { + this.assertSummaryDocument( + "/summary?family=000C5F55BD4814B917CC474BD537F1A3B33CCE2A", 2, + null, 0, null); + } + + @Test() + public void testFamilyFerrari458() { + this.assertSummaryDocument( + "/summary?family=001C13B3A55A71B977CA65EC85539D79C653A3FC", 2, + null, 0, null); + } + + @Test() + public void testFamilyTimMayTribute() { + this.assertSummaryDocument( + "/summary?family=0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", 1, + null, 0, null); + } + + @Test() + public void testFamilyBridgegummy() { + this.assertSummaryDocument( + "/summary?family=0000831B236DFF73D409AD17B40E2A728A53994F", 0, + null, 0, null); + } + + @Test() + public void testFamily39Characters() { + this.assertErrorStatusCode( + "/summary?family=00000000000000000000000000000000000000", 400); + } }
diff --git a/web/protocol.html b/web/protocol.html index 3142880..0c191bf 100644 --- a/web/protocol.html +++ b/web/protocol.html @@ -358,6 +358,23 @@ Comparisons are case-insensitive. </p> </li>
+<li> +<font color="blue"><b>family</b></font> +<p> +Return only the relay whose fingerprint matches the parameter value and +all relays that this relay has listed in its family by fingerprint and +that in turn have listed this relay in their family by fingerprint. +If relays have listed other relays in their family by nickname, those +family relationships will not be considered, regardless of whether they +have the Named flag or not. +The provided relay fingerprint must consist of 40 hex characters where +case does not matter, and it must not be hashed using SHA-1. +Bridges are not contained in the result, regardless of whether they define +a family. +<font color="blue">Added on July 4, 2014.</font> +</p> +</li> + </ul>
<p>