[tor-commits] [onionoo/master] Add new contact parameter to the servlet.

commit fd810e6c6b5ee46977142901105e35290806dcf0 Author: Karsten Loesing karsten.loesing@gmx.net Date: Wed Jul 17 09:55:49 2013 +0200

Add new contact parameter to the servlet.

Implements #5255. --- src/org/torproject/onionoo/DetailsDataWriter.java | 23 ++++- src/org/torproject/onionoo/NodeDataWriter.java | 4 +- src/org/torproject/onionoo/NodeStatus.java | 72 +++++++++---- src/org/torproject/onionoo/ResourceServlet.java | 53 +++++++++- .../torproject/onionoo/ResourceServletTest.java | 107 ++++++++++++++------ web/index.html | 11 ++ 6 files changed, 212 insertions(+), 58 deletions(-)

diff --git a/src/org/torproject/onionoo/DetailsDataWriter.java b/src/org/torproject/onionoo/DetailsDataWriter.java index aa511c2..51ca2e9 100644 --- a/src/org/torproject/onionoo/DetailsDataWriter.java +++ b/src/org/torproject/onionoo/DetailsDataWriter.java @@ -426,6 +426,10 @@ public class DetailsDataWriter implements DescriptorListener { return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\'", "'"); }

+ private static String unescapeJSON(String s) { + return StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\'")); + } + private void updateRelayDetailsFiles( SortedSet<String> remainingDetailsFiles) { SimpleDateFormat dateTimeFormat = new SimpleDateFormat( @@ -588,12 +592,29 @@ public class DetailsDataWriter implements DescriptorListener { sb.append("]"); }

- /* Append descriptor-specific part from details status file. */ + /* Append descriptor-specific part from details status file, and + * update contact in node status. */ DetailsStatus detailsStatus = this.documentStore.retrieve( DetailsStatus.class, false, fingerprint); if (detailsStatus != null && detailsStatus.documentString.length() > 0) { sb.append(",\n" + detailsStatus.documentString); + String contact = null; + Scanner s = new Scanner(detailsStatus.documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + if (!line.startsWith(""contact":")) { + continue; + } + int start = ""contact":"".length(), end = line.length() - 1; + if (line.endsWith(",")) { + end--; + } + contact = unescapeJSON(line.substring(start, end)); + break; + } + s.close(); + entry.setContact(contact); }

/* Finish details string. */ diff --git a/src/org/torproject/onionoo/NodeDataWriter.java b/src/org/torproject/onionoo/NodeDataWriter.java index 723dd16..b1e9b5c 100644 --- a/src/org/torproject/onionoo/NodeDataWriter.java +++ b/src/org/torproject/onionoo/NodeDataWriter.java @@ -82,7 +82,7 @@ public class NodeDataWriter implements DescriptorListener { fingerprint, address, orAddressesAndPorts, null, validAfterMillis, orPort, dirPort, relayFlags, consensusWeight, null, null, -1L, defaultPolicy, portList, validAfterMillis, - validAfterMillis, null); + validAfterMillis, null, null); if (this.knownNodes.containsKey(fingerprint)) { this.knownNodes.get(fingerprint).update(newNodeStatus); } else { @@ -112,7 +112,7 @@ public class NodeDataWriter implements DescriptorListener { NodeStatus newNodeStatus = new NodeStatus(false, nickname, fingerprint, address, orAddressesAndPorts, null, publishedMillis, orPort, dirPort, relayFlags, -1L, "??", null, - -1L, null, null, publishedMillis, -1L, null); + -1L, null, null, publishedMillis, -1L, null, null); if (this.knownNodes.containsKey(fingerprint)) { this.knownNodes.get(fingerprint).update(newNodeStatus); } else { diff --git a/src/org/torproject/onionoo/NodeStatus.java b/src/org/torproject/onionoo/NodeStatus.java index 4f207a0..e549336 100644 --- a/src/org/torproject/onionoo/NodeStatus.java +++ b/src/org/torproject/onionoo/NodeStatus.java @@ -55,13 +55,14 @@ public class NodeStatus extends Document { private String defaultPolicy; private String portList; private SortedMap<Long, Set<String>> lastAddresses; + private String contact; public NodeStatus(boolean isRelay, String nickname, String fingerprint, String address, SortedSet<String> orAddressesAndPorts, SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, int dirPort, SortedSet<String> relayFlags, long consensusWeight, String countryCode, String hostName, long lastRdnsLookup, String defaultPolicy, String portList, long firstSeenMillis, - long lastChangedAddresses, String aSNumber) { + long lastChangedAddresses, String aSNumber, String contact) { this.isRelay = isRelay; this.nickname = nickname; this.fingerprint = fingerprint; @@ -106,13 +107,14 @@ public class NodeStatus extends Document { addresses.addAll(orAddressesAndPorts); this.lastAddresses.put(lastChangedAddresses, addresses); this.aSNumber = aSNumber; + this.contact = contact; }

public static NodeStatus fromString(String documentString) { boolean isRelay = false; String nickname = null, fingerprint = null, address = null, countryCode = null, hostName = null, defaultPolicy = null, - portList = null, aSNumber = null; + portList = null, aSNumber = null, contact = null; SortedSet<String> orAddressesAndPorts = null, exitAddresses = null, relayFlags = null; long lastSeenMillis = -1L, consensusWeight = -1L, @@ -123,7 +125,8 @@ public class NodeStatus extends Document { SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String[] parts = documentString.trim().split(" "); + String separator = documentString.contains("\t") ? "\t" : " "; + String[] parts = documentString.trim().split(separator); isRelay = parts[0].equals("r"); if (parts.length < 9) { System.err.println("Too few space-separated values in line '" @@ -193,6 +196,9 @@ public class NodeStatus extends Document { if (parts.length > 19) { aSNumber = parts[19]; } + if (parts.length > 20) { + contact = parts[20]; + } } catch (NumberFormatException e) { System.err.println("Number format exception while parsing node " + "status line '" + documentString + "': " + e.getMessage() @@ -215,7 +221,7 @@ public class NodeStatus extends Document { fingerprint, address, orAddressesAndPorts, exitAddresses, lastSeenMillis, orPort, dirPort, relayFlags, consensusWeight, countryCode, hostName, lastRdnsLookup, defaultPolicy, portList, - firstSeenMillis, lastChangedAddresses, aSNumber); + firstSeenMillis, lastChangedAddresses, aSNumber, contact); return newNodeStatus; }

@@ -234,6 +240,7 @@ public class NodeStatus extends Document { this.defaultPolicy = newNodeStatus.defaultPolicy; this.portList = newNodeStatus.portList; this.aSNumber = newNodeStatus.aSNumber; + this.contact = newNodeStatus.contact; } if (this.isRelay && newNodeStatus.isRelay) { this.lastAddresses.putAll(newNodeStatus.lastAddresses); @@ -244,13 +251,13 @@ public class NodeStatus extends Document {

public String toString() { SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); + "yyyy-MM-dd\tHH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); StringBuilder sb = new StringBuilder(); sb.append(this.isRelay ? "r" : "b"); - sb.append(" " + this.nickname); - sb.append(" " + this.fingerprint); - sb.append(" " + this.address + ";"); + sb.append("\t" + this.nickname); + sb.append("\t" + this.fingerprint); + sb.append("\t" + this.address + ";"); int written = 0; for (String orAddressAndPort : this.orAddressesAndPorts) { sb.append((written++ > 0 ? "+" : "") + orAddressAndPort); @@ -263,32 +270,34 @@ public class NodeStatus extends Document { + exitAddress); } } - sb.append(" " + dateTimeFormat.format(this.lastSeenMillis)); - sb.append(" " + this.orPort); - sb.append(" " + this.dirPort + " "); + sb.append("\t" + dateTimeFormat.format(this.lastSeenMillis)); + sb.append("\t" + this.orPort); + sb.append("\t" + this.dirPort + "\t"); written = 0; for (String relayFlag : this.relayFlags) { sb.append((written++ > 0 ? "," : "") + relayFlag); } if (this.isRelay) { - sb.append(" " + String.valueOf(this.consensusWeight)); - sb.append(" " + (this.countryCode != null ? this.countryCode : "??")); - sb.append(" " + (this.hostName != null ? this.hostName : "null")); - sb.append(" " + String.valueOf(this.lastRdnsLookup)); - sb.append(" " + (this.defaultPolicy != null ? this.defaultPolicy + sb.append("\t" + String.valueOf(this.consensusWeight)); + sb.append("\t" + + (this.countryCode != null ? this.countryCode : "??")); + sb.append("\t" + (this.hostName != null ? this.hostName : "null")); + sb.append("\t" + String.valueOf(this.lastRdnsLookup)); + sb.append("\t" + (this.defaultPolicy != null ? this.defaultPolicy : "null")); - sb.append(" " + (this.portList != null ? this.portList : "null")); + sb.append("\t" + (this.portList != null ? this.portList : "null")); } else { - sb.append(" -1 ?? null -1 null null"); + sb.append("\t-1\t??\tnull\t-1\tnull\tnull"); } - sb.append(" " + dateTimeFormat.format(this.firstSeenMillis)); + sb.append("\t" + dateTimeFormat.format(this.firstSeenMillis)); if (this.isRelay) { - sb.append(" " + dateTimeFormat.format( + sb.append("\t" + dateTimeFormat.format( this.getLastChangedOrAddress())); - sb.append(" " + (this.aSNumber != null ? this.aSNumber : "null")); + sb.append("\t" + (this.aSNumber != null ? this.aSNumber : "null")); } else { - sb.append(" null null null"); + sb.append("\tnull\tnull\tnull"); } + sb.append("\t" + (this.contact != null ? this.contact : "")); return sb.toString(); }

@@ -480,5 +489,24 @@ public class NodeStatus extends Document { } return lastChangedAddressesMillis; } + public void setContact(String contact) { + if (contact == null) { + this.contact = null; + } else { + contact = contact.toLowerCase(); + StringBuilder sb = new StringBuilder(); + for (char c : contact.toCharArray()) { + if (c >= 32 && c < 127) { + sb.append(c); + } else { + sb.append(" "); + } + } + this.contact = sb.toString(); + } + } + public String getContact() { + return this.contact; + } }

diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java index 110b123..47d4a93 100644 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ b/src/org/torproject/onionoo/ResourceServlet.java @@ -59,7 +59,8 @@ public class ResourceServlet extends HttpServlet { private Map<String, String> relayFingerprintSummaryLines = null, bridgeFingerprintSummaryLines = null; private Map<String, Set<String>> relaysByCountryCode = null, - relaysByASNumber = null, relaysByFlag = null; + relaysByASNumber = null, relaysByFlag = null, + relaysByContact = null; private SortedMap<Integer, Set<String>> relaysByFirstSeenDays = null, bridgesByFirstSeenDays = null, relaysByLastSeenDays = null, bridgesByLastSeenDays = null; @@ -90,7 +91,8 @@ public class ResourceServlet extends HttpServlet { Map<String, Set<String>> relaysByCountryCode = new HashMap<String, Set<String>>(), relaysByASNumber = new HashMap<String, Set<String>>(), - relaysByFlag = new HashMap<String, Set<String>>(); + relaysByFlag = new HashMap<String, Set<String>>(), + relaysByContact = new HashMap<String, Set<String>>(); SortedMap<Integer, Set<String>> relaysByFirstSeenDays = new TreeMap<Integer, Set<String>>(), bridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(), @@ -182,6 +184,12 @@ public class ResourceServlet extends HttpServlet { relaysByLastSeenDays.get(daysSinceLastSeen).add(fingerprint); relaysByLastSeenDays.get(daysSinceLastSeen).add( hashedFingerprint); + String contact = entry.getContact(); + if (!relaysByContact.containsKey(contact)) { + relaysByContact.put(contact, new HashSet<String>()); + } + relaysByContact.get(contact).add(fingerprint); + relaysByContact.get(contact).add(hashedFingerprint); } Collections.sort(orderRelaysByConsensusWeight); relaysByConsensusWeight = new ArrayList<String>(); @@ -224,6 +232,7 @@ public class ResourceServlet extends HttpServlet { this.relaysByCountryCode = relaysByCountryCode; this.relaysByASNumber = relaysByASNumber; this.relaysByFlag = relaysByFlag; + this.relaysByContact = relaysByContact; this.relaysByFirstSeenDays = relaysByFirstSeenDays; this.relaysByLastSeenDays = relaysByLastSeenDays; this.bridgesByFirstSeenDays = bridgesByFirstSeenDays; @@ -374,7 +383,7 @@ public class ResourceServlet extends HttpServlet { * parameters. */ Set<String> knownParameters = new HashSet<String>(Arrays.asList(( "type,running,search,lookup,country,as,flag,first_seen_days," - + "last_seen_days,order,limit,offset").split(","))); + + "last_seen_days,contact,order,limit,offset").split(","))); for (String parameterKey : parameterMap.keySet()) { if (!knownParameters.contains(parameterKey)) { response.sendError(HttpServletResponse.SC_BAD_REQUEST); @@ -485,6 +494,15 @@ public class ResourceServlet extends HttpServlet { this.filterNodesByDays(filteredBridges, this.bridgesByLastSeenDays, days); } + if (parameterMap.containsKey("contact")) { + String[] contactParts = this.parseContactParameter( + parameterMap.get("contact")); + if (contactParts == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + this.filterByContact(filteredRelays, filteredBridges, contactParts); + }

/* Re-order and limit results. */ List<String> orderedRelays = new ArrayList<String>(); @@ -667,6 +685,15 @@ public class ResourceServlet extends HttpServlet { return new int[] { x, y }; }

+ private String[] parseContactParameter(String parameter) { + for (char c : parameter.toCharArray()) { + if (c < 32 || c >= 127) { + return null; + } + } + return parameter.split(" "); + } + private void filterByType(Map<String, String> filteredRelays, Map<String, String> filteredBridges, boolean relaysRequested) { if (relaysRequested) { @@ -874,6 +901,26 @@ public class ResourceServlet extends HttpServlet { } }

+ private void filterByContact(Map<String, String> filteredRelays, + Map<String, String> filteredBridges, String[] contactParts) { + Set<String> removeRelays = new HashSet<String>(); + for (Map.Entry<String, Set<String>> e : + this.relaysByContact.entrySet()) { + String contact = e.getKey(); + for (String contactPart : contactParts) { + if (contact == null || + !contact.contains(contactPart.toLowerCase())) { + removeRelays.addAll(e.getValue()); + break; + } + } + } + for (String fingerprint : removeRelays) { + filteredRelays.remove(fingerprint); + } + filteredBridges.clear(); + } + private void writeRelays(List<String> relays, PrintWriter pw, String resourceType) { pw.write("{"relays_published":"" + this.relaysPublishedString diff --git a/test/org/torproject/onionoo/ResourceServletTest.java b/test/org/torproject/onionoo/ResourceServletTest.java index 5be2fba..2d131b4 100644 --- a/test/org/torproject/onionoo/ResourceServletTest.java +++ b/test/org/torproject/onionoo/ResourceServletTest.java @@ -110,36 +110,41 @@ public class ResourceServletTest { Map<String, String[]> parameterMap) { this.tempOutDir = tempOutDir; this.relays = new TreeMap<String, String>(); - this.relays.put("000C5F55", "r TorkaZ " - + "000C5F55BD4814B917CC474BD537F1A3B33CCE2A " - + "62.216.201.221;;62.216.201.222+62.216.201.223 " - + "2013-04-19 05:00:00 9001 0 Running,Valid 20 de null -1 " - + "reject 1-65535 2013-04-18 05:00:00 2013-04-19 05:00:00 " - + "AS8767"); - this.relays.put("001C13B3", "r Ferrari458 " - + "001C13B3A55A71B977CA65EC85539D79C653A3FC " - + "68.38.171.200;[2001:4f8:3:2e::51]:9001; " - + "2013-04-24 12:00:00 9001 9030 " - + "Fast,Named,Running,V2Dir,Valid 1140 us " - + "c-68-38-171-200.hsd1.pa.comcast.net 1366805763009 reject " - + "1-65535 2013-02-12 16:00:00 2013-02-26 18:00:00 AS7922"); - this.relays.put("0025C136", "r TimMayTribute " - + "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B 89.69.68.246;; " - + "2013-04-22 20:00:00 9001 9030 " - + "Fast,Running,Unnamed,V2Dir,Valid 63 a1 null -1 reject " - + "1-65535 2013-04-16 18:00:00 2013-04-16 18:00:00 AS6830"); - this.bridges.put("0000831B", "b ec2bridgercc7f31fe " - + "0000831B236DFF73D409AD17B40E2A728A53994F 10.199.7.176;; " - + "2013-04-21 18:07:03 443 0 Valid -1 ?? null -1 null null " - + "2013-04-20 15:37:04 null null null"); - this.bridges.put("0002D9BD", "b Unnamed " - + "0002D9BDBBC230BD9C78FF502A16E0033EF87E0C 10.0.52.84;; " - + "2013-04-20 17:37:04 443 0 Valid -1 ?? null -1 null " - + "null 2013-04-14 07:07:05 null null null"); - this.bridges.put("0010D49C", "b gummy " - + "1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756 10.63.169.98;; " - + "2013-04-24 01:07:04 9001 0 Running,Valid -1 ?? null -1 null " - + "null 2013-01-16 21:07:04 null null null"); + this.relays.put("000C5F55", "r\tTorkaZ\t" + + "000C5F55BD4814B917CC474BD537F1A3B33CCE2A\t" + + "62.216.201.221;;62.216.201.222+62.216.201.223\t" + + "2013-04-19\t05:00:00\t9001\t0\tRunning,Valid\t20\tde\tnull\t" + + "-1\treject\t1-65535\t2013-04-18\t05:00:00\t" + + "2013-04-19\t05:00:00\tAS8767\ttorkaz <klaus dot zufall at " + + "gmx dot de> fb-token:np5_g_83jmf="); + this.relays.put("001C13B3", "r\tFerrari458\t" + + "001C13B3A55A71B977CA65EC85539D79C653A3FC\t" + + "68.38.171.200;[2001:4f8:3:2e::51]:9001;\t" + + "2013-04-24\t12:00:00\t9001\t9030\t" + + "Fast,Named,Running,V2Dir,Valid\t1140\tus\t" + + "c-68-38-171-200.hsd1.pa.comcast.net\t1366805763009\treject\t" + + "1-65535\t2013-02-12\t16:00:00\t2013-02-26\t18:00:00\t" + + "AS7922\t"); + this.relays.put("0025C136", "r\tTimMayTribute\t" + + "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B\t89.69.68.246;;\t" + + "2013-04-22\t20:00:00\t9001\t9030\t" + + "Fast,Running,Unnamed,V2Dir,Valid\t63\ta1\tnull\t-1\treject\t" + + "1-65535\t2013-04-16\t18:00:00\t2013-04-16\t18:00:00\t" + + "AS6830\t1024D/51E2A1C7 steven j. murdoch " + + "tor+steven.murdoch@cl.cam.ac.uk fb-token:5sr_k_zs2wm="); + this.bridges.put("0000831B", "b\tec2bridgercc7f31fe\t" + + "0000831B236DFF73D409AD17B40E2A728A53994F\t10.199.7.176;;\t" + + "2013-04-21\t18:07:03\t443\t0\tValid\t-1\t??\tnull\t-1\t" + + "null\tnull\t2013-04-20\t15:37:04\tnull\tnull\tnull\tnull"); + this.bridges.put("0002D9BD", "b\tUnnamed\t" + + "0002D9BDBBC230BD9C78FF502A16E0033EF87E0C\t10.0.52.84;;\t" + + "2013-04-20\t17:37:04\t443\t0\tValid\t-1\t??\tnull\t-1\t" + + "null\tnull\t2013-04-14\t07:07:05\tnull\tnull\tnull\tnull"); + this.bridges.put("0010D49C", "b\tgummy\t" + + "1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756\t10.63.169.98;;\t" + + "2013-04-24\t01:07:04\t9001\t0\tRunning,Valid\t-1\t??\tnull\t" + + "-1\tnull\tnull\t2013-01-16\t21:07:04\tnull\tnull\tnull\t" + + "null"); this.request = new TestingHttpServletRequestWrapper(requestURI, parameterMap); this.response = new TestingHttpServletResponseWrapper(); @@ -987,6 +992,48 @@ public class ResourceServletTest { }

@Test() + public void testContactSteven() { + ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + "/summary?contact=Steven", 1, null, 0, null); + } + + @Test() + public void testContactStevenMurdoch() { + ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + "/summary?contact=Steven Murdoch", 1, null, 0, null); + } + + @Test() + public void testContactMurdochSteven() { + ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + "/summary?contact=Murdoch Steven", 1, null, 0, null); + } + + @Test() + public void testContactStevenDotMurdoch() { + ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + "/summary?contact=Steven.Murdoch", 1, null, 0, null); + } + + @Test() + public void testContactFbTokenFive() { + ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + "/summary?contact=fb-token:5sR_K_zs2wM=", 1, null, 0, null); + } + + @Test() + public void testContactFbToken() { + ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + "/summary?contact=<fb-token:", 2, null, 0, null); + } + + @Test() + public void testContactDash() { + ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, + "/summary?contact=-", 2, null, 0, null); + } + + @Test() public void testOrderConsensusWeightAscending() { ResourceServletTestHelper.assertSummaryDocument(this.tempOutDir, "/summary?order=consensus_weight", 3, diff --git a/web/index.html b/web/index.html index a3bcae8..2655975 100755 --- a/web/index.html +++ b/web/index.html @@ -687,6 +687,17 @@ Note that relays and bridges that haven't been running in the past week are never included in results, so that setting x to 8 or higher will always lead to an empty result set. </td></tr> +<tr><td><b><font color="blue">contact</font></b></td><td>Return only +relays with the parameter value +matching (part of) the contact line. +If the parameter value contains spaces, only relays are returned which +contain all space-separated parts in their contact line. +Only printable ASCII characters are permitted in the parameter value, +some of which need to be percent-encoded (# as %23, % as %25, &#38; as +%26, + as %2B, and / as %2F). +Comparisons are case-insensitive. +<font color="blue">Added on July 19, 2013.</font> +</td></tr> </table> <p>Relay and/or bridge documents in the response can be ordered and limited by providing further parameters.