commit ef51f1b4d6a44699177c2c09d19bfa8e6c2bac9e Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Tue Apr 9 05:32:19 2013 +0200 Add two new parameters "as" and "flag". Also fix a potential bug in the servlet's filtering and sorting code. It's unclear whether this really was a bug, but let's clean up the code just in case. --- src/org/torproject/onionoo/CurrentNodes.java | 29 +++-- src/org/torproject/onionoo/Node.java | 3 +- src/org/torproject/onionoo/ResourceServlet.java | 153 +++++++++++++++++++---- web/index.html | 16 +++ 4 files changed, 168 insertions(+), 33 deletions(-) diff --git a/src/org/torproject/onionoo/CurrentNodes.java b/src/org/torproject/onionoo/CurrentNodes.java index d98eb4e..f2bbac7 100644 --- a/src/org/torproject/onionoo/CurrentNodes.java +++ b/src/org/torproject/onionoo/CurrentNodes.java @@ -124,20 +124,24 @@ public class CurrentNodes { lastChangedAddresses = dateTimeFormat.parse(parts[17] + " " + parts[18]).getTime(); } + String aSNumber = null; + if (parts.length > 19) { + aSNumber = parts[19]; + } if (isRelay) { this.addRelay(nickname, fingerprint, address, orAddressesAndPorts, exitAddresses, publishedOrValidAfterMillis, orPort, dirPort, relayFlags, consensusWeight, countryCode, hostName, lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, - lastChangedAddresses); + lastChangedAddresses, aSNumber); } else { this.addBridge(nickname, fingerprint, address, orAddressesAndPorts, exitAddresses, publishedOrValidAfterMillis, orPort, dirPort, relayFlags, consensusWeight, countryCode, hostName, lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, - lastChangedAddresses); + lastChangedAddresses, aSNumber); } } br.close(); @@ -212,13 +216,15 @@ public class CurrentNodes { entry.getFirstSeenMillis()); String lastChangedAddresses = dateTimeFormat.format( entry.getLastChangedOrAddress()); + String aSNumber = entry.getASNumber() != null + ? entry.getASNumber() : "null"; bw.write("r " + nickname + " " + fingerprint + " " + addressesBuilder.toString() + " " + lastSeen + " " + orPort + " " + dirPort + " " + flagsBuilder.toString() + " " + consensusWeight + " " + countryCode + " " + hostName + " " + String.valueOf(lastRdnsLookup) + " " + defaultPolicy + " " + portList + " " + firstSeen + " " + lastChangedAddresses - + "\n"); + + " " + aSNumber + "\n"); } for (Node entry : this.currentBridges.values()) { String nickname = entry.getNickname(); @@ -246,7 +252,8 @@ public class CurrentNodes { bw.write("b " + nickname + " " + fingerprint + " " + addressesBuilder.toString() + " " + published + " " + orPort + " " + dirPort + " " + flagsBuilder.toString() - + " -1 ?? null -1 null null " + firstSeen + " null null\n"); + + " -1 ?? null -1 null null " + firstSeen + " null null " + + "null\n"); } bw.close(); } catch (IOException e) { @@ -317,7 +324,7 @@ public class CurrentNodes { this.addRelay(nickname, fingerprint, address, orAddressesAndPorts, null, validAfterMillis, orPort, dirPort, relayFlags, consensusWeight, null, null, -1L, defaultPolicy, portList, - validAfterMillis, validAfterMillis); + validAfterMillis, validAfterMillis, null); } if (this.lastValidAfterMillis == validAfterMillis) { this.lastBandwidthWeights = consensus.getBandwidthWeights(); @@ -330,7 +337,7 @@ public class CurrentNodes { int dirPort, SortedSet<String> relayFlags, long consensusWeight, String countryCode, String hostName, long lastRdnsLookup, String defaultPolicy, String portList, long firstSeenMillis, - long lastChangedAddresses) { + long lastChangedAddresses, String aSNumber) { /* Remember addresses and OR/dir ports that the relay advertised at * the given time. */ SortedMap<Long, Set<String>> lastAddresses = @@ -376,7 +383,7 @@ public class CurrentNodes { orAddressesAndPorts, exitAddresses, lastSeenMillis, orPort, dirPort, relayFlags, consensusWeight, countryCode, hostName, lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, - lastAddresses); + lastAddresses, aSNumber); this.currentRelays.put(fingerprint, entry); /* If this entry comes from a new consensus, update our global last * valid-after time. */ @@ -769,7 +776,7 @@ public class CurrentNodes { SortedSet<String> relayFlags = entry.getFlags(); this.addBridge(nickname, fingerprint, address, orAddressesAndPorts, null, publishedMillis, orPort, dirPort, relayFlags, -1, "??", - null, -1L, null, null, publishedMillis, -1L); + null, -1L, null, null, publishedMillis, -1L, null); } } @@ -779,7 +786,7 @@ public class CurrentNodes { int dirPort, SortedSet<String> relayFlags, long consensusWeight, String countryCode, String hostname, long lastRdnsLookup, String defaultPolicy, String portList, long firstSeenMillis, - long lastChangedAddresses) { + long lastChangedAddresses, String aSNumber) { /* See if there's already an entry for this bridge. */ if (this.currentBridges.containsKey(fingerprint)) { Node existingEntry = this.currentBridges.get(fingerprint); @@ -797,6 +804,7 @@ public class CurrentNodes { countryCode = existingEntry.getCountryCode(); defaultPolicy = existingEntry.getDefaultPolicy(); portList = existingEntry.getPortList(); + aSNumber = existingEntry.getASNumber(); } /* Update relay-history fields. */ firstSeenMillis = Math.min(firstSeenMillis, @@ -806,7 +814,8 @@ public class CurrentNodes { Node entry = new Node(nickname, fingerprint, address, orAddressesAndPorts, exitAddresses, lastSeenMillis, orPort, dirPort, relayFlags, consensusWeight, countryCode, hostname, - lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, null); + lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, null, + aSNumber); this.currentBridges.put(fingerprint, entry); /* If this entry comes from a new status, update our global last * published time. */ diff --git a/src/org/torproject/onionoo/Node.java b/src/org/torproject/onionoo/Node.java index 0cadd38..35d81c3 100644 --- a/src/org/torproject/onionoo/Node.java +++ b/src/org/torproject/onionoo/Node.java @@ -54,7 +54,7 @@ public class Node { int dirPort, SortedSet<String> relayFlags, long consensusWeight, String countryCode, String hostName, long lastRdnsLookup, String defaultPolicy, String portList, long firstSeenMillis, - SortedMap<Long, Set<String>> lastAddresses) { + SortedMap<Long, Set<String>> lastAddresses, String aSNumber) { this.nickname = nickname; this.fingerprint = fingerprint; try { @@ -89,6 +89,7 @@ public class Node { this.portList = portList; this.firstSeenMillis = firstSeenMillis; this.lastAddresses = lastAddresses; + this.aSNumber = aSNumber; } public String getFingerprint() { return this.fingerprint; diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java index 0fa9174..19dc752 100644 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ b/src/org/torproject/onionoo/ResourceServlet.java @@ -45,18 +45,25 @@ public class ResourceServlet extends HttpServlet { long summaryFileLastModified = -1L; boolean readSummaryFile = false; private String relaysPublishedString, bridgesPublishedString; - private List<String> relaysByConsensusWeight = new ArrayList<String>(); - private Map<String, String> - relayFingerprintSummaryLines = new HashMap<String, String>(), - bridgeFingerprintSummaryLines = new HashMap<String, String>(); - private Map<String, Set<String>> relaysByCountryCode = - new HashMap<String, Set<String>>(); + private List<String> relaysByConsensusWeight = null; + private Map<String, String> relayFingerprintSummaryLines = null, + bridgeFingerprintSummaryLines = null; + private Map<String, Set<String>> relaysByCountryCode = null, + relaysByASNumber = null, relaysByFlag = null; private void readSummaryFile() { if (!summaryFile.exists()) { readSummaryFile = false; return; } if (summaryFile.lastModified() > this.summaryFileLastModified) { + List<String> relaysByConsensusWeight = new ArrayList<String>(); + Map<String, String> + relayFingerprintSummaryLines = new HashMap<String, String>(), + bridgeFingerprintSummaryLines = new HashMap<String, String>(); + Map<String, Set<String>> + relaysByCountryCode = new HashMap<String, Set<String>>(), + relaysByASNumber = new HashMap<String, Set<String>>(), + relaysByFlag = new HashMap<String, Set<String>>(); CurrentNodes cn = new CurrentNodes(); cn.readRelaySearchDataFile(this.summaryFile); cn.setRelayRunningBits(); @@ -74,38 +81,57 @@ public class ResourceServlet extends HttpServlet { String hashedFingerprint = entry.getHashedFingerprint(). toUpperCase(); String line = this.formatRelaySummaryLine(entry); - this.relayFingerprintSummaryLines.put(fingerprint, line); - this.relayFingerprintSummaryLines.put(hashedFingerprint, line); + relayFingerprintSummaryLines.put(fingerprint, line); + relayFingerprintSummaryLines.put(hashedFingerprint, line); long consensusWeight = entry.getConsensusWeight(); orderRelaysByConsensusWeight.add(String.format("%020d %s", consensusWeight, fingerprint)); orderRelaysByConsensusWeight.add(String.format("%020d %s", consensusWeight, hashedFingerprint)); - String countryCode = entry.getCountryCode(); - if (countryCode == null) { - countryCode = "??"; + if (entry.getCountryCode() != null) { + String countryCode = entry.getCountryCode(); + if (!relaysByCountryCode.containsKey(countryCode)) { + relaysByCountryCode.put(countryCode, new HashSet<String>()); + } + relaysByCountryCode.get(countryCode).add(fingerprint); + relaysByCountryCode.get(countryCode).add(hashedFingerprint); + } + if (entry.getASNumber() != null) { + String aSNumber = entry.getASNumber(); + if (!relaysByASNumber.containsKey(aSNumber)) { + relaysByASNumber.put(aSNumber, new HashSet<String>()); + } + relaysByASNumber.get(aSNumber).add(fingerprint); + relaysByASNumber.get(aSNumber).add(hashedFingerprint); } - if (!this.relaysByCountryCode.containsKey(countryCode)) { - this.relaysByCountryCode.put(countryCode, - new HashSet<String>()); + for (String flag : entry.getRelayFlags()) { + String flagLowerCase = flag.toLowerCase(); + if (!relaysByFlag.containsKey(flagLowerCase)) { + relaysByFlag.put(flagLowerCase, new HashSet<String>()); + } + relaysByFlag.get(flagLowerCase).add(fingerprint); + relaysByFlag.get(flagLowerCase).add(hashedFingerprint); } - this.relaysByCountryCode.get(countryCode).add(fingerprint); - this.relaysByCountryCode.get(countryCode).add(hashedFingerprint); } Collections.sort(orderRelaysByConsensusWeight); - this.relaysByConsensusWeight = new ArrayList<String>(); + relaysByConsensusWeight = new ArrayList<String>(); for (String relay : orderRelaysByConsensusWeight) { - this.relaysByConsensusWeight.add(relay.split(" ")[1]); + relaysByConsensusWeight.add(relay.split(" ")[1]); } for (Node entry : cn.getCurrentBridges().values()) { String hashedFingerprint = entry.getFingerprint().toUpperCase(); String hashedHashedFingerprint = entry.getHashedFingerprint(). toUpperCase(); String line = this.formatBridgeSummaryLine(entry); - this.bridgeFingerprintSummaryLines.put(hashedFingerprint, line); - this.bridgeFingerprintSummaryLines.put(hashedHashedFingerprint, - line); + bridgeFingerprintSummaryLines.put(hashedFingerprint, line); + bridgeFingerprintSummaryLines.put(hashedHashedFingerprint, line); } + this.relaysByConsensusWeight = relaysByConsensusWeight; + this.relayFingerprintSummaryLines = relayFingerprintSummaryLines; + this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines; + this.relaysByCountryCode = relaysByCountryCode; + this.relaysByASNumber = relaysByASNumber; + this.relaysByFlag = relaysByFlag; } this.summaryFileLastModified = summaryFile.lastModified(); this.readSummaryFile = true; @@ -185,7 +211,7 @@ public class ResourceServlet extends HttpServlet { /* Make sure that the request doesn't contain any unknown * parameters. */ Set<String> knownParameters = new HashSet<String>(Arrays.asList( - "type,running,search,lookup,country,order,limit,offset". + "type,running,search,lookup,country,as,flag,order,limit,offset". split(","))); for (String parameterKey : parameterMap.keySet()) { if (!knownParameters.contains(parameterKey)) { @@ -253,6 +279,25 @@ public class ResourceServlet extends HttpServlet { this.filterByCountryCode(filteredRelays, filteredBridges, countryCodeParameter); } + if (parameterMap.containsKey("as")) { + String aSNumberParameter = this.parseASNumberParameter( + parameterMap.get("as")); + if (aSNumberParameter == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + this.filterByASNumber(filteredRelays, filteredBridges, + aSNumberParameter); + } + if (parameterMap.containsKey("flag")) { + String flagParameter = this.parseFlagParameter( + parameterMap.get("flag")); + if (flagParameter == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + this.filterByFlag(filteredRelays, filteredBridges, flagParameter); + } /* Re-order and limit results. */ List<String> orderedRelays = new ArrayList<String>(); @@ -390,6 +435,24 @@ public class ResourceServlet extends HttpServlet { return parameter; } + private static Pattern aSNumberParameterPattern = + Pattern.compile("^[asAS]{0,2}[0-9]{1,10}$"); + private String parseASNumberParameter(String parameter) { + if (!aSNumberParameterPattern.matcher(parameter).matches()) { + return null; + } + return parameter; + } + + private static Pattern flagPattern = + Pattern.compile("^[a-zA-Z]{1,20}$"); + private String parseFlagParameter(String parameter) { + if (!flagPattern.matcher(parameter).matches()) { + return null; + } + return parameter; + } + private void filterByType(Map<String, String> filteredRelays, Map<String, String> filteredBridges, boolean relaysRequested) { if (relaysRequested) { @@ -540,6 +603,52 @@ public class ResourceServlet extends HttpServlet { filteredBridges.clear(); } + private void filterByASNumber(Map<String, String> filteredRelays, + Map<String, String> filteredBridges, String aSNumberParameter) { + String aSNumber = aSNumberParameter.toUpperCase(); + if (!aSNumber.startsWith("AS")) { + aSNumber = "AS" + aSNumber; + } + if (!this.relaysByASNumber.containsKey(aSNumber)) { + filteredRelays.clear(); + } else { + Set<String> relaysWithASNumber = + this.relaysByASNumber.get(aSNumber); + Set<String> removeRelays = new HashSet<String>(); + for (Map.Entry<String, String> e : filteredRelays.entrySet()) { + String fingerprint = e.getKey(); + if (!relaysWithASNumber.contains(fingerprint)) { + removeRelays.add(fingerprint); + } + } + for (String fingerprint : removeRelays) { + filteredRelays.remove(fingerprint); + } + } + filteredBridges.clear(); + } + + private void filterByFlag(Map<String, String> filteredRelays, + Map<String, String> filteredBridges, String flagParameter) { + String flag = flagParameter.toLowerCase(); + if (!this.relaysByFlag.containsKey(flag)) { + filteredRelays.clear(); + } else { + Set<String> relaysWithFlag = this.relaysByFlag.get(flag); + Set<String> removeRelays = new HashSet<String>(); + for (Map.Entry<String, String> e : filteredRelays.entrySet()) { + String fingerprint = e.getKey(); + if (!relaysWithFlag.contains(fingerprint)) { + removeRelays.add(fingerprint); + } + } + for (String fingerprint : removeRelays) { + filteredRelays.remove(fingerprint); + } + } + filteredBridges.clear(); + } + private void writeRelays(List<String> relays, PrintWriter pw, String resourceType) { pw.write("{\"relays_published\":\"" + this.relaysPublishedString diff --git a/web/index.html b/web/index.html index e3bbdb7..152cec6 100755 --- a/web/index.html +++ b/web/index.html @@ -660,6 +660,22 @@ Lookups are case-insensitive. given country as identified by a two-letter country code. Filtering by country code is case-insensitive. </td></tr> +<tr><td><b><font color="blue">as</font></b></td> +<td>Return only relays which are located in the +given autonomous system (AS) as identified by the AS number (with or +without preceding "AS" part). +Filtering by AS number is case-insensitive. +<font color="blue">Added on April 9.</font> +</td></tr> +<tr><td><b><font color="blue">flag</font></b></td> +<td>Return only relays or bridges which have the +given relay flag assigned by the directory authorities or the bridge +authority. +Note that if the flag parameter is specified more than once, only the +first parameter value will be considered. +Filtering by flag is case-insensitive. +<font color="blue">Added on April 9.</font> +</td></tr> </table> <p>Relay and/or bridge documents in the response can be ordered and limited by providing further parameters.