November 2017 - tor-commits - lists.torproject.org

[onionoo/master] Add "unreachable_or_addresses" field.
by karsten＠torproject.org 17 Nov '17

17 Nov '17

commit 7aca1ca996bcfd75a275a799fd42a29c8cb0080b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 18 09:20:33 2017 +0200 Add "unreachable_or_addresses" field. Implements #21637. --- CHANGELOG.md | 2 ++ src/main/java/org/torproject/onionoo/docs/DetailsDocument.java | 10 ++++++++++ src/main/java/org/torproject/onionoo/docs/DetailsStatus.java | 10 ++++++++++ .../torproject/onionoo/updater/… [View More]NodeDetailsStatusUpdater.java | 1 + .../org/torproject/onionoo/writer/DetailsDocumentWriter.java | 10 ++++++++++ 5 files changed, 33 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76243ef..c48075e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ - Skip unrecognized descriptors when importing archives rather than aborting the entire import. - Add new "host_name" parameter to filter by host name. + - Add new "unreachable_or_addresses" field with declared but + unreachable OR addresses. # Changes in version 4.2-1.6.1 - 2017-10-26 diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index 97fc703..bde3cae 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -473,5 +473,15 @@ public class DetailsDocument extends Document { public Boolean getMeasured() { return this.measured; } + + private List<String> unreachable_or_addresses; + + public void setUnreachableOrAddresses(List<String> unreachableOrAddresses) { + this.unreachable_or_addresses = unreachableOrAddresses; + } + + public List<String> getUnreachableOrAddresses() { + return this.unreachable_or_addresses; + } } diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index faec117..77ff75f 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -530,5 +530,15 @@ public class DetailsStatus extends Document { public String getHostName() { return unescapeJson(this.host_name); } + + private List<String> advertised_or_addresses; + + public void setAdvertisedOrAddresses(List<String> advertisedOrAddresses) { + this.advertised_or_addresses = advertisedOrAddresses; + } + + public List<String> getAdvertisedOrAddresses() { + return this.advertised_or_addresses; + } } diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 73e826a..fa3f3af 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -199,6 +199,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } detailsStatus.setHibernating(descriptor.isHibernating() ? true : null); + detailsStatus.setAdvertisedOrAddresses(descriptor.getOrAddresses()); this.documentStore.store(detailsStatus, fingerprint); } diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index e3f638d..9538cba 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -155,6 +155,16 @@ public class DetailsDocumentWriter implements DocumentWriter { exitAddressesWithoutOrAddresses)); } detailsDocument.setMeasured(detailsStatus.getMeasured()); + List<String> unreachableOrAddresses = new ArrayList<>(); + if (null != detailsStatus.getAdvertisedOrAddresses()) { + unreachableOrAddresses.addAll(detailsStatus.getAdvertisedOrAddresses()); + } + if (null != detailsStatus.getOrAddressesAndPorts()) { + unreachableOrAddresses.removeAll(detailsStatus.getOrAddressesAndPorts()); + } + if (!unreachableOrAddresses.isEmpty()) { + detailsDocument.setUnreachableOrAddresses(unreachableOrAddresses); + } this.documentStore.store(detailsDocument, fingerprint); } [View Less]

1 0

[onionoo/master] Use full variable names in SummaryDocument.
by karsten＠torproject.org 17 Nov '17

17 Nov '17

commit 0c9e9b812f2e068cae896e823c92a20c829f3519 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Oct 30 17:25:33 2017 +0100 Use full variable names in SummaryDocument. --- .../torproject/onionoo/docs/SummaryDocument.java | 147 +++++++++++++-------- 1 file changed, 90 insertions(+), 57 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index f33100d..cca5ab9 … [View More]100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -3,6 +3,8 @@ package org.torproject.onionoo.docs; +import com.google.gson.annotations.Expose; +import com.google.gson.annotations.SerializedName; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Hex; @@ -16,20 +18,23 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.regex.Pattern; -@SuppressWarnings("checkstyle:membername") public class SummaryDocument extends Document { - private boolean t; + @Expose + @SerializedName("t") + private boolean isRelay; public void setRelay(boolean isRelay) { - this.t = isRelay; + this.isRelay = isRelay; } public boolean isRelay() { - return this.t; + return this.isRelay; } - private String f; + @Expose + @SerializedName("f") + private String fingerprint; /** Sets the fingerprint to the given 40 hex characters and clears * SHA1-hashed and base64 fingerprints, so that they are re-computed at @@ -42,14 +47,14 @@ public class SummaryDocument extends Document { + "' is not a valid fingerprint."); } } - this.f = fingerprint; + this.fingerprint = fingerprint; this.hashedFingerprint = null; this.base64Fingerprint = null; this.fingerprintSortedHexBlocks = null; } public String getFingerprint() { - return this.f; + return this.fingerprint; } private transient String hashedFingerprint = null; @@ -57,10 +62,10 @@ public class SummaryDocument extends Document { /** Returns the SHA1-hashed fingerprint, or <code>null</code> if no * fingerprint is set. */ public String getHashedFingerprint() { - if (this.hashedFingerprint == null && this.f != null) { + if (this.hashedFingerprint == null && this.fingerprint != null) { try { this.hashedFingerprint = DigestUtils.sha1Hex(Hex.decodeHex( - this.f.toCharArray())).toUpperCase(); + this.fingerprint.toCharArray())).toUpperCase(); } catch (DecoderException e) { /* Format tested in setFingerprint(). */ } @@ -73,10 +78,10 @@ public class SummaryDocument extends Document { /** Returns the base64-encoded fingerprint, or <code>null</code> if no * fingerprint is set. */ public String getBase64Fingerprint() { - if (this.base64Fingerprint == null && this.f != null) { + if (this.base64Fingerprint == null && this.fingerprint != null) { try { this.base64Fingerprint = Base64.encodeBase64String(Hex.decodeHex( - this.f.toCharArray())).replaceAll("=", ""); + this.fingerprint.toCharArray())).replaceAll("=", ""); } catch (DecoderException e) { /* Format tested in setFingerprint(). */ } @@ -90,8 +95,8 @@ public class SummaryDocument extends Document { * characters from the fingerprint, or <code>null</code> if no * fingerprint is set. */ public String[] getFingerprintSortedHexBlocks() { - if (this.fingerprintSortedHexBlocks == null && this.f != null) { - String fingerprint = this.f.toUpperCase(); + if (this.fingerprintSortedHexBlocks == null && this.fingerprint != null) { + String fingerprint = this.fingerprint.toUpperCase(); String[] fingerprintSortedHexBlocks = new String[fingerprint.length() / 4]; for (int i = 0; i < fingerprint.length(); i += 4) { @@ -104,29 +109,33 @@ public class SummaryDocument extends Document { return this.fingerprintSortedHexBlocks; } - private String n; + @Expose + @SerializedName("n") + private String nickname; @SuppressWarnings("checkstyle:javadocmethod") public void setNickname(String nickname) { if (nickname == null || nickname.equals("Unnamed")) { - this.n = null; + this.nickname = null; } else { - this.n = nickname; + this.nickname = nickname; } } public String getNickname() { - return this.n == null ? "Unnamed" : this.n; + return this.nickname == null ? "Unnamed" : this.nickname; } - private String[] ad; + @Expose + @SerializedName("ad") + private String[] addresses; public void setAddresses(List<String> addresses) { - this.ad = this.collectionToStringArray(addresses); + this.addresses = this.collectionToStringArray(addresses); } public List<String> getAddresses() { - return this.stringArrayToList(this.ad); + return this.stringArrayToList(this.addresses); } private String[] collectionToStringArray( @@ -160,133 +169,157 @@ public class SummaryDocument extends Document { return sortedSet; } - private String cc; + @Expose + @SerializedName("cc") + private String countryCode; public void setCountryCode(String countryCode) { - this.cc = countryCode; + this.countryCode = countryCode; } public String getCountryCode() { - return this.cc; + return this.countryCode; } - private String as; + @Expose + @SerializedName("as") + private String asNumber; public void setAsNumber(String asNumber) { - this.as = asNumber; + this.asNumber = asNumber; } public String getAsNumber() { - return this.as; + return this.asNumber; } - private String fs; + @Expose + @SerializedName("fs") + private String firstSeenMillis; public void setFirstSeenMillis(long firstSeenMillis) { - this.fs = DateTimeHelper.format(firstSeenMillis); + this.firstSeenMillis = DateTimeHelper.format(firstSeenMillis); } public long getFirstSeenMillis() { - return DateTimeHelper.parse(this.fs); + return DateTimeHelper.parse(this.firstSeenMillis); } - private String ls; + @Expose + @SerializedName("ls") + private String lastSeenMillis; public void setLastSeenMillis(long lastSeenMillis) { - this.ls = DateTimeHelper.format(lastSeenMillis); + this.lastSeenMillis = DateTimeHelper.format(lastSeenMillis); } public long getLastSeenMillis() { - return DateTimeHelper.parse(this.ls); + return DateTimeHelper.parse(this.lastSeenMillis); } - private String[] rf; + @Expose + @SerializedName("rf") + private String[] relayFlags; public void setRelayFlags(SortedSet<String> relayFlags) { - this.rf = this.collectionToStringArray(relayFlags); + this.relayFlags = this.collectionToStringArray(relayFlags); } public SortedSet<String> getRelayFlags() { - return this.stringArrayToSortedSet(this.rf); + return this.stringArrayToSortedSet(this.relayFlags); } - private long cw; + @Expose + @SerializedName("cw") + private long consensusWeight; public void setConsensusWeight(long consensusWeight) { - this.cw = consensusWeight; + this.consensusWeight = consensusWeight; } public long getConsensusWeight() { - return this.cw; + return this.consensusWeight; } - private boolean r; + @Expose + @SerializedName("r") + private boolean running; public void setRunning(boolean isRunning) { - this.r = isRunning; + this.running = isRunning; } public boolean isRunning() { - return this.r; + return this.running; } - private String c; + @Expose + @SerializedName("c") + private String contact; @SuppressWarnings("checkstyle:javadocmethod") public void setContact(String contact) { if (contact != null && contact.length() == 0) { - this.c = null; + this.contact = null; } else { - this.c = contact; + this.contact = contact; } } public String getContact() { - return this.c; + return this.contact; } /* This attribute can go away once all Onionoo services had their hourly * updater write effective families to summary documents at least once. * Remove this code after September 8, 2015. */ - private String[] ff; + @Expose + @SerializedName("ff") + private String[] familyFingerprints; public void setFamilyFingerprints( SortedSet<String> familyFingerprints) { - this.ff = this.collectionToStringArray(familyFingerprints); + this.familyFingerprints = this.collectionToStringArray(familyFingerprints); } public SortedSet<String> getFamilyFingerprints() { - return this.stringArrayToSortedSet(this.ff); + return this.stringArrayToSortedSet(this.familyFingerprints); } - private String[] ef; + @Expose + @SerializedName("ef") + private String[] effectiveFamily; public void setEffectiveFamily(SortedSet<String> effectiveFamily) { - this.ef = this.collectionToStringArray(effectiveFamily); + this.effectiveFamily = this.collectionToStringArray(effectiveFamily); } public SortedSet<String> getEffectiveFamily() { - return this.stringArrayToSortedSet(this.ef); + return this.stringArrayToSortedSet(this.effectiveFamily); } - private String v; + @Expose + @SerializedName("v") + private String version; public void setVersion(String version) { - this.v = version; + this.version = version; } public String getVersion() { - return this.v; + return this.version; } - private String h; + @Expose + @SerializedName("h") + private String hostName; public void setHostName(String hostName) { - this.h = hostName; + this.hostName = hostName; } public String getHostName() { - return this.h; + return this.hostName; } /* The familyFingerprints parameter can go away after September 8, 2015. [View Less]

1 0

[onionoo/master] Add new "host_name" parameter to filter by host name.
by karsten＠torproject.org 17 Nov '17

17 Nov '17

commit d5d0edc9aab3d301b228c2b177daf172d93827b5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Oct 17 14:48:26 2017 +0200 Add new "host_name" parameter to filter by host name. Implements #16553. --- CHANGELOG.md | 1 + .../org/torproject/onionoo/docs/DocumentStore.java | 3 +- .../org/torproject/onionoo/docs/NodeStatus.java | 15 ++++ .../torproject/onionoo/docs/SummaryDocument.java | 13 +++- .../org/torproject/… [View More]onionoo/server/NodeIndex.java | 10 +++ .../org/torproject/onionoo/server/NodeIndexer.java | 11 +++ .../torproject/onionoo/server/RequestHandler.java | 26 +++++++ .../torproject/onionoo/server/ResourceServlet.java | 22 +++++- .../onionoo/updater/NodeDetailsStatusUpdater.java | 1 + .../onionoo/writer/SummaryDocumentWriter.java | 4 +- .../onionoo/docs/SummaryDocumentTest.java | 3 +- .../onionoo/server/ResourceServletTest.java | 79 ++++++++++++++++++++-- .../server/SummaryDocumentComparatorTest.java | 3 +- 13 files changed, 179 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12b0741..76243ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Support quoted qualified search terms. - Skip unrecognized descriptors when importing archives rather than aborting the entire import. + - Add new "host_name" parameter to filter by host name. # Changes in version 4.2-1.6.1 - 2017-10-26 diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 7819d0f..57d4165 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -439,10 +439,11 @@ public class DocumentStore { long lastSeenMillis = -1L; long consensusWeight = -1L; long firstSeenMillis = -1L; + String hostName = null; SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - asNumber, contact, family, family, version); + asNumber, contact, family, family, version, hostName); return summaryDocument; } diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 24546dd..3271ccc 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -368,6 +368,16 @@ public class NodeStatus extends Document { /* Reverse DNS lookup result */ + private String hostName; + + public void setHostName(String hostName) { + this.hostName = hostName; + } + + public String getHostName() { + return this.hostName; + } + private long lastRdnsLookup = -1L; public void setLastRdnsLookup(long lastRdnsLookup) { @@ -561,6 +571,9 @@ public class NodeStatus extends Document { if (parts.length >= 24 && !parts[23].isEmpty()) { nodeStatus.setVersion(parts[23]); } + if (parts.length >= 25 && !parts[24].isEmpty()) { + nodeStatus.setHostName(parts[24]); + } return nodeStatus; } catch (NumberFormatException e) { log.error("Number format exception while parsing node " @@ -627,6 +640,8 @@ public class NodeStatus extends Document { .append(StringUtils.join(this.getIndirectFamily(), ";")); sb.append("\t") .append((this.getVersion() != null ? this.getVersion() : "")); + sb.append("\t") + .append((this.getHostName() != null ? this.getHostName() : "")); return sb.toString(); } } diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index 527f91d..f33100d 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -279,6 +279,16 @@ public class SummaryDocument extends Document { return this.v; } + private String h; + + public void setHostName(String hostName) { + this.h = hostName; + } + + public String getHostName() { + return this.h; + } + /* The familyFingerprints parameter can go away after September 8, 2015. * See above. */ /** Instantiates a summary document with all given properties. */ @@ -287,7 +297,7 @@ public class SummaryDocument extends Document { boolean running, SortedSet<String> relayFlags, long consensusWeight, String countryCode, long firstSeenMillis, String asNumber, String contact, SortedSet<String> familyFingerprints, - SortedSet<String> effectiveFamily, String version) { + SortedSet<String> effectiveFamily, String version, String hostName) { this.setRelay(isRelay); this.setNickname(nickname); this.setFingerprint(fingerprint); @@ -303,6 +313,7 @@ public class SummaryDocument extends Document { this.setFamilyFingerprints(familyFingerprints); this.setEffectiveFamily(effectiveFamily); this.setVersion(version); + this.setHostName(hostName); } } diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java index dd4bd4d..126cd5c 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java @@ -179,5 +179,15 @@ class NodeIndex { public Map<String, Set<String>> getRelaysByVersion() { return this.relaysByVersion; } + + private Map<String, Set<String>> relaysByHostName; + + public void setRelaysByHostName(Map<String, Set<String>> relaysByHostName) { + this.relaysByHostName = relaysByHostName; + } + + public Map<String, Set<String>> getRelaysByHostName() { + return this.relaysByHostName; + } } diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index 5411d0e..d609f63 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -154,6 +154,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { Map<String, Set<String>> newRelaysByContact = new HashMap<>(); Map<String, Set<String>> newRelaysByFamily = new HashMap<>(); Map<String, Set<String>> newRelaysByVersion = new HashMap<>(); + Map<String, Set<String>> newRelaysByHostName = new HashMap<>(); SortedMap<Integer, Set<String>> newRelaysByFirstSeenDays = new TreeMap<>(); SortedMap<Integer, Set<String>> newBridgesByFirstSeenDays = new TreeMap<>(); SortedMap<Integer, Set<String>> newRelaysByLastSeenDays = new TreeMap<>(); @@ -258,6 +259,15 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByVersion.get(version).add(fingerprint); newRelaysByVersion.get(version).add(hashedFingerprint); } + String hostName = entry.getHostName(); + if (null != hostName) { + String hostNameLowerCase = hostName.toLowerCase(); + if (!newRelaysByHostName.containsKey(hostNameLowerCase)) { + newRelaysByHostName.put(hostNameLowerCase, new HashSet<>()); + } + newRelaysByHostName.get(hostNameLowerCase).add(fingerprint); + newRelaysByHostName.get(hostNameLowerCase).add(hashedFingerprint); + } } /* This loop can go away once all Onionoo services had their hourly * updater write effective families to summary documents at least @@ -332,6 +342,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { newNodeIndex.setRelaysPublishedMillis(relaysLastValidAfterMillis); newNodeIndex.setBridgesPublishedMillis(bridgesLastPublishedMillis); newNodeIndex.setRelaysByVersion(newRelaysByVersion); + newNodeIndex.setRelaysByHostName(newRelaysByHostName); synchronized (this) { this.lastIndexed = updateStatusMillis; this.latestNodeIndex = newNodeIndex; diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java index 46c2f54..23af60b 100644 --- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java +++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java @@ -97,6 +97,12 @@ public class RequestHandler { this.version = version; } + private String hostName; + + public void setHostName(String hostName) { + this.hostName = hostName; + } + private String[] order; public void setOrder(String[] order) { @@ -165,6 +171,7 @@ public class RequestHandler { this.filterByContact(); this.filterByFamily(); this.filterByVersion(); + this.filterByHostName(); this.order(); this.offset(); this.limit(); @@ -537,6 +544,25 @@ public class RequestHandler { this.filteredBridges.clear(); } + private void filterByHostName() { + if (this.hostName == null) { + /* Not filtering by host name. */ + return; + } + String hostName = this.hostName.toLowerCase(); + Set<String> removeRelays = new HashSet<>(this.filteredRelays.keySet()); + for (Map.Entry<String, Set<String>> e : + this.nodeIndex.getRelaysByHostName().entrySet()) { + if (e.getKey().endsWith(hostName)) { + removeRelays.removeAll(e.getValue()); + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + this.filteredBridges.clear(); + } + private void order() { List<SummaryDocument> uniqueRelays = new ArrayList<>(); List<SummaryDocument> uniqueBridges = new ArrayList<>(); diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java index 2fff913..f1521e2 100644 --- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java +++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java @@ -68,7 +68,7 @@ public class ResourceServlet extends HttpServlet { private static Set<String> knownParameters = new HashSet<>( Arrays.asList(("type,running,search,lookup,fingerprint,country,as," + "flag,first_seen_days,last_seen_days,contact,order,limit," - + "offset,fields,family,version").split(","))); + + "offset,fields,family,version,host_name").split(","))); private static Set<String> illegalSearchQualifiers = new HashSet<>(Arrays.asList(("search,fingerprint,order,limit," @@ -292,6 +292,15 @@ public class ResourceServlet extends HttpServlet { } rh.setVersion(versionParameter); } + if (parameterMap.containsKey("host_name")) { + String hostNameParameter = this.parseHostNameParameter( + parameterMap.get("host_name")); + if (null == hostNameParameter) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setHostName(hostNameParameter); + } if (parameterMap.containsKey("order")) { String[] order = this.parseOrderParameter(parameterMap.get("order")); if (order == null) { @@ -584,5 +593,16 @@ public class ResourceServlet extends HttpServlet { } return parameter; } + + private static Pattern hostNameParameterPattern = + Pattern.compile("^[0-9A-Za-z_\\.\\-]+$"); + + private String parseHostNameParameter(String parameter) { + if (!hostNameParameterPattern.matcher(parameter).matches()) { + /* Host name contains illegal character(s). */ + return null; + } + return parameter; + } } diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 04a04fa..73e826a 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -849,6 +849,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, if (this.rdnsLookupResults.containsKey(fingerprint)) { String hostName = this.rdnsLookupResults.get(fingerprint); detailsStatus.setHostName(hostName); + nodeStatus.setHostName(hostName); nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); } diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index 4364aad..faf54fa 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -92,10 +92,12 @@ public class SummaryDocumentWriter implements DocumentWriter { SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily(); String nickname = nodeStatus.getNickname(); String version = nodeStatus.getVersion(); + String hostName = nodeStatus.getHostName(); SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - asNumber, contact, declaredFamily, effectiveFamily, version); + asNumber, contact, declaredFamily, effectiveFamily, version, + hostName); if (this.documentStore.store(summaryDocument, fingerprint)) { this.writtenDocuments++; } diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java index 9d7625a..8c45ed2 100644 --- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java +++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java @@ -26,7 +26,8 @@ public class SummaryDocumentTest { new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( - new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null); + new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, + null); } @Test() diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java index e41eecc..2720f7a 100644 --- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java +++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java @@ -143,7 +143,7 @@ public class ResourceServletTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), - "0.2.3.25"); + "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de"); org.torproject.onionoo.docs.SummaryDocument relayFerrari458 = new org.torproject.onionoo.docs.SummaryDocument(true, "Ferrari458", "001C13B3A55A71B977CA65EC85539D79C653A3FC", Arrays.asList( @@ -155,7 +155,8 @@ public class ResourceServletTest { new TreeSet<String>(Arrays.asList(new String[] { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), new TreeSet<>(Arrays.asList(new String[] { - "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null); + "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null, + "c-68-38-171-200.hsd1.in.comcast.net"); this.relays = new TreeMap<>(); this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A", relayTorkaZ); @@ -171,7 +172,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830", "1024d/51e2a1c7 \"steven j. murdoch\" " + "<tor+steven.murdoch(a)cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>", - new TreeSet<String>(), new TreeSet<String>(), "0.2.3.25"); + new TreeSet<String>(), new TreeSet<String>(), "0.2.3.25", null); this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", relayTimMayTribute); this.bridges = new TreeMap<>(); @@ -182,7 +183,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-21 18:07:03"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, - null, null, null); + null, null, null, null); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", bridgeec2bridgercc7f31fe); org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed = @@ -192,7 +193,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-20 17:37:04"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, - null, null, null); + null, null, null, null); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", bridgeUnnamed); org.torproject.onionoo.docs.SummaryDocument bridgegummy = @@ -203,7 +204,7 @@ public class ResourceServletTest { new TreeSet<>(Arrays.asList(new String[] { "Running", "Valid" })), -1L, null, DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null, - null, null); + null, null, null); this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", bridgegummy); } @@ -1623,5 +1624,71 @@ public class ResourceServletTest { /* This is also correct when comparing strings. */ this.assertErrorStatusCode("/summary?version=*", 400); } + + @Test + public void testHostNameDe() { + this.assertSummaryDocument("/summary?host_name=de", 1, null, 0, null); + } + + @Test + public void testHostNameE() { + this.assertSummaryDocument("/summary?host_name=e", 1, null, 0, null); + } + + @Test + public void testHostNameDotDe() { + this.assertSummaryDocument("/summary?host_name=.de", 1, null, 0, null); + } + + @Test + public void testHostNameOnlineDe() { + this.assertSummaryDocument("/summary?host_name=online.de", 1, null, 0, + null); + } + + @Test + public void testHostNameOnlineDeSomeCapitalized() { + this.assertSummaryDocument("/summary?host_name=onLiNe.dE", 1, null, 0, + null); + } + + @Test + public void testHostNameOnline() { + this.assertSummaryDocument("/summary?host_name=online", 0, null, 0, null); + } + + @Test + public void testHostNameTorkaZFull() { + this.assertSummaryDocument( + "/summary?host_name=ppp-62-216-201-221.dynamic.mnet-online.de", + 1, null, 0, null); + } + + @Test + public void testHostNameTorkaZSub() { + this.assertSummaryDocument( + "/summary?host_name=sub.ppp-62-216-201-221.dynamic.mnet-online.de", + 0, null, 0, null); + } + + @Test + public void testHostNameNet() { + this.assertSummaryDocument("/summary?host_name=net", 1, null, 0, null); + } + + @Test + public void testHostNameCom() { + this.assertSummaryDocument("/summary?host_name=com", 0, null, 0, null); + } + + @Test + public void testHostNameDollar() { + this.assertErrorStatusCode("/summary?host_name=$", 400); + } + + @Test + public void testHostNameUmlaut() { + this.assertErrorStatusCode("/summary?host_name=äöü", 400); + } } diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java index fb3f5b3..38d94fa 100644 --- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java +++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java @@ -40,7 +40,8 @@ public class SummaryDocumentComparatorTest { new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( - new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null); + new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, + null); } /** Some values for running all comparison types. */ [View Less]

1 0

[metrics-web/master] Update news.json to version 193 of doc/MetricsTimeline.
by karsten＠torproject.org 17 Nov '17

17 Nov '17

commit 50ea265c16ff2224834187784cbe3af72c4836c9 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Nov 17 09:20:41 2017 +0100 Update news.json to version 193 of doc/MetricsTimeline. --- website/src/main/resources/etc/news.json | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/website/src/main/resources/etc/news.json b/website/src/main/resources/etc/news.json index e054182..704eac0 100644 --- a/website/src/main/resources/etc/news.json +++ … [View More]

1 0

[translation/tails-onioncircuits] Update translations for tails-onioncircuits
by translation＠torproject.org 16 Nov '17

16 Nov '17

commit d3ce515f2f927f1a64109a7197c04220b168f5ad Author: Translation commit bot <translation(a)torproject.org> Date: Thu Nov 16 21:49:04 2017 +0000 Update translations for tails-onioncircuits --- hi/onioncircuits.pot | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hi/onioncircuits.pot b/hi/onioncircuits.pot index 3af806873..983f8287b 100644 --- a/hi/onioncircuits.pot +++ b/hi/onioncircuits.pot @@ -8,7 +8,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n"… [View More]

1 0

[translation/tails-misc] Update translations for tails-misc
by translation＠torproject.org 16 Nov '17

16 Nov '17

commit cd7185c876206be4b3b02301667b79cb6182a7cd Author: Translation commit bot <translation(a)torproject.org> Date: Thu Nov 16 21:47:07 2017 +0000 Update translations for tails-misc --- hi.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hi.po b/hi.po index b3570748e..b357ac0fb 100644 --- a/hi.po +++ b/hi.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2017-09-13 20:10+0200\n" -"PO-… [View More]

1 0

[translation/mat-gui] Update translations for mat-gui
by translation＠torproject.org 16 Nov '17

16 Nov '17

commit d5376f6f7406e49eddb8bb036d2199e83e8477fd Author: Translation commit bot <translation(a)torproject.org> Date: Thu Nov 16 21:46:28 2017 +0000 Update translations for mat-gui --- hi.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hi.po b/hi.po index 7309f2e92..523f6df3a 100644 --- a/hi.po +++ b/hi.po @@ -10,7 +10,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2016-02-10 23:06+0100\n" -"PO-Revision-… [View More]

1 0

[translation/tails-misc] Update translations for tails-misc
by translation＠torproject.org 16 Nov '17

16 Nov '17

commit 42c1510548d31c3dfd1fb6c766e220311b65ca58 Author: Translation commit bot <translation(a)torproject.org> Date: Thu Nov 16 21:17:09 2017 +0000 Update translations for tails-misc --- hi.po | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hi.po b/hi.po index 70c28620e..b3570748e 100644 --- a/hi.po +++ b/hi.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2017-09-13 20:10+0200\n" -"PO-… [View More]

1 0

[translation/mat-gui] Update translations for mat-gui
by translation＠torproject.org 16 Nov '17

16 Nov '17

commit 55c2cad387c848fd3815ee37aba54e7d0492cdc3 Author: Translation commit bot <translation(a)torproject.org> Date: Thu Nov 16 21:16:30 2017 +0000 Update translations for mat-gui --- hi.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hi.po b/hi.po index bd1977e62..7309f2e92 100644 --- a/hi.po +++ b/hi.po @@ -10,7 +10,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2016-02-10 23:06+0100\n" -"PO-Revision-… [View More]

1 0

[webwml/staging] Move language switch to the top
by hiro＠torproject.org 16 Nov '17

16 Nov '17

commit b90c065d630276688f55b97ca26ab513b63aa58f Author: hiromipaw <hiro(a)torproject.org> Date: Thu Nov 16 21:03:12 2017 +0100 Move language switch to the top --- include/dlhead.wmi | 39 +++++++++++++++++++++++++++++++++++- include/foot.wmi | 52 +----------------------------------------------- include/perl-globals.wmi | 2 +- 3 files changed, 40 insertions(+), 53 deletions(-) diff --git a/include/dlhead.wmi b/include/dlhead.wmi index 15c35317..59794458 100644 --- … [View More]a/include/dlhead.wmi +++ b/include/dlhead.wmi @@ -114,7 +114,44 @@ docdir: $(DOCROOT)/; dir: $dir; base: $base -> $(DOCROOT)/$dir/$lang/$base.wml), } } }:> - </ul> + # LANGUAGE SWITCH CGI + <li> + # this is a cgi trampoline to bounce us to the right page + # alternately, if the client supports javascript we can redirect that way + # noscript does not block onclick but clients may have disabled javascript completely + # + # for this to work we need to know the relative path from the document root + # to the current directory that wml is in. + <form action="$(DOCROOT)/cgi-bin/languageswitch.cgi"> + <select name="Language" id="lang"> + <:{ + #import "perl-globals.wmi"; + use Cwd; + use Cwd 'abs_path'; + use File::Spec; + my $urlbase = File::Spec->abs2rel(getcwd(),abs_path("$(DOCROOT)")); + my $LANGUAGES; + my $page = $WML_SRC_FILENAME; + opendir(DIR, getcwd()) or die $!; + # take a look and see what language support we have + while (my $file = readdir(DIR)) { + if (-d $file) { + if (-e "$file/$page") { + my $url = $urlbase.'/'.$WML_SRC_BASENAME.'.html.'.$file; + my $url2js = $WML_SRC_BASENAME.'.html.'.$file; + print '<option value="'.$url.'" onclick="window.location=\''.$url2js.'\'">'.$LANGUAGES{$file}.'</option>'."\n" unless not defined $LANGUAGES{$file}; + } + } + } + closedir(DIR); + }:> + </select> + <input class="go" type="submit" name="submit" value="Go"> + </form> + </li> + # + </ul> + </div>  # end navigation generation diff --git a/include/foot.wmi b/include/foot.wmi index 12f30cea..85116848 100644 --- a/include/foot.wmi +++ b/include/foot.wmi @@ -65,57 +65,7 @@ </div>  -  - <div class="col wider"> - <h4>Languages</h4> - <: if (has_translations()) { :> - <p> - This page is also available in the following languages: - <: print list_translations() :>.<br /> - How to set <a href="http://www.debian.org/intro/cn#howtoset">the default document language</a>. - </p> - <: }; :> - </div> - -# LANGUAGE SWITCH CGI - <div class="col wider"> - <h4>Languages</h4> - # this is a cgi trampoline to bounce us to the right page - # alternately, if the client supports javascript we can redirect that way - # noscript does not block onclick but clients may have disabled javascript completely - # - # for this to work we need to know the relative path from the document root - # to the current directory that wml is in. - <form action="$(DOCROOT)/cgi-bin/languageswitch.cgi"> - <select name="Language" id="lang"> - <:{ - #import "perl-globals.wmi"; - use Cwd; - use Cwd 'abs_path'; - use File::Spec; - my $urlbase = File::Spec->abs2rel(getcwd(),abs_path("$(DOCROOT)")); - my $LANGUAGES; - my $page = $WML_SRC_FILENAME; - opendir(DIR, getcwd()) or die $!; - # take a look and see what language support we have - while (my $file = readdir(DIR)) { - if (-d $file) { - if (-e "$file/$page") { - my $url = $urlbase.'/'.$WML_SRC_BASENAME.'.html.'.$file; - my $url2js = $WML_SRC_BASENAME.'.html.'.$file; - print '<option value="'.$url.'" onclick="window.location=\''.$url2js.'\'">'.$LANGUAGES{$file}.'</option>'."\n" unless not defined $LANGUAGES{$file}; - } - } - } - closedir(DIR); - }:> - </select> - <input class="go" type="submit" name="submit" value="Go"> - </form> - <p>Questions on this? Visit <a href="http://www.debian.org/intro/cn#howtoset">how to set the default document language</a>.</p> - </div> -# - </div> +</div>  </div> diff --git a/include/perl-globals.wmi b/include/perl-globals.wmi index b639483d..7a5fd76b 100644 --- a/include/perl-globals.wmi +++ b/include/perl-globals.wmi @@ -10,7 +10,7 @@ 'da' => "Danish", 'de' => "Deutsch", 'en' => "English", - 'es' => "español", + 'es' => "Español", 'et' => "Estonian", 'fa' => "فارسی (Fārsī)", 'fi' => "suomi", [View Less]

1 0