November 2017 - tor-commits - lists.torproject.org

[onionoo/master] Add tests for "unreachable_or_addresses" field.
by karsten＠torproject.org 17 Nov '17

17 Nov '17

commit 550d60e01d4e44137739fc6d27c16863793d5a7f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Nov 16 21:33:56 2017 +0100 Add tests for "unreachable_or_addresses" field. --- .../onionoo/writer/DetailsDocumentWriterTest.java | 100 +++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/src/test/java/org/torproject/onionoo/writer/DetailsDocumentWriterTest.java b/src/test/java/org/torproject/onionoo/writer/DetailsDocumentWriterTest.java new file mode 100644 index 0000000..3356a9c --- /dev/null +++ b/src/test/java/org/torproject/onionoo/writer/DetailsDocumentWriterTest.java @@ -0,0 +1,100 @@ +package org.torproject.onionoo.writer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import org.torproject.onionoo.docs.DetailsDocument; +import org.torproject.onionoo.docs.DetailsStatus; +import org.torproject.onionoo.docs.DocumentStoreFactory; +import org.torproject.onionoo.docs.DummyDocumentStore; + +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; +import java.util.TreeSet; + +public class DetailsDocumentWriterTest { + + private DummyDocumentStore documentStore; + + @Before + public void createDummyDocumentStore() { + this.documentStore = new DummyDocumentStore(); + DocumentStoreFactory.setDocumentStore(this.documentStore); + } + + @Test + public void testNoDetailsStatuses() { + DetailsDocumentWriter writer = new DetailsDocumentWriter(); + writer.writeDocuments(); + assertEquals("Without statuses, no documents should be written.", 0, + this.documentStore.getPerformedStoreOperations()); + } + + private static final String GABELMOO_FINGERPRINT = + "F2044413DAC2E02E3D6BCF4735A19BCA1DE97281"; + + private static final String GABELMOO_OR_ADDRESS = + "[2001:638:a000:4140::ffff:189]:443]"; + + @Test + public void testAdvertisedAndReachableOrAddress() { + DetailsStatus status = new DetailsStatus(); + status.setRelay(true); + status.setAdvertisedOrAddresses(Arrays.asList(GABELMOO_OR_ADDRESS)); + status.setOrAddressesAndPorts(new TreeSet<>(Arrays.asList( + GABELMOO_OR_ADDRESS))); + this.documentStore.addDocument(status, GABELMOO_FINGERPRINT); + DetailsDocumentWriter writer = new DetailsDocumentWriter(); + writer.writeDocuments(); + assertEquals("One document should be written.", 1, + this.documentStore.getPerformedStoreOperations()); + DetailsDocument document = this.documentStore.getDocument( + DetailsDocument.class, GABELMOO_FINGERPRINT); + assertNotNull("There should be a document for the given fingerprint.", + document); + assertNull("Document should not contain any unreachable OR addresses.", + document.getUnreachableOrAddresses()); + } + + @Test + public void testUnadvertisedButSomehowReachableOrAddress() { + DetailsStatus status = new DetailsStatus(); + status.setRelay(true); + status.setOrAddressesAndPorts(new TreeSet<>(Arrays.asList( + GABELMOO_OR_ADDRESS))); + this.documentStore.addDocument(status, GABELMOO_FINGERPRINT); + DetailsDocumentWriter writer = new DetailsDocumentWriter(); + writer.writeDocuments(); + assertEquals("One document should be written.", 1, + this.documentStore.getPerformedStoreOperations()); + DetailsDocument document = this.documentStore.getDocument( + DetailsDocument.class, GABELMOO_FINGERPRINT); + assertNotNull("There should be a document for the given fingerprint.", + document); + assertNull("Document should not contain unreachable OR addresses.", + document.getUnreachableOrAddresses()); + } + + @Test + public void testAdvertisedButUnreachableOrAddress() { + DetailsStatus status = new DetailsStatus(); + status.setRelay(true); + status.setAdvertisedOrAddresses(Arrays.asList(GABELMOO_OR_ADDRESS)); + this.documentStore.addDocument(status, GABELMOO_FINGERPRINT); + DetailsDocumentWriter writer = new DetailsDocumentWriter(); + writer.writeDocuments(); + assertEquals("One document should be written.", 1, + this.documentStore.getPerformedStoreOperations()); + DetailsDocument document = this.documentStore.getDocument( + DetailsDocument.class, GABELMOO_FINGERPRINT); + assertNotNull("There should be a document for the given fingerprint.", + document); + assertEquals("Document should contain one unreachable OR address.", + Arrays.asList(GABELMOO_OR_ADDRESS), + document.getUnreachableOrAddresses()); + } +} +

1 0

[onionoo/master] Add "unreachable_or_addresses" field.
by karsten＠torproject.org 17 Nov '17

17 Nov '17

commit 7aca1ca996bcfd75a275a799fd42a29c8cb0080b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Oct 18 09:20:33 2017 +0200 Add "unreachable_or_addresses" field. Implements #21637. --- CHANGELOG.md | 2 ++ src/main/java/org/torproject/onionoo/docs/DetailsDocument.java | 10 ++++++++++ src/main/java/org/torproject/onionoo/docs/DetailsStatus.java | 10 ++++++++++ .../torproject/onionoo/updater/NodeDetailsStatusUpdater.java | 1 + .../org/torproject/onionoo/writer/DetailsDocumentWriter.java | 10 ++++++++++ 5 files changed, 33 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76243ef..c48075e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ - Skip unrecognized descriptors when importing archives rather than aborting the entire import. - Add new "host_name" parameter to filter by host name. + - Add new "unreachable_or_addresses" field with declared but + unreachable OR addresses. # Changes in version 4.2-1.6.1 - 2017-10-26 diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index 97fc703..bde3cae 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -473,5 +473,15 @@ public class DetailsDocument extends Document { public Boolean getMeasured() { return this.measured; } + + private List<String> unreachable_or_addresses; + + public void setUnreachableOrAddresses(List<String> unreachableOrAddresses) { + this.unreachable_or_addresses = unreachableOrAddresses; + } + + public List<String> getUnreachableOrAddresses() { + return this.unreachable_or_addresses; + } } diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index faec117..77ff75f 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -530,5 +530,15 @@ public class DetailsStatus extends Document { public String getHostName() { return unescapeJson(this.host_name); } + + private List<String> advertised_or_addresses; + + public void setAdvertisedOrAddresses(List<String> advertisedOrAddresses) { + this.advertised_or_addresses = advertisedOrAddresses; + } + + public List<String> getAdvertisedOrAddresses() { + return this.advertised_or_addresses; + } } diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 73e826a..fa3f3af 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -199,6 +199,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } detailsStatus.setHibernating(descriptor.isHibernating() ? true : null); + detailsStatus.setAdvertisedOrAddresses(descriptor.getOrAddresses()); this.documentStore.store(detailsStatus, fingerprint); } diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index e3f638d..9538cba 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -155,6 +155,16 @@ public class DetailsDocumentWriter implements DocumentWriter { exitAddressesWithoutOrAddresses)); } detailsDocument.setMeasured(detailsStatus.getMeasured()); + List<String> unreachableOrAddresses = new ArrayList<>(); + if (null != detailsStatus.getAdvertisedOrAddresses()) { + unreachableOrAddresses.addAll(detailsStatus.getAdvertisedOrAddresses()); + } + if (null != detailsStatus.getOrAddressesAndPorts()) { + unreachableOrAddresses.removeAll(detailsStatus.getOrAddressesAndPorts()); + } + if (!unreachableOrAddresses.isEmpty()) { + detailsDocument.setUnreachableOrAddresses(unreachableOrAddresses); + } this.documentStore.store(detailsDocument, fingerprint); }

1 0

[onionoo/master] Use full variable names in SummaryDocument.
by karsten＠torproject.org 17 Nov '17

17 Nov '17

commit 0c9e9b812f2e068cae896e823c92a20c829f3519 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Oct 30 17:25:33 2017 +0100 Use full variable names in SummaryDocument. --- .../torproject/onionoo/docs/SummaryDocument.java | 147 +++++++++++++-------- 1 file changed, 90 insertions(+), 57 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index f33100d..cca5ab9 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -3,6 +3,8 @@ package org.torproject.onionoo.docs; +import com.google.gson.annotations.Expose; +import com.google.gson.annotations.SerializedName; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Base64; import org.apache.commons.codec.binary.Hex; @@ -16,20 +18,23 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.regex.Pattern; -@SuppressWarnings("checkstyle:membername") public class SummaryDocument extends Document { - private boolean t; + @Expose + @SerializedName("t") + private boolean isRelay; public void setRelay(boolean isRelay) { - this.t = isRelay; + this.isRelay = isRelay; } public boolean isRelay() { - return this.t; + return this.isRelay; } - private String f; + @Expose + @SerializedName("f") + private String fingerprint; /** Sets the fingerprint to the given 40 hex characters and clears * SHA1-hashed and base64 fingerprints, so that they are re-computed at @@ -42,14 +47,14 @@ public class SummaryDocument extends Document { + "' is not a valid fingerprint."); } } - this.f = fingerprint; + this.fingerprint = fingerprint; this.hashedFingerprint = null; this.base64Fingerprint = null; this.fingerprintSortedHexBlocks = null; } public String getFingerprint() { - return this.f; + return this.fingerprint; } private transient String hashedFingerprint = null; @@ -57,10 +62,10 @@ public class SummaryDocument extends Document { /** Returns the SHA1-hashed fingerprint, or <code>null</code> if no * fingerprint is set. */ public String getHashedFingerprint() { - if (this.hashedFingerprint == null && this.f != null) { + if (this.hashedFingerprint == null && this.fingerprint != null) { try { this.hashedFingerprint = DigestUtils.sha1Hex(Hex.decodeHex( - this.f.toCharArray())).toUpperCase(); + this.fingerprint.toCharArray())).toUpperCase(); } catch (DecoderException e) { /* Format tested in setFingerprint(). */ } @@ -73,10 +78,10 @@ public class SummaryDocument extends Document { /** Returns the base64-encoded fingerprint, or <code>null</code> if no * fingerprint is set. */ public String getBase64Fingerprint() { - if (this.base64Fingerprint == null && this.f != null) { + if (this.base64Fingerprint == null && this.fingerprint != null) { try { this.base64Fingerprint = Base64.encodeBase64String(Hex.decodeHex( - this.f.toCharArray())).replaceAll("=", ""); + this.fingerprint.toCharArray())).replaceAll("=", ""); } catch (DecoderException e) { /* Format tested in setFingerprint(). */ } @@ -90,8 +95,8 @@ public class SummaryDocument extends Document { * characters from the fingerprint, or <code>null</code> if no * fingerprint is set. */ public String[] getFingerprintSortedHexBlocks() { - if (this.fingerprintSortedHexBlocks == null && this.f != null) { - String fingerprint = this.f.toUpperCase(); + if (this.fingerprintSortedHexBlocks == null && this.fingerprint != null) { + String fingerprint = this.fingerprint.toUpperCase(); String[] fingerprintSortedHexBlocks = new String[fingerprint.length() / 4]; for (int i = 0; i < fingerprint.length(); i += 4) { @@ -104,29 +109,33 @@ public class SummaryDocument extends Document { return this.fingerprintSortedHexBlocks; } - private String n; + @Expose + @SerializedName("n") + private String nickname; @SuppressWarnings("checkstyle:javadocmethod") public void setNickname(String nickname) { if (nickname == null || nickname.equals("Unnamed")) { - this.n = null; + this.nickname = null; } else { - this.n = nickname; + this.nickname = nickname; } } public String getNickname() { - return this.n == null ? "Unnamed" : this.n; + return this.nickname == null ? "Unnamed" : this.nickname; } - private String[] ad; + @Expose + @SerializedName("ad") + private String[] addresses; public void setAddresses(List<String> addresses) { - this.ad = this.collectionToStringArray(addresses); + this.addresses = this.collectionToStringArray(addresses); } public List<String> getAddresses() { - return this.stringArrayToList(this.ad); + return this.stringArrayToList(this.addresses); } private String[] collectionToStringArray( @@ -160,133 +169,157 @@ public class SummaryDocument extends Document { return sortedSet; } - private String cc; + @Expose + @SerializedName("cc") + private String countryCode; public void setCountryCode(String countryCode) { - this.cc = countryCode; + this.countryCode = countryCode; } public String getCountryCode() { - return this.cc; + return this.countryCode; } - private String as; + @Expose + @SerializedName("as") + private String asNumber; public void setAsNumber(String asNumber) { - this.as = asNumber; + this.asNumber = asNumber; } public String getAsNumber() { - return this.as; + return this.asNumber; } - private String fs; + @Expose + @SerializedName("fs") + private String firstSeenMillis; public void setFirstSeenMillis(long firstSeenMillis) { - this.fs = DateTimeHelper.format(firstSeenMillis); + this.firstSeenMillis = DateTimeHelper.format(firstSeenMillis); } public long getFirstSeenMillis() { - return DateTimeHelper.parse(this.fs); + return DateTimeHelper.parse(this.firstSeenMillis); } - private String ls; + @Expose + @SerializedName("ls") + private String lastSeenMillis; public void setLastSeenMillis(long lastSeenMillis) { - this.ls = DateTimeHelper.format(lastSeenMillis); + this.lastSeenMillis = DateTimeHelper.format(lastSeenMillis); } public long getLastSeenMillis() { - return DateTimeHelper.parse(this.ls); + return DateTimeHelper.parse(this.lastSeenMillis); } - private String[] rf; + @Expose + @SerializedName("rf") + private String[] relayFlags; public void setRelayFlags(SortedSet<String> relayFlags) { - this.rf = this.collectionToStringArray(relayFlags); + this.relayFlags = this.collectionToStringArray(relayFlags); } public SortedSet<String> getRelayFlags() { - return this.stringArrayToSortedSet(this.rf); + return this.stringArrayToSortedSet(this.relayFlags); } - private long cw; + @Expose + @SerializedName("cw") + private long consensusWeight; public void setConsensusWeight(long consensusWeight) { - this.cw = consensusWeight; + this.consensusWeight = consensusWeight; } public long getConsensusWeight() { - return this.cw; + return this.consensusWeight; } - private boolean r; + @Expose + @SerializedName("r") + private boolean running; public void setRunning(boolean isRunning) { - this.r = isRunning; + this.running = isRunning; } public boolean isRunning() { - return this.r; + return this.running; } - private String c; + @Expose + @SerializedName("c") + private String contact; @SuppressWarnings("checkstyle:javadocmethod") public void setContact(String contact) { if (contact != null && contact.length() == 0) { - this.c = null; + this.contact = null; } else { - this.c = contact; + this.contact = contact; } } public String getContact() { - return this.c; + return this.contact; } /* This attribute can go away once all Onionoo services had their hourly * updater write effective families to summary documents at least once. * Remove this code after September 8, 2015. */ - private String[] ff; + @Expose + @SerializedName("ff") + private String[] familyFingerprints; public void setFamilyFingerprints( SortedSet<String> familyFingerprints) { - this.ff = this.collectionToStringArray(familyFingerprints); + this.familyFingerprints = this.collectionToStringArray(familyFingerprints); } public SortedSet<String> getFamilyFingerprints() { - return this.stringArrayToSortedSet(this.ff); + return this.stringArrayToSortedSet(this.familyFingerprints); } - private String[] ef; + @Expose + @SerializedName("ef") + private String[] effectiveFamily; public void setEffectiveFamily(SortedSet<String> effectiveFamily) { - this.ef = this.collectionToStringArray(effectiveFamily); + this.effectiveFamily = this.collectionToStringArray(effectiveFamily); } public SortedSet<String> getEffectiveFamily() { - return this.stringArrayToSortedSet(this.ef); + return this.stringArrayToSortedSet(this.effectiveFamily); } - private String v; + @Expose + @SerializedName("v") + private String version; public void setVersion(String version) { - this.v = version; + this.version = version; } public String getVersion() { - return this.v; + return this.version; } - private String h; + @Expose + @SerializedName("h") + private String hostName; public void setHostName(String hostName) { - this.h = hostName; + this.hostName = hostName; } public String getHostName() { - return this.h; + return this.hostName; } /* The familyFingerprints parameter can go away after September 8, 2015.

1 0

[onionoo/master] Add new "host_name" parameter to filter by host name.
by karsten＠torproject.org 17 Nov '17

17 Nov '17

commit d5d0edc9aab3d301b228c2b177daf172d93827b5 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Oct 17 14:48:26 2017 +0200 Add new "host_name" parameter to filter by host name. Implements #16553. --- CHANGELOG.md | 1 + .../org/torproject/onionoo/docs/DocumentStore.java | 3 +- .../org/torproject/onionoo/docs/NodeStatus.java | 15 ++++ .../torproject/onionoo/docs/SummaryDocument.java | 13 +++- .../org/torproject/onionoo/server/NodeIndex.java | 10 +++ .../org/torproject/onionoo/server/NodeIndexer.java | 11 +++ .../torproject/onionoo/server/RequestHandler.java | 26 +++++++ .../torproject/onionoo/server/ResourceServlet.java | 22 +++++- .../onionoo/updater/NodeDetailsStatusUpdater.java | 1 + .../onionoo/writer/SummaryDocumentWriter.java | 4 +- .../onionoo/docs/SummaryDocumentTest.java | 3 +- .../onionoo/server/ResourceServletTest.java | 79 ++++++++++++++++++++-- .../server/SummaryDocumentComparatorTest.java | 3 +- 13 files changed, 179 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12b0741..76243ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Support quoted qualified search terms. - Skip unrecognized descriptors when importing archives rather than aborting the entire import. + - Add new "host_name" parameter to filter by host name. # Changes in version 4.2-1.6.1 - 2017-10-26 diff --git a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java index 7819d0f..57d4165 100644 --- a/src/main/java/org/torproject/onionoo/docs/DocumentStore.java +++ b/src/main/java/org/torproject/onionoo/docs/DocumentStore.java @@ -439,10 +439,11 @@ public class DocumentStore { long lastSeenMillis = -1L; long consensusWeight = -1L; long firstSeenMillis = -1L; + String hostName = null; SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - asNumber, contact, family, family, version); + asNumber, contact, family, family, version, hostName); return summaryDocument; } diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index 24546dd..3271ccc 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -368,6 +368,16 @@ public class NodeStatus extends Document { /* Reverse DNS lookup result */ + private String hostName; + + public void setHostName(String hostName) { + this.hostName = hostName; + } + + public String getHostName() { + return this.hostName; + } + private long lastRdnsLookup = -1L; public void setLastRdnsLookup(long lastRdnsLookup) { @@ -561,6 +571,9 @@ public class NodeStatus extends Document { if (parts.length >= 24 && !parts[23].isEmpty()) { nodeStatus.setVersion(parts[23]); } + if (parts.length >= 25 && !parts[24].isEmpty()) { + nodeStatus.setHostName(parts[24]); + } return nodeStatus; } catch (NumberFormatException e) { log.error("Number format exception while parsing node " @@ -627,6 +640,8 @@ public class NodeStatus extends Document { .append(StringUtils.join(this.getIndirectFamily(), ";")); sb.append("\t") .append((this.getVersion() != null ? this.getVersion() : "")); + sb.append("\t") + .append((this.getHostName() != null ? this.getHostName() : "")); return sb.toString(); } } diff --git a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java index 527f91d..f33100d 100644 --- a/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/SummaryDocument.java @@ -279,6 +279,16 @@ public class SummaryDocument extends Document { return this.v; } + private String h; + + public void setHostName(String hostName) { + this.h = hostName; + } + + public String getHostName() { + return this.h; + } + /* The familyFingerprints parameter can go away after September 8, 2015. * See above. */ /** Instantiates a summary document with all given properties. */ @@ -287,7 +297,7 @@ public class SummaryDocument extends Document { boolean running, SortedSet<String> relayFlags, long consensusWeight, String countryCode, long firstSeenMillis, String asNumber, String contact, SortedSet<String> familyFingerprints, - SortedSet<String> effectiveFamily, String version) { + SortedSet<String> effectiveFamily, String version, String hostName) { this.setRelay(isRelay); this.setNickname(nickname); this.setFingerprint(fingerprint); @@ -303,6 +313,7 @@ public class SummaryDocument extends Document { this.setFamilyFingerprints(familyFingerprints); this.setEffectiveFamily(effectiveFamily); this.setVersion(version); + this.setHostName(hostName); } } diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndex.java b/src/main/java/org/torproject/onionoo/server/NodeIndex.java index dd4bd4d..126cd5c 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndex.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndex.java @@ -179,5 +179,15 @@ class NodeIndex { public Map<String, Set<String>> getRelaysByVersion() { return this.relaysByVersion; } + + private Map<String, Set<String>> relaysByHostName; + + public void setRelaysByHostName(Map<String, Set<String>> relaysByHostName) { + this.relaysByHostName = relaysByHostName; + } + + public Map<String, Set<String>> getRelaysByHostName() { + return this.relaysByHostName; + } } diff --git a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java index 5411d0e..d609f63 100644 --- a/src/main/java/org/torproject/onionoo/server/NodeIndexer.java +++ b/src/main/java/org/torproject/onionoo/server/NodeIndexer.java @@ -154,6 +154,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { Map<String, Set<String>> newRelaysByContact = new HashMap<>(); Map<String, Set<String>> newRelaysByFamily = new HashMap<>(); Map<String, Set<String>> newRelaysByVersion = new HashMap<>(); + Map<String, Set<String>> newRelaysByHostName = new HashMap<>(); SortedMap<Integer, Set<String>> newRelaysByFirstSeenDays = new TreeMap<>(); SortedMap<Integer, Set<String>> newBridgesByFirstSeenDays = new TreeMap<>(); SortedMap<Integer, Set<String>> newRelaysByLastSeenDays = new TreeMap<>(); @@ -258,6 +259,15 @@ public class NodeIndexer implements ServletContextListener, Runnable { newRelaysByVersion.get(version).add(fingerprint); newRelaysByVersion.get(version).add(hashedFingerprint); } + String hostName = entry.getHostName(); + if (null != hostName) { + String hostNameLowerCase = hostName.toLowerCase(); + if (!newRelaysByHostName.containsKey(hostNameLowerCase)) { + newRelaysByHostName.put(hostNameLowerCase, new HashSet<>()); + } + newRelaysByHostName.get(hostNameLowerCase).add(fingerprint); + newRelaysByHostName.get(hostNameLowerCase).add(hashedFingerprint); + } } /* This loop can go away once all Onionoo services had their hourly * updater write effective families to summary documents at least @@ -332,6 +342,7 @@ public class NodeIndexer implements ServletContextListener, Runnable { newNodeIndex.setRelaysPublishedMillis(relaysLastValidAfterMillis); newNodeIndex.setBridgesPublishedMillis(bridgesLastPublishedMillis); newNodeIndex.setRelaysByVersion(newRelaysByVersion); + newNodeIndex.setRelaysByHostName(newRelaysByHostName); synchronized (this) { this.lastIndexed = updateStatusMillis; this.latestNodeIndex = newNodeIndex; diff --git a/src/main/java/org/torproject/onionoo/server/RequestHandler.java b/src/main/java/org/torproject/onionoo/server/RequestHandler.java index 46c2f54..23af60b 100644 --- a/src/main/java/org/torproject/onionoo/server/RequestHandler.java +++ b/src/main/java/org/torproject/onionoo/server/RequestHandler.java @@ -97,6 +97,12 @@ public class RequestHandler { this.version = version; } + private String hostName; + + public void setHostName(String hostName) { + this.hostName = hostName; + } + private String[] order; public void setOrder(String[] order) { @@ -165,6 +171,7 @@ public class RequestHandler { this.filterByContact(); this.filterByFamily(); this.filterByVersion(); + this.filterByHostName(); this.order(); this.offset(); this.limit(); @@ -537,6 +544,25 @@ public class RequestHandler { this.filteredBridges.clear(); } + private void filterByHostName() { + if (this.hostName == null) { + /* Not filtering by host name. */ + return; + } + String hostName = this.hostName.toLowerCase(); + Set<String> removeRelays = new HashSet<>(this.filteredRelays.keySet()); + for (Map.Entry<String, Set<String>> e : + this.nodeIndex.getRelaysByHostName().entrySet()) { + if (e.getKey().endsWith(hostName)) { + removeRelays.removeAll(e.getValue()); + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + this.filteredBridges.clear(); + } + private void order() { List<SummaryDocument> uniqueRelays = new ArrayList<>(); List<SummaryDocument> uniqueBridges = new ArrayList<>(); diff --git a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java index 2fff913..f1521e2 100644 --- a/src/main/java/org/torproject/onionoo/server/ResourceServlet.java +++ b/src/main/java/org/torproject/onionoo/server/ResourceServlet.java @@ -68,7 +68,7 @@ public class ResourceServlet extends HttpServlet { private static Set<String> knownParameters = new HashSet<>( Arrays.asList(("type,running,search,lookup,fingerprint,country,as," + "flag,first_seen_days,last_seen_days,contact,order,limit," - + "offset,fields,family,version").split(","))); + + "offset,fields,family,version,host_name").split(","))); private static Set<String> illegalSearchQualifiers = new HashSet<>(Arrays.asList(("search,fingerprint,order,limit," @@ -292,6 +292,15 @@ public class ResourceServlet extends HttpServlet { } rh.setVersion(versionParameter); } + if (parameterMap.containsKey("host_name")) { + String hostNameParameter = this.parseHostNameParameter( + parameterMap.get("host_name")); + if (null == hostNameParameter) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setHostName(hostNameParameter); + } if (parameterMap.containsKey("order")) { String[] order = this.parseOrderParameter(parameterMap.get("order")); if (order == null) { @@ -584,5 +593,16 @@ public class ResourceServlet extends HttpServlet { } return parameter; } + + private static Pattern hostNameParameterPattern = + Pattern.compile("^[0-9A-Za-z_\\.\\-]+$"); + + private String parseHostNameParameter(String parameter) { + if (!hostNameParameterPattern.matcher(parameter).matches()) { + /* Host name contains illegal character(s). */ + return null; + } + return parameter; + } } diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 04a04fa..73e826a 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -849,6 +849,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, if (this.rdnsLookupResults.containsKey(fingerprint)) { String hostName = this.rdnsLookupResults.get(fingerprint); detailsStatus.setHostName(hostName); + nodeStatus.setHostName(hostName); nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); } diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index 4364aad..faf54fa 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -92,10 +92,12 @@ public class SummaryDocumentWriter implements DocumentWriter { SortedSet<String> effectiveFamily = nodeStatus.getEffectiveFamily(); String nickname = nodeStatus.getNickname(); String version = nodeStatus.getVersion(); + String hostName = nodeStatus.getHostName(); SummaryDocument summaryDocument = new SummaryDocument(isRelay, nickname, fingerprint, addresses, lastSeenMillis, running, relayFlags, consensusWeight, countryCode, firstSeenMillis, - asNumber, contact, declaredFamily, effectiveFamily, version); + asNumber, contact, declaredFamily, effectiveFamily, version, + hostName); if (this.documentStore.store(summaryDocument, fingerprint)) { this.writtenDocuments++; } diff --git a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java index 9d7625a..8c45ed2 100644 --- a/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java +++ b/src/test/java/org/torproject/onionoo/docs/SummaryDocumentTest.java @@ -26,7 +26,8 @@ public class SummaryDocumentTest { new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( - new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null); + new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, + null); } @Test() diff --git a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java index e41eecc..2720f7a 100644 --- a/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java +++ b/src/test/java/org/torproject/onionoo/server/ResourceServletTest.java @@ -143,7 +143,7 @@ public class ResourceServletTest { "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), - "0.2.3.25"); + "0.2.3.25", "ppp-62-216-201-221.dynamic.mnet-online.de"); org.torproject.onionoo.docs.SummaryDocument relayFerrari458 = new org.torproject.onionoo.docs.SummaryDocument(true, "Ferrari458", "001C13B3A55A71B977CA65EC85539D79C653A3FC", Arrays.asList( @@ -155,7 +155,8 @@ public class ResourceServletTest { new TreeSet<String>(Arrays.asList(new String[] { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), new TreeSet<>(Arrays.asList(new String[] { - "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null); + "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })), null, + "c-68-38-171-200.hsd1.in.comcast.net"); this.relays = new TreeMap<>(); this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A", relayTorkaZ); @@ -171,7 +172,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-16 18:00:00"), "AS6830", "1024d/51e2a1c7 \"steven j. murdoch\" " + "<tor+steven.murdoch(a)cl.cam.ac.uk> <fb-token:5sr_k_zs2wm=>", - new TreeSet<String>(), new TreeSet<String>(), "0.2.3.25"); + new TreeSet<String>(), new TreeSet<String>(), "0.2.3.25", null); this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", relayTimMayTribute); this.bridges = new TreeMap<>(); @@ -182,7 +183,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-21 18:07:03"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, - null, null, null); + null, null, null, null); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", bridgeec2bridgercc7f31fe); org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed = @@ -192,7 +193,7 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-04-20 17:37:04"), false, new TreeSet<>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, - null, null, null); + null, null, null, null); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", bridgeUnnamed); org.torproject.onionoo.docs.SummaryDocument bridgegummy = @@ -203,7 +204,7 @@ public class ResourceServletTest { new TreeSet<>(Arrays.asList(new String[] { "Running", "Valid" })), -1L, null, DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null, - null, null); + null, null, null); this.bridges.put("1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", bridgegummy); } @@ -1623,5 +1624,71 @@ public class ResourceServletTest { /* This is also correct when comparing strings. */ this.assertErrorStatusCode("/summary?version=*", 400); } + + @Test + public void testHostNameDe() { + this.assertSummaryDocument("/summary?host_name=de", 1, null, 0, null); + } + + @Test + public void testHostNameE() { + this.assertSummaryDocument("/summary?host_name=e", 1, null, 0, null); + } + + @Test + public void testHostNameDotDe() { + this.assertSummaryDocument("/summary?host_name=.de", 1, null, 0, null); + } + + @Test + public void testHostNameOnlineDe() { + this.assertSummaryDocument("/summary?host_name=online.de", 1, null, 0, + null); + } + + @Test + public void testHostNameOnlineDeSomeCapitalized() { + this.assertSummaryDocument("/summary?host_name=onLiNe.dE", 1, null, 0, + null); + } + + @Test + public void testHostNameOnline() { + this.assertSummaryDocument("/summary?host_name=online", 0, null, 0, null); + } + + @Test + public void testHostNameTorkaZFull() { + this.assertSummaryDocument( + "/summary?host_name=ppp-62-216-201-221.dynamic.mnet-online.de", + 1, null, 0, null); + } + + @Test + public void testHostNameTorkaZSub() { + this.assertSummaryDocument( + "/summary?host_name=sub.ppp-62-216-201-221.dynamic.mnet-online.de", + 0, null, 0, null); + } + + @Test + public void testHostNameNet() { + this.assertSummaryDocument("/summary?host_name=net", 1, null, 0, null); + } + + @Test + public void testHostNameCom() { + this.assertSummaryDocument("/summary?host_name=com", 0, null, 0, null); + } + + @Test + public void testHostNameDollar() { + this.assertErrorStatusCode("/summary?host_name=$", 400); + } + + @Test + public void testHostNameUmlaut() { + this.assertErrorStatusCode("/summary?host_name=äöü", 400); + } } diff --git a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java index fb3f5b3..38d94fa 100644 --- a/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java +++ b/src/test/java/org/torproject/onionoo/server/SummaryDocumentComparatorTest.java @@ -40,7 +40,8 @@ public class SummaryDocumentComparatorTest { new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })), new TreeSet<>(Arrays.asList( - new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null); + new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC" })), null, + null); } /** Some values for running all comparison types. */

1 0

[metrics-web/master] Update news.json to version 193 of doc/MetricsTimeline.
by karsten＠torproject.org 17 Nov '17

17 Nov '17

commit 50ea265c16ff2224834187784cbe3af72c4836c9 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Nov 17 09:20:41 2017 +0100 Update news.json to version 193 of doc/MetricsTimeline. --- website/src/main/resources/etc/news.json | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/website/src/main/resources/etc/news.json b/website/src/main/resources/etc/news.json index e054182..704eac0 100644 --- a/website/src/main/resources/etc/news.json +++ b/website/src/main/resources/etc/news.json @@ -2759,6 +2759,17 @@ ] }, { + "start": "2017-11-15", + "end": "2017-11-15", + "protocols": [ + "meek" + ], + "description": "Outage of meek-amazon bridge.", + "links": [ + "<a href=\"https://bugs.torproject.org/24284\">ticket</a>" + ] + }, + { "start": "2016-02-24", "places": [ "tm" @@ -3227,7 +3238,7 @@ "protocols": [ "obfs4" ], - "description": "About a 35% increased in obfs4 users before returning to normal. Happened in many countries. Other transports do not have it.", + "description": "About a 35% increase in obfs4 users before returning to normal. Happened in many countries. Other transports do not have it.", "links": [ "<a href=\"https://metrics.torproject.org/userstats-bridge-transport.html?start=2017-0…">graph</a>" ],

1 0

[translation/tails-onioncircuits] Update translations for tails-onioncircuits
by translation＠torproject.org 16 Nov '17

16 Nov '17

commit d3ce515f2f927f1a64109a7197c04220b168f5ad Author: Translation commit bot <translation(a)torproject.org> Date: Thu Nov 16 21:49:04 2017 +0000 Update translations for tails-onioncircuits --- hi/onioncircuits.pot | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hi/onioncircuits.pot b/hi/onioncircuits.pot index 3af806873..983f8287b 100644 --- a/hi/onioncircuits.pot +++ b/hi/onioncircuits.pot @@ -8,7 +8,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2017-08-03 13:00+0000\n" -"PO-Revision-Date: 2017-09-22 21:53+0000\n" +"PO-Revision-Date: 2017-11-16 21:19+0000\n" "Last-Translator: carolyn <carolyn(a)anhalt.org>\n" "Language-Team: Hindi (http://www.transifex.com/otf/torproject/language/hi/)\n" "MIME-Version: 1.0\n" @@ -43,7 +43,7 @@ msgstr "" #: ../onioncircuits:317 msgid "..." -msgstr "" +msgstr "..." #: ../onioncircuits:343 #, c-format

1 0

[translation/tails-misc] Update translations for tails-misc
by translation＠torproject.org 16 Nov '17

16 Nov '17

commit cd7185c876206be4b3b02301667b79cb6182a7cd Author: Translation commit bot <translation(a)torproject.org> Date: Thu Nov 16 21:47:07 2017 +0000 Update translations for tails-misc --- hi.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hi.po b/hi.po index b3570748e..b357ac0fb 100644 --- a/hi.po +++ b/hi.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2017-09-13 20:10+0200\n" -"PO-Revision-Date: 2017-11-16 21:04+0000\n" +"PO-Revision-Date: 2017-11-16 21:19+0000\n" "Last-Translator: carolyn <carolyn(a)anhalt.org>\n" "Language-Team: Hindi (http://www.transifex.com/otf/torproject/language/hi/)\n" "MIME-Version: 1.0\n"

1 0

[translation/mat-gui] Update translations for mat-gui
by translation＠torproject.org 16 Nov '17

16 Nov '17

commit d5376f6f7406e49eddb8bb036d2199e83e8477fd Author: Translation commit bot <translation(a)torproject.org> Date: Thu Nov 16 21:46:28 2017 +0000 Update translations for mat-gui --- hi.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hi.po b/hi.po index 7309f2e92..523f6df3a 100644 --- a/hi.po +++ b/hi.po @@ -10,7 +10,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2016-02-10 23:06+0100\n" -"PO-Revision-Date: 2017-11-16 21:14+0000\n" +"PO-Revision-Date: 2017-11-16 21:19+0000\n" "Last-Translator: carolyn <carolyn(a)anhalt.org>\n" "Language-Team: Hindi (http://www.transifex.com/otf/torproject/language/hi/)\n" "MIME-Version: 1.0\n"

1 0

[translation/tails-misc] Update translations for tails-misc
by translation＠torproject.org 16 Nov '17

16 Nov '17

commit 42c1510548d31c3dfd1fb6c766e220311b65ca58 Author: Translation commit bot <translation(a)torproject.org> Date: Thu Nov 16 21:17:09 2017 +0000 Update translations for tails-misc --- hi.po | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hi.po b/hi.po index 70c28620e..b3570748e 100644 --- a/hi.po +++ b/hi.po @@ -9,7 +9,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2017-09-13 20:10+0200\n" -"PO-Revision-Date: 2017-09-28 14:40+0000\n" +"PO-Revision-Date: 2017-11-16 21:04+0000\n" "Last-Translator: carolyn <carolyn(a)anhalt.org>\n" "Language-Team: Hindi (http://www.transifex.com/otf/torproject/language/hi/)\n" "MIME-Version: 1.0\n" @@ -70,7 +70,7 @@ msgstr "" #: config/chroot_local-includes/usr/share/gnome-shell/extensions/shutdown-helper@tails.boum.org/extension.js:71 msgid "Restart" -msgstr "" +msgstr "पुनः आरंभ करें" #: config/chroot_local-includes/usr/share/gnome-shell/extensions/shutdown-helper@tails.boum.org/extension.js:74 msgid "Power Off"

1 0

[translation/mat-gui] Update translations for mat-gui
by translation＠torproject.org 16 Nov '17

16 Nov '17

commit 55c2cad387c848fd3815ee37aba54e7d0492cdc3 Author: Translation commit bot <translation(a)torproject.org> Date: Thu Nov 16 21:16:30 2017 +0000 Update translations for mat-gui --- hi.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hi.po b/hi.po index bd1977e62..7309f2e92 100644 --- a/hi.po +++ b/hi.po @@ -10,7 +10,7 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2016-02-10 23:06+0100\n" -"PO-Revision-Date: 2017-09-19 17:08+0000\n" +"PO-Revision-Date: 2017-11-16 21:14+0000\n" "Last-Translator: carolyn <carolyn(a)anhalt.org>\n" "Language-Team: Hindi (http://www.transifex.com/otf/torproject/language/hi/)\n" "MIME-Version: 1.0\n"

1 0