commit e630a086cf900918cf18a26f181f2b93fe7f033b Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue May 13 17:49:05 2014 +0200
Fix how we're processing exit lists.
This commit really contains three fixes:
1. We were previously processing exit lists when writing details documents. That had the downside that we only included exit addresses when we parsed new exit lists. Now we're storing exit list entries together with the scan time in details statuses and include them in details documents until they are older than 24 hours.
2. We now make sure that the latest exit addresses make it into node statuses and are not reset when there is no exit list to parse. This fix is pretty similar to 0dc7436.
3. We now clarify in the specification that the `search` parameter includes exit addresses.
Altogether, fixes #11066. --- .../torproject/onionoo/DetailsDocumentWriter.java | 83 +++++-------------- src/org/torproject/onionoo/DetailsStatus.java | 8 ++ .../onionoo/NodeDetailsStatusUpdater.java | 84 +++++++++++++++++++- src/org/torproject/onionoo/NodeStatus.java | 1 - web/index.html | 2 + 5 files changed, 112 insertions(+), 66 deletions(-)
diff --git a/src/org/torproject/onionoo/DetailsDocumentWriter.java b/src/org/torproject/onionoo/DetailsDocumentWriter.java index 73caf24..a70efd0 100644 --- a/src/org/torproject/onionoo/DetailsDocumentWriter.java +++ b/src/org/torproject/onionoo/DetailsDocumentWriter.java @@ -3,20 +3,13 @@ package org.torproject.onionoo; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Scanner; -import java.util.Set; import java.util.SortedSet; import java.util.TreeSet;
-import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.ExitList; -import org.torproject.descriptor.ExitListEntry; - -public class DetailsDocumentWriter implements DescriptorListener, - FingerprintListener, DocumentWriter { +public class DetailsDocumentWriter implements FingerprintListener, + DocumentWriter {
private DescriptorSource descriptorSource;
@@ -28,42 +21,9 @@ public class DetailsDocumentWriter implements DescriptorListener, this.descriptorSource = ApplicationFactory.getDescriptorSource(); this.documentStore = ApplicationFactory.getDocumentStore(); this.now = ApplicationFactory.getTime().currentTimeMillis(); - this.registerDescriptorListeners(); this.registerFingerprintListeners(); }
- private void registerDescriptorListeners() { - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.EXIT_LISTS); - } - - public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof ExitList) { - this.processExitList((ExitList) descriptor); - } - } - - private Map<String, Set<ExitListEntry>> exitListEntries = - new HashMap<String, Set<ExitListEntry>>(); - - /* TODO Processing descriptors should really be done in - * NodeDetailsStatusUpdater, not here. This is also a bug, because - * we're only considering newly published exit lists. */ - private void processExitList(ExitList exitList) { - for (ExitListEntry exitListEntry : exitList.getExitListEntries()) { - if (exitListEntry.getScanMillis() < - this.now - 24L * 60L * 60L * 1000L) { - continue; - } - String fingerprint = exitListEntry.getFingerprint(); - if (!this.exitListEntries.containsKey(fingerprint)) { - this.exitListEntries.put(fingerprint, - new HashSet<ExitListEntry>()); - } - this.exitListEntries.get(fingerprint).add(exitListEntry); - } - } - private void registerFingerprintListeners() { this.descriptorSource.registerFingerprintListener(this, DescriptorType.RELAY_CONSENSUSES); @@ -164,26 +124,8 @@ public class DetailsDocumentWriter implements DescriptorListener, detailsDocument.setRecommendedVersion( entry.getRecommendedVersion());
- /* Add exit addresses if at least one of them is distinct from the - * onion-routing addresses. */ - SortedSet<String> exitAddresses = new TreeSet<String>(); - if (this.exitListEntries.containsKey(fingerprint)) { - for (ExitListEntry exitListEntry : - this.exitListEntries.get(fingerprint)) { - String exitAddress = exitListEntry.getExitAddress(); - if (exitAddress.length() > 0 && - !entry.getAddress().equals(exitAddress) && - !entry.getOrAddresses().contains(exitAddress)) { - exitAddresses.add(exitAddress.toLowerCase()); - } - } - } - if (!exitAddresses.isEmpty()) { - detailsDocument.setExitAddresses(new ArrayList<String>( - exitAddresses)); - } - - /* Append descriptor-specific part from details status file. */ + /* Append descriptor-specific part and exit addresses from details + * status file. */ DetailsStatus detailsStatus = this.documentStore.retrieve( DetailsStatus.class, true, fingerprint); if (detailsStatus != null) { @@ -204,6 +146,23 @@ public class DetailsDocumentWriter implements DescriptorListener, detailsDocument.setExitPolicyV6Summary( detailsStatus.getExitPolicyV6Summary()); detailsDocument.setHibernating(detailsStatus.getHibernating()); + if (detailsStatus.getExitAddresses() != null) { + SortedSet<String> exitAddresses = new TreeSet<String>(); + for (Map.Entry<String, Long> e : + detailsStatus.getExitAddresses().entrySet()) { + String exitAddress = e.getKey().toLowerCase(); + long scanMillis = e.getValue(); + if (!entry.getAddress().equals(exitAddress) && + !entry.getOrAddresses().contains(exitAddress) && + scanMillis >= this.now - DateTimeHelper.ONE_DAY) { + exitAddresses.add(exitAddress); + } + } + if (!exitAddresses.isEmpty()) { + detailsDocument.setExitAddresses(new ArrayList<String>( + exitAddresses)); + } + } }
/* Write details file to disk. */ diff --git a/src/org/torproject/onionoo/DetailsStatus.java b/src/org/torproject/onionoo/DetailsStatus.java index 1a497e4..5a552cd 100644 --- a/src/org/torproject/onionoo/DetailsStatus.java +++ b/src/org/torproject/onionoo/DetailsStatus.java @@ -130,4 +130,12 @@ class DetailsStatus extends Document { public String getPoolAssignment() { return this.pool_assignment; } + + private Map<String, Long> exit_addresses; + public void setExitAddresses(Map<String, Long> exitAddresses) { + this.exit_addresses = exitAddresses; + } + public Map<String, Long> getExitAddresses() { + return this.exit_addresses; + } } diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java index afc72bb..97cc2ab 100644 --- a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java +++ b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java @@ -16,6 +16,8 @@ import java.util.TreeSet; import org.torproject.descriptor.BridgeNetworkStatus; import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.ExitList; +import org.torproject.descriptor.ExitListEntry; import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.descriptor.ServerDescriptor; @@ -86,6 +88,8 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, this.processBridgeServerDescriptor((ServerDescriptor) descriptor); } else if (descriptor instanceof BridgePoolAssignment) { this.processBridgePoolAssignment((BridgePoolAssignment) descriptor); + } else if (descriptor instanceof ExitList) { + this.processExitList((ExitList) descriptor); } }
@@ -98,8 +102,9 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, Logger.printStatusTime("Started reverse domain name lookups"); this.lookUpCitiesAndASes(); Logger.printStatusTime("Looked up cities and ASes"); - this.setRunningBitsAndContacts(); - Logger.printStatusTime("Set running bits and contacts"); + this.setRunningBitsContactsAndExitAddresses(); + Logger.printStatusTime("Set running bits, contacts, and exit " + + "addresses"); this.calculatePathSelectionProbabilities(); Logger.printStatusTime("Calculated path selection probabilities"); this.finishReverseDomainNameLookups(); @@ -111,6 +116,8 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, * all node statuses and an out/summary with only recent ones? this.writeOutSummary(); Logger.printStatusTime("Wrote out summary");*/ + this.updateDetailsStatuses(); + Logger.printStatusTime("Updated exit addresses in details statuses"); }
private void processRelayNetworkStatusConsensus( @@ -208,7 +215,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } }
- private void setRunningBitsAndContacts() { + private void setRunningBitsContactsAndExitAddresses() { for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { String fingerprint = e.getKey(); NodeStatus node = e.getValue(); @@ -221,6 +228,14 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, DetailsStatus.class, true, fingerprint); if (detailsStatus != null) { node.setContact(detailsStatus.getContact()); + if (detailsStatus.getExitAddresses() != null) { + for (Map.Entry<String, Long> ea : + detailsStatus.getExitAddresses().entrySet()) { + if (ea.getValue() >= this.now - DateTimeHelper.ONE_DAY) { + node.addExitAddress(ea.getKey()); + } + } + } } } if (!node.isRelay() && node.getRelayFlags().contains("Running") && @@ -396,6 +411,31 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } }
+ private Map<String, Map<String, Long>> exitListEntries = + new HashMap<String, Map<String, Long>>(); + + private void processExitList(ExitList exitList) { + for (ExitListEntry exitListEntry : exitList.getExitListEntries()) { + String fingerprint = exitListEntry.getFingerprint(); + if (exitListEntry.getScanMillis() < + this.now - DateTimeHelper.ONE_DAY) { + continue; + } + if (!this.exitListEntries.containsKey(fingerprint)) { + this.exitListEntries.put(fingerprint, + new HashMap<String, Long>()); + } + String exitAddress = exitListEntry.getExitAddress(); + long scanMillis = exitListEntry.getScanMillis(); + if (!this.exitListEntries.get(fingerprint).containsKey(exitAddress) + || this.exitListEntries.get(fingerprint).get(exitAddress) + < scanMillis) { + this.exitListEntries.get(fingerprint).put(exitAddress, + scanMillis); + } + } + } + private void startReverseDomainNameLookups() { Map<String, Long> addressLastLookupTimes = new HashMap<String, Long>(); @@ -534,6 +574,44 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } }
+ private void updateDetailsStatuses() { + SortedSet<String> fingerprints = new TreeSet<String>(); + fingerprints.addAll(this.exitListEntries.keySet()); + for (String fingerprint : fingerprints) { + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + if (detailsStatus == null) { + detailsStatus = new DetailsStatus(); + } + Map<String, Long> exitAddresses = new HashMap<String, Long>(); + if (detailsStatus.getExitAddresses() != null) { + for (Map.Entry<String, Long> e : + detailsStatus.getExitAddresses().entrySet()) { + if (e.getValue() >= this.now - DateTimeHelper.ONE_DAY) { + exitAddresses.put(e.getKey(), e.getValue()); + } + } + } + if (this.exitListEntries.containsKey(fingerprint)) { + for (Map.Entry<String, Long> e : + this.exitListEntries.get(fingerprint).entrySet()) { + if (!exitAddresses.containsKey(e.getKey()) || + exitAddresses.get(e.getKey()) < e.getValue()) { + exitAddresses.put(e.getKey(), e.getValue()); + } + } + } + if (this.knownNodes.containsKey(fingerprint)) { + for (String orAddress : + this.knownNodes.get(fingerprint).getOrAddresses()) { + this.exitListEntries.remove(orAddress); + } + } + detailsStatus.setExitAddresses(exitAddresses); + this.documentStore.store(detailsStatus, fingerprint); + } + } + public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + Logger.formatDecimalNumber( diff --git a/src/org/torproject/onionoo/NodeStatus.java b/src/org/torproject/onionoo/NodeStatus.java index 781d6db..c829f7f 100644 --- a/src/org/torproject/onionoo/NodeStatus.java +++ b/src/org/torproject/onionoo/NodeStatus.java @@ -482,7 +482,6 @@ public class NodeStatus extends Document { this.nickname = newNodeStatus.nickname; this.address = newNodeStatus.address; this.orAddressesAndPorts = newNodeStatus.orAddressesAndPorts; - this.exitAddresses = newNodeStatus.exitAddresses; this.lastSeenMillis = newNodeStatus.lastSeenMillis; this.orPort = newNodeStatus.orPort; this.dirPort = newNodeStatus.dirPort; diff --git a/web/index.html b/web/index.html index c941f4e..ebb675a 100644 --- a/web/index.html +++ b/web/index.html @@ -270,6 +270,8 @@ Return only relays with the parameter value matching (part of a) nickname, (possibly $-prefixed) beginning of a fingerprint, or beginning of an IP address, and bridges with (part of a) nickname or (possibly $-prefixed) beginning of a hashed fingerprint. +Searches by relay IP address include all known addresses used for onion +routing and for exiting to the Internet. Searches for beginnings of IP addresses are performed on textual representations of canonical IP address forms, so that searches using CIDR notation or non-canonical forms will return empty results.