commit 96b098c0c345b83a8dfd276bf0a2e2ae6d749f1d
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Thu Jul 17 09:20:09 2014 +0200
Reorder methods in NodeDetailsStatusUpdater.
---
.../onionoo/NodeDetailsStatusUpdater.java | 456 ++++++++++----------
1 file changed, 228 insertions(+), 228 deletions(-)
diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java
index 5c253bd..e898638 100644
--- a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java
+++ b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java
@@ -77,41 +77,95 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
}
public void processDescriptor(Descriptor descriptor, boolean relay) {
- if (descriptor instanceof RelayNetworkStatusConsensus) {
+ if (descriptor instanceof ServerDescriptor && relay) {
+ this.processRelayServerDescriptor((ServerDescriptor) descriptor);
+ } else if (descriptor instanceof ExitList) {
+ this.processExitList((ExitList) descriptor);
+ } else if (descriptor instanceof RelayNetworkStatusConsensus) {
this.processRelayNetworkStatusConsensus(
(RelayNetworkStatusConsensus) descriptor);
- } else if (descriptor instanceof ServerDescriptor && relay) {
- this.processRelayServerDescriptor((ServerDescriptor) descriptor);
- } else if (descriptor instanceof BridgeNetworkStatus) {
- this.processBridgeNetworkStatus((BridgeNetworkStatus) descriptor);
} else if (descriptor instanceof ServerDescriptor && !relay) {
this.processBridgeServerDescriptor((ServerDescriptor) descriptor);
} else if (descriptor instanceof BridgePoolAssignment) {
this.processBridgePoolAssignment((BridgePoolAssignment) descriptor);
- } else if (descriptor instanceof ExitList) {
- this.processExitList((ExitList) descriptor);
+ } else if (descriptor instanceof BridgeNetworkStatus) {
+ this.processBridgeNetworkStatus((BridgeNetworkStatus) descriptor);
}
}
- public void updateStatuses() {
- this.readStatusSummary();
- Logger.printStatusTime("Read status summary");
- this.setCurrentNodes();
- Logger.printStatusTime("Set current node fingerprints");
- this.startReverseDomainNameLookups();
- Logger.printStatusTime("Started reverse domain name lookups");
- this.lookUpCitiesAndASes();
- Logger.printStatusTime("Looked up cities and ASes");
- this.setDescriptorPartsOfNodeStatus();
- Logger.printStatusTime("Set descriptor parts of node statuses.");
- this.calculatePathSelectionProbabilities();
- Logger.printStatusTime("Calculated path selection probabilities");
- this.finishReverseDomainNameLookups();
- Logger.printStatusTime("Finished reverse domain name lookups");
- this.writeStatusSummary();
- Logger.printStatusTime("Wrote status summary");
- this.updateDetailsStatuses();
- Logger.printStatusTime("Updated exit addresses in details statuses");
+ private void processRelayServerDescriptor(
+ ServerDescriptor descriptor) {
+ String fingerprint = descriptor.getFingerprint();
+ DetailsStatus detailsStatus = this.documentStore.retrieve(
+ DetailsStatus.class, true, fingerprint);
+ String publishedDateTime =
+ DateTimeHelper.format(descriptor.getPublishedMillis());
+ if (detailsStatus == null) {
+ detailsStatus = new DetailsStatus();
+ } else if (detailsStatus.getDescPublished() != null &&
+ publishedDateTime.compareTo(
+ detailsStatus.getDescPublished()) < 0) {
+ return;
+ }
+ String lastRestartedString = DateTimeHelper.format(
+ descriptor.getPublishedMillis() - descriptor.getUptime()
+ * DateTimeHelper.ONE_SECOND);
+ int bandwidthRate = descriptor.getBandwidthRate();
+ int bandwidthBurst = descriptor.getBandwidthBurst();
+ int observedBandwidth = descriptor.getBandwidthObserved();
+ int advertisedBandwidth = Math.min(bandwidthRate,
+ Math.min(bandwidthBurst, observedBandwidth));
+ detailsStatus.setDescPublished(publishedDateTime);
+ detailsStatus.setLastRestarted(lastRestartedString);
+ detailsStatus.setBandwidthRate(bandwidthRate);
+ detailsStatus.setBandwidthBurst(bandwidthBurst);
+ detailsStatus.setObservedBandwidth(observedBandwidth);
+ detailsStatus.setAdvertisedBandwidth(advertisedBandwidth);
+ detailsStatus.setExitPolicy(descriptor.getExitPolicyLines());
+ detailsStatus.setContact(descriptor.getContact());
+ detailsStatus.setPlatform(descriptor.getPlatform());
+ detailsStatus.setFamily(descriptor.getFamilyEntries());
+ if (descriptor.getIpv6DefaultPolicy() != null &&
+ (descriptor.getIpv6DefaultPolicy().equals("accept") ||
+ descriptor.getIpv6DefaultPolicy().equals("reject")) &&
+ descriptor.getIpv6PortList() != null) {
+ Map<String, List<String>> exitPolicyV6Summary =
+ new HashMap<String, List<String>>();
+ List<String> portsOrPortRanges = Arrays.asList(
+ descriptor.getIpv6PortList().split(","));
+ exitPolicyV6Summary.put(descriptor.getIpv6DefaultPolicy(),
+ portsOrPortRanges);
+ detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary);
+ }
+ if (descriptor.isHibernating()) {
+ detailsStatus.setHibernating(true);
+ }
+ this.documentStore.store(detailsStatus, fingerprint);
+ }
+
+ private Map<String, Map<String, Long>> exitListEntries =
+ new HashMap<String, Map<String, Long>>();
+
+ private void processExitList(ExitList exitList) {
+ for (ExitListEntry exitListEntry : exitList.getExitListEntries()) {
+ String fingerprint = exitListEntry.getFingerprint();
+ if (exitListEntry.getScanMillis() <
+ this.now - DateTimeHelper.ONE_DAY) {
+ continue;
+ }
+ if (!this.exitListEntries.containsKey(fingerprint)) {
+ this.exitListEntries.put(fingerprint,
+ new HashMap<String, Long>());
+ }
+ String exitAddress = exitListEntry.getExitAddress();
+ long scanMillis = exitListEntry.getScanMillis();
+ if (!this.exitListEntries.get(fingerprint).containsKey(exitAddress)
+ || this.exitListEntries.get(fingerprint).get(exitAddress)
+ < scanMillis) {
+ this.exitListEntries.get(fingerprint).put(exitAddress,
+ scanMillis);
+ }
+ }
}
private void processRelayNetworkStatusConsensus(
@@ -161,6 +215,51 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
}
}
+ private void processBridgeServerDescriptor(
+ ServerDescriptor descriptor) {
+ String fingerprint = descriptor.getFingerprint();
+ DetailsStatus detailsStatus = this.documentStore.retrieve(
+ DetailsStatus.class, true, fingerprint);
+ String publishedDateTime =
+ DateTimeHelper.format(descriptor.getPublishedMillis());
+ if (detailsStatus == null) {
+ detailsStatus = new DetailsStatus();
+ } else if (detailsStatus.getDescPublished() != null &&
+ publishedDateTime.compareTo(
+ detailsStatus.getDescPublished()) < 0) {
+ return;
+ }
+ String lastRestartedString = DateTimeHelper.format(
+ descriptor.getPublishedMillis() - descriptor.getUptime()
+ * DateTimeHelper.ONE_SECOND);
+ int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(),
+ Math.min(descriptor.getBandwidthBurst(),
+ descriptor.getBandwidthObserved()));
+ detailsStatus.setDescPublished(publishedDateTime);
+ detailsStatus.setLastRestarted(lastRestartedString);
+ detailsStatus.setAdvertisedBandwidth(advertisedBandwidth);
+ detailsStatus.setPlatform(descriptor.getPlatform());
+ this.documentStore.store(detailsStatus, fingerprint);
+ }
+
+ private void processBridgePoolAssignment(
+ BridgePoolAssignment bridgePoolAssignment) {
+ for (Map.Entry<String, String> e :
+ bridgePoolAssignment.getEntries().entrySet()) {
+ String fingerprint = e.getKey();
+ String details = e.getValue();
+ DetailsStatus detailsStatus = this.documentStore.retrieve(
+ DetailsStatus.class, true, fingerprint);
+ if (detailsStatus == null) {
+ detailsStatus = new DetailsStatus();
+ } else if (details.equals(detailsStatus.getPoolAssignment())) {
+ continue;
+ }
+ detailsStatus.setPoolAssignment(details);
+ this.documentStore.store(detailsStatus, fingerprint);
+ }
+ }
+
private void processBridgeNetworkStatus(BridgeNetworkStatus status) {
long publishedMillis = status.getPublishedMillis();
if (publishedMillis > this.bridgesLastPublishedMillis) {
@@ -188,6 +287,27 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
this.bridgeStatusesProcessed++;
}
+ public void updateStatuses() {
+ this.readStatusSummary();
+ Logger.printStatusTime("Read status summary");
+ this.setCurrentNodes();
+ Logger.printStatusTime("Set current node fingerprints");
+ this.startReverseDomainNameLookups();
+ Logger.printStatusTime("Started reverse domain name lookups");
+ this.lookUpCitiesAndASes();
+ Logger.printStatusTime("Looked up cities and ASes");
+ this.setDescriptorPartsOfNodeStatus();
+ Logger.printStatusTime("Set descriptor parts of node statuses.");
+ this.calculatePathSelectionProbabilities();
+ Logger.printStatusTime("Calculated path selection probabilities");
+ this.finishReverseDomainNameLookups();
+ Logger.printStatusTime("Finished reverse domain name lookups");
+ this.writeStatusSummary();
+ Logger.printStatusTime("Wrote status summary");
+ this.updateDetailsStatuses();
+ Logger.printStatusTime("Updated exit addresses in details statuses");
+ }
+
private void readStatusSummary() {
SortedSet<String> fingerprints = this.documentStore.list(
NodeStatus.class, true);
@@ -209,49 +329,38 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
}
}
- private void setDescriptorPartsOfNodeStatus() {
+ private void setCurrentNodes() {
+ long cutoff = Math.max(this.relaysLastValidAfterMillis,
+ this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L;
+ SortedMap<String, NodeStatus> currentNodes =
+ new TreeMap<String, NodeStatus>();
for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) {
- String fingerprint = e.getKey();
- NodeStatus node = e.getValue();
- if (node.isRelay()) {
- if (node.getRelayFlags().contains("Running") &&
- node.getLastSeenMillis() == this.relaysLastValidAfterMillis) {
- node.setRunning(true);
- }
- DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, true, fingerprint);
- if (detailsStatus != null) {
- node.setContact(detailsStatus.getContact());
- if (detailsStatus.getExitAddresses() != null) {
- for (Map.Entry<String, Long> ea :
- detailsStatus.getExitAddresses().entrySet()) {
- if (ea.getValue() >= this.now - DateTimeHelper.ONE_DAY) {
- node.addExitAddress(ea.getKey());
- }
- }
- }
- if (detailsStatus.getFamily() != null &&
- !detailsStatus.getFamily().isEmpty()) {
- SortedSet<String> familyFingerprints = new TreeSet<String>();
- for (String familyMember : detailsStatus.getFamily()) {
- if (familyMember.startsWith("$") &&
- familyMember.length() == 41) {
- familyFingerprints.add(familyMember.substring(1));
- }
- }
- if (!familyFingerprints.isEmpty()) {
- node.setFamilyFingerprints(familyFingerprints);
- }
- }
- }
+ if (e.getValue().getLastSeenMillis() >= cutoff) {
+ currentNodes.put(e.getKey(), e.getValue());
}
- if (!node.isRelay() && node.getRelayFlags().contains("Running") &&
- node.getLastSeenMillis() == this.bridgesLastPublishedMillis) {
- node.setRunning(true);
+ }
+ this.relays = new TreeMap<String, NodeStatus>();
+ this.bridges = new TreeMap<String, NodeStatus>();
+ for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) {
+ if (e.getValue().isRelay()) {
+ this.relays.put(e.getKey(), e.getValue());
+ } else {
+ this.bridges.put(e.getKey(), e.getValue());
}
}
}
+ private void startReverseDomainNameLookups() {
+ Map<String, Long> addressLastLookupTimes =
+ new HashMap<String, Long>();
+ for (NodeStatus relay : relays.values()) {
+ addressLastLookupTimes.put(relay.getAddress(),
+ relay.getLastRdnsLookup());
+ }
+ this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes);
+ this.reverseDomainNameResolver.startReverseDomainNameLookups();
+ }
+
private void lookUpCitiesAndASes() {
SortedSet<String> addressStrings = new TreeSet<String>();
for (NodeStatus node : this.knownNodes.values()) {
@@ -285,174 +394,45 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
}
}
- private void writeStatusSummary() {
+ private void setDescriptorPartsOfNodeStatus() {
for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) {
- this.documentStore.store(e.getValue(), e.getKey());
- }
- }
-
- private void processRelayServerDescriptor(
- ServerDescriptor descriptor) {
- String fingerprint = descriptor.getFingerprint();
- DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, true, fingerprint);
- String publishedDateTime =
- DateTimeHelper.format(descriptor.getPublishedMillis());
- if (detailsStatus == null) {
- detailsStatus = new DetailsStatus();
- } else if (detailsStatus.getDescPublished() != null &&
- publishedDateTime.compareTo(
- detailsStatus.getDescPublished()) < 0) {
- return;
- }
- String lastRestartedString = DateTimeHelper.format(
- descriptor.getPublishedMillis() - descriptor.getUptime()
- * DateTimeHelper.ONE_SECOND);
- int bandwidthRate = descriptor.getBandwidthRate();
- int bandwidthBurst = descriptor.getBandwidthBurst();
- int observedBandwidth = descriptor.getBandwidthObserved();
- int advertisedBandwidth = Math.min(bandwidthRate,
- Math.min(bandwidthBurst, observedBandwidth));
- detailsStatus.setDescPublished(publishedDateTime);
- detailsStatus.setLastRestarted(lastRestartedString);
- detailsStatus.setBandwidthRate(bandwidthRate);
- detailsStatus.setBandwidthBurst(bandwidthBurst);
- detailsStatus.setObservedBandwidth(observedBandwidth);
- detailsStatus.setAdvertisedBandwidth(advertisedBandwidth);
- detailsStatus.setExitPolicy(descriptor.getExitPolicyLines());
- detailsStatus.setContact(descriptor.getContact());
- detailsStatus.setPlatform(descriptor.getPlatform());
- detailsStatus.setFamily(descriptor.getFamilyEntries());
- if (descriptor.getIpv6DefaultPolicy() != null &&
- (descriptor.getIpv6DefaultPolicy().equals("accept") ||
- descriptor.getIpv6DefaultPolicy().equals("reject")) &&
- descriptor.getIpv6PortList() != null) {
- Map<String, List<String>> exitPolicyV6Summary =
- new HashMap<String, List<String>>();
- List<String> portsOrPortRanges = Arrays.asList(
- descriptor.getIpv6PortList().split(","));
- exitPolicyV6Summary.put(descriptor.getIpv6DefaultPolicy(),
- portsOrPortRanges);
- detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary);
- }
- if (descriptor.isHibernating()) {
- detailsStatus.setHibernating(true);
- }
- this.documentStore.store(detailsStatus, fingerprint);
- }
-
- private void processBridgeServerDescriptor(
- ServerDescriptor descriptor) {
- String fingerprint = descriptor.getFingerprint();
- DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, true, fingerprint);
- String publishedDateTime =
- DateTimeHelper.format(descriptor.getPublishedMillis());
- if (detailsStatus == null) {
- detailsStatus = new DetailsStatus();
- } else if (detailsStatus.getDescPublished() != null &&
- publishedDateTime.compareTo(
- detailsStatus.getDescPublished()) < 0) {
- return;
- }
- String lastRestartedString = DateTimeHelper.format(
- descriptor.getPublishedMillis() - descriptor.getUptime()
- * DateTimeHelper.ONE_SECOND);
- int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(),
- Math.min(descriptor.getBandwidthBurst(),
- descriptor.getBandwidthObserved()));
- detailsStatus.setDescPublished(publishedDateTime);
- detailsStatus.setLastRestarted(lastRestartedString);
- detailsStatus.setAdvertisedBandwidth(advertisedBandwidth);
- detailsStatus.setPlatform(descriptor.getPlatform());
- this.documentStore.store(detailsStatus, fingerprint);
- }
-
- private void processBridgePoolAssignment(
- BridgePoolAssignment bridgePoolAssignment) {
- for (Map.Entry<String, String> e :
- bridgePoolAssignment.getEntries().entrySet()) {
String fingerprint = e.getKey();
- String details = e.getValue();
- DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, true, fingerprint);
- if (detailsStatus == null) {
- detailsStatus = new DetailsStatus();
- } else if (details.equals(detailsStatus.getPoolAssignment())) {
- continue;
- }
- detailsStatus.setPoolAssignment(details);
- this.documentStore.store(detailsStatus, fingerprint);
- }
- }
-
- private void setCurrentNodes() {
- long cutoff = Math.max(this.relaysLastValidAfterMillis,
- this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L;
- SortedMap<String, NodeStatus> currentNodes =
- new TreeMap<String, NodeStatus>();
- for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) {
- if (e.getValue().getLastSeenMillis() >= cutoff) {
- currentNodes.put(e.getKey(), e.getValue());
- }
- }
- this.relays = new TreeMap<String, NodeStatus>();
- this.bridges = new TreeMap<String, NodeStatus>();
- for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) {
- if (e.getValue().isRelay()) {
- this.relays.put(e.getKey(), e.getValue());
- } else {
- this.bridges.put(e.getKey(), e.getValue());
- }
- }
- }
-
- private Map<String, Map<String, Long>> exitListEntries =
- new HashMap<String, Map<String, Long>>();
-
- private void processExitList(ExitList exitList) {
- for (ExitListEntry exitListEntry : exitList.getExitListEntries()) {
- String fingerprint = exitListEntry.getFingerprint();
- if (exitListEntry.getScanMillis() <
- this.now - DateTimeHelper.ONE_DAY) {
- continue;
- }
- if (!this.exitListEntries.containsKey(fingerprint)) {
- this.exitListEntries.put(fingerprint,
- new HashMap<String, Long>());
- }
- String exitAddress = exitListEntry.getExitAddress();
- long scanMillis = exitListEntry.getScanMillis();
- if (!this.exitListEntries.get(fingerprint).containsKey(exitAddress)
- || this.exitListEntries.get(fingerprint).get(exitAddress)
- < scanMillis) {
- this.exitListEntries.get(fingerprint).put(exitAddress,
- scanMillis);
+ NodeStatus node = e.getValue();
+ if (node.isRelay()) {
+ if (node.getRelayFlags().contains("Running") &&
+ node.getLastSeenMillis() == this.relaysLastValidAfterMillis) {
+ node.setRunning(true);
+ }
+ DetailsStatus detailsStatus = this.documentStore.retrieve(
+ DetailsStatus.class, true, fingerprint);
+ if (detailsStatus != null) {
+ node.setContact(detailsStatus.getContact());
+ if (detailsStatus.getExitAddresses() != null) {
+ for (Map.Entry<String, Long> ea :
+ detailsStatus.getExitAddresses().entrySet()) {
+ if (ea.getValue() >= this.now - DateTimeHelper.ONE_DAY) {
+ node.addExitAddress(ea.getKey());
+ }
+ }
+ }
+ if (detailsStatus.getFamily() != null &&
+ !detailsStatus.getFamily().isEmpty()) {
+ SortedSet<String> familyFingerprints = new TreeSet<String>();
+ for (String familyMember : detailsStatus.getFamily()) {
+ if (familyMember.startsWith("$") &&
+ familyMember.length() == 41) {
+ familyFingerprints.add(familyMember.substring(1));
+ }
+ }
+ if (!familyFingerprints.isEmpty()) {
+ node.setFamilyFingerprints(familyFingerprints);
+ }
+ }
+ }
}
- }
- }
-
- private void startReverseDomainNameLookups() {
- Map<String, Long> addressLastLookupTimes =
- new HashMap<String, Long>();
- for (NodeStatus relay : relays.values()) {
- addressLastLookupTimes.put(relay.getAddress(),
- relay.getLastRdnsLookup());
- }
- this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes);
- this.reverseDomainNameResolver.startReverseDomainNameLookups();
- }
-
- private void finishReverseDomainNameLookups() {
- this.reverseDomainNameResolver.finishReverseDomainNameLookups();
- Map<String, String> lookupResults =
- this.reverseDomainNameResolver.getLookupResults();
- long startedRdnsLookups =
- this.reverseDomainNameResolver.getLookupStartMillis();
- for (NodeStatus relay : relays.values()) {
- if (lookupResults.containsKey(relay.getAddress())) {
- relay.setHostName(lookupResults.get(relay.getAddress()));
- relay.setLastRdnsLookup(startedRdnsLookups);
+ if (!node.isRelay() && node.getRelayFlags().contains("Running") &&
+ node.getLastSeenMillis() == this.bridgesLastPublishedMillis) {
+ node.setRunning(true);
}
}
}
@@ -570,6 +550,26 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
}
}
+ private void finishReverseDomainNameLookups() {
+ this.reverseDomainNameResolver.finishReverseDomainNameLookups();
+ Map<String, String> lookupResults =
+ this.reverseDomainNameResolver.getLookupResults();
+ long startedRdnsLookups =
+ this.reverseDomainNameResolver.getLookupStartMillis();
+ for (NodeStatus relay : relays.values()) {
+ if (lookupResults.containsKey(relay.getAddress())) {
+ relay.setHostName(lookupResults.get(relay.getAddress()));
+ relay.setLastRdnsLookup(startedRdnsLookups);
+ }
+ }
+ }
+
+ private void writeStatusSummary() {
+ for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) {
+ this.documentStore.store(e.getValue(), e.getKey());
+ }
+ }
+
private void updateDetailsStatuses() {
SortedSet<String> fingerprints = new TreeSet<String>();
fingerprints.addAll(this.exitListEntries.keySet());
