commit 96b098c0c345b83a8dfd276bf0a2e2ae6d749f1d Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Thu Jul 17 09:20:09 2014 +0200 Reorder methods in NodeDetailsStatusUpdater. --- .../onionoo/NodeDetailsStatusUpdater.java | 456 ++++++++++---------- 1 file changed, 228 insertions(+), 228 deletions(-) diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java index 5c253bd..e898638 100644 --- a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java +++ b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java @@ -77,41 +77,95 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof RelayNetworkStatusConsensus) { + if (descriptor instanceof ServerDescriptor && relay) { + this.processRelayServerDescriptor((ServerDescriptor) descriptor); + } else if (descriptor instanceof ExitList) { + this.processExitList((ExitList) descriptor); + } else if (descriptor instanceof RelayNetworkStatusConsensus) { this.processRelayNetworkStatusConsensus( (RelayNetworkStatusConsensus) descriptor); - } else if (descriptor instanceof ServerDescriptor && relay) { - this.processRelayServerDescriptor((ServerDescriptor) descriptor); - } else if (descriptor instanceof BridgeNetworkStatus) { - this.processBridgeNetworkStatus((BridgeNetworkStatus) descriptor); } else if (descriptor instanceof ServerDescriptor && !relay) { this.processBridgeServerDescriptor((ServerDescriptor) descriptor); } else if (descriptor instanceof BridgePoolAssignment) { this.processBridgePoolAssignment((BridgePoolAssignment) descriptor); - } else if (descriptor instanceof ExitList) { - this.processExitList((ExitList) descriptor); + } else if (descriptor instanceof BridgeNetworkStatus) { + this.processBridgeNetworkStatus((BridgeNetworkStatus) descriptor); } } - public void updateStatuses() { - this.readStatusSummary(); - Logger.printStatusTime("Read status summary"); - this.setCurrentNodes(); - Logger.printStatusTime("Set current node fingerprints"); - this.startReverseDomainNameLookups(); - Logger.printStatusTime("Started reverse domain name lookups"); - this.lookUpCitiesAndASes(); - Logger.printStatusTime("Looked up cities and ASes"); - this.setDescriptorPartsOfNodeStatus(); - Logger.printStatusTime("Set descriptor parts of node statuses."); - this.calculatePathSelectionProbabilities(); - Logger.printStatusTime("Calculated path selection probabilities"); - this.finishReverseDomainNameLookups(); - Logger.printStatusTime("Finished reverse domain name lookups"); - this.writeStatusSummary(); - Logger.printStatusTime("Wrote status summary"); - this.updateDetailsStatuses(); - Logger.printStatusTime("Updated exit addresses in details statuses"); + private void processRelayServerDescriptor( + ServerDescriptor descriptor) { + String fingerprint = descriptor.getFingerprint(); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + String publishedDateTime = + DateTimeHelper.format(descriptor.getPublishedMillis()); + if (detailsStatus == null) { + detailsStatus = new DetailsStatus(); + } else if (detailsStatus.getDescPublished() != null && + publishedDateTime.compareTo( + detailsStatus.getDescPublished()) < 0) { + return; + } + String lastRestartedString = DateTimeHelper.format( + descriptor.getPublishedMillis() - descriptor.getUptime() + * DateTimeHelper.ONE_SECOND); + int bandwidthRate = descriptor.getBandwidthRate(); + int bandwidthBurst = descriptor.getBandwidthBurst(); + int observedBandwidth = descriptor.getBandwidthObserved(); + int advertisedBandwidth = Math.min(bandwidthRate, + Math.min(bandwidthBurst, observedBandwidth)); + detailsStatus.setDescPublished(publishedDateTime); + detailsStatus.setLastRestarted(lastRestartedString); + detailsStatus.setBandwidthRate(bandwidthRate); + detailsStatus.setBandwidthBurst(bandwidthBurst); + detailsStatus.setObservedBandwidth(observedBandwidth); + detailsStatus.setAdvertisedBandwidth(advertisedBandwidth); + detailsStatus.setExitPolicy(descriptor.getExitPolicyLines()); + detailsStatus.setContact(descriptor.getContact()); + detailsStatus.setPlatform(descriptor.getPlatform()); + detailsStatus.setFamily(descriptor.getFamilyEntries()); + if (descriptor.getIpv6DefaultPolicy() != null && + (descriptor.getIpv6DefaultPolicy().equals("accept") || + descriptor.getIpv6DefaultPolicy().equals("reject")) && + descriptor.getIpv6PortList() != null) { + Map<String, List<String>> exitPolicyV6Summary = + new HashMap<String, List<String>>(); + List<String> portsOrPortRanges = Arrays.asList( + descriptor.getIpv6PortList().split(",")); + exitPolicyV6Summary.put(descriptor.getIpv6DefaultPolicy(), + portsOrPortRanges); + detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary); + } + if (descriptor.isHibernating()) { + detailsStatus.setHibernating(true); + } + this.documentStore.store(detailsStatus, fingerprint); + } + + private Map<String, Map<String, Long>> exitListEntries = + new HashMap<String, Map<String, Long>>(); + + private void processExitList(ExitList exitList) { + for (ExitListEntry exitListEntry : exitList.getExitListEntries()) { + String fingerprint = exitListEntry.getFingerprint(); + if (exitListEntry.getScanMillis() < + this.now - DateTimeHelper.ONE_DAY) { + continue; + } + if (!this.exitListEntries.containsKey(fingerprint)) { + this.exitListEntries.put(fingerprint, + new HashMap<String, Long>()); + } + String exitAddress = exitListEntry.getExitAddress(); + long scanMillis = exitListEntry.getScanMillis(); + if (!this.exitListEntries.get(fingerprint).containsKey(exitAddress) + || this.exitListEntries.get(fingerprint).get(exitAddress) + < scanMillis) { + this.exitListEntries.get(fingerprint).put(exitAddress, + scanMillis); + } + } } private void processRelayNetworkStatusConsensus( @@ -161,6 +215,51 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } } + private void processBridgeServerDescriptor( + ServerDescriptor descriptor) { + String fingerprint = descriptor.getFingerprint(); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + String publishedDateTime = + DateTimeHelper.format(descriptor.getPublishedMillis()); + if (detailsStatus == null) { + detailsStatus = new DetailsStatus(); + } else if (detailsStatus.getDescPublished() != null && + publishedDateTime.compareTo( + detailsStatus.getDescPublished()) < 0) { + return; + } + String lastRestartedString = DateTimeHelper.format( + descriptor.getPublishedMillis() - descriptor.getUptime() + * DateTimeHelper.ONE_SECOND); + int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(), + Math.min(descriptor.getBandwidthBurst(), + descriptor.getBandwidthObserved())); + detailsStatus.setDescPublished(publishedDateTime); + detailsStatus.setLastRestarted(lastRestartedString); + detailsStatus.setAdvertisedBandwidth(advertisedBandwidth); + detailsStatus.setPlatform(descriptor.getPlatform()); + this.documentStore.store(detailsStatus, fingerprint); + } + + private void processBridgePoolAssignment( + BridgePoolAssignment bridgePoolAssignment) { + for (Map.Entry<String, String> e : + bridgePoolAssignment.getEntries().entrySet()) { + String fingerprint = e.getKey(); + String details = e.getValue(); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + if (detailsStatus == null) { + detailsStatus = new DetailsStatus(); + } else if (details.equals(detailsStatus.getPoolAssignment())) { + continue; + } + detailsStatus.setPoolAssignment(details); + this.documentStore.store(detailsStatus, fingerprint); + } + } + private void processBridgeNetworkStatus(BridgeNetworkStatus status) { long publishedMillis = status.getPublishedMillis(); if (publishedMillis > this.bridgesLastPublishedMillis) { @@ -188,6 +287,27 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, this.bridgeStatusesProcessed++; } + public void updateStatuses() { + this.readStatusSummary(); + Logger.printStatusTime("Read status summary"); + this.setCurrentNodes(); + Logger.printStatusTime("Set current node fingerprints"); + this.startReverseDomainNameLookups(); + Logger.printStatusTime("Started reverse domain name lookups"); + this.lookUpCitiesAndASes(); + Logger.printStatusTime("Looked up cities and ASes"); + this.setDescriptorPartsOfNodeStatus(); + Logger.printStatusTime("Set descriptor parts of node statuses."); + this.calculatePathSelectionProbabilities(); + Logger.printStatusTime("Calculated path selection probabilities"); + this.finishReverseDomainNameLookups(); + Logger.printStatusTime("Finished reverse domain name lookups"); + this.writeStatusSummary(); + Logger.printStatusTime("Wrote status summary"); + this.updateDetailsStatuses(); + Logger.printStatusTime("Updated exit addresses in details statuses"); + } + private void readStatusSummary() { SortedSet<String> fingerprints = this.documentStore.list( NodeStatus.class, true); @@ -209,49 +329,38 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } } - private void setDescriptorPartsOfNodeStatus() { + private void setCurrentNodes() { + long cutoff = Math.max(this.relaysLastValidAfterMillis, + this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L; + SortedMap<String, NodeStatus> currentNodes = + new TreeMap<String, NodeStatus>(); for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus node = e.getValue(); - if (node.isRelay()) { - if (node.getRelayFlags().contains("Running") && - node.getLastSeenMillis() == this.relaysLastValidAfterMillis) { - node.setRunning(true); - } - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus != null) { - node.setContact(detailsStatus.getContact()); - if (detailsStatus.getExitAddresses() != null) { - for (Map.Entry<String, Long> ea : - detailsStatus.getExitAddresses().entrySet()) { - if (ea.getValue() >= this.now - DateTimeHelper.ONE_DAY) { - node.addExitAddress(ea.getKey()); - } - } - } - if (detailsStatus.getFamily() != null && - !detailsStatus.getFamily().isEmpty()) { - SortedSet<String> familyFingerprints = new TreeSet<String>(); - for (String familyMember : detailsStatus.getFamily()) { - if (familyMember.startsWith("$") && - familyMember.length() == 41) { - familyFingerprints.add(familyMember.substring(1)); - } - } - if (!familyFingerprints.isEmpty()) { - node.setFamilyFingerprints(familyFingerprints); - } - } - } + if (e.getValue().getLastSeenMillis() >= cutoff) { + currentNodes.put(e.getKey(), e.getValue()); } - if (!node.isRelay() && node.getRelayFlags().contains("Running") && - node.getLastSeenMillis() == this.bridgesLastPublishedMillis) { - node.setRunning(true); + } + this.relays = new TreeMap<String, NodeStatus>(); + this.bridges = new TreeMap<String, NodeStatus>(); + for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) { + if (e.getValue().isRelay()) { + this.relays.put(e.getKey(), e.getValue()); + } else { + this.bridges.put(e.getKey(), e.getValue()); } } } + private void startReverseDomainNameLookups() { + Map<String, Long> addressLastLookupTimes = + new HashMap<String, Long>(); + for (NodeStatus relay : relays.values()) { + addressLastLookupTimes.put(relay.getAddress(), + relay.getLastRdnsLookup()); + } + this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes); + this.reverseDomainNameResolver.startReverseDomainNameLookups(); + } + private void lookUpCitiesAndASes() { SortedSet<String> addressStrings = new TreeSet<String>(); for (NodeStatus node : this.knownNodes.values()) { @@ -285,174 +394,45 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } } - private void writeStatusSummary() { + private void setDescriptorPartsOfNodeStatus() { for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { - this.documentStore.store(e.getValue(), e.getKey()); - } - } - - private void processRelayServerDescriptor( - ServerDescriptor descriptor) { - String fingerprint = descriptor.getFingerprint(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - String publishedDateTime = - DateTimeHelper.format(descriptor.getPublishedMillis()); - if (detailsStatus == null) { - detailsStatus = new DetailsStatus(); - } else if (detailsStatus.getDescPublished() != null && - publishedDateTime.compareTo( - detailsStatus.getDescPublished()) < 0) { - return; - } - String lastRestartedString = DateTimeHelper.format( - descriptor.getPublishedMillis() - descriptor.getUptime() - * DateTimeHelper.ONE_SECOND); - int bandwidthRate = descriptor.getBandwidthRate(); - int bandwidthBurst = descriptor.getBandwidthBurst(); - int observedBandwidth = descriptor.getBandwidthObserved(); - int advertisedBandwidth = Math.min(bandwidthRate, - Math.min(bandwidthBurst, observedBandwidth)); - detailsStatus.setDescPublished(publishedDateTime); - detailsStatus.setLastRestarted(lastRestartedString); - detailsStatus.setBandwidthRate(bandwidthRate); - detailsStatus.setBandwidthBurst(bandwidthBurst); - detailsStatus.setObservedBandwidth(observedBandwidth); - detailsStatus.setAdvertisedBandwidth(advertisedBandwidth); - detailsStatus.setExitPolicy(descriptor.getExitPolicyLines()); - detailsStatus.setContact(descriptor.getContact()); - detailsStatus.setPlatform(descriptor.getPlatform()); - detailsStatus.setFamily(descriptor.getFamilyEntries()); - if (descriptor.getIpv6DefaultPolicy() != null && - (descriptor.getIpv6DefaultPolicy().equals("accept") || - descriptor.getIpv6DefaultPolicy().equals("reject")) && - descriptor.getIpv6PortList() != null) { - Map<String, List<String>> exitPolicyV6Summary = - new HashMap<String, List<String>>(); - List<String> portsOrPortRanges = Arrays.asList( - descriptor.getIpv6PortList().split(",")); - exitPolicyV6Summary.put(descriptor.getIpv6DefaultPolicy(), - portsOrPortRanges); - detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary); - } - if (descriptor.isHibernating()) { - detailsStatus.setHibernating(true); - } - this.documentStore.store(detailsStatus, fingerprint); - } - - private void processBridgeServerDescriptor( - ServerDescriptor descriptor) { - String fingerprint = descriptor.getFingerprint(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - String publishedDateTime = - DateTimeHelper.format(descriptor.getPublishedMillis()); - if (detailsStatus == null) { - detailsStatus = new DetailsStatus(); - } else if (detailsStatus.getDescPublished() != null && - publishedDateTime.compareTo( - detailsStatus.getDescPublished()) < 0) { - return; - } - String lastRestartedString = DateTimeHelper.format( - descriptor.getPublishedMillis() - descriptor.getUptime() - * DateTimeHelper.ONE_SECOND); - int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(), - Math.min(descriptor.getBandwidthBurst(), - descriptor.getBandwidthObserved())); - detailsStatus.setDescPublished(publishedDateTime); - detailsStatus.setLastRestarted(lastRestartedString); - detailsStatus.setAdvertisedBandwidth(advertisedBandwidth); - detailsStatus.setPlatform(descriptor.getPlatform()); - this.documentStore.store(detailsStatus, fingerprint); - } - - private void processBridgePoolAssignment( - BridgePoolAssignment bridgePoolAssignment) { - for (Map.Entry<String, String> e : - bridgePoolAssignment.getEntries().entrySet()) { String fingerprint = e.getKey(); - String details = e.getValue(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus == null) { - detailsStatus = new DetailsStatus(); - } else if (details.equals(detailsStatus.getPoolAssignment())) { - continue; - } - detailsStatus.setPoolAssignment(details); - this.documentStore.store(detailsStatus, fingerprint); - } - } - - private void setCurrentNodes() { - long cutoff = Math.max(this.relaysLastValidAfterMillis, - this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L; - SortedMap<String, NodeStatus> currentNodes = - new TreeMap<String, NodeStatus>(); - for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { - if (e.getValue().getLastSeenMillis() >= cutoff) { - currentNodes.put(e.getKey(), e.getValue()); - } - } - this.relays = new TreeMap<String, NodeStatus>(); - this.bridges = new TreeMap<String, NodeStatus>(); - for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) { - if (e.getValue().isRelay()) { - this.relays.put(e.getKey(), e.getValue()); - } else { - this.bridges.put(e.getKey(), e.getValue()); - } - } - } - - private Map<String, Map<String, Long>> exitListEntries = - new HashMap<String, Map<String, Long>>(); - - private void processExitList(ExitList exitList) { - for (ExitListEntry exitListEntry : exitList.getExitListEntries()) { - String fingerprint = exitListEntry.getFingerprint(); - if (exitListEntry.getScanMillis() < - this.now - DateTimeHelper.ONE_DAY) { - continue; - } - if (!this.exitListEntries.containsKey(fingerprint)) { - this.exitListEntries.put(fingerprint, - new HashMap<String, Long>()); - } - String exitAddress = exitListEntry.getExitAddress(); - long scanMillis = exitListEntry.getScanMillis(); - if (!this.exitListEntries.get(fingerprint).containsKey(exitAddress) - || this.exitListEntries.get(fingerprint).get(exitAddress) - < scanMillis) { - this.exitListEntries.get(fingerprint).put(exitAddress, - scanMillis); + NodeStatus node = e.getValue(); + if (node.isRelay()) { + if (node.getRelayFlags().contains("Running") && + node.getLastSeenMillis() == this.relaysLastValidAfterMillis) { + node.setRunning(true); + } + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + if (detailsStatus != null) { + node.setContact(detailsStatus.getContact()); + if (detailsStatus.getExitAddresses() != null) { + for (Map.Entry<String, Long> ea : + detailsStatus.getExitAddresses().entrySet()) { + if (ea.getValue() >= this.now - DateTimeHelper.ONE_DAY) { + node.addExitAddress(ea.getKey()); + } + } + } + if (detailsStatus.getFamily() != null && + !detailsStatus.getFamily().isEmpty()) { + SortedSet<String> familyFingerprints = new TreeSet<String>(); + for (String familyMember : detailsStatus.getFamily()) { + if (familyMember.startsWith("$") && + familyMember.length() == 41) { + familyFingerprints.add(familyMember.substring(1)); + } + } + if (!familyFingerprints.isEmpty()) { + node.setFamilyFingerprints(familyFingerprints); + } + } + } } - } - } - - private void startReverseDomainNameLookups() { - Map<String, Long> addressLastLookupTimes = - new HashMap<String, Long>(); - for (NodeStatus relay : relays.values()) { - addressLastLookupTimes.put(relay.getAddress(), - relay.getLastRdnsLookup()); - } - this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes); - this.reverseDomainNameResolver.startReverseDomainNameLookups(); - } - - private void finishReverseDomainNameLookups() { - this.reverseDomainNameResolver.finishReverseDomainNameLookups(); - Map<String, String> lookupResults = - this.reverseDomainNameResolver.getLookupResults(); - long startedRdnsLookups = - this.reverseDomainNameResolver.getLookupStartMillis(); - for (NodeStatus relay : relays.values()) { - if (lookupResults.containsKey(relay.getAddress())) { - relay.setHostName(lookupResults.get(relay.getAddress())); - relay.setLastRdnsLookup(startedRdnsLookups); + if (!node.isRelay() && node.getRelayFlags().contains("Running") && + node.getLastSeenMillis() == this.bridgesLastPublishedMillis) { + node.setRunning(true); } } } @@ -570,6 +550,26 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } } + private void finishReverseDomainNameLookups() { + this.reverseDomainNameResolver.finishReverseDomainNameLookups(); + Map<String, String> lookupResults = + this.reverseDomainNameResolver.getLookupResults(); + long startedRdnsLookups = + this.reverseDomainNameResolver.getLookupStartMillis(); + for (NodeStatus relay : relays.values()) { + if (lookupResults.containsKey(relay.getAddress())) { + relay.setHostName(lookupResults.get(relay.getAddress())); + relay.setLastRdnsLookup(startedRdnsLookups); + } + } + } + + private void writeStatusSummary() { + for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { + this.documentStore.store(e.getValue(), e.getKey()); + } + } + private void updateDetailsStatuses() { SortedSet<String> fingerprints = new TreeSet<String>(); fingerprints.addAll(this.exitListEntries.keySet());