commit 0dabab48c6792579498692503afd8076b219662e Author: Karsten Loesing karsten.loesing@gmx.net Date: Fri Jun 28 19:39:12 2013 +0200

Refactor rDNS code and add more execution stats. --- src/org/torproject/onionoo/DetailsDataWriter.java | 117 +++---------- src/org/torproject/onionoo/DocumentStore.java | 4 + src/org/torproject/onionoo/LookupService.java | 20 +++ src/org/torproject/onionoo/Main.java | 48 +++--- src/org/torproject/onionoo/NodeDataWriter.java | 31 ++++ .../onionoo/ReverseDomainNameResolver.java | 178 ++++++++++++++++++++ 6 files changed, 273 insertions(+), 125 deletions(-)

diff --git a/src/org/torproject/onionoo/DetailsDataWriter.java b/src/org/torproject/onionoo/DetailsDataWriter.java index 3d49169..e4ccb78 100644 --- a/src/org/torproject/onionoo/DetailsDataWriter.java +++ b/src/org/torproject/onionoo/DetailsDataWriter.java @@ -2,8 +2,6 @@ * See LICENSE for licensing information */ package org.torproject.onionoo;

-import java.net.InetAddress; -import java.net.UnknownHostException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -38,6 +36,8 @@ public class DetailsDataWriter {

private DescriptorSource descriptorSource;

+ private ReverseDomainNameResolver reverseDomainNameResolver; + private DocumentStore documentStore;

private SortedMap<String, NodeStatus> relays; @@ -45,8 +45,10 @@ public class DetailsDataWriter { private SortedMap<String, NodeStatus> bridges;

public DetailsDataWriter(DescriptorSource descriptorSource, + ReverseDomainNameResolver reverseDomainNameResolver, DocumentStore documentStore) { this.descriptorSource = descriptorSource; + this.reverseDomainNameResolver = reverseDomainNameResolver; this.documentStore = documentStore; }

@@ -63,109 +65,28 @@ public class DetailsDataWriter { } }

- private static final long RDNS_LOOKUP_MAX_REQUEST_MILLIS = 10L * 1000L; - private static final long RDNS_LOOKUP_MAX_DURATION_MILLIS = 5L * 60L - * 1000L; - private static final long RDNS_LOOKUP_MAX_AGE_MILLIS = 12L * 60L * 60L - * 1000L; - private static final int RDNS_LOOKUP_WORKERS_NUM = 5; - private Set<String> rdnsLookupJobs; - private Map<String, String> rdnsLookupResults; - private long startedRdnsLookups; - private List<RdnsLookupWorker> rdnsLookupWorkers; public void startReverseDomainNameLookups() { - this.startedRdnsLookups = System.currentTimeMillis(); - this.rdnsLookupJobs = new HashSet<String>(); + Map<String, Long> addressLastLookupTimes = + new HashMap<String, Long>(); for (NodeStatus relay : relays.values()) { - if (relay.getLastRdnsLookup() < this.startedRdnsLookups - - RDNS_LOOKUP_MAX_AGE_MILLIS) { - this.rdnsLookupJobs.add(relay.getAddress()); - } - } - this.rdnsLookupResults = new HashMap<String, String>(); - this.rdnsLookupWorkers = new ArrayList<RdnsLookupWorker>(); - for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) { - RdnsLookupWorker rdnsLookupWorker = new RdnsLookupWorker(); - this.rdnsLookupWorkers.add(rdnsLookupWorker); - rdnsLookupWorker.setDaemon(true); - rdnsLookupWorker.start(); + addressLastLookupTimes.put(relay.getAddress(), + relay.getLastRdnsLookup()); } + this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes); + this.reverseDomainNameResolver.startReverseDomainNameLookups(); }

public void finishReverseDomainNameLookups() { - for (RdnsLookupWorker rdnsLookupWorker : this.rdnsLookupWorkers) { - try { - rdnsLookupWorker.join(); - } catch (InterruptedException e) { - /* This is not something that we can take care of. Just leave the - * worker thread alone. */ - } - } - synchronized (this.rdnsLookupResults) { - for (NodeStatus relay : relays.values()) { - if (this.rdnsLookupResults.containsKey(relay.getAddress())) { - relay.setHostName(this.rdnsLookupResults.get( - relay.getAddress())); - relay.setLastRdnsLookup(this.startedRdnsLookups); - } - } - } - } - - private class RdnsLookupWorker extends Thread { - public void run() { - while (System.currentTimeMillis() - RDNS_LOOKUP_MAX_DURATION_MILLIS - <= startedRdnsLookups) { - String rdnsLookupJob = null; - synchronized (rdnsLookupJobs) { - for (String job : rdnsLookupJobs) { - rdnsLookupJob = job; - rdnsLookupJobs.remove(job); - break; - } - } - if (rdnsLookupJob == null) { - break; - } - RdnsLookupRequest request = new RdnsLookupRequest(this, - rdnsLookupJob); - request.setDaemon(true); - request.start(); - try { - Thread.sleep(RDNS_LOOKUP_MAX_REQUEST_MILLIS); - } catch (InterruptedException e) { - /* Getting interrupted should be the default case. */ - } - String hostName = request.getHostName(); - if (hostName != null) { - synchronized (rdnsLookupResults) { - rdnsLookupResults.put(rdnsLookupJob, hostName); - } - } - } - } - } - - private class RdnsLookupRequest extends Thread { - RdnsLookupWorker parent; - String address, hostName; - public RdnsLookupRequest(RdnsLookupWorker parent, String address) { - this.parent = parent; - this.address = address; - } - public void run() { - try { - String result = InetAddress.getByName(this.address).getHostName(); - synchronized (this) { - this.hostName = result; - } - } catch (UnknownHostException e) { - /* We'll try again the next time. */ + this.reverseDomainNameResolver.finishReverseDomainNameLookups(); + Map<String, String> lookupResults = + this.reverseDomainNameResolver.getLookupResults(); + long startedRdnsLookups = + this.reverseDomainNameResolver.getLookupStartMillis(); + for (NodeStatus relay : relays.values()) { + if (lookupResults.containsKey(relay.getAddress())) { + relay.setHostName(lookupResults.get(relay.getAddress())); + relay.setLastRdnsLookup(startedRdnsLookups); } - this.parent.interrupt(); - } - public synchronized String getHostName() { - return hostName; } }

diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index 829f35f..11d7d1b 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -374,6 +374,10 @@ public class DocumentStore { }

public void flushDocumentCache() { + /* Write cached node statuses to disk, and write update file + * containing current time. It's important to write the update file + * now, not earlier, because the front-end should not read new node + * statuses until all details, bandwidths, and weights are ready. */ if (this.listedArchivedNodeStatuses) { this.writeNodeStatuses(false); this.writeNodeStatuses(true); diff --git a/src/org/torproject/onionoo/LookupService.java b/src/org/torproject/onionoo/LookupService.java index 4071e26..3791443 100644 --- a/src/org/torproject/onionoo/LookupService.java +++ b/src/org/torproject/onionoo/LookupService.java @@ -379,6 +379,10 @@ public class LookupService { lookupResults.put(addressString, lookupResult); }

+ /* Keep statistics. */ + this.addressesLookedUp += addressStrings.size(); + this.addressesResolved += lookupResults.size(); + return lookupResults; }

@@ -392,4 +396,20 @@ public class LookupService { String aSNumber; String aSName; } + + int addressesLookedUp = 0, addressesResolved = 0; + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + formatDecimalNumber(addressesLookedUp) + + " addresses looked up\n"); + sb.append(" " + formatDecimalNumber(addressesResolved) + + " addresses resolved\n"); + return sb.toString(); + } + + //TODO This method should go into a utility class. + private static String formatDecimalNumber(long decimalNumber) { + return String.format("%,d", decimalNumber); + } } diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 2232d81..3fca6f0 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -10,19 +10,17 @@ import java.util.SortedMap; public class Main { public static void main(String[] args) {

- printStatus("Initializing descriptor source."); + printStatus("Initializing."); DescriptorSource dso = new DescriptorSource(new File("in"), new File("status")); printStatusTime("Initialized descriptor source"); - - printStatus("Initializing document store."); DocumentStore ds = new DocumentStore(new File("status"), new File("out")); printStatusTime("Initialized document store"); - - printStatus("Initializing lookup service."); LookupService ls = new LookupService(new File("geoip")); printStatusTime("Initialized Geoip lookup service"); + ReverseDomainNameResolver rdnr = new ReverseDomainNameResolver(); + printStatusTime("Initialized reverse domain name resolver");

printStatus("Updating internal node list."); NodeDataWriter ndw = new NodeDataWriter(dso, ls, ds); @@ -38,13 +36,14 @@ public class Main { printStatusTime("Set running bits"); ndw.writeStatusSummary(); printStatusTime("Wrote status summary"); + ndw.writeOutSummary(); + printStatusTime("Wrote out summary"); SortedMap<String, NodeStatus> currentNodes = ndw.getCurrentNodes(); SortedMap<String, Integer> lastBandwidthWeights = ndw.getLastBandwidthWeights(); - // TODO Could write statistics here, too.

printStatus("Updating detail data."); - DetailsDataWriter ddw = new DetailsDataWriter(dso, ds); + DetailsDataWriter ddw = new DetailsDataWriter(dso, rdnr, ds); // TODO Instead of using ndw's currentNodes and lastBandwidthWeights, // parse statuses once again, keeping separate parse history. Allows // us to run ndw and ddw in parallel in the future. Alternatively, @@ -67,7 +66,6 @@ public class Main { printStatusTime("Finished reverse domain name lookups"); ddw.writeOutDetails(); printStatusTime("Wrote detail data files"); - // TODO Could write statistics here, too.

printStatus("Updating bandwidth data."); BandwidthDataWriter bdw = new BandwidthDataWriter(dso, ds); @@ -81,7 +79,6 @@ public class Main { // future. bdw.deleteObsoleteBandwidthFiles(); printStatusTime("Deleted obsolete bandwidth files"); - // TODO Could write statistics here, too.

printStatus("Updating weights data."); WeightsDataWriter wdw = new WeightsDataWriter(dso, ds); @@ -98,27 +95,24 @@ public class Main { // which allows us to run ndw and wdw in parallel in the future. wdw.deleteObsoleteWeightsDataFiles(); printStatusTime("Deleted obsolete weights files"); - // TODO Could write statistics here, too. - - printStatus("Updating summary data."); - ndw.writeOutSummary(); - printStatusTime("Wrote out summary"); - // TODO Could write statistics here, too. - - // TODO "Shut down" lookup service and write statistics about number - // of (successfully) looked up addresses.

- printStatus("Shutting down descriptor source."); + printStatus("Shutting down."); dso.writeHistoryFiles(); printStatusTime("Wrote parse histories"); - printStatistics(dso.getStatsString()); - printStatusTime("Shut down descriptor source"); - - printStatus("Shutting down document store."); ds.flushDocumentCache(); printStatusTime("Flushed document cache"); - printStatistics(ds.getStatsString()); - printStatusTime("Shut down document store"); + + printStatus("Gathering statistics."); + printStatistics("Node data writer", ndw.getStatsString()); + /* TODO Add statistics to remaining *Writers. */ + //printStatistics("Details data writer", ddw.getStatsString()); + //printStatistics("Bandwidth data writer", bdw.getStatsString()); + //printStatistics("Weights data writer", wdw.getStatsString()); + printStatistics("Descriptor source", dso.getStatsString()); + printStatistics("Document store", ds.getStatsString()); + printStatistics("GeoIP lookup service", ls.getStatsString()); + printStatistics("Reverse domain name resolver", + rdnr.getStatsString());

printStatus("Terminating."); } @@ -131,8 +125,8 @@ public class Main { printedLastStatusMessage = System.currentTimeMillis(); }

- private static void printStatistics(String message) { - System.out.print(" Statistics:\n" + message); + private static void printStatistics(String component, String message) { + System.out.print(" " + component + " statistics:\n" + message); }

private static void printStatusTime(String message) { diff --git a/src/org/torproject/onionoo/NodeDataWriter.java b/src/org/torproject/onionoo/NodeDataWriter.java index 9cfbb21..4afc595 100644 --- a/src/org/torproject/onionoo/NodeDataWriter.java +++ b/src/org/torproject/onionoo/NodeDataWriter.java @@ -33,6 +33,9 @@ public class NodeDataWriter {

private SortedMap<String, Integer> lastBandwidthWeights = null;

+ private int relaysUpdated = 0, relaysAdded = 0, + relayConsensusesProcessed = 0, bridgesUpdated = 0, + bridgesAdded = 0, bridgeStatusesProcessed = 0; public NodeDataWriter(DescriptorSource descriptorSource, LookupService lookupService, DocumentStore documentStore) { this.descriptorSource = descriptorSource; @@ -115,10 +118,13 @@ public class NodeDataWriter { validAfterMillis, null); if (this.knownNodes.containsKey(fingerprint)) { this.knownNodes.get(fingerprint).update(newNodeStatus); + this.relaysUpdated++; } else { this.knownNodes.put(fingerprint, newNodeStatus); + this.relaysAdded++; } } + this.relayConsensusesProcessed++; if (this.relaysLastValidAfterMillis == validAfterMillis) { this.lastBandwidthWeights = consensus.getBandwidthWeights(); } @@ -195,10 +201,13 @@ public class NodeDataWriter { -1L, null, null, publishedMillis, -1L, null); if (this.knownNodes.containsKey(fingerprint)) { this.knownNodes.get(fingerprint).update(newNodeStatus); + this.bridgesUpdated++; } else { this.knownNodes.put(fingerprint, newNodeStatus); + this.bridgesAdded++; } } + this.bridgeStatusesProcessed++; }

public void writeStatusSummary() { @@ -233,5 +242,27 @@ public class NodeDataWriter { public SortedMap<String, Integer> getLastBandwidthWeights() { return this.lastBandwidthWeights; } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + formatDecimalNumber(relayConsensusesProcessed) + + " relay consensuses processed\n"); + sb.append(" " + formatDecimalNumber(relaysUpdated) + + " relays updated\n"); + sb.append(" " + formatDecimalNumber(relaysAdded) + + " relays added\n"); + sb.append(" " + formatDecimalNumber(bridgeStatusesProcessed) + + " bridge statuses processed\n"); + sb.append(" " + formatDecimalNumber(bridgesUpdated) + + " bridges updated\n"); + sb.append(" " + formatDecimalNumber(bridgesAdded) + + " bridges added\n"); + return sb.toString(); + } + + //TODO This method should go into a utility class. + private static String formatDecimalNumber(long decimalNumber) { + return String.format("%,d", decimalNumber); + } }

diff --git a/src/org/torproject/onionoo/ReverseDomainNameResolver.java b/src/org/torproject/onionoo/ReverseDomainNameResolver.java new file mode 100644 index 0000000..b04ad0a --- /dev/null +++ b/src/org/torproject/onionoo/ReverseDomainNameResolver.java @@ -0,0 +1,178 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ReverseDomainNameResolver { + + private class RdnsLookupWorker extends Thread { + public void run() { + while (System.currentTimeMillis() - RDNS_LOOKUP_MAX_DURATION_MILLIS + <= startedRdnsLookups) { + String rdnsLookupJob = null; + synchronized (rdnsLookupJobs) { + for (String job : rdnsLookupJobs) { + rdnsLookupJob = job; + rdnsLookupJobs.remove(job); + break; + } + } + if (rdnsLookupJob == null) { + break; + } + RdnsLookupRequest request = new RdnsLookupRequest(this, + rdnsLookupJob); + request.setDaemon(true); + request.start(); + try { + Thread.sleep(RDNS_LOOKUP_MAX_REQUEST_MILLIS); + } catch (InterruptedException e) { + /* Getting interrupted should be the default case. */ + } + String hostName = request.getHostName(); + if (hostName != null) { + synchronized (rdnsLookupResults) { + rdnsLookupResults.put(rdnsLookupJob, hostName); + } + } + long lookupMillis = request.getLookupMillis(); + if (lookupMillis >= 0L) { + synchronized (rdnsLookupMillis) { + rdnsLookupMillis.add(lookupMillis); + } + } + } + } + } + + private class RdnsLookupRequest extends Thread { + RdnsLookupWorker parent; + String address, hostName; + long lookupStartedMillis = -1L, lookupCompletedMillis = -1L; + public RdnsLookupRequest(RdnsLookupWorker parent, String address) { + this.parent = parent; + this.address = address; + } + public void run() { + this.lookupStartedMillis = System.currentTimeMillis(); + try { + String result = InetAddress.getByName(this.address).getHostName(); + synchronized (this) { + this.hostName = result; + } + } catch (UnknownHostException e) { + /* We'll try again the next time. */ + } + this.lookupCompletedMillis = System.currentTimeMillis(); + this.parent.interrupt(); + } + public synchronized String getHostName() { + return hostName; + } + public synchronized long getLookupMillis() { + return this.lookupCompletedMillis - this.lookupStartedMillis; + } + } + + private static final long RDNS_LOOKUP_MAX_REQUEST_MILLIS = 10L * 1000L; + private static final long RDNS_LOOKUP_MAX_DURATION_MILLIS = 5L * 60L + * 1000L; + private static final long RDNS_LOOKUP_MAX_AGE_MILLIS = 12L * 60L * 60L + * 1000L; + private static final int RDNS_LOOKUP_WORKERS_NUM = 5; + + private Map<String, Long> addressLastLookupTimes; + + private Set<String> rdnsLookupJobs; + + private Map<String, String> rdnsLookupResults; + + private List<Long> rdnsLookupMillis; + + private long startedRdnsLookups; + + private List<RdnsLookupWorker> rdnsLookupWorkers; + + public void setAddresses(Map<String, Long> addressLastLookupTimes) { + this.addressLastLookupTimes = addressLastLookupTimes; + } + + public void startReverseDomainNameLookups() { + this.startedRdnsLookups = System.currentTimeMillis(); + this.rdnsLookupJobs = new HashSet<String>(); + for (Map.Entry<String, Long> e : + this.addressLastLookupTimes.entrySet()) { + if (e.getValue() < this.startedRdnsLookups + - RDNS_LOOKUP_MAX_AGE_MILLIS) { + this.rdnsLookupJobs.add(e.getKey()); + } + } + this.rdnsLookupResults = new HashMap<String, String>(); + this.rdnsLookupMillis = new ArrayList<Long>(); + this.rdnsLookupWorkers = new ArrayList<RdnsLookupWorker>(); + for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) { + RdnsLookupWorker rdnsLookupWorker = new RdnsLookupWorker(); + this.rdnsLookupWorkers.add(rdnsLookupWorker); + rdnsLookupWorker.setDaemon(true); + rdnsLookupWorker.start(); + } + } + + public void finishReverseDomainNameLookups() { + for (RdnsLookupWorker rdnsLookupWorker : this.rdnsLookupWorkers) { + try { + rdnsLookupWorker.join(); + } catch (InterruptedException e) { + /* This is not something that we can take care of. Just leave the + * worker thread alone. */ + } + } + } + + public Map<String, String> getLookupResults() { + synchronized (this.rdnsLookupResults) { + return new HashMap<String, String>(this.rdnsLookupResults); + } + } + + public long getLookupStartMillis() { + return this.startedRdnsLookups; + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + formatDecimalNumber(rdnsLookupMillis.size()) + + " lookups performed\n"); + if (rdnsLookupMillis.size() > 0) { + Collections.sort(rdnsLookupMillis); + sb.append(" " + formatMillis(rdnsLookupMillis.get(0)) + + " minimum lookup time\n"); + sb.append(" " + formatMillis(rdnsLookupMillis.get( + rdnsLookupMillis.size() / 2)) + " median lookup time\n"); + sb.append(" " + formatMillis(rdnsLookupMillis.get( + rdnsLookupMillis.size() - 1)) + " maximum lookup time\n"); + } + return sb.toString(); + } + + //TODO This method should go into a utility class. + private static String formatDecimalNumber(long decimalNumber) { + return String.format("%,d", decimalNumber); + } + + // TODO This method should go into a utility class. + private static String formatMillis(long millis) { + return String.format("%02d:%02d.%03d minutes", + millis / (1000L * 60L), (millis / 1000L) % 60L, millis % 1000L); + } +} +