commit 9ae78a5b7e0ccb8cc81215fd74a215a30ce3142d Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Mon Jul 1 08:56:57 2013 +0200 Move all descriptor parsing to a single execution phase. --- src/org/torproject/onionoo/Main.java | 53 ++++++++++++++---------- src/org/torproject/onionoo/NodeDataWriter.java | 3 +- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 3fca6f0..9caac06 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -21,17 +21,41 @@ public class Main { printStatusTime("Initialized Geoip lookup service"); ReverseDomainNameResolver rdnr = new ReverseDomainNameResolver(); printStatusTime("Initialized reverse domain name resolver"); - - printStatus("Updating internal node list."); NodeDataWriter ndw = new NodeDataWriter(dso, ls, ds); - ndw.readStatusSummary(); - printStatusTime("Read status summary"); + printStatusTime("Initialized node data writer"); + DetailsDataWriter ddw = new DetailsDataWriter(dso, rdnr, ds); + printStatusTime("Initialized details data writer"); + BandwidthDataWriter bdw = new BandwidthDataWriter(dso, ds); + printStatusTime("Initialized bandwidth data writer"); + WeightsDataWriter wdw = new WeightsDataWriter(dso, ds); + printStatusTime("Initialized weights data writer"); + + // TODO Instead of creating nine, partly overlapping descriptor + // queues, register for descriptor type and let DescriptorSource + // parse everything just once. + printStatus("Reading descriptors."); ndw.readRelayNetworkConsensuses(); printStatusTime("Read network status consensuses"); - ndw.lookUpCitiesAndASes(); - printStatusTime("Looked up cities and ASes"); ndw.readBridgeNetworkStatuses(); printStatusTime("Read bridge network statuses"); + ddw.readRelayServerDescriptors(); + printStatusTime("Read relay server descriptors"); + ddw.readExitLists(); + printStatusTime("Read exit lists"); + ddw.readBridgeServerDescriptors(); + printStatusTime("Read bridge server descriptors"); + ddw.readBridgePoolAssignments(); + printStatusTime("Read bridge-pool assignments"); + bdw.readExtraInfoDescriptors(); + printStatusTime("Read extra-info descriptors"); + wdw.readRelayServerDescriptors(); + printStatusTime("Read relay server descriptors"); + wdw.readRelayNetworkConsensuses(); + printStatusTime("Read relay network consensuses"); + + printStatus("Updating internal node list."); + ndw.lookUpCitiesAndASes(); + printStatusTime("Looked up cities and ASes"); ndw.setRunningBits(); printStatusTime("Set running bits"); ndw.writeStatusSummary(); @@ -43,7 +67,6 @@ public class Main { ndw.getLastBandwidthWeights(); printStatus("Updating detail data."); - DetailsDataWriter ddw = new DetailsDataWriter(dso, rdnr, ds); // TODO Instead of using ndw's currentNodes and lastBandwidthWeights, // parse statuses once again, keeping separate parse history. Allows // us to run ndw and ddw in parallel in the future. Alternatively, @@ -52,27 +75,16 @@ public class Main { printStatusTime("Set current node fingerprints"); ddw.startReverseDomainNameLookups(); printStatusTime("Started reverse domain name lookups"); - ddw.readRelayServerDescriptors(); - printStatusTime("Read relay server descriptors"); ddw.calculatePathSelectionProbabilities(lastBandwidthWeights); printStatusTime("Calculated path selection probabilities"); - ddw.readExitLists(); - printStatusTime("Read exit lists"); - ddw.readBridgeServerDescriptors(); - printStatusTime("Read bridge server descriptors"); - ddw.readBridgePoolAssignments(); - printStatusTime("Read bridge-pool assignments"); ddw.finishReverseDomainNameLookups(); printStatusTime("Finished reverse domain name lookups"); ddw.writeOutDetails(); printStatusTime("Wrote detail data files"); printStatus("Updating bandwidth data."); - BandwidthDataWriter bdw = new BandwidthDataWriter(dso, ds); bdw.setCurrentNodes(currentNodes); printStatusTime("Set current node fingerprints"); - bdw.readExtraInfoDescriptors(); - printStatusTime("Read extra-info descriptors"); // TODO Evaluate overhead of not deleting obsolete bandwidth files. // An advantage would be that we don't need ndw's currentNodes // anymore, which allows us to run ndw and bdw in parallel in the @@ -81,13 +93,8 @@ public class Main { printStatusTime("Deleted obsolete bandwidth files"); printStatus("Updating weights data."); - WeightsDataWriter wdw = new WeightsDataWriter(dso, ds); wdw.setCurrentNodes(currentNodes); printStatusTime("Set current node fingerprints"); - wdw.readRelayServerDescriptors(); - printStatusTime("Read relay server descriptors"); - wdw.readRelayNetworkConsensuses(); - printStatusTime("Read relay network consensuses"); wdw.writeWeightsDataFiles(); printStatusTime("Wrote weights data files"); // TODO Evaluate overhead of not deleting obsolete weights files. An diff --git a/src/org/torproject/onionoo/NodeDataWriter.java b/src/org/torproject/onionoo/NodeDataWriter.java index 4afc595..b2d7e32 100644 --- a/src/org/torproject/onionoo/NodeDataWriter.java +++ b/src/org/torproject/onionoo/NodeDataWriter.java @@ -41,9 +41,10 @@ public class NodeDataWriter { this.descriptorSource = descriptorSource; this.lookupService = lookupService; this.documentStore = documentStore; + this.readStatusSummary(); } - public void readStatusSummary() { + private void readStatusSummary() { SortedSet<String> fingerprints = this.documentStore.list( NodeStatus.class, true); for (String fingerprint : fingerprints) {