commit a4e80c3fd65c1ef19f59e3baa302b9814a21b9cb Author: Karsten Loesing karsten.loesing@gmx.net Date: Mon Nov 25 09:23:58 2013 +0100

Use unified DataWriter interface for *DataWriters. --- .../torproject/onionoo/BandwidthDataWriter.java | 16 +++++- src/org/torproject/onionoo/DataWriter.java | 11 ++++ src/org/torproject/onionoo/Main.java | 51 ++++++------------ src/org/torproject/onionoo/NodeDataWriter.java | 54 +++++++++++++++----- src/org/torproject/onionoo/WeightsDataWriter.java | 25 +++++++-- 5 files changed, 103 insertions(+), 54 deletions(-)

diff --git a/src/org/torproject/onionoo/BandwidthDataWriter.java b/src/org/torproject/onionoo/BandwidthDataWriter.java index 6a96b98..4291bdd 100644 --- a/src/org/torproject/onionoo/BandwidthDataWriter.java +++ b/src/org/torproject/onionoo/BandwidthDataWriter.java @@ -34,7 +34,8 @@ import org.torproject.descriptor.ExtraInfoDescriptor; * last 3 days in the bandwidth document may not be equivalent to the last * 3 days as of publishing the document, but that's something clients can * work around. */ -public class BandwidthDataWriter implements DescriptorListener { +public class BandwidthDataWriter implements DataWriter, + DescriptorListener {

private DescriptorSource descriptorSource;

@@ -68,6 +69,14 @@ public class BandwidthDataWriter implements DescriptorListener { } }

+ public void updateStatuses() { + /* Status files are already updated while processing descriptors. */ + } + + public void updateDocuments() { + /* Document files are already updated while processing descriptors. */ + } + private void parseDescriptor(ExtraInfoDescriptor descriptor) { String fingerprint = descriptor.getFingerprint(); boolean updateHistory = false; @@ -356,5 +365,10 @@ public class BandwidthDataWriter implements DescriptorListener { } return result; } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } }

diff --git a/src/org/torproject/onionoo/DataWriter.java b/src/org/torproject/onionoo/DataWriter.java new file mode 100644 index 0000000..c374153 --- /dev/null +++ b/src/org/torproject/onionoo/DataWriter.java @@ -0,0 +1,11 @@ +package org.torproject.onionoo; + +public interface DataWriter { + + public void updateStatuses(); + + public void updateDocuments(); + + public String getStatsString(); +} + diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 84e7e15..a7e9461 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -36,6 +36,7 @@ public class Main { Logger.printStatusTime("Initialized bandwidth data writer"); WeightsDataWriter wdw = new WeightsDataWriter(dso, ds, t); Logger.printStatusTime("Initialized weights data writer"); + DataWriter[] dws = new DataWriter[] { ndw, bdw, wdw };

Logger.printStatus("Reading descriptors."); dso.readRelayNetworkConsensuses(); @@ -55,37 +56,15 @@ public class Main { dso.readBridgePoolAssignments(); Logger.printStatusTime("Read bridge-pool assignments");

- Logger.printStatus("Updating internal node list."); - ndw.readStatusSummary(); - Logger.printStatusTime("Read status summary"); - ndw.setCurrentNodes(); - Logger.printStatusTime("Set current node fingerprints"); - ndw.startReverseDomainNameLookups(); - Logger.printStatusTime("Started reverse domain name lookups"); - ndw.lookUpCitiesAndASes(); - Logger.printStatusTime("Looked up cities and ASes"); - ndw.setRunningBits(); - Logger.printStatusTime("Set running bits"); - ndw.calculatePathSelectionProbabilities(); - Logger.printStatusTime("Calculated path selection probabilities"); - ndw.finishReverseDomainNameLookups(); - Logger.printStatusTime("Finished reverse domain name lookups"); - ndw.writeStatusSummary(); - Logger.printStatusTime("Wrote status summary"); - ndw.writeOutSummary(); - Logger.printStatusTime("Wrote out summary"); - ndw.writeOutDetails(); - Logger.printStatusTime("Wrote detail data files"); - - Logger.printStatus("Updating bandwidth data."); + Logger.printStatus("Updating internal status files."); + for (DataWriter dw : dws) { + dw.updateStatuses(); + }

- Logger.printStatus("Updating weights data."); - wdw.updateWeightsHistories(); - Logger.printStatusTime("Updated weights histories"); - wdw.updateWeightsStatuses(); - Logger.printStatusTime("Updated weights status files"); - wdw.writeWeightsDataFiles(); - Logger.printStatusTime("Wrote weights document files"); + Logger.printStatus("Updating document files."); + for (DataWriter dw : dws) { + dw.updateDocuments(); + }

Logger.printStatus("Shutting down."); dso.writeHistoryFiles(); @@ -94,11 +73,13 @@ public class Main { Logger.printStatusTime("Flushed document cache");

Logger.printStatus("Gathering statistics."); - Logger.printStatistics("Node data writer", ndw.getStatsString()); - /* TODO Add statistics to remaining *Writers. */ - //printStatistics("Details data writer", ddw.getStatsString()); - //printStatistics("Bandwidth data writer", bdw.getStatsString()); - //printStatistics("Weights data writer", wdw.getStatsString()); + for (DataWriter dw : dws) { + String statsString = dw.getStatsString(); + if (statsString != null) { + Logger.printStatistics(dw.getClass().getSimpleName(), + statsString); + } + } Logger.printStatistics("Descriptor source", dso.getStatsString()); Logger.printStatistics("Document store", ds.getStatsString()); Logger.printStatistics("GeoIP lookup service", ls.getStatsString()); diff --git a/src/org/torproject/onionoo/NodeDataWriter.java b/src/org/torproject/onionoo/NodeDataWriter.java index 64176c3..72def10 100644 --- a/src/org/torproject/onionoo/NodeDataWriter.java +++ b/src/org/torproject/onionoo/NodeDataWriter.java @@ -33,7 +33,7 @@ import org.torproject.onionoo.LookupService.LookupResult; * The parts of details files coming from server descriptors always come * from the last known descriptor of a relay or bridge, not from the * descriptor that was last referenced in a network status. */ -public class NodeDataWriter implements DescriptorListener { +public class NodeDataWriter implements DataWriter, DescriptorListener {

private DescriptorSource descriptorSource;

@@ -89,12 +89,12 @@ public class NodeDataWriter implements DescriptorListener {

public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof RelayNetworkStatusConsensus) { - updateRelayNetworkStatusConsensus( + this.updateRelayNetworkStatusConsensus( (RelayNetworkStatusConsensus) descriptor); } else if (descriptor instanceof ServerDescriptor && relay) { this.processRelayServerDescriptor((ServerDescriptor) descriptor); } else if (descriptor instanceof BridgeNetworkStatus) { - updateBridgeNetworkStatus((BridgeNetworkStatus) descriptor); + this.updateBridgeNetworkStatus((BridgeNetworkStatus) descriptor); } else if (descriptor instanceof ServerDescriptor && !relay) { this.processBridgeServerDescriptor((ServerDescriptor) descriptor); } else if (descriptor instanceof BridgePoolAssignment) { @@ -104,6 +104,32 @@ public class NodeDataWriter implements DescriptorListener { } }

+ public void updateStatuses() { + this.readStatusSummary(); + Logger.printStatusTime("Read status summary"); + this.setCurrentNodes(); + Logger.printStatusTime("Set current node fingerprints"); + this.startReverseDomainNameLookups(); + Logger.printStatusTime("Started reverse domain name lookups"); + this.lookUpCitiesAndASes(); + Logger.printStatusTime("Looked up cities and ASes"); + this.setRunningBits(); + Logger.printStatusTime("Set running bits"); + this.calculatePathSelectionProbabilities(); + Logger.printStatusTime("Calculated path selection probabilities"); + this.finishReverseDomainNameLookups(); + Logger.printStatusTime("Finished reverse domain name lookups"); + this.writeStatusSummary(); + Logger.printStatusTime("Wrote status summary"); + this.writeOutSummary(); + Logger.printStatusTime("Wrote out summary"); + } + + public void updateDocuments() { + this.writeOutDetails(); + Logger.printStatusTime("Wrote detail data files"); + } + private void updateRelayNetworkStatusConsensus( RelayNetworkStatusConsensus consensus) { long validAfterMillis = consensus.getValidAfterMillis(); @@ -167,7 +193,7 @@ public class NodeDataWriter implements DescriptorListener { this.bridgeStatusesProcessed++; }

- public void readStatusSummary() { + private void readStatusSummary() { SortedSet<String> fingerprints = this.documentStore.list( NodeStatus.class, true); for (String fingerprint : fingerprints) { @@ -188,7 +214,7 @@ public class NodeDataWriter implements DescriptorListener { } }

- public void setRunningBits() { + private void setRunningBits() { for (NodeStatus node : this.knownNodes.values()) { if (node.isRelay() && node.getRelayFlags().contains("Running") && node.getLastSeenMillis() == this.relaysLastValidAfterMillis) { @@ -201,7 +227,7 @@ public class NodeDataWriter implements DescriptorListener { } }

- public void lookUpCitiesAndASes() { + private void lookUpCitiesAndASes() { SortedSet<String> addressStrings = new TreeSet<String>(); for (NodeStatus node : this.knownNodes.values()) { if (node.isRelay()) { @@ -234,11 +260,11 @@ public class NodeDataWriter implements DescriptorListener { } }

- public void writeStatusSummary() { + private void writeStatusSummary() { this.writeSummary(true); }

- public void writeOutSummary() { + private void writeOutSummary() { this.writeSummary(false); }

@@ -250,7 +276,7 @@ public class NodeDataWriter implements DescriptorListener { } }

- public SortedMap<String, NodeStatus> getCurrentNodes() { + private SortedMap<String, NodeStatus> getCurrentNodes() { long cutoff = Math.max(this.relaysLastValidAfterMillis, this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L; SortedMap<String, NodeStatus> currentNodes = @@ -432,7 +458,7 @@ public class NodeDataWriter implements DescriptorListener { } }

- public void setCurrentNodes() { + private void setCurrentNodes() { SortedMap<String, NodeStatus> currentNodes = this.getCurrentNodes(); this.relays = new TreeMap<String, NodeStatus>(); this.bridges = new TreeMap<String, NodeStatus>(); @@ -445,7 +471,7 @@ public class NodeDataWriter implements DescriptorListener { } }

- public void startReverseDomainNameLookups() { + private void startReverseDomainNameLookups() { Map<String, Long> addressLastLookupTimes = new HashMap<String, Long>(); for (NodeStatus relay : relays.values()) { @@ -456,7 +482,7 @@ public class NodeDataWriter implements DescriptorListener { this.reverseDomainNameResolver.startReverseDomainNameLookups(); }

- public void finishReverseDomainNameLookups() { + private void finishReverseDomainNameLookups() { this.reverseDomainNameResolver.finishReverseDomainNameLookups(); Map<String, String> lookupResults = this.reverseDomainNameResolver.getLookupResults(); @@ -470,7 +496,7 @@ public class NodeDataWriter implements DescriptorListener { } }

- public void calculatePathSelectionProbabilities() { + private void calculatePathSelectionProbabilities() { boolean consensusContainsBandwidthWeights = false; double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0, wmd = 0.0, wee = 0.0, wed = 0.0; @@ -598,7 +624,7 @@ public class NodeDataWriter implements DescriptorListener { } }

- public void writeOutDetails() { + private void writeOutDetails() { this.updateRelayDetailsFiles(); this.updateBridgeDetailsFiles(); } diff --git a/src/org/torproject/onionoo/WeightsDataWriter.java b/src/org/torproject/onionoo/WeightsDataWriter.java index d95e19c..40c85ed 100644 --- a/src/org/torproject/onionoo/WeightsDataWriter.java +++ b/src/org/torproject/onionoo/WeightsDataWriter.java @@ -25,7 +25,7 @@ import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.descriptor.ServerDescriptor;

-public class WeightsDataWriter implements DescriptorListener { +public class WeightsDataWriter implements DataWriter, DescriptorListener {

private DescriptorSource descriptorSource;

@@ -57,6 +57,18 @@ public class WeightsDataWriter implements DescriptorListener { } }

+ public void updateStatuses() { + this.updateWeightsHistories(); + Logger.printStatusTime("Updated weights histories"); + this.updateWeightsStatuses(); + Logger.printStatusTime("Updated weights status files"); + } + + public void updateDocuments() { + this.writeWeightsDataFiles(); + Logger.printStatusTime("Wrote weights document files"); + } + private Set<RelayNetworkStatusConsensus> consensuses = new HashSet<RelayNetworkStatusConsensus>();

@@ -94,7 +106,7 @@ public class WeightsDataWriter implements DescriptorListener { this.descriptorDigestsByFingerprint.get(fingerprint).add(digest); }

- public void updateWeightsHistories() { + private void updateWeightsHistories() { for (RelayNetworkStatusConsensus consensus : this.consensuses) { long validAfterMillis = consensus.getValidAfterMillis(), freshUntilMillis = consensus.getFreshUntilMillis(); @@ -438,7 +450,7 @@ public class WeightsDataWriter implements DescriptorListener { this.documentStore.store(weightsStatus, fingerprint); }

- public void writeWeightsDataFiles() { + private void writeWeightsDataFiles() { for (String fingerprint : this.updateWeightsDocuments) { SortedMap<long[], double[]> history = this.readHistoryFromDisk(fingerprint); @@ -598,12 +610,17 @@ public class WeightsDataWriter implements DescriptorListener { } }

- public void updateWeightsStatuses() { + private void updateWeightsStatuses() { for (String fingerprint : this.updateWeightsStatuses) { SortedMap<long[], double[]> history = this.readHistoryFromDisk(fingerprint); this.writeHistoryToDisk(fingerprint, history); } } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } }