[tor-commits] [onionoo/master] Use unified DataWriter interface for *DataWriters.
karsten at torproject.org
karsten at torproject.org
Thu Nov 28 14:07:23 UTC 2013
commit a4e80c3fd65c1ef19f59e3baa302b9814a21b9cb
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Mon Nov 25 09:23:58 2013 +0100
Use unified DataWriter interface for *DataWriters.
---
.../torproject/onionoo/BandwidthDataWriter.java | 16 +++++-
src/org/torproject/onionoo/DataWriter.java | 11 ++++
src/org/torproject/onionoo/Main.java | 51 ++++++------------
src/org/torproject/onionoo/NodeDataWriter.java | 54 +++++++++++++++-----
src/org/torproject/onionoo/WeightsDataWriter.java | 25 +++++++--
5 files changed, 103 insertions(+), 54 deletions(-)
diff --git a/src/org/torproject/onionoo/BandwidthDataWriter.java b/src/org/torproject/onionoo/BandwidthDataWriter.java
index 6a96b98..4291bdd 100644
--- a/src/org/torproject/onionoo/BandwidthDataWriter.java
+++ b/src/org/torproject/onionoo/BandwidthDataWriter.java
@@ -34,7 +34,8 @@ import org.torproject.descriptor.ExtraInfoDescriptor;
* last 3 days in the bandwidth document may not be equivalent to the last
* 3 days as of publishing the document, but that's something clients can
* work around. */
-public class BandwidthDataWriter implements DescriptorListener {
+public class BandwidthDataWriter implements DataWriter,
+ DescriptorListener {
private DescriptorSource descriptorSource;
@@ -68,6 +69,14 @@ public class BandwidthDataWriter implements DescriptorListener {
}
}
+ public void updateStatuses() {
+ /* Status files are already updated while processing descriptors. */
+ }
+
+ public void updateDocuments() {
+ /* Document files are already updated while processing descriptors. */
+ }
+
private void parseDescriptor(ExtraInfoDescriptor descriptor) {
String fingerprint = descriptor.getFingerprint();
boolean updateHistory = false;
@@ -356,5 +365,10 @@ public class BandwidthDataWriter implements DescriptorListener {
}
return result;
}
+
+ public String getStatsString() {
+ /* TODO Add statistics string. */
+ return null;
+ }
}
diff --git a/src/org/torproject/onionoo/DataWriter.java b/src/org/torproject/onionoo/DataWriter.java
new file mode 100644
index 0000000..c374153
--- /dev/null
+++ b/src/org/torproject/onionoo/DataWriter.java
@@ -0,0 +1,11 @@
+package org.torproject.onionoo;
+
+public interface DataWriter {
+
+ public void updateStatuses();
+
+ public void updateDocuments();
+
+ public String getStatsString();
+}
+
diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java
index 84e7e15..a7e9461 100644
--- a/src/org/torproject/onionoo/Main.java
+++ b/src/org/torproject/onionoo/Main.java
@@ -36,6 +36,7 @@ public class Main {
Logger.printStatusTime("Initialized bandwidth data writer");
WeightsDataWriter wdw = new WeightsDataWriter(dso, ds, t);
Logger.printStatusTime("Initialized weights data writer");
+ DataWriter[] dws = new DataWriter[] { ndw, bdw, wdw };
Logger.printStatus("Reading descriptors.");
dso.readRelayNetworkConsensuses();
@@ -55,37 +56,15 @@ public class Main {
dso.readBridgePoolAssignments();
Logger.printStatusTime("Read bridge-pool assignments");
- Logger.printStatus("Updating internal node list.");
- ndw.readStatusSummary();
- Logger.printStatusTime("Read status summary");
- ndw.setCurrentNodes();
- Logger.printStatusTime("Set current node fingerprints");
- ndw.startReverseDomainNameLookups();
- Logger.printStatusTime("Started reverse domain name lookups");
- ndw.lookUpCitiesAndASes();
- Logger.printStatusTime("Looked up cities and ASes");
- ndw.setRunningBits();
- Logger.printStatusTime("Set running bits");
- ndw.calculatePathSelectionProbabilities();
- Logger.printStatusTime("Calculated path selection probabilities");
- ndw.finishReverseDomainNameLookups();
- Logger.printStatusTime("Finished reverse domain name lookups");
- ndw.writeStatusSummary();
- Logger.printStatusTime("Wrote status summary");
- ndw.writeOutSummary();
- Logger.printStatusTime("Wrote out summary");
- ndw.writeOutDetails();
- Logger.printStatusTime("Wrote detail data files");
-
- Logger.printStatus("Updating bandwidth data.");
+ Logger.printStatus("Updating internal status files.");
+ for (DataWriter dw : dws) {
+ dw.updateStatuses();
+ }
- Logger.printStatus("Updating weights data.");
- wdw.updateWeightsHistories();
- Logger.printStatusTime("Updated weights histories");
- wdw.updateWeightsStatuses();
- Logger.printStatusTime("Updated weights status files");
- wdw.writeWeightsDataFiles();
- Logger.printStatusTime("Wrote weights document files");
+ Logger.printStatus("Updating document files.");
+ for (DataWriter dw : dws) {
+ dw.updateDocuments();
+ }
Logger.printStatus("Shutting down.");
dso.writeHistoryFiles();
@@ -94,11 +73,13 @@ public class Main {
Logger.printStatusTime("Flushed document cache");
Logger.printStatus("Gathering statistics.");
- Logger.printStatistics("Node data writer", ndw.getStatsString());
- /* TODO Add statistics to remaining *Writers. */
- //printStatistics("Details data writer", ddw.getStatsString());
- //printStatistics("Bandwidth data writer", bdw.getStatsString());
- //printStatistics("Weights data writer", wdw.getStatsString());
+ for (DataWriter dw : dws) {
+ String statsString = dw.getStatsString();
+ if (statsString != null) {
+ Logger.printStatistics(dw.getClass().getSimpleName(),
+ statsString);
+ }
+ }
Logger.printStatistics("Descriptor source", dso.getStatsString());
Logger.printStatistics("Document store", ds.getStatsString());
Logger.printStatistics("GeoIP lookup service", ls.getStatsString());
diff --git a/src/org/torproject/onionoo/NodeDataWriter.java b/src/org/torproject/onionoo/NodeDataWriter.java
index 64176c3..72def10 100644
--- a/src/org/torproject/onionoo/NodeDataWriter.java
+++ b/src/org/torproject/onionoo/NodeDataWriter.java
@@ -33,7 +33,7 @@ import org.torproject.onionoo.LookupService.LookupResult;
* The parts of details files coming from server descriptors always come
* from the last known descriptor of a relay or bridge, not from the
* descriptor that was last referenced in a network status. */
-public class NodeDataWriter implements DescriptorListener {
+public class NodeDataWriter implements DataWriter, DescriptorListener {
private DescriptorSource descriptorSource;
@@ -89,12 +89,12 @@ public class NodeDataWriter implements DescriptorListener {
public void processDescriptor(Descriptor descriptor, boolean relay) {
if (descriptor instanceof RelayNetworkStatusConsensus) {
- updateRelayNetworkStatusConsensus(
+ this.updateRelayNetworkStatusConsensus(
(RelayNetworkStatusConsensus) descriptor);
} else if (descriptor instanceof ServerDescriptor && relay) {
this.processRelayServerDescriptor((ServerDescriptor) descriptor);
} else if (descriptor instanceof BridgeNetworkStatus) {
- updateBridgeNetworkStatus((BridgeNetworkStatus) descriptor);
+ this.updateBridgeNetworkStatus((BridgeNetworkStatus) descriptor);
} else if (descriptor instanceof ServerDescriptor && !relay) {
this.processBridgeServerDescriptor((ServerDescriptor) descriptor);
} else if (descriptor instanceof BridgePoolAssignment) {
@@ -104,6 +104,32 @@ public class NodeDataWriter implements DescriptorListener {
}
}
+ public void updateStatuses() {
+ this.readStatusSummary();
+ Logger.printStatusTime("Read status summary");
+ this.setCurrentNodes();
+ Logger.printStatusTime("Set current node fingerprints");
+ this.startReverseDomainNameLookups();
+ Logger.printStatusTime("Started reverse domain name lookups");
+ this.lookUpCitiesAndASes();
+ Logger.printStatusTime("Looked up cities and ASes");
+ this.setRunningBits();
+ Logger.printStatusTime("Set running bits");
+ this.calculatePathSelectionProbabilities();
+ Logger.printStatusTime("Calculated path selection probabilities");
+ this.finishReverseDomainNameLookups();
+ Logger.printStatusTime("Finished reverse domain name lookups");
+ this.writeStatusSummary();
+ Logger.printStatusTime("Wrote status summary");
+ this.writeOutSummary();
+ Logger.printStatusTime("Wrote out summary");
+ }
+
+ public void updateDocuments() {
+ this.writeOutDetails();
+ Logger.printStatusTime("Wrote detail data files");
+ }
+
private void updateRelayNetworkStatusConsensus(
RelayNetworkStatusConsensus consensus) {
long validAfterMillis = consensus.getValidAfterMillis();
@@ -167,7 +193,7 @@ public class NodeDataWriter implements DescriptorListener {
this.bridgeStatusesProcessed++;
}
- public void readStatusSummary() {
+ private void readStatusSummary() {
SortedSet<String> fingerprints = this.documentStore.list(
NodeStatus.class, true);
for (String fingerprint : fingerprints) {
@@ -188,7 +214,7 @@ public class NodeDataWriter implements DescriptorListener {
}
}
- public void setRunningBits() {
+ private void setRunningBits() {
for (NodeStatus node : this.knownNodes.values()) {
if (node.isRelay() && node.getRelayFlags().contains("Running") &&
node.getLastSeenMillis() == this.relaysLastValidAfterMillis) {
@@ -201,7 +227,7 @@ public class NodeDataWriter implements DescriptorListener {
}
}
- public void lookUpCitiesAndASes() {
+ private void lookUpCitiesAndASes() {
SortedSet<String> addressStrings = new TreeSet<String>();
for (NodeStatus node : this.knownNodes.values()) {
if (node.isRelay()) {
@@ -234,11 +260,11 @@ public class NodeDataWriter implements DescriptorListener {
}
}
- public void writeStatusSummary() {
+ private void writeStatusSummary() {
this.writeSummary(true);
}
- public void writeOutSummary() {
+ private void writeOutSummary() {
this.writeSummary(false);
}
@@ -250,7 +276,7 @@ public class NodeDataWriter implements DescriptorListener {
}
}
- public SortedMap<String, NodeStatus> getCurrentNodes() {
+ private SortedMap<String, NodeStatus> getCurrentNodes() {
long cutoff = Math.max(this.relaysLastValidAfterMillis,
this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L;
SortedMap<String, NodeStatus> currentNodes =
@@ -432,7 +458,7 @@ public class NodeDataWriter implements DescriptorListener {
}
}
- public void setCurrentNodes() {
+ private void setCurrentNodes() {
SortedMap<String, NodeStatus> currentNodes = this.getCurrentNodes();
this.relays = new TreeMap<String, NodeStatus>();
this.bridges = new TreeMap<String, NodeStatus>();
@@ -445,7 +471,7 @@ public class NodeDataWriter implements DescriptorListener {
}
}
- public void startReverseDomainNameLookups() {
+ private void startReverseDomainNameLookups() {
Map<String, Long> addressLastLookupTimes =
new HashMap<String, Long>();
for (NodeStatus relay : relays.values()) {
@@ -456,7 +482,7 @@ public class NodeDataWriter implements DescriptorListener {
this.reverseDomainNameResolver.startReverseDomainNameLookups();
}
- public void finishReverseDomainNameLookups() {
+ private void finishReverseDomainNameLookups() {
this.reverseDomainNameResolver.finishReverseDomainNameLookups();
Map<String, String> lookupResults =
this.reverseDomainNameResolver.getLookupResults();
@@ -470,7 +496,7 @@ public class NodeDataWriter implements DescriptorListener {
}
}
- public void calculatePathSelectionProbabilities() {
+ private void calculatePathSelectionProbabilities() {
boolean consensusContainsBandwidthWeights = false;
double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0,
wmd = 0.0, wee = 0.0, wed = 0.0;
@@ -598,7 +624,7 @@ public class NodeDataWriter implements DescriptorListener {
}
}
- public void writeOutDetails() {
+ private void writeOutDetails() {
this.updateRelayDetailsFiles();
this.updateBridgeDetailsFiles();
}
diff --git a/src/org/torproject/onionoo/WeightsDataWriter.java b/src/org/torproject/onionoo/WeightsDataWriter.java
index d95e19c..40c85ed 100644
--- a/src/org/torproject/onionoo/WeightsDataWriter.java
+++ b/src/org/torproject/onionoo/WeightsDataWriter.java
@@ -25,7 +25,7 @@ import org.torproject.descriptor.NetworkStatusEntry;
import org.torproject.descriptor.RelayNetworkStatusConsensus;
import org.torproject.descriptor.ServerDescriptor;
-public class WeightsDataWriter implements DescriptorListener {
+public class WeightsDataWriter implements DataWriter, DescriptorListener {
private DescriptorSource descriptorSource;
@@ -57,6 +57,18 @@ public class WeightsDataWriter implements DescriptorListener {
}
}
+ public void updateStatuses() {
+ this.updateWeightsHistories();
+ Logger.printStatusTime("Updated weights histories");
+ this.updateWeightsStatuses();
+ Logger.printStatusTime("Updated weights status files");
+ }
+
+ public void updateDocuments() {
+ this.writeWeightsDataFiles();
+ Logger.printStatusTime("Wrote weights document files");
+ }
+
private Set<RelayNetworkStatusConsensus> consensuses =
new HashSet<RelayNetworkStatusConsensus>();
@@ -94,7 +106,7 @@ public class WeightsDataWriter implements DescriptorListener {
this.descriptorDigestsByFingerprint.get(fingerprint).add(digest);
}
- public void updateWeightsHistories() {
+ private void updateWeightsHistories() {
for (RelayNetworkStatusConsensus consensus : this.consensuses) {
long validAfterMillis = consensus.getValidAfterMillis(),
freshUntilMillis = consensus.getFreshUntilMillis();
@@ -438,7 +450,7 @@ public class WeightsDataWriter implements DescriptorListener {
this.documentStore.store(weightsStatus, fingerprint);
}
- public void writeWeightsDataFiles() {
+ private void writeWeightsDataFiles() {
for (String fingerprint : this.updateWeightsDocuments) {
SortedMap<long[], double[]> history =
this.readHistoryFromDisk(fingerprint);
@@ -598,12 +610,17 @@ public class WeightsDataWriter implements DescriptorListener {
}
}
- public void updateWeightsStatuses() {
+ private void updateWeightsStatuses() {
for (String fingerprint : this.updateWeightsStatuses) {
SortedMap<long[], double[]> history =
this.readHistoryFromDisk(fingerprint);
this.writeHistoryToDisk(fingerprint, history);
}
}
+
+ public String getStatsString() {
+ /* TODO Add statistics string. */
+ return null;
+ }
}
More information about the tor-commits
mailing list