[tor-commits] [onionoo/master] Add advertised_bandwidth and consensus_weight graphs.
karsten at torproject.org
karsten at torproject.org
Wed Jul 9 08:27:13 UTC 2014
commit 99e841366e9f56489915cd886367ccfaf3dfdc64
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Sun Jun 8 19:56:26 2014 +0200
Add advertised_bandwidth and consensus_weight graphs.
Weights documents contain advertised_bandwidth_fraction and
consensus_weight_fraction graphs containing, well, fractions of the
relay's value compared to the network total.
This commit adds two new graphs showing the respective absolute values, to
distinguish effects of things happening on a relay from those happening to
the overall Tor network.
This change further allows us to spot changes to a relays' absolute values
over time, like relays increasing their advertised bandwidth by more than
128 KB/s.
Also fixed a few things while re-processing the archives:
- Older consensuses contained non-"Running" relays, which caused a
NullPointerException.
- Older consensuses did not contain "w" lines and/or no bandwidth
weights, either of which we didn't handle very well.
- Whenever we didn't find a server descriptor we silently assumed that
advertised bandwidth is 0. Now we're handling this as missing data and
not including intervals in advertised_bandwidth_fraction and the new
advertised_bandwidth.
Implements #11388.
---
src/org/torproject/onionoo/WeightsDocument.java | 12 ++
.../torproject/onionoo/WeightsDocumentWriter.java | 12 +-
src/org/torproject/onionoo/WeightsStatus.java | 8 +-
.../torproject/onionoo/WeightsStatusUpdater.java | 118 ++++++++++++--------
web/protocol.html | 24 ++++
5 files changed, 124 insertions(+), 50 deletions(-)
diff --git a/src/org/torproject/onionoo/WeightsDocument.java b/src/org/torproject/onionoo/WeightsDocument.java
index ff1d5f4..6dddf9d 100644
--- a/src/org/torproject/onionoo/WeightsDocument.java
+++ b/src/org/torproject/onionoo/WeightsDocument.java
@@ -40,5 +40,17 @@ class WeightsDocument extends Document {
Map<String, GraphHistory> exitProbability) {
this.exit_probability = exitProbability;
}
+
+ private Map<String, GraphHistory> advertised_bandwidth;
+ public void setAdvertisedBandwidth(
+ Map<String, GraphHistory> advertisedBandwidth) {
+ this.advertised_bandwidth = advertisedBandwidth;
+ }
+
+ private Map<String, GraphHistory> consensus_weight;
+ public void setConsensusWeight(
+ Map<String, GraphHistory> consensusWeight) {
+ this.consensus_weight = consensusWeight;
+ }
}
diff --git a/src/org/torproject/onionoo/WeightsDocumentWriter.java b/src/org/torproject/onionoo/WeightsDocumentWriter.java
index 74108c9..2e0d465 100644
--- a/src/org/torproject/onionoo/WeightsDocumentWriter.java
+++ b/src/org/torproject/onionoo/WeightsDocumentWriter.java
@@ -98,6 +98,10 @@ public class WeightsDocumentWriter implements FingerprintListener,
this.compileGraphType(history, 3));
weightsDocument.setExitProbability(
this.compileGraphType(history, 4));
+ weightsDocument.setAdvertisedBandwidth(
+ this.compileGraphType(history, 5));
+ weightsDocument.setConsensusWeight(
+ this.compileGraphType(history, 6));
return weightsDocument;
}
@@ -141,9 +145,11 @@ public class WeightsDocumentWriter implements FingerprintListener,
totalMillis = 0L;
intervalStartMillis += dataPointInterval;
}
- totalWeightTimesMillis += weight
- * ((double) (endMillis - startMillis));
- totalMillis += (endMillis - startMillis);
+ if (weight >= 0.0) {
+ totalWeightTimesMillis += weight
+ * ((double) (endMillis - startMillis));
+ totalMillis += (endMillis - startMillis);
+ }
}
dataPoints.add(totalMillis * 5L < dataPointInterval
? -1.0 : totalWeightTimesMillis / (double) totalMillis);
diff --git a/src/org/torproject/onionoo/WeightsStatus.java b/src/org/torproject/onionoo/WeightsStatus.java
index 872fd00..f5eacd9 100644
--- a/src/org/torproject/onionoo/WeightsStatus.java
+++ b/src/org/torproject/onionoo/WeightsStatus.java
@@ -40,7 +40,7 @@ class WeightsStatus extends Document {
advertisedBandwidth);
continue;
}
- if (parts.length != 9) {
+ if (parts.length != 9 && parts.length != 11) {
System.err.println("Illegal line '" + line + "' in weights "
+ "status file. Skipping this line.");
continue;
@@ -65,7 +65,11 @@ class WeightsStatus extends Document {
Double.parseDouble(parts[5]),
Double.parseDouble(parts[6]),
Double.parseDouble(parts[7]),
- Double.parseDouble(parts[8]) };
+ Double.parseDouble(parts[8]), -1.0, -1.0 };
+ if (parts.length == 11) {
+ weights[5] = Double.parseDouble(parts[9]);
+ weights[6] = Double.parseDouble(parts[10]);
+ }
this.history.put(interval, weights);
}
s.close();
diff --git a/src/org/torproject/onionoo/WeightsStatusUpdater.java b/src/org/torproject/onionoo/WeightsStatusUpdater.java
index 9b45f95..24c98ff 100644
--- a/src/org/torproject/onionoo/WeightsStatusUpdater.java
+++ b/src/org/torproject/onionoo/WeightsStatusUpdater.java
@@ -101,6 +101,7 @@ public class WeightsStatusUpdater implements DescriptorListener,
private SortedMap<String, double[]> calculatePathSelectionProbabilities(
RelayNetworkStatusConsensus consensus) {
+ boolean containsBandwidthWeights = false;
double wgg = 1.0, wgd = 1.0, wmg = 1.0, wmm = 1.0, wme = 1.0,
wmd = 1.0, wee = 1.0, wed = 1.0;
SortedMap<String, Integer> bandwidthWeights =
@@ -118,6 +119,7 @@ public class WeightsStatusUpdater implements DescriptorListener,
wmd = ((double) bandwidthWeights.get("Wmd")) / 10000.0;
wee = ((double) bandwidthWeights.get("Wee")) / 10000.0;
wed = ((double) bandwidthWeights.get("Wed")) / 10000.0;
+ containsBandwidthWeights = true;
}
}
SortedMap<String, Double>
@@ -137,11 +139,7 @@ public class WeightsStatusUpdater implements DescriptorListener,
if (!relay.getFlags().contains("Running")) {
continue;
}
- boolean isExit = relay.getFlags().contains("Exit") &&
- !relay.getFlags().contains("BadExit");
- boolean isGuard = relay.getFlags().contains("Guard");
String digest = relay.getDescriptor().toUpperCase();
- double advertisedBandwidth = 0.0;
WeightsStatus weightsStatus = this.documentStore.retrieve(
WeightsStatus.class, true, fingerprint);
if (weightsStatus != null &&
@@ -151,53 +149,83 @@ public class WeightsStatusUpdater implements DescriptorListener,
* descriptors are parsed before consensuses, so we're sure that
* if there's a server descriptor for this relay, it'll be
* contained in the weights status file by now. */
- advertisedBandwidth =
+ double advertisedBandwidth =
(double) weightsStatus.getAdvertisedBandwidths().get(digest);
+ advertisedBandwidths.put(fingerprint, advertisedBandwidth);
+ totalAdvertisedBandwidth += advertisedBandwidth;
}
- double consensusWeight = (double) relay.getBandwidth();
- double guardWeight = (double) relay.getBandwidth();
- double middleWeight = (double) relay.getBandwidth();
- double exitWeight = (double) relay.getBandwidth();
- if (isGuard && isExit) {
- guardWeight *= wgd;
- middleWeight *= wmd;
- exitWeight *= wed;
- } else if (isGuard) {
- guardWeight *= wgg;
- middleWeight *= wmg;
- exitWeight = 0.0;
- } else if (isExit) {
- guardWeight = 0.0;
- middleWeight *= wme;
- exitWeight *= wee;
- } else {
- guardWeight = 0.0;
- middleWeight *= wmm;
- exitWeight = 0.0;
+ if (relay.getBandwidth() >= 0L) {
+ double consensusWeight = (double) relay.getBandwidth();
+ consensusWeights.put(fingerprint, consensusWeight);
+ totalConsensusWeight += consensusWeight;
+ if (containsBandwidthWeights) {
+ double guardWeight = (double) relay.getBandwidth();
+ double middleWeight = (double) relay.getBandwidth();
+ double exitWeight = (double) relay.getBandwidth();
+ boolean isExit = relay.getFlags().contains("Exit") &&
+ !relay.getFlags().contains("BadExit");
+ boolean isGuard = relay.getFlags().contains("Guard");
+ if (isGuard && isExit) {
+ guardWeight *= wgd;
+ middleWeight *= wmd;
+ exitWeight *= wed;
+ } else if (isGuard) {
+ guardWeight *= wgg;
+ middleWeight *= wmg;
+ exitWeight = 0.0;
+ } else if (isExit) {
+ guardWeight = 0.0;
+ middleWeight *= wme;
+ exitWeight *= wee;
+ } else {
+ guardWeight = 0.0;
+ middleWeight *= wmm;
+ exitWeight = 0.0;
+ }
+ guardWeights.put(fingerprint, guardWeight);
+ middleWeights.put(fingerprint, middleWeight);
+ exitWeights.put(fingerprint, exitWeight);
+ totalGuardWeight += guardWeight;
+ totalMiddleWeight += middleWeight;
+ totalExitWeight += exitWeight;
+ }
}
- advertisedBandwidths.put(fingerprint, advertisedBandwidth);
- consensusWeights.put(fingerprint, consensusWeight);
- guardWeights.put(fingerprint, guardWeight);
- middleWeights.put(fingerprint, middleWeight);
- exitWeights.put(fingerprint, exitWeight);
- totalAdvertisedBandwidth += advertisedBandwidth;
- totalConsensusWeight += consensusWeight;
- totalGuardWeight += guardWeight;
- totalMiddleWeight += middleWeight;
- totalExitWeight += exitWeight;
}
SortedMap<String, double[]> pathSelectionProbabilities =
new TreeMap<String, double[]>();
- for (NetworkStatusEntry relay :
- consensus.getStatusEntries().values()) {
- String fingerprint = relay.getFingerprint();
- double[] probabilities = new double[] {
- advertisedBandwidths.get(fingerprint)
- / totalAdvertisedBandwidth,
- consensusWeights.get(fingerprint) / totalConsensusWeight,
- guardWeights.get(fingerprint) / totalGuardWeight,
- middleWeights.get(fingerprint) / totalMiddleWeight,
- exitWeights.get(fingerprint) / totalExitWeight };
+ SortedSet<String> fingerprints = new TreeSet<String>();
+ fingerprints.addAll(consensusWeights.keySet());
+ fingerprints.addAll(advertisedBandwidths.keySet());
+ for (String fingerprint : fingerprints) {
+ double[] probabilities = new double[] { -1.0, -1.0, -1.0, -1.0,
+ -1.0, -1.0, -1.0 };
+ if (consensusWeights.containsKey(fingerprint) &&
+ totalConsensusWeight > 0.0) {
+ probabilities[1] = consensusWeights.get(fingerprint) /
+ totalConsensusWeight;
+ probabilities[6] = consensusWeights.get(fingerprint);
+ }
+ if (guardWeights.containsKey(fingerprint) &&
+ totalGuardWeight > 0.0) {
+ probabilities[2] = guardWeights.get(fingerprint) /
+ totalGuardWeight;
+ }
+ if (middleWeights.containsKey(fingerprint) &&
+ totalMiddleWeight > 0.0) {
+ probabilities[3] = middleWeights.get(fingerprint) /
+ totalMiddleWeight;
+ }
+ if (exitWeights.containsKey(fingerprint) &&
+ totalExitWeight > 0.0) {
+ probabilities[4] = exitWeights.get(fingerprint) /
+ totalExitWeight;
+ }
+ if (advertisedBandwidths.containsKey(fingerprint) &&
+ totalAdvertisedBandwidth > 0.0) {
+ probabilities[0] = advertisedBandwidths.get(fingerprint)
+ / totalAdvertisedBandwidth;
+ probabilities[5] = advertisedBandwidths.get(fingerprint);
+ }
pathSelectionProbabilities.put(fingerprint, probabilities);
}
return pathSelectionProbabilities;
diff --git a/web/protocol.html b/web/protocol.html
index 0c191bf..2d51621 100644
--- a/web/protocol.html
+++ b/web/protocol.html
@@ -1715,6 +1715,30 @@ The specification of this history object is similar to that in the
</p>
</li>
+<li>
+<font color="blue"><b>advertised_bandwidth</b></font>
+<code class="typeof">object</code>
+<span class="required-false">optional</span>
+<p>
+History object containing the absolute advertised bandwidth of this relay.
+The specification of this history object is similar to that in the
+<strong>advertised_bandwidth_fraction</strong> field above.
+<font color="blue">Added on June 18, 2014.</font>
+</p>
+</li>
+
+<li>
+<font color="blue"><b>consensus_weight</b></font>
+<code class="typeof">object</code>
+<span class="required-false">optional</span>
+<p>
+History object containing the absolute consensus weight of this relay.
+The specification of this history object is similar to that in the
+<strong>advertised_bandwidth_fraction</strong> field above.
+<font color="blue">Added on June 18, 2014.</font>
+</p>
+</li>
+
</ul>
</div> <!-- box -->
More information about the tor-commits
mailing list