commit 99e841366e9f56489915cd886367ccfaf3dfdc64 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Sun Jun 8 19:56:26 2014 +0200 Add advertised_bandwidth and consensus_weight graphs. Weights documents contain advertised_bandwidth_fraction and consensus_weight_fraction graphs containing, well, fractions of the relay's value compared to the network total. This commit adds two new graphs showing the respective absolute values, to distinguish effects of things happening on a relay from those happening to the overall Tor network. This change further allows us to spot changes to a relays' absolute values over time, like relays increasing their advertised bandwidth by more than 128 KB/s. Also fixed a few things while re-processing the archives: - Older consensuses contained non-"Running" relays, which caused a NullPointerException. - Older consensuses did not contain "w" lines and/or no bandwidth weights, either of which we didn't handle very well. - Whenever we didn't find a server descriptor we silently assumed that advertised bandwidth is 0. Now we're handling this as missing data and not including intervals in advertised_bandwidth_fraction and the new advertised_bandwidth. Implements #11388. --- src/org/torproject/onionoo/WeightsDocument.java | 12 ++ .../torproject/onionoo/WeightsDocumentWriter.java | 12 +- src/org/torproject/onionoo/WeightsStatus.java | 8 +- .../torproject/onionoo/WeightsStatusUpdater.java | 118 ++++++++++++-------- web/protocol.html | 24 ++++ 5 files changed, 124 insertions(+), 50 deletions(-) diff --git a/src/org/torproject/onionoo/WeightsDocument.java b/src/org/torproject/onionoo/WeightsDocument.java index ff1d5f4..6dddf9d 100644 --- a/src/org/torproject/onionoo/WeightsDocument.java +++ b/src/org/torproject/onionoo/WeightsDocument.java @@ -40,5 +40,17 @@ class WeightsDocument extends Document { Map<String, GraphHistory> exitProbability) { this.exit_probability = exitProbability; } + + private Map<String, GraphHistory> advertised_bandwidth; + public void setAdvertisedBandwidth( + Map<String, GraphHistory> advertisedBandwidth) { + this.advertised_bandwidth = advertisedBandwidth; + } + + private Map<String, GraphHistory> consensus_weight; + public void setConsensusWeight( + Map<String, GraphHistory> consensusWeight) { + this.consensus_weight = consensusWeight; + } } diff --git a/src/org/torproject/onionoo/WeightsDocumentWriter.java b/src/org/torproject/onionoo/WeightsDocumentWriter.java index 74108c9..2e0d465 100644 --- a/src/org/torproject/onionoo/WeightsDocumentWriter.java +++ b/src/org/torproject/onionoo/WeightsDocumentWriter.java @@ -98,6 +98,10 @@ public class WeightsDocumentWriter implements FingerprintListener, this.compileGraphType(history, 3)); weightsDocument.setExitProbability( this.compileGraphType(history, 4)); + weightsDocument.setAdvertisedBandwidth( + this.compileGraphType(history, 5)); + weightsDocument.setConsensusWeight( + this.compileGraphType(history, 6)); return weightsDocument; } @@ -141,9 +145,11 @@ public class WeightsDocumentWriter implements FingerprintListener, totalMillis = 0L; intervalStartMillis += dataPointInterval; } - totalWeightTimesMillis += weight - * ((double) (endMillis - startMillis)); - totalMillis += (endMillis - startMillis); + if (weight >= 0.0) { + totalWeightTimesMillis += weight + * ((double) (endMillis - startMillis)); + totalMillis += (endMillis - startMillis); + } } dataPoints.add(totalMillis * 5L < dataPointInterval ? -1.0 : totalWeightTimesMillis / (double) totalMillis); diff --git a/src/org/torproject/onionoo/WeightsStatus.java b/src/org/torproject/onionoo/WeightsStatus.java index 872fd00..f5eacd9 100644 --- a/src/org/torproject/onionoo/WeightsStatus.java +++ b/src/org/torproject/onionoo/WeightsStatus.java @@ -40,7 +40,7 @@ class WeightsStatus extends Document { advertisedBandwidth); continue; } - if (parts.length != 9) { + if (parts.length != 9 && parts.length != 11) { System.err.println("Illegal line '" + line + "' in weights " + "status file. Skipping this line."); continue; @@ -65,7 +65,11 @@ class WeightsStatus extends Document { Double.parseDouble(parts[5]), Double.parseDouble(parts[6]), Double.parseDouble(parts[7]), - Double.parseDouble(parts[8]) }; + Double.parseDouble(parts[8]), -1.0, -1.0 }; + if (parts.length == 11) { + weights[5] = Double.parseDouble(parts[9]); + weights[6] = Double.parseDouble(parts[10]); + } this.history.put(interval, weights); } s.close(); diff --git a/src/org/torproject/onionoo/WeightsStatusUpdater.java b/src/org/torproject/onionoo/WeightsStatusUpdater.java index 9b45f95..24c98ff 100644 --- a/src/org/torproject/onionoo/WeightsStatusUpdater.java +++ b/src/org/torproject/onionoo/WeightsStatusUpdater.java @@ -101,6 +101,7 @@ public class WeightsStatusUpdater implements DescriptorListener, private SortedMap<String, double[]> calculatePathSelectionProbabilities( RelayNetworkStatusConsensus consensus) { + boolean containsBandwidthWeights = false; double wgg = 1.0, wgd = 1.0, wmg = 1.0, wmm = 1.0, wme = 1.0, wmd = 1.0, wee = 1.0, wed = 1.0; SortedMap<String, Integer> bandwidthWeights = @@ -118,6 +119,7 @@ public class WeightsStatusUpdater implements DescriptorListener, wmd = ((double) bandwidthWeights.get("Wmd")) / 10000.0; wee = ((double) bandwidthWeights.get("Wee")) / 10000.0; wed = ((double) bandwidthWeights.get("Wed")) / 10000.0; + containsBandwidthWeights = true; } } SortedMap<String, Double> @@ -137,11 +139,7 @@ public class WeightsStatusUpdater implements DescriptorListener, if (!relay.getFlags().contains("Running")) { continue; } - boolean isExit = relay.getFlags().contains("Exit") && - !relay.getFlags().contains("BadExit"); - boolean isGuard = relay.getFlags().contains("Guard"); String digest = relay.getDescriptor().toUpperCase(); - double advertisedBandwidth = 0.0; WeightsStatus weightsStatus = this.documentStore.retrieve( WeightsStatus.class, true, fingerprint); if (weightsStatus != null && @@ -151,53 +149,83 @@ public class WeightsStatusUpdater implements DescriptorListener, * descriptors are parsed before consensuses, so we're sure that * if there's a server descriptor for this relay, it'll be * contained in the weights status file by now. */ - advertisedBandwidth = + double advertisedBandwidth = (double) weightsStatus.getAdvertisedBandwidths().get(digest); + advertisedBandwidths.put(fingerprint, advertisedBandwidth); + totalAdvertisedBandwidth += advertisedBandwidth; } - double consensusWeight = (double) relay.getBandwidth(); - double guardWeight = (double) relay.getBandwidth(); - double middleWeight = (double) relay.getBandwidth(); - double exitWeight = (double) relay.getBandwidth(); - if (isGuard && isExit) { - guardWeight *= wgd; - middleWeight *= wmd; - exitWeight *= wed; - } else if (isGuard) { - guardWeight *= wgg; - middleWeight *= wmg; - exitWeight = 0.0; - } else if (isExit) { - guardWeight = 0.0; - middleWeight *= wme; - exitWeight *= wee; - } else { - guardWeight = 0.0; - middleWeight *= wmm; - exitWeight = 0.0; + if (relay.getBandwidth() >= 0L) { + double consensusWeight = (double) relay.getBandwidth(); + consensusWeights.put(fingerprint, consensusWeight); + totalConsensusWeight += consensusWeight; + if (containsBandwidthWeights) { + double guardWeight = (double) relay.getBandwidth(); + double middleWeight = (double) relay.getBandwidth(); + double exitWeight = (double) relay.getBandwidth(); + boolean isExit = relay.getFlags().contains("Exit") && + !relay.getFlags().contains("BadExit"); + boolean isGuard = relay.getFlags().contains("Guard"); + if (isGuard && isExit) { + guardWeight *= wgd; + middleWeight *= wmd; + exitWeight *= wed; + } else if (isGuard) { + guardWeight *= wgg; + middleWeight *= wmg; + exitWeight = 0.0; + } else if (isExit) { + guardWeight = 0.0; + middleWeight *= wme; + exitWeight *= wee; + } else { + guardWeight = 0.0; + middleWeight *= wmm; + exitWeight = 0.0; + } + guardWeights.put(fingerprint, guardWeight); + middleWeights.put(fingerprint, middleWeight); + exitWeights.put(fingerprint, exitWeight); + totalGuardWeight += guardWeight; + totalMiddleWeight += middleWeight; + totalExitWeight += exitWeight; + } } - advertisedBandwidths.put(fingerprint, advertisedBandwidth); - consensusWeights.put(fingerprint, consensusWeight); - guardWeights.put(fingerprint, guardWeight); - middleWeights.put(fingerprint, middleWeight); - exitWeights.put(fingerprint, exitWeight); - totalAdvertisedBandwidth += advertisedBandwidth; - totalConsensusWeight += consensusWeight; - totalGuardWeight += guardWeight; - totalMiddleWeight += middleWeight; - totalExitWeight += exitWeight; } SortedMap<String, double[]> pathSelectionProbabilities = new TreeMap<String, double[]>(); - for (NetworkStatusEntry relay : - consensus.getStatusEntries().values()) { - String fingerprint = relay.getFingerprint(); - double[] probabilities = new double[] { - advertisedBandwidths.get(fingerprint) - / totalAdvertisedBandwidth, - consensusWeights.get(fingerprint) / totalConsensusWeight, - guardWeights.get(fingerprint) / totalGuardWeight, - middleWeights.get(fingerprint) / totalMiddleWeight, - exitWeights.get(fingerprint) / totalExitWeight }; + SortedSet<String> fingerprints = new TreeSet<String>(); + fingerprints.addAll(consensusWeights.keySet()); + fingerprints.addAll(advertisedBandwidths.keySet()); + for (String fingerprint : fingerprints) { + double[] probabilities = new double[] { -1.0, -1.0, -1.0, -1.0, + -1.0, -1.0, -1.0 }; + if (consensusWeights.containsKey(fingerprint) && + totalConsensusWeight > 0.0) { + probabilities[1] = consensusWeights.get(fingerprint) / + totalConsensusWeight; + probabilities[6] = consensusWeights.get(fingerprint); + } + if (guardWeights.containsKey(fingerprint) && + totalGuardWeight > 0.0) { + probabilities[2] = guardWeights.get(fingerprint) / + totalGuardWeight; + } + if (middleWeights.containsKey(fingerprint) && + totalMiddleWeight > 0.0) { + probabilities[3] = middleWeights.get(fingerprint) / + totalMiddleWeight; + } + if (exitWeights.containsKey(fingerprint) && + totalExitWeight > 0.0) { + probabilities[4] = exitWeights.get(fingerprint) / + totalExitWeight; + } + if (advertisedBandwidths.containsKey(fingerprint) && + totalAdvertisedBandwidth > 0.0) { + probabilities[0] = advertisedBandwidths.get(fingerprint) + / totalAdvertisedBandwidth; + probabilities[5] = advertisedBandwidths.get(fingerprint); + } pathSelectionProbabilities.put(fingerprint, probabilities); } return pathSelectionProbabilities; diff --git a/web/protocol.html b/web/protocol.html index 0c191bf..2d51621 100644 --- a/web/protocol.html +++ b/web/protocol.html @@ -1715,6 +1715,30 @@ The specification of this history object is similar to that in the </p> </li> +<li> +<font color="blue"><b>advertised_bandwidth</b></font> +<code class="typeof">object</code> +<span class="required-false">optional</span> +<p> +History object containing the absolute advertised bandwidth of this relay. +The specification of this history object is similar to that in the +<strong>advertised_bandwidth_fraction</strong> field above. +<font color="blue">Added on June 18, 2014.</font> +</p> +</li> + +<li> +<font color="blue"><b>consensus_weight</b></font> +<code class="typeof">object</code> +<span class="required-false">optional</span> +<p> +History object containing the absolute consensus weight of this relay. +The specification of this history object is similar to that in the +<strong>advertised_bandwidth_fraction</strong> field above. +<font color="blue">Added on June 18, 2014.</font> +</p> +</li> + </ul> </div> <!-- box -->