commit 3ae9b18fdaa99b36b6448853f200d7014d5d75ac Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Mon Nov 17 15:21:37 2014 +0100 Remove advertised bandwidth graphs from weights. Implements #13674. --- .../torproject/onionoo/docs/WeightsDocument.java | 14 ----- .../org/torproject/onionoo/docs/WeightsStatus.java | 28 +++------- .../onionoo/updater/DescriptorSource.java | 3 +- .../onionoo/updater/WeightsStatusUpdater.java | 54 +------------------- .../onionoo/writer/WeightsDocumentWriter.java | 4 -- web/protocol.html | 44 ++++++++-------- 6 files changed, 34 insertions(+), 113 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java b/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java index 104b661..64ce713 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsDocument.java @@ -13,13 +13,6 @@ public class WeightsDocument extends Document { } @SuppressWarnings("unused") - private Map<String, GraphHistory> advertised_bandwidth_fraction; - public void setAdvertisedBandwidthFraction( - Map<String, GraphHistory> advertisedBandwidthFraction) { - this.advertised_bandwidth_fraction = advertisedBandwidthFraction; - } - - @SuppressWarnings("unused") private Map<String, GraphHistory> consensus_weight_fraction; public void setConsensusWeightFraction( Map<String, GraphHistory> consensusWeightFraction) { @@ -48,13 +41,6 @@ public class WeightsDocument extends Document { } @SuppressWarnings("unused") - private Map<String, GraphHistory> advertised_bandwidth; - public void setAdvertisedBandwidth( - Map<String, GraphHistory> advertisedBandwidth) { - this.advertised_bandwidth = advertisedBandwidth; - } - - @SuppressWarnings("unused") private Map<String, GraphHistory> consensus_weight; public void setConsensusWeight( Map<String, GraphHistory> consensusWeight) { diff --git a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java index e20112e..0e4eff8 100644 --- a/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/WeightsStatus.java @@ -1,7 +1,6 @@ package org.torproject.onionoo.docs; import java.util.Comparator; -import java.util.HashMap; import java.util.Map; import java.util.Scanner; import java.util.SortedMap; @@ -37,22 +36,14 @@ public class WeightsStatus extends Document { return this.history; } - private Map<String, Integer> advertisedBandwidths = - new HashMap<String, Integer>(); - public Map<String, Integer> getAdvertisedBandwidths() { - return this.advertisedBandwidths; - } - public void fromDocumentString(String documentString) { Scanner s = new Scanner(documentString); while (s.hasNextLine()) { String line = s.nextLine(); String[] parts = line.split(" "); if (parts.length == 2) { - String descriptorDigest = parts[0]; - int advertisedBandwidth = Integer.parseInt(parts[1]); - this.advertisedBandwidths.put(descriptorDigest, - advertisedBandwidth); + /* Skip lines containing descriptor digest and advertised + * bandwidth. */ continue; } if (parts.length != 9 && parts.length != 11) { @@ -75,14 +66,12 @@ public class WeightsStatus extends Document { break; } long[] interval = new long[] { validAfterMillis, freshUntilMillis }; - double[] weights = new double[] { - Double.parseDouble(parts[4]), + double[] weights = new double[] { -1.0, Double.parseDouble(parts[5]), Double.parseDouble(parts[6]), Double.parseDouble(parts[7]), Double.parseDouble(parts[8]), -1.0, -1.0 }; if (parts.length == 11) { - weights[5] = Double.parseDouble(parts[9]); weights[6] = Double.parseDouble(parts[10]); } this.history.put(interval, weights); @@ -176,17 +165,16 @@ public class WeightsStatus extends Document { public String toDocumentString() { StringBuilder sb = new StringBuilder(); - for (Map.Entry<String, Integer> e : - this.advertisedBandwidths.entrySet()) { - sb.append(e.getKey() + " " + String.valueOf(e.getValue()) + "\n"); - } for (Map.Entry<long[], double[]> e : history.entrySet()) { long[] fresh = e.getKey(); double[] weights = e.getValue(); sb.append(DateTimeHelper.format(fresh[0]) + " " + DateTimeHelper.format(fresh[1])); - for (double weight : weights) { - sb.append(String.format(" %.12f", weight)); + for (int i = 0; i < weights.length; i++) { + sb.append(" "); + if (i != 0 && i != 5) { + sb.append(String.format("%.12f", weights[i])); + } } sb.append("\n"); } diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java index 644f503..30001f8 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java @@ -106,7 +106,8 @@ public class DescriptorSource { public void readDescriptors() { /* Careful when changing the order of parsing descriptor types! The - * various status updaters may base assumptions on this order. */ + * various status updaters may base assumptions on this order. With + * #13674 being implemented, this may not be the case anymore. */ log.debug("Reading " + DescriptorType.RELAY_SERVER_DESCRIPTORS + " ..."); diff --git a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java index 2be213f..b6c3bd7 100644 --- a/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/WeightsStatusUpdater.java @@ -12,7 +12,6 @@ import java.util.TreeSet; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; -import org.torproject.descriptor.ServerDescriptor; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; import org.torproject.onionoo.docs.WeightsStatus; @@ -33,14 +32,10 @@ public class WeightsStatusUpdater implements DescriptorListener, private void registerDescriptorListeners() { this.descriptorSource.registerDescriptorListener(this, DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); } public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof ServerDescriptor) { - this.processRelayServerDescriptor((ServerDescriptor) descriptor); - } else if (descriptor instanceof RelayNetworkStatusConsensus) { + if (descriptor instanceof RelayNetworkStatusConsensus) { this.processRelayNetworkConsensus( (RelayNetworkStatusConsensus) descriptor); } @@ -60,25 +55,6 @@ public class WeightsStatusUpdater implements DescriptorListener, pathSelectionWeights); } - private void processRelayServerDescriptor( - ServerDescriptor serverDescriptor) { - String digest = serverDescriptor.getServerDescriptorDigest(). - toUpperCase(); - int advertisedBandwidth = Math.min(Math.min( - serverDescriptor.getBandwidthBurst(), - serverDescriptor.getBandwidthObserved()), - serverDescriptor.getBandwidthRate()); - String fingerprint = serverDescriptor.getFingerprint(); - WeightsStatus weightsStatus = this.documentStore.retrieve( - WeightsStatus.class, true, fingerprint); - if (weightsStatus == null) { - weightsStatus = new WeightsStatus(); - } - weightsStatus.getAdvertisedBandwidths().put(digest, - advertisedBandwidth); - this.documentStore.store(weightsStatus, fingerprint); -} - private void updateWeightsHistory(long validAfterMillis, long freshUntilMillis, SortedMap<String, double[]> pathSelectionWeights) { @@ -125,12 +101,10 @@ public class WeightsStatusUpdater implements DescriptorListener, } } SortedMap<String, Double> - advertisedBandwidths = new TreeMap<String, Double>(), consensusWeights = new TreeMap<String, Double>(), guardWeights = new TreeMap<String, Double>(), middleWeights = new TreeMap<String, Double>(), exitWeights = new TreeMap<String, Double>(); - double totalAdvertisedBandwidth = 0.0; double totalConsensusWeight = 0.0; double totalGuardWeight = 0.0; double totalMiddleWeight = 0.0; @@ -141,21 +115,6 @@ public class WeightsStatusUpdater implements DescriptorListener, if (!relay.getFlags().contains("Running")) { continue; } - String digest = relay.getDescriptor().toUpperCase(); - WeightsStatus weightsStatus = this.documentStore.retrieve( - WeightsStatus.class, true, fingerprint); - if (weightsStatus != null && - weightsStatus.getAdvertisedBandwidths() != null && - weightsStatus.getAdvertisedBandwidths().containsKey(digest)) { - /* Read advertised bandwidth from weights status file. Server - * descriptors are parsed before consensuses, so we're sure that - * if there's a server descriptor for this relay, it'll be - * contained in the weights status file by now. */ - double advertisedBandwidth = - (double) weightsStatus.getAdvertisedBandwidths().get(digest); - advertisedBandwidths.put(fingerprint, advertisedBandwidth); - totalAdvertisedBandwidth += advertisedBandwidth; - } if (relay.getBandwidth() >= 0L) { double consensusWeight = (double) relay.getBandwidth(); consensusWeights.put(fingerprint, consensusWeight); @@ -195,10 +154,7 @@ public class WeightsStatusUpdater implements DescriptorListener, } SortedMap<String, double[]> pathSelectionProbabilities = new TreeMap<String, double[]>(); - SortedSet<String> fingerprints = new TreeSet<String>(); - fingerprints.addAll(consensusWeights.keySet()); - fingerprints.addAll(advertisedBandwidths.keySet()); - for (String fingerprint : fingerprints) { + for (String fingerprint : consensusWeights.keySet()) { double[] probabilities = new double[] { -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0 }; if (consensusWeights.containsKey(fingerprint) && @@ -222,12 +178,6 @@ public class WeightsStatusUpdater implements DescriptorListener, probabilities[4] = exitWeights.get(fingerprint) / totalExitWeight; } - if (advertisedBandwidths.containsKey(fingerprint) && - totalAdvertisedBandwidth > 0.0) { - probabilities[0] = advertisedBandwidths.get(fingerprint) - / totalAdvertisedBandwidth; - probabilities[5] = advertisedBandwidths.get(fingerprint); - } pathSelectionProbabilities.put(fingerprint, probabilities); } return pathSelectionProbabilities; diff --git a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java index 0d70465..00c529e 100644 --- a/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/WeightsDocumentWriter.java @@ -105,8 +105,6 @@ public class WeightsDocumentWriter implements FingerprintListener, SortedMap<long[], double[]> history) { WeightsDocument weightsDocument = new WeightsDocument(); weightsDocument.setFingerprint(fingerprint); - weightsDocument.setAdvertisedBandwidthFraction( - this.compileGraphType(history, 0)); weightsDocument.setConsensusWeightFraction( this.compileGraphType(history, 1)); weightsDocument.setGuardProbability( @@ -115,8 +113,6 @@ public class WeightsDocumentWriter implements FingerprintListener, this.compileGraphType(history, 3)); weightsDocument.setExitProbability( this.compileGraphType(history, 4)); - weightsDocument.setAdvertisedBandwidth( - this.compileGraphType(history, 5)); weightsDocument.setConsensusWeight( this.compileGraphType(history, 6)); return weightsDocument; diff --git a/web/protocol.html b/web/protocol.html index f5dc87a..6885c00 100644 --- a/web/protocol.html +++ b/web/protocol.html @@ -1762,7 +1762,7 @@ hexadecimal characters. </li> <li> -<b>advertised_bandwidth_fraction</b> +<b><font color="red">advertised_bandwidth_fraction</font></b> <code class="typeof">object</code> <span class="required-false">optional</span> <p> @@ -1771,19 +1771,7 @@ of this relay compared to the total advertised bandwidth in the network. If there were no bandwidth authorities, this fraction would be a very rough approximation of the probability of this relay to be selected by clients. -Keys are string representation of the time period covered by the graph -history object. -Keys are fixed strings <strong>"1_week"</strong>, -<strong>"1_month"</strong>, <strong>"3_months"</strong>, -<strong>"1_year"</strong>, and <strong>"5_years"</strong>. -Keys refer to the last known weights history of a relay, not to the time -when the weights document was published. -A graph history object is only contained if the time period it covers is -not already contained in another graph history object with shorter time -period and higher data resolution. -The unit is path-selection probability. -Contained graph history objects may contain null values if the relay was -running less than 20% of a given time period. +<font color="red">This field was removed on November 16, 2014.</font> </p> </li> @@ -1797,8 +1785,19 @@ fraction of this relay's consensus weight compared to the sum of all consensus weights in the network. This fraction is a very rough approximation of the probability of this relay to be selected by clients. -The specification of this history object is similar to that in the -<strong>advertised_bandwidth_fraction</strong> field above. +Keys are string representation of the time period covered by the graph +history object. +Keys are fixed strings <strong>"1_week"</strong>, +<strong>"1_month"</strong>, <strong>"3_months"</strong>, +<strong>"1_year"</strong>, and <strong>"5_years"</strong>. +Keys refer to the last known weights history of a relay, not to the time +when the weights document was published. +A graph history object is only contained if the time period it covers is +not already contained in another graph history object with shorter time +period and higher data resolution. +The unit is path-selection probability. +Contained graph history objects may contain null values if the relay was +running less than 20% of a given time period. </p> </li> @@ -1814,7 +1813,7 @@ and bandwidth weights in the consensus. Path selection depends on more factors, so that this probability can only be an approximation. The specification of this history object is similar to that in the -<strong>advertised_bandwidth_fraction</strong> field above. +<strong>consensus_weight_fraction</strong> field above. </p> </li> @@ -1830,7 +1829,7 @@ and bandwidth weights in the consensus. Path selection depends on more factors, so that this probability can only be an approximation. The specification of this history object is similar to that in the -<strong>advertised_bandwidth_fraction</strong> field above. +<strong>consensus_weight_fraction</strong> field above. </p> </li> @@ -1846,18 +1845,19 @@ and bandwidth weights in the consensus. Path selection depends on more factors, so that this probability can only be an approximation. The specification of this history object is similar to that in the -<strong>advertised_bandwidth_fraction</strong> field above. +<strong>consensus_weight_fraction</strong> field above. </p> </li> <li> -<b>advertised_bandwidth</b> +<b><font color="red">advertised_bandwidth</font></b> <code class="typeof">object</code> <span class="required-false">optional</span> <p> History object containing the absolute advertised bandwidth of this relay. The specification of this history object is similar to that in the -<strong>advertised_bandwidth_fraction</strong> field above. +<strong>consensus_weight_fraction</strong> field above. +<font color="red">This field was removed on November 16, 2014.</font> </p> </li> @@ -1868,7 +1868,7 @@ The specification of this history object is similar to that in the <p> History object containing the absolute consensus weight of this relay. The specification of this history object is similar to that in the -<strong>advertised_bandwidth_fraction</strong> field above. +<strong>consensus_weight_fraction</strong> field above. </p> </li>