commit 7713b17fa2cc79656ca51dea25a1ebe17e9e5f34
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Tue Jul 24 16:16:37 2012 +0200
Add a new document type for path-selection probabilities.
---
etc/web.xml.template | 4 +
src/org/torproject/onionoo/Main.java | 8 +
src/org/torproject/onionoo/ResourceServlet.java | 38 ++
src/org/torproject/onionoo/WeightsDataWriter.java | 547 +++++++++++++++++++++
web/index.html | 109 ++++
5 files changed, 706 insertions(+), 0 deletions(-)
diff --git a/etc/web.xml.template b/etc/web.xml.template
index b2c3178..b7fbf9e 100644
--- a/etc/web.xml.template
+++ b/etc/web.xml.template
@@ -28,6 +28,10 @@
<servlet-name>Resource</servlet-name>
<url-pattern>/bandwidth</url-pattern>
</servlet-mapping>
+ <servlet-mapping>
+ <servlet-name>Resource</servlet-name>
+ <url-pattern>/weights</url-pattern>
+ </servlet-mapping>
</web-app>
diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java
index 175176c..fb11899 100644
--- a/src/org/torproject/onionoo/Main.java
+++ b/src/org/torproject/onionoo/Main.java
@@ -38,6 +38,14 @@ public class Main {
bdw.readExtraInfoDescriptors();
bdw.deleteObsoleteBandwidthFiles();
+ printStatus("Updating weights data.");
+ WeightsDataWriter wdw = new WeightsDataWriter();
+ wdw.setCurrentRelays(cn.getCurrentRelays());
+ wdw.readRelayServerDescriptors();
+ wdw.readRelayNetworkConsensuses();
+ wdw.writeWeightsDataFiles();
+ wdw.deleteObsoleteWeightsDataFiles();
+
printStatus("Updating summary data.");
cn.writeRelaySearchDataFile();
diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java
index 7fddbea..512b7ca 100644
--- a/src/org/torproject/onionoo/ResourceServlet.java
+++ b/src/org/torproject/onionoo/ResourceServlet.java
@@ -166,6 +166,8 @@ public class ResourceServlet extends HttpServlet {
resourceType = "details";
} else if (uri.startsWith("/bandwidth")) {
resourceType = "bandwidth";
+ } else if (uri.startsWith("/weights")) {
+ resourceType = "weights";
} else {
response.sendError(HttpServletResponse.SC_BAD_REQUEST);
return;
@@ -578,6 +580,8 @@ public class ResourceServlet extends HttpServlet {
return this.writeDetailsLines(summaryLine);
} else if (resourceType.equals("bandwidth")) {
return this.writeBandwidthLines(summaryLine);
+ } else if (resourceType.equals("weights")) {
+ return this.writeWeightsLines(summaryLine);
} else {
return "";
}
@@ -670,5 +674,39 @@ public class ResourceServlet extends HttpServlet {
return "";
}
}
+
+ private String writeWeightsLines(String summaryLine) {
+ String fingerprint = null;
+ if (summaryLine.contains("\"f\":\"")) {
+ fingerprint = summaryLine.substring(summaryLine.indexOf(
+ "\"f\":\"") + "\"f\":\"".length());
+ } else {
+ return "";
+ }
+ fingerprint = fingerprint.substring(0, 40);
+ File weightsFile = new File(this.outDirString + "weights/"
+ + fingerprint);
+ StringBuilder sb = new StringBuilder();
+ String weightsLines = null;
+ if (weightsFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ weightsFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ sb.append(line + "\n");
+ }
+ br.close();
+ weightsLines = sb.toString();
+ } catch (IOException e) {
+ }
+ }
+ if (weightsLines != null) {
+ weightsLines = weightsLines.substring(0, weightsLines.length() - 1);
+ return weightsLines;
+ } else {
+ return "";
+ }
+ }
}
diff --git a/src/org/torproject/onionoo/WeightsDataWriter.java b/src/org/torproject/onionoo/WeightsDataWriter.java
new file mode 100644
index 0000000..d3a37fc
--- /dev/null
+++ b/src/org/torproject/onionoo/WeightsDataWriter.java
@@ -0,0 +1,547 @@
+/* Copyright 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorFile;
+import org.torproject.descriptor.DescriptorReader;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.NetworkStatusEntry;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import org.torproject.descriptor.ServerDescriptor;
+
+public class WeightsDataWriter {
+
+ private SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ public WeightsDataWriter() {
+ this.dateTimeFormat.setLenient(false);
+ this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ }
+
+ private SortedSet<String> currentFingerprints = new TreeSet<String>();
+ public void setCurrentRelays(SortedMap<String, Node> currentRelays) {
+ this.currentFingerprints.addAll(currentRelays.keySet());
+ }
+
+ /* Read advertised bandwidths of all server descriptors in
+ * in/relay-descriptors/server-descriptors/ to memory. Ideally, we'd
+ * skip descriptors that we read before and obtain their advertised
+ * bandwidths from some temp file. This approach should do for now,
+ * though. */
+ private Map<String, Integer> advertisedBandwidths =
+ new HashMap<String, Integer>();
+ public void readRelayServerDescriptors() {
+ DescriptorReader reader =
+ DescriptorSourceFactory.createDescriptorReader();
+ reader.addDirectory(new File(
+ "in/relay-descriptors/server-descriptors"));
+ Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors();
+ while (descriptorFiles.hasNext()) {
+ DescriptorFile descriptorFile = descriptorFiles.next();
+ if (descriptorFile.getDescriptors() != null) {
+ for (Descriptor descriptor : descriptorFile.getDescriptors()) {
+ if (descriptor instanceof ServerDescriptor) {
+ ServerDescriptor serverDescriptor =
+ (ServerDescriptor) descriptor;
+ String digest = serverDescriptor.getServerDescriptorDigest().
+ toUpperCase();
+ int advertisedBandwidth = Math.min(Math.min(
+ serverDescriptor.getBandwidthBurst(),
+ serverDescriptor.getBandwidthObserved()),
+ serverDescriptor.getBandwidthRate());
+ this.advertisedBandwidths.put(digest, advertisedBandwidth);
+ }
+ }
+ }
+ }
+ }
+
+ public void readRelayNetworkConsensuses() {
+ DescriptorReader reader =
+ DescriptorSourceFactory.createDescriptorReader();
+ reader.addDirectory(new File("in/relay-descriptors/consensuses"));
+ reader.setExcludeFiles(new File(
+ "status/weights-relay-consensus-history"));
+ Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors();
+ while (descriptorFiles.hasNext()) {
+ DescriptorFile descriptorFile = descriptorFiles.next();
+ if (descriptorFile.getDescriptors() != null) {
+ for (Descriptor descriptor : descriptorFile.getDescriptors()) {
+ if (descriptor instanceof RelayNetworkStatusConsensus) {
+ RelayNetworkStatusConsensus consensus =
+ (RelayNetworkStatusConsensus) descriptor;
+ long validAfterMillis = consensus.getValidAfterMillis(),
+ freshUntilMillis = consensus.getFreshUntilMillis();
+ SortedMap<String, double[]> pathSelectionWeights =
+ this.calculatePathSelectionProbabilities(consensus);
+ for (Map.Entry<String, double[]> e :
+ pathSelectionWeights.entrySet()) {
+ String fingerprint = e.getKey();
+ double[] weights = e.getValue();
+ this.addToHistory(fingerprint, validAfterMillis,
+ freshUntilMillis, weights);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private SortedMap<String, double[]> calculatePathSelectionProbabilities(
+ RelayNetworkStatusConsensus consensus) {
+ double wgg = 1.0, wgd = 1.0, wmg = 1.0, wmm = 1.0, wme = 1.0,
+ wmd = 1.0, wee = 1.0, wed = 1.0;
+ SortedMap<String, Integer> bandwidthWeights =
+ consensus.getBandwidthWeights();
+ if (bandwidthWeights != null) {
+ SortedSet<String> missingWeightKeys = new TreeSet<String>(
+ Arrays.asList("Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(",")));
+ missingWeightKeys.removeAll(bandwidthWeights.keySet());
+ if (missingWeightKeys.isEmpty()) {
+ wgg = ((double) bandwidthWeights.get("Wgg")) / 10000.0;
+ wgd = ((double) bandwidthWeights.get("Wgd")) / 10000.0;
+ wmg = ((double) bandwidthWeights.get("Wmg")) / 10000.0;
+ wmm = ((double) bandwidthWeights.get("Wmm")) / 10000.0;
+ wme = ((double) bandwidthWeights.get("Wme")) / 10000.0;
+ wmd = ((double) bandwidthWeights.get("Wmd")) / 10000.0;
+ wee = ((double) bandwidthWeights.get("Wee")) / 10000.0;
+ wed = ((double) bandwidthWeights.get("Wed")) / 10000.0;
+ }
+ }
+ SortedMap<String, Double>
+ advertisedBandwidths = new TreeMap<String, Double>(),
+ consensusWeights = new TreeMap<String, Double>(),
+ guardWeights = new TreeMap<String, Double>(),
+ middleWeights = new TreeMap<String, Double>(),
+ exitWeights = new TreeMap<String, Double>();
+ double totalAdvertisedBandwidth = 0.0;
+ double totalConsensusWeight = 0.0;
+ double totalGuardWeight = 0.0;
+ double totalMiddleWeight = 0.0;
+ double totalExitWeight = 0.0;
+ for (NetworkStatusEntry relay :
+ consensus.getStatusEntries().values()) {
+ String fingerprint = relay.getFingerprint();
+ if (!relay.getFlags().contains("Running")) {
+ continue;
+ }
+ boolean isExit = relay.getFlags().contains("Exit") &&
+ !relay.getFlags().contains("BadExit");
+ boolean isGuard = relay.getFlags().contains("Guard");
+ String serverDescriptorDigest = relay.getDescriptor().
+ toUpperCase();
+ double advertisedBandwidth = 0.0;
+ if (this.advertisedBandwidths.containsKey(
+ serverDescriptorDigest)) {
+ advertisedBandwidth = (double) this.advertisedBandwidths.get(
+ serverDescriptorDigest);
+ }
+ double consensusWeight = (double) relay.getBandwidth();
+ double guardWeight = (double) relay.getBandwidth();
+ double middleWeight = (double) relay.getBandwidth();
+ double exitWeight = (double) relay.getBandwidth();
+ if (isGuard && isExit) {
+ guardWeight *= wgd;
+ middleWeight *= wmd;
+ exitWeight *= wed;
+ } else if (isGuard) {
+ guardWeight *= wgg;
+ middleWeight *= wmg;
+ exitWeight = 0.0;
+ } else if (isExit) {
+ guardWeight = 0.0;
+ middleWeight *= wme;
+ exitWeight *= wee;
+ } else {
+ guardWeight = 0.0;
+ middleWeight *= wmm;
+ exitWeight = 0.0;
+ }
+ advertisedBandwidths.put(fingerprint, advertisedBandwidth);
+ consensusWeights.put(fingerprint, consensusWeight);
+ guardWeights.put(fingerprint, guardWeight);
+ middleWeights.put(fingerprint, middleWeight);
+ exitWeights.put(fingerprint, exitWeight);
+ totalAdvertisedBandwidth += advertisedBandwidth;
+ totalConsensusWeight += consensusWeight;
+ totalGuardWeight += guardWeight;
+ totalMiddleWeight += middleWeight;
+ totalExitWeight += exitWeight;
+ }
+ SortedMap<String, double[]> pathSelectionProbabilities =
+ new TreeMap<String, double[]>();
+ for (NetworkStatusEntry relay :
+ consensus.getStatusEntries().values()) {
+ String fingerprint = relay.getFingerprint();
+ double[] probabilities = new double[] {
+ advertisedBandwidths.get(fingerprint)
+ / totalAdvertisedBandwidth,
+ consensusWeights.get(fingerprint) / totalConsensusWeight,
+ guardWeights.get(fingerprint) / totalGuardWeight,
+ middleWeights.get(fingerprint) / totalMiddleWeight,
+ exitWeights.get(fingerprint) / totalExitWeight };
+ pathSelectionProbabilities.put(fingerprint, probabilities);
+ }
+ return pathSelectionProbabilities;
+ }
+
+ private void addToHistory(String fingerprint, long validAfterMillis,
+ long freshUntilMillis, double[] weights) {
+ SortedMap<long[], double[]> history =
+ this.readHistoryFromDisk(fingerprint);
+ long[] interval = new long[] { validAfterMillis, freshUntilMillis };
+ if ((history.headMap(interval).isEmpty() ||
+ history.headMap(interval).lastKey()[1] <= validAfterMillis) &&
+ (history.tailMap(interval).isEmpty() ||
+ history.tailMap(interval).firstKey()[0] >= freshUntilMillis)) {
+ history.put(interval, weights);
+ history = this.compressHistory(history);
+ this.writeHistoryToDisk(fingerprint, history);
+ }
+ }
+
+ private SortedMap<long[], double[]> readHistoryFromDisk(
+ String fingerprint) {
+ SortedMap<long[], double[]> history =
+ new TreeMap<long[], double[]>(new Comparator<long[]>() {
+ public int compare(long[] a, long[] b) {
+ return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;
+ }
+ });
+ File historyFile = new File("status/weights", fingerprint);
+ if (historyFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ historyFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(" ");
+ if (parts.length != 9) {
+ System.err.println("Illegal line '" + line + "' in history "
+ + "file '" + historyFile.getAbsolutePath()
+ + "'. Skipping this line.");
+ continue;
+ }
+ long validAfterMillis = this.dateTimeFormat.parse(parts[0]
+ + " " + parts[1]).getTime();
+ long freshUntilMillis = this.dateTimeFormat.parse(parts[2]
+ + " " + parts[3]).getTime();
+ long[] interval = new long[] { validAfterMillis,
+ freshUntilMillis };
+ double[] weights = new double[] {
+ Double.parseDouble(parts[4]),
+ Double.parseDouble(parts[5]),
+ Double.parseDouble(parts[6]),
+ Double.parseDouble(parts[7]),
+ Double.parseDouble(parts[8]) };
+ history.put(interval, weights);
+ }
+ br.close();
+ } catch (ParseException e) {
+ System.err.println("Could not parse timestamp while reading "
+ + "history file '" + historyFile.getAbsolutePath()
+ + "'. Skipping.");
+ } catch (IOException e) {
+ System.err.println("Could not read history file '"
+ + historyFile.getAbsolutePath() + "'. Skipping.");
+ }
+ }
+ return history;
+ }
+
+ private long now = System.currentTimeMillis();
+ private SortedMap<long[], double[]> compressHistory(
+ SortedMap<long[], double[]> history) {
+ SortedMap<long[], double[]> compressedHistory =
+ new TreeMap<long[], double[]>(history.comparator());
+ long lastStartMillis = 0L, lastEndMillis = 0L;
+ double[] lastWeights = null;
+ for (Map.Entry<long[], double[]> e : history.entrySet()) {
+ long startMillis = e.getKey()[0], endMillis = e.getKey()[1];
+ double[] weights = e.getValue();
+ long intervalLengthMillis;
+ if (this.now - endMillis <= 7L * 24L * 60L * 60L * 1000L) {
+ intervalLengthMillis = 60L * 60L * 1000L;
+ } else if (this.now - endMillis <= 31L * 24L * 60L * 60L * 1000L) {
+ intervalLengthMillis = 4L * 60L * 60L * 1000L;
+ } else if (this.now - endMillis <= 92L * 24L * 60L * 60L * 1000L) {
+ intervalLengthMillis = 12L * 60L * 60L * 1000L;
+ } else if (this.now - endMillis <= 366L * 24L * 60L * 60L * 1000L) {
+ intervalLengthMillis = 2L * 24L * 60L * 60L * 1000L;
+ } else {
+ intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L;
+ }
+ if (lastEndMillis == startMillis &&
+ (lastEndMillis / intervalLengthMillis) ==
+ (endMillis / intervalLengthMillis)) {
+ double lastIntervalInHours = (double) ((lastEndMillis
+ - lastStartMillis) / 60L * 60L * 1000L);
+ double currentIntervalInHours = (double) ((endMillis
+ - startMillis) / 60L * 60L * 1000L);
+ double newIntervalInHours = (double) ((endMillis
+ - lastStartMillis) / 60L * 60L * 1000L);
+ for (int i = 0; i < lastWeights.length; i++) {
+ lastWeights[i] *= lastIntervalInHours;
+ lastWeights[i] += weights[i] * currentIntervalInHours;
+ lastWeights[i] /= newIntervalInHours;
+ }
+ lastEndMillis = endMillis;
+ } else {
+ if (lastStartMillis > 0L) {
+ compressedHistory.put(new long[] { lastStartMillis,
+ lastEndMillis }, lastWeights);
+ }
+ lastStartMillis = startMillis;
+ lastEndMillis = endMillis;
+ lastWeights = weights;
+ }
+ }
+ if (lastStartMillis > 0L) {
+ compressedHistory.put(new long[] { lastStartMillis, lastEndMillis },
+ lastWeights);
+ }
+ return compressedHistory;
+ }
+
+ private void writeHistoryToDisk(String fingerprint,
+ SortedMap<long[], double[]> history) {
+ File historyFile = new File("status/weights", fingerprint);
+ try {
+ historyFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(historyFile));
+ for (Map.Entry<long[], double[]> e : history.entrySet()) {
+ long[] fresh = e.getKey();
+ double[] weights = e.getValue();
+ bw.write(this.dateTimeFormat.format(fresh[0]) + " "
+ + this.dateTimeFormat.format(fresh[1]));
+ for (double weight : weights) {
+ bw.write(String.format(" %.12f", weight));
+ }
+ bw.write("\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ System.err.println("Could not write weights file '"
+ + historyFile.getAbsolutePath() + "'. Skipping.");
+ }
+ }
+
+ private File weightsFileDirectory = new File("out/weights");
+ public void writeWeightsDataFiles() {
+ for (String fingerprint : this.currentFingerprints) {
+ SortedMap<long[], double[]> history =
+ this.readHistoryFromDisk(fingerprint);
+ if (history.isEmpty() || history.lastKey()[1] < this.now
+ - 7L * 24L * 60L * 60L * 1000L) {
+ /* Don't write weights data file to disk. */
+ continue;
+ }
+ String historyString = this.formatHistoryString(fingerprint,
+ history);
+ File weightsFile = new File(weightsFileDirectory, fingerprint);
+ try {
+ weightsFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ weightsFile));
+ bw.write(historyString);
+ bw.close();
+ } catch (IOException e) {
+ System.err.println("Could not write weights data file '"
+ + weightsFile.getAbsolutePath() + "'. Skipping.");
+ }
+ }
+ }
+
+ private String[] graphTypes = new String[] {
+ "advertised_bandwidth_fraction",
+ "consensus_weight_fraction",
+ "guard_probability",
+ "middle_probability",
+ "exit_probability"
+ };
+
+ private String[] graphNames = new String[] {
+ "1_week",
+ "1_month",
+ "3_months",
+ "1_year",
+ "5_years" };
+
+ private long[] graphIntervals = new long[] {
+ 7L * 24L * 60L * 60L * 1000L,
+ 31L * 24L * 60L * 60L * 1000L,
+ 92L * 24L * 60L * 60L * 1000L,
+ 366L * 24L * 60L * 60L * 1000L,
+ 5L * 366L * 24L * 60L * 60L * 1000L };
+
+ private long[] dataPointIntervals = new long[] {
+ 60L * 60L * 1000L,
+ 4L * 60L * 60L * 1000L,
+ 12L * 60L * 60L * 1000L,
+ 2L * 24L * 60L * 60L * 1000L,
+ 10L * 24L * 60L * 60L * 1000L };
+
+ private String formatHistoryString(String fingerprint,
+ SortedMap<long[], double[]> history) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("{\"fingerprint\":\"" + fingerprint + "\"");
+ for (int graphTypeIndex = 0; graphTypeIndex < this.graphTypes.length;
+ graphTypeIndex++) {
+ String graphType = this.graphTypes[graphTypeIndex];
+ sb.append(",\n\"" + graphType + "\":{");
+ int graphIntervalsWritten = 0;
+ for (int graphIntervalIndex = 0; graphIntervalIndex <
+ this.graphIntervals.length; graphIntervalIndex++) {
+ String timeline = this.formatTimeline(graphTypeIndex,
+ graphIntervalIndex, history);
+ if (timeline != null) {
+ sb.append((graphIntervalsWritten++ > 0 ? "," : "") + "\n"
+ + timeline);
+ }
+ }
+ sb.append("}");
+ }
+ sb.append("\n}\n");
+ return sb.toString();
+ }
+
+ private String formatTimeline(int graphTypeIndex,
+ int graphIntervalIndex, SortedMap<long[], double[]> history) {
+ String graphName = this.graphNames[graphIntervalIndex];
+ long graphInterval = this.graphIntervals[graphIntervalIndex];
+ long dataPointInterval =
+ this.dataPointIntervals[graphIntervalIndex];
+ List<Double> dataPoints = new ArrayList<Double>();
+ long intervalStartMillis = ((this.now - graphInterval)
+ / dataPointInterval) * dataPointInterval;
+ long totalMillis = 0L;
+ double totalWeightTimesMillis = 0.0;
+ for (Map.Entry<long[], double[]> e : history.entrySet()) {
+ long startMillis = e.getKey()[0], endMillis = e.getKey()[1];
+ double weight = e.getValue()[graphTypeIndex];
+ if (endMillis < intervalStartMillis) {
+ continue;
+ }
+ while ((intervalStartMillis / dataPointInterval) !=
+ (endMillis / dataPointInterval)) {
+ dataPoints.add(totalMillis * 5L < dataPointInterval
+ ? -1.0 : totalWeightTimesMillis / (double) totalMillis);
+ totalWeightTimesMillis = 0.0;
+ totalMillis = 0L;
+ intervalStartMillis += dataPointInterval;
+ }
+ totalWeightTimesMillis += weight
+ * ((double) (endMillis - startMillis));
+ totalMillis += (endMillis - startMillis);
+ }
+ dataPoints.add(totalMillis * 5L < dataPointInterval
+ ? -1.0 : totalWeightTimesMillis / (double) totalMillis);
+ double maxValue = 0.0;
+ int firstNonNullIndex = -1, lastNonNullIndex = -1;
+ for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
+ dataPointIndex++) {
+ double dataPoint = dataPoints.get(dataPointIndex);
+ if (dataPoint >= 0.0) {
+ if (firstNonNullIndex < 0) {
+ firstNonNullIndex = dataPointIndex;
+ }
+ lastNonNullIndex = dataPointIndex;
+ if (dataPoint > maxValue) {
+ maxValue = dataPoint;
+ }
+ }
+ }
+ if (firstNonNullIndex < 0) {
+ return null;
+ }
+ long firstDataPointMillis = (((this.now - graphInterval)
+ / dataPointInterval) + firstNonNullIndex) * dataPointInterval
+ + dataPointInterval / 2L;
+ if (graphIntervalIndex > 0 && firstDataPointMillis >=
+ this.now - graphIntervals[graphIntervalIndex - 1]) {
+ /* Skip weights history object, because it doesn't contain
+ * anything new that wasn't already contained in the last
+ * weights history object(s). */
+ return null;
+ }
+ long lastDataPointMillis = firstDataPointMillis
+ + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
+ double factor = ((double) maxValue) / 999.0;
+ int count = lastNonNullIndex - firstNonNullIndex + 1;
+ StringBuilder sb = new StringBuilder();
+ sb.append("\"" + graphName + "\":{"
+ + "\"first\":\""
+ + this.dateTimeFormat.format(firstDataPointMillis) + "\","
+ + "\"last\":\""
+ + this.dateTimeFormat.format(lastDataPointMillis) + "\","
+ + "\"interval\":" + String.valueOf(dataPointInterval / 1000L)
+ + ",\"factor\":" + String.format(Locale.US, "%.9f", factor)
+ + ",\"count\":" + String.valueOf(count) + ",\"values\":[");
+ int dataPointsWritten = 0, previousNonNullIndex = -2;
+ boolean foundTwoAdjacentDataPoints = false;
+ for (int dataPointIndex = firstNonNullIndex; dataPointIndex <=
+ lastNonNullIndex; dataPointIndex++) {
+ double dataPoint = dataPoints.get(dataPointIndex);
+ if (dataPoint >= 0.0) {
+ if (dataPointIndex - previousNonNullIndex == 1) {
+ foundTwoAdjacentDataPoints = true;
+ }
+ previousNonNullIndex = dataPointIndex;
+ }
+ sb.append((dataPointsWritten++ > 0 ? "," : "")
+ + (dataPoint < 0.0 ? "null" :
+ String.valueOf((long) ((dataPoint * 999.0) / maxValue))));
+ }
+ sb.append("]}");
+ if (foundTwoAdjacentDataPoints) {
+ return sb.toString();
+ } else {
+ return null;
+ }
+ }
+
+ public void deleteObsoleteWeightsDataFiles() {
+ SortedMap<String, File> obsoleteWeightsFiles =
+ new TreeMap<String, File>();
+ if (weightsFileDirectory.exists() &&
+ weightsFileDirectory.isDirectory()) {
+ for (File file : weightsFileDirectory.listFiles()) {
+ if (file.getName().length() == 40) {
+ obsoleteWeightsFiles.put(file.getName(), file);
+ }
+ }
+ }
+ for (String fingerprint : this.currentFingerprints) {
+ if (obsoleteWeightsFiles.containsKey(fingerprint)) {
+ obsoleteWeightsFiles.remove(fingerprint);
+ }
+ }
+ for (File weightsFile : obsoleteWeightsFiles.values()) {
+ weightsFile.delete();
+ }
+ }
+}
+
diff --git a/web/index.html b/web/index.html
index 0578843..b956822 100755
--- a/web/index.html
+++ b/web/index.html
@@ -616,6 +616,109 @@ fingerprints in the URL.
</tr>
</table>
<br>
+<h3>Weights documents</h3>
+<p>Weights documents contain aggregate statistics of a relay's probability
+to be selected by clients for building paths.
+<font color="blue">Added document type on July 24, 2012.</font>
+Weights documents contain different time intervals and are available for
+all relays that have been running in the past week.
+Weights documents contain the following fields:
+<ul>
+<li><b>"relays_published":</b> UTC timestamp (YYYY-MM-DD hh:mm:ss) when
+the last contained relay network status consensus started being valid.
+Indicates how recent the relay weights documents in this document are.
+Required field.</li>
+<li><b>"relays":</b> Array of objects representing relay weights
+documents.
+Required field.
+Each array object contains the following key-value pairs:
+<ul>
+<li><b>"fingerprint":</b> Relay fingerprint consisting of 40 upper-case
+hexadecimal characters.
+Required field.</li>
+<li><b>"advertised_bandwidth_fraction":</b> History object containing
+relative advertised bandwidth of this relay compared to the total
+advertised bandwidth in the network.
+If there were no bandwidth authorities, this fraction would be a very
+rough approximation of the probability of this relay to be selected by
+clients.
+Optional field.
+Keys are string representation of the time period covered by the weights
+history object.
+Keys are fixed strings <i>"1_week"</i>, <i>"1_month"</i>,
+<i>"3_months"</i>, <i>"1_year"</i>, and <i>"5_years"</i>.
+Keys refer to the last known weights history of a relay, not to the time
+when the weights document was published.
+A weights history object is only contained if the time period it covers
+is not already contained in another weights history object with shorter
+time period and higher data resolution.
+Each weights history object contains the following key-value pairs:
+<ul>
+<li><b>"first":</b> UTC timestamp (YYYY-MM-DD hh:mm:ss) of the first data
+data point in the weights history.
+Required field.</li>
+<li><b>"last":</b> UTC timestamp (YYYY-MM-DD hh:mm:ss) of the last data
+data point in the weights history.
+Required field.</li>
+<li><b>"interval":</b> Time interval between two data points in seconds.
+Required field.</li>
+<li><b>"factor":</b> Factor by which subsequent weights values need to
+be multiplied to get the path-selection probability.
+The idea is that contained weights values are normalized to a range from 0
+to 999 to reduce document size while still providing sufficient detail for
+both slow and fast relays.
+Required field.</li>
+<li><b>"count":</b> Number of provided data points, included mostly for
+debugging purposes.
+Can also be derived from the number of elements in the subsequent array.
+Optional field.</li>
+<li><b>"values":</b> Array of normalized weights values.
+May contain null values if the relay was running less than 20% of a given
+time period.
+Only includes non-null values for series of at least two subsequent data
+points to enable drawing of line graphs.
+Required field.</li>
+</ul>
+</li>
+<li><b>"consensus_weight_fraction":</b> History object containing the
+fraction of this relay's consensus weight compared to the sum of all
+consensus weights in the network.
+This fraction is a very rough approximation of the probability of this
+relay to be selected by clients.
+Optional field.
+The specification of this history object is similar to that in the
+<i>advertised_bandwidth_fraction</i> field above.</li>
+<li><b>"guard_probability":</b> History object containing the probability
+of this relay to be selected for the guard position.
+This probability is calculated based on consensus weights, relay flags,
+and bandwidth weights in the consensus.
+Path selection depends on more factors, so that this probability can only
+be an approximation.
+Optional field.
+The specification of this history object is similar to that in the
+<i>advertised_bandwidth_fraction</i> field above.</li>
+<li><b>"middle_probability":</b> History object containing the probability
+of this relay to be selected for the middle position.
+This probability is calculated based on consensus weights, relay flags,
+and bandwidth weights in the consensus.
+Path selection depends on more factors, so that this probability can only
+be an approximation.
+Optional field.
+The specification of this history object is similar to that in the
+<i>advertised_bandwidth_fraction</i> field above.</li>
+<li><b>"exit_probability":</b> History object containing the probability
+of this relay to be selected for the exit position.
+This probability is calculated based on consensus weights, relay flags,
+and bandwidth weights in the consensus.
+Path selection depends on more factors, so that this probability can only
+be an approximation.
+Optional field.
+The specification of this history object is similar to that in the
+<i>advertised_bandwidth_fraction</i> field above.</li>
+</ul>
+</li>
+</ul>
+<br>
<h3>Methods</h3>
<p>The following methods each return a single document containing zero or
more relay and/or bridge documents.</p>
@@ -642,6 +745,12 @@ or that have been running in the past week.
currently running or that have been running in the past week.
</td>
</tr>
+<tr>
+<td><b>GET weights</b></td>
+<td>Return weights documents of all relays and bridges that are currently
+running or that have been running in the past week.
+</td>
+</tr>
</table>
<p>Each of the methods above can be parameterized to select only a subset
of relay and/or bridge documents to be included in the response.</p>
