commit 99f87e820289d1fdba7d6362cb8bb84693f09c94
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Thu Sep 29 17:21:43 2016 +0200
Remove module on disagreement among dirauths.
Implements #20227.
---
.../org/torproject/metrics/collectdescs/Main.java | 1 -
modules/disagreement/.gitignore | 2 -
modules/disagreement/build.xml | 38 --
.../org/torproject/metrics/disagreement/Main.java | 642 ---------------------
shared/bin/20-run-disagreement-stats.sh | 5 -
shared/bin/99-copy-stats-files.sh | 1 -
website/etc/metrics.json | 4 +-
7 files changed, 2 insertions(+), 691 deletions(-)
diff --git a/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java b/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java
index 88955b4..8be2e0b 100644
--- a/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java
+++ b/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java
@@ -24,7 +24,6 @@ public class Main {
"/recent/relay-descriptors/consensuses/",
"/recent/relay-descriptors/extra-infos/",
"/recent/relay-descriptors/server-descriptors/",
- "/recent/relay-descriptors/votes/",
"/recent/torperf/"
}, 0L, new File("../../shared/in"), true);
}
diff --git a/modules/disagreement/.gitignore b/modules/disagreement/.gitignore
deleted file mode 100644
index bb84c80..0000000
--- a/modules/disagreement/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-stats/
-
diff --git a/modules/disagreement/build.xml b/modules/disagreement/build.xml
deleted file mode 100644
index aed679b..0000000
--- a/modules/disagreement/build.xml
+++ /dev/null
@@ -1,38 +0,0 @@
-<project default="run" name="disagreement" basedir=".">
-
- <property name="disagreement-sources" value="src/main/java"/>
- <property name="disagreement-libs" value="../../shared/lib"/>
- <property name="disagreement-classes" value="classes"/>
- <path id="classpath">
- <pathelement path="${disagreement-classes}"/>
- <fileset dir="${disagreement-libs}">
- <include name="descriptor-1.2.0.jar"/>
- <include name="commons-compress-1.9.jar"/>
- <include name="xz-1.0.jar"/>
- </fileset>
- </path>
-
- <target name="compile">
- <mkdir dir="${disagreement-classes}"/>
- <javac destdir="${disagreement-classes}"
- srcdir="${disagreement-sources}"
- source="1.7"
- target="1.7"
- debug="true"
- deprecation="true"
- optimize="false"
- failonerror="true"
- includeantruntime="false">
- <classpath refid="classpath"/>
- </javac>
- </target>
-
- <target name="run" depends="compile">
- <java fork="true"
- maxmemory="2g"
- classname="org.torproject.metrics.disagreement.Main">
- <classpath refid="classpath"/>
- </java>
- </target>
-</project>
-
diff --git a/modules/disagreement/src/main/java/org/torproject/metrics/disagreement/Main.java b/modules/disagreement/src/main/java/org/torproject/metrics/disagreement/Main.java
deleted file mode 100644
index 05159a3..0000000
--- a/modules/disagreement/src/main/java/org/torproject/metrics/disagreement/Main.java
+++ /dev/null
@@ -1,642 +0,0 @@
-/* Copyright 2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.metrics.disagreement;
-
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
-import org.torproject.descriptor.DescriptorReader;
-import org.torproject.descriptor.DescriptorSourceFactory;
-import org.torproject.descriptor.NetworkStatusEntry;
-import org.torproject.descriptor.RelayNetworkStatusVote;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.LineNumberReader;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TimeZone;
-import java.util.TreeMap;
-
-/* Read all relay network status votes from the in/ subdirectory with a
- * valid-after time of 12:00:00, extract attributes like relay flags or
- * bandwidth measurements that the directory authorities assigned to
- * relays, and output aggregate statistics on disagreement among the
- * directory authorities.
- *
- * When initializing from descriptor archives, put tarballs in the in/
- * subdirectory, run this code, and then move the tarballs away.
- * Otherwise, tarballs will be re-processed in each subsequent execution.
- *
- * Recent descriptors can stay in the in/ subdirectory and be re-processed
- * in each execution. */
-public class Main {
-
- /** Creates a new instance of this class that executes this
- * data-processing module. */
- public static void main(String[] args) throws Exception {
- new Main().run();
- }
-
- /** Executes this data-processing module. */
- public void run() throws Exception {
- readResults();
- readDescriptors();
- aggregate();
- writeResults();
- }
-
- /* We're processing a lot of strings, including authority identities,
- * relay fingerprints, and attributes like relay flags. It would be
- * wasteful to store more than one instance of these strings in memory.
- * We also want to store them as part of long integers below.
- *
- * That's why we're resolving all strings to integers and keeping maps
- * from string to integers. In case of authority identities and relay
- * fingerprints we don't need to resolve integers back to strings, but
- * in case of attributes we need to put attribute strings into the
- * output file, so we're also keeping a list of attribute strings in
- * insertion order. */
- private Map<String, Integer> authorityIndexes =
- new HashMap<String, Integer>();
- private Map<String, Integer> fingerprintIndexes =
- new HashMap<String, Integer>();
- private Map<String, Integer> attributeIndexes =
- new HashMap<String, Integer>();
- private List<String> attributeStrings = new ArrayList<String>();
-
- /** Initializes this class. */
- public Main() {
-
- /* Initialize maps from strings to integers and back by adding the
- * empty string as 0-th element. This is necessary, because we want
- * to be able to treat 0 as special case below. Once we add non-empty
- * strings to maps, they'll be indexed starting at 1. */
- this.getAuthorityIndexForString("");
- this.getFingerprintIndexForString("");
- this.getAttributeIndexForString("");
- }
-
- /* Resolve the given authority string to its authority index, possibly
- * after adding it to the mapping if it wasn't contained before. */
- protected int getAuthorityIndexForString(String authorityString) {
- return putToMapsAndReturnIndex(this.authorityIndexes, null,
- authorityString);
- }
-
- /* Resolve the given fingerprint string to its fingerprint index,
- * possibly after adding it to the mapping if it wasn't contained
- * before. */
- protected int getFingerprintIndexForString(String fingerprintString) {
- return putToMapsAndReturnIndex(this.fingerprintIndexes, null,
- fingerprintString);
- }
-
- /* Resolve the given attribute string to its attribute index, possibly
- * after adding it to the mapping if it wasn't contained before. */
- protected int getAttributeIndexForString(String attributeString) {
- return putToMapsAndReturnIndex(this.attributeIndexes,
- this.attributeStrings, attributeString);
- }
-
- /* Resolve the given attribute index to its attribute string, or return
- * null if this attribute index doesn't exist. */
- protected String getAttributeStringForIndex(int attributeIndex) {
- if (attributeIndex < 0 || attributeIndex >= attributeStrings.size()) {
- return null;
- } else {
- return this.attributeStrings.get(attributeIndex);
- }
- }
-
- /* Helper method: Return an index for a given string and possibly put
- * the string into the map and corresponding list if it was not
- * contained before. */
- protected int putToMapsAndReturnIndex(
- Map<String, Integer> stringToIntMap, List<String> strings,
- String string) {
- Integer index = stringToIntMap.get(string);
- if (index == null) {
- if (strings != null) {
- strings.add(string);
- }
- stringToIntMap.put(string, stringToIntMap.size());
- }
- return stringToIntMap.get(string);
- }
-
- /* The following code is heavily optimized towards low memory usage,
- * because we're potentially processing a lot of data, especially when
- * initializing from descriptor archives.
- *
- * We need to be able to store weeks or even months of votes in memory,
- * which is really a lot of data. The reason is that we need all votes
- * published at the same hour to be present when running aggregations,
- * but we cannot rely on the order of incoming votes. That's why we
- * have to process votes in two steps: first, we extract everything we
- * need from votes and store it in memory, and second, we aggregate what
- * we have in memory and output aggregate results.
- *
- * We're going rather low-level here by converting each attribute that
- * an authority assigned to a relay at a certain vote valid-after time
- * into a single 64-bit signed long integer. We're doing this by
- * subdividing (most of) the available bits into ranges for the
- * different parts we want to store. As an added requirement we're
- * arranging ranges in a way that we can later process long values in
- * sorted order without keeping much state. Ranges are:
- * - 22 bits for the valid-after time in half hours since 1970-01-01
- * 00:00:00 (which won't overflow until year 2209),
- * - 8 bits for the attribute index (which allows up to 253 different
- * relay flags in addition to reserved 0, "Listed", and
- * "Measured"),
- * - 24 bits for the fingerprint index (which allows over 16 million
- * different relay fingerprints in the current execution), and
- * - 6 bits for the authority index (which allows up to 63 different
- * authorities in addition to the reserved 0).
- *
- * If any of these numbers overflows during the execution, we'll detect
- * that, suggest to process less data at once, and exit with an error.
- * Still, it should be possible to process months of data in a single
- * execution. For example, one set of votes published at the same
- * valid-after hour in December 2015 required to keep 420,000 long
- * values in memory, which is roughly 3.2 MiB plus list overhead. */
- protected List<Long> assignments = new ArrayList<Long>();
-
- protected static final int VALIDAFTER_LEN = 22;
-
- protected static final int ATTRIBUTE_LEN = 8;
-
- protected static final int FINGERPRINT_LEN = 24;
-
- protected static final int AUTHORITY_LEN = 6;
-
- protected static final int VALIDAFTER_SHIFT = ATTRIBUTE_LEN
- + FINGERPRINT_LEN + AUTHORITY_LEN;
-
- protected static final int ATTRIBUTE_SHIFT = FINGERPRINT_LEN
- + AUTHORITY_LEN;
-
- protected static final int FINGERPRINT_SHIFT = AUTHORITY_LEN;
-
- protected static final int AUTHORITY_SHIFT = 0;
-
- /* Define some constants for timestamp math. */
- protected static final long HALF_HOUR = 30L * 60L * 1000L;
-
- protected static final long ONE_HOUR = 2L * HALF_HOUR;
-
- protected static final long HALF_DAY = 12L * ONE_HOUR;
-
- protected static final long ONE_DAY = 2L * HALF_DAY;
-
- /* Convert the given valid-after time in milliseconds, attribute index,
- * fingerprint index, and authority index to a long integer following
- * the conversion rules stated above. Return -1 in case of
- * overflows. */
- protected static long convertToLongValue(long validAfterMillis,
- int attributeIndex, int fingerprintIndex, int authorityIndex) {
- long validAfterHalfHours = validAfterMillis / HALF_HOUR;
- if (validAfterHalfHours < 0L
- || validAfterHalfHours >= (1L << VALIDAFTER_LEN)) {
- return -1;
- }
- if (attributeIndex < 0 || attributeIndex >= (1 << ATTRIBUTE_LEN)) {
- return -1;
- }
- if (fingerprintIndex < 0
- || fingerprintIndex >= (1 << FINGERPRINT_LEN)) {
- return -1;
- }
- if (authorityIndex < 0 || authorityIndex >= (1 << AUTHORITY_LEN)) {
- return -1;
- }
- long longValue = (validAfterHalfHours << VALIDAFTER_SHIFT)
- + ((long) attributeIndex << ATTRIBUTE_SHIFT)
- + ((long) fingerprintIndex << FINGERPRINT_SHIFT)
- + ((long) authorityIndex << AUTHORITY_SHIFT);
- return longValue;
- }
-
- /* Extract the valid-after time in milliseconds from the given long
- * integer value. */
- protected static long extractValidAfterMillisFromLongValue(
- long longValue) {
- return (longValue >> VALIDAFTER_SHIFT) * HALF_HOUR;
- }
-
- /* Extract the attribute index from the given long integer value. */
- protected static int extractAttributeIndexFromLongValue(
- long longValue) {
- return (int) ((longValue >> ATTRIBUTE_SHIFT) % (1 << ATTRIBUTE_LEN));
- }
-
- /* Extract the fingerprint index from the given long integer value. */
- protected static int extractFingerprintIndexFromLongValue(
- long longValue) {
- return (int) ((longValue >> FINGERPRINT_SHIFT)
- % (1 << FINGERPRINT_LEN));
- }
-
- /* Extract the authority index from the given long integer value. */
- protected static int extractAuthorityIndexFromLongValue(
- long longValue) {
- return (int) ((longValue >> AUTHORITY_SHIFT) % (1 << AUTHORITY_LEN));
- }
-
- /* Keep all aggregated results in memory, so that we easily merge new
- * results obtained in the current execution.
- *
- * Another reason for keeping all results in memory is that we may have
- * processed some votes from a given valid-after time before, and now we
- * need to decide whether we keep old results or replace them with new
- * results. We decide this by using results that are based on more
- * votes, which could be old or new results.
- *
- * Map keys are valid-after times formatted as strings, map values are
- * sorted lists of all subsequent columns starting with that valid-after
- * time. */
- protected SortedMap<String, List<String>> results =
- new TreeMap<String, List<String>>();
-
- /* Store all results in this .csv file. */
- protected File resultsFile = new File("stats/disagreement.csv");
-
- /* Use the following .csv header line for results. */
- protected String resultsHeaderLine =
- "validafter,attribute,votes,required,max,relays";
-
- /* Read results from the previous execution to memory and do some minor
- * validation of the file format while doing so. Return immediately
- * without error if the file does not exist (yet). */
- private void readResults() throws Exception {
- if (!this.resultsFile.exists()) {
- return;
- }
- LineNumberReader lnr = new LineNumberReader(new BufferedReader(
- new FileReader(this.resultsFile)));
- String line;
- if ((line = lnr.readLine()) == null
- || !line.equals(this.resultsHeaderLine)) {
- lnr.close();
- throw new IOException("Unexpected line " + lnr.getLineNumber()
- + " in " + this.resultsFile + ".");
- }
- while ((line = lnr.readLine()) != null) {
- if (!line.contains(",")) {
- lnr.close();
- throw new IOException("Cannot parse invalid line "
- + lnr.getLineNumber() + " in " + this.resultsFile + ".");
- }
- int indexOfFirstComma = line.indexOf(",");
- String validafter = line.substring(0, indexOfFirstComma);
- String otherColumns = line.substring(indexOfFirstComma);
- if (!this.results.containsKey(validafter)) {
- this.results.put(validafter, new ArrayList<String>());
- }
- this.results.get(validafter).add(otherColumns);
- }
- lnr.close();
- }
-
- /* Write results to the results file. More precisely, write them to a
- * temporary file and rename it to the target file to avoid failing half
- * way through and losing the previous results file. */
- private void writeResults() throws Exception {
- this.resultsFile.getParentFile().mkdirs();
- File resultsTmpFile = new File(this.resultsFile + ".tmp");
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- resultsTmpFile));
- bw.write(this.resultsHeaderLine + "\n");
- for (Map.Entry<String, List<String>> e : this.results.entrySet()) {
- String validafter = e.getKey();
- for (String otherColumns : e.getValue()) {
- bw.write(validafter + otherColumns + "\n");
- }
- }
- bw.close();
- resultsTmpFile.renameTo(this.resultsFile);
- }
-
- /* Read relay network status votes from this directory. */
- protected File[] inDirectories = new File[] {
- new File("../../shared/in/archive/relay-descriptors/votes"),
- new File("../../shared/in/recent/relay-descriptors/votes")
- };
-
- /* Read relay network status votes from disk and extract all relevant
- * pieces from them. */
- protected void readDescriptors() throws Exception {
- DescriptorReader descriptorReader =
- DescriptorSourceFactory.createDescriptorReader();
- descriptorReader.setMaxDescriptorFilesInQueue(5);
- for (File inDirectory : this.inDirectories) {
- descriptorReader.addDirectory(inDirectory);
- }
- Iterator<DescriptorFile> descriptorFiles =
- descriptorReader.readDescriptors();
- while (descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = descriptorFiles.next();
- for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- if ((descriptor instanceof RelayNetworkStatusVote)) {
- RelayNetworkStatusVote vote =
- (RelayNetworkStatusVote) descriptor;
- processVote(vote);
- }
- }
- }
- }
-
- private static final String LISTED_ATTRIBUTE = "Listed";
-
- private static final String MEASURED_ATTRIBUTE = "Measured";
-
- /* Process a single relay network status vote. */
- private void processVote(RelayNetworkStatusVote vote) throws Exception {
- long validAfterMillis = vote.getValidAfterMillis();
- if (validAfterMillis % ONE_DAY != HALF_DAY) {
- /* Only process votes with a valid-after time of 12:00:00 as a means
- * to reduce the overall amount of data. */
- return;
- }
-
- /* Use the authority's identity to distinguish votes. */
- String authorityIdentity = vote.getIdentity();
-
- /* Collect a set of all attributes that the authority assigns in this
- * vote, which includes all known flags, the general "Listed"
- * attribute for listing relays, and possibly the "Measured" attribute
- * for bandwidth-measured relays. */
- Set<String> knownAttributes = new HashSet<String>(
- vote.getKnownFlags());
- knownAttributes.add(LISTED_ATTRIBUTE);
-
- /* Go through all status entries in this vote and remember which
- * attributes this authority assigns to which relays. */
- for (NetworkStatusEntry entry
- : vote.getStatusEntries().values()) {
-
- /* Use the relay's fingerprint to distinguish relays. */
- String fingerprint = entry.getFingerprint();
-
- /* Compile a set of all attributes assigned to this relay, including
- * all relay flags, "Listed", and possibly "Measured". */
- Set<String> attributes = new HashSet<String>(entry.getFlags());
- attributes.add(LISTED_ATTRIBUTE);
- if (entry.getMeasured() >= 0L) {
- attributes.add(MEASURED_ATTRIBUTE);
- knownAttributes.add(MEASURED_ATTRIBUTE);
- }
-
- /* Remember all attributes assigned to this relay. */
- this.addAssignedAttributes(validAfterMillis, attributes,
- fingerprint, authorityIdentity);
- }
-
- /* Remember all attributes assigned by the authority in this vote. */
- this.addKnownAttributes(validAfterMillis, knownAttributes,
- authorityIdentity);
-
- /* Remember all fingerprints voted on by the authority at the given
- * valid-after time. */
- this.addKnownFingerprints(validAfterMillis,
- vote.getStatusEntries().keySet());
- }
-
- /* Remember all attributes assigned to a given relay by an authority at
- * a given valid-after time. These are converted to long integers
- * with all components being non-zero. */
- protected void addAssignedAttributes(long validAfterMillis,
- Set<String> attributes, String fingerprint,
- String authorityIdentity) throws Exception {
- int fingerprintIndex = this.getFingerprintIndexForString(fingerprint);
- int authorityIndex = this.getAuthorityIndexForString(
- authorityIdentity);
- for (String attribute : attributes) {
- int attributeIndex = this.getAttributeIndexForString(attribute);
- long longValue = convertToLongValue(validAfterMillis,
- attributeIndex, fingerprintIndex, authorityIndex);
- if (longValue < 0L) {
- throw new Exception("Could not convert vote data to the internal "
- + "format. Try processing fewer votes at once.");
- }
- this.assignments.add(longValue);
- }
- }
-
- /* Remember all attributes voted on by an authority at a given
- * valid-after time. These are converted to long integers with
- * fingerprint component being zero. */
- protected void addKnownAttributes(long validAfterMillis,
- Set<String> knownAttributes, String authorityIdentity)
- throws Exception {
- int authorityIndex = this.getAuthorityIndexForString(
- authorityIdentity);
- for (String attribute : knownAttributes) {
- int attributeIndex = this.getAttributeIndexForString(attribute);
- long longValue = convertToLongValue(validAfterMillis,
- attributeIndex, 0, authorityIndex);
- if (longValue < 0L) {
- throw new Exception("Could not convert vote data to the internal "
- + "format. Try processing fewer votes at once.");
- }
- this.assignments.add(longValue);
- }
- }
-
- /* Remember all fingerprints known by an authority at a given
- * valid-after time. These are converted to long integers with
- * attribute and authority component being zero. */
- protected void addKnownFingerprints(long validAfterMillis,
- Set<String> knownFingerprints) throws Exception {
- for (String fingerprint : knownFingerprints) {
- int fingerprintIndex = this.getFingerprintIndexForString(
- fingerprint);
- long longValue = convertToLongValue(validAfterMillis, 0,
- fingerprintIndex, 0);
- if (longValue < 0L) {
- throw new Exception("Could not convert vote data to the internal "
- + "format. Try processing fewer votes at once.");
- }
- this.assignments.add(longValue);
- }
- }
-
- /* Aggregate everything we extracted from votes earlier into statistics
- * on disagreement among directory authorities. */
- protected void aggregate() {
-
- /* Initialize a date format for formatting valid-after times. */
- DateFormat dateTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss", Locale.US);
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-
- /* Sort long integer values and append the largest possible long
- * value, so that we can process values in the following order
- * (columns are valid-after time, attribute, fingerprint, authority;
- * + stands for a value > 0, 0 stands for 0):
- * (+, 0, +, 0): all fingerprints known at a given valid-after time
- * (+, +, 0, +): all attributes known by a given authority
- * (+, +, +, +): attributes assigned to relays by authority
- * ...
- * (MAX_VALUE): end-of-list marker */
- Collections.sort(this.assignments);
- this.assignments.add(Long.MAX_VALUE);
-
- /* Remember long value and some of its components from the last
- * iteration. */
- long lastLongValue = -1L;
- long lastValidAfterMillis = -1L;
- int lastAttributeIndex = -1;
- int lastFingerprintIndex = -1;
-
- /* Keep a list of all output lines for a single valid-after time. */
- List<String> outputLines = new ArrayList<String>();
-
- /* Keep counters for the number of fingerprints seen at a valid-after
- * time, the number of authorities voting on an attribute, and the
- * number of votes that a relay received for a given attribute. */
- int knownFingerprintsByAllAuthorities = 0;
- int authoritiesVotingOnAttribute = 0;
- int votesForAttribute = 0;
-
- /* Keep counters of relays receiving a given number of votes on an
- * attribute. The number at element i is the number of relays
- * receiving i votes. */
- int[] relays = new int[(1 << AUTHORITY_LEN)];
-
- /* Go through all long values in ascending order. */
- for (long longValue : this.assignments) {
-
- /* Skip duplicate values. */
- if (lastLongValue == longValue) {
- continue;
- }
-
- /* If we're looking at attributes assigned to relays and just moved
- * from one non-zero fingerprint to the next, we need to wrap up
- * results for the last fingerprint before moving on. */
- int fingerprintIndex = extractFingerprintIndexFromLongValue(
- longValue);
- if (lastAttributeIndex > 0 && lastFingerprintIndex > 0
- && lastFingerprintIndex != fingerprintIndex) {
-
- /* This relay received at least one vote for the given attribute,
- * or otherwise it wouldn't be contained in the list of long
- * values. Increment the counter for the number of votes we
- * counted for this attribute, and then reset that counter. */
- relays[0]--;
- relays[votesForAttribute]++;
- votesForAttribute = 0;
- }
-
- /* If we're looking at attributes and just moved from one non-zero
- * attribute to the next, we need to wrap up results for the last
- * attribute before moving on. And if we just moved to the first
- * attribute, initialize counters. */
- int attributeIndex = extractAttributeIndexFromLongValue(longValue);
- if (lastAttributeIndex >= 0
- && lastAttributeIndex != attributeIndex) {
-
- /* If we just finished a non-zero attribute, wrap it up.
- * Determine the number of votes required for getting into the
- * consensus, which is typically the majority of votes, except for
- * the "Measured" attribute, where it's set to 3. Format one
- * output line for each possible number of votes, from 0 to the
- * total number of authorities voting on the attribute. */
- if (lastAttributeIndex > 0) {
- String lastAttribute = this.getAttributeStringForIndex(
- lastAttributeIndex);
- int required = lastAttribute.equals(MEASURED_ATTRIBUTE) ? 3
- : (authoritiesVotingOnAttribute / 2) + 1;
- for (int i = 0; i <= authoritiesVotingOnAttribute; i++) {
- outputLines.add(String.format(",%s,%d,%d,%d,%d",
- lastAttribute, i, required, authoritiesVotingOnAttribute,
- relays[i]));
- }
- }
-
- /* Reset counters and initialize the bucket at index 0 with the
- * total number of fingerprints known at this valid-after time. */
- authoritiesVotingOnAttribute = 0;
- relays = new int[(1 << AUTHORITY_LEN)];
- relays[0] = knownFingerprintsByAllAuthorities;
- }
-
- /* If we just moved from one valid-after time to the next, we need
- * to wrap up results for the last valid-after time before moving
- * on. */
- long validAfterMillis = extractValidAfterMillisFromLongValue(
- longValue);
- if (lastValidAfterMillis >= 0L
- && lastValidAfterMillis < validAfterMillis) {
-
- /* Check if results already contain lines for this valid-after
- * time. If so, only replace them with new results lines if there
- * are more new lines than old lines. The rationale is that more
- * lines are very likely based on more votes, and we want to
- * include as many votes as possible in the aggregation. */
- String validAfterString = dateTimeFormat.format(
- lastValidAfterMillis);
- if (!this.results.containsKey(validAfterString)
- || this.results.get(validAfterString).size()
- < outputLines.size()) {
-
- /* Sort results lines, and then put them in. */
- Collections.sort(outputLines);
- this.results.put(validAfterString, outputLines);
- }
-
- /* Prepare for aggregating votes from the next valid-after
- * time. */
- outputLines = new ArrayList<String>();
- knownFingerprintsByAllAuthorities = 0;
- }
-
- /* If we reached our end-of-list marker, stop here. */
- if (longValue == Long.MAX_VALUE) {
- break;
- }
-
- /* Look at the current indexes and increment one of the three
- * counters. If this value doesn't contain an attribute index, it
- * was put in for counting all known fingerprints at this
- * valid-after time. */
- if (attributeIndex == 0) {
- knownFingerprintsByAllAuthorities++;
-
- /* Otherwise, if this value doesn't contain a fingerprint index, it
- * was put in for counting authorities voting on a given attribute
- * at the current valid-after time. */
- } else if (fingerprintIndex == 0) {
- authoritiesVotingOnAttribute++;
-
- /* Otherwise, if both indexes are non-zero, this value was put in to
- * count how many authorities assign the attribute to this relay at
- * this valid-after time. */
- } else {
- votesForAttribute++;
- }
-
- /* Prepare moving on to the next value. */
- lastLongValue = longValue;
- lastValidAfterMillis = validAfterMillis;
- lastAttributeIndex = attributeIndex;
- lastFingerprintIndex = fingerprintIndex;
- }
- }
-}
-
diff --git a/shared/bin/20-run-disagreement-stats.sh b/shared/bin/20-run-disagreement-stats.sh
deleted file mode 100755
index 9abe1e2..0000000
--- a/shared/bin/20-run-disagreement-stats.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-cd modules/disagreement/
-ant | grep "\[java\]"
-cd ../../
-
diff --git a/shared/bin/99-copy-stats-files.sh b/shared/bin/99-copy-stats-files.sh
index 5ced875..504216a 100755
--- a/shared/bin/99-copy-stats-files.sh
+++ b/shared/bin/99-copy-stats-files.sh
@@ -6,5 +6,4 @@ cp -a modules/advbwdist/stats/advbwdist.csv shared/stats/
cp -a modules/hidserv/stats/hidserv.csv shared/stats/
cp -a modules/clients/stats/clients.csv shared/stats/
cp -a modules/clients/stats/userstats-combined.csv shared/stats/
-cp -a modules/disagreement/stats/disagreement.csv shared/stats/
diff --git a/website/etc/metrics.json b/website/etc/metrics.json
index 64633ca..ffec652 100644
--- a/website/etc/metrics.json
+++ b/website/etc/metrics.json
@@ -870,13 +870,13 @@
},
{
"id": "disagreement-data",
- "title": "Disagreement among the directory authorities",
+ "title": "Disagreement among the directory authorities (deprecated)",
"tags": [
"Relays"
],
"type": "Data",
"level": "Advanced",
- "description": "<p>The following data file contains statistics about agreement or disagreement among the <a href=\"about.html#directory-authority\">directory authorities</a>. Once per hour the directory authorities exchange votes with their view of the <a href=\"about.html#relay\">relays</a> in the network including attributes like <a href=\"about.html#relay-flag\">relay flags</a> or bandwidth measurements. This data file includes counts of relays by number of directory authorities assigning them a given attribute.</p>",
+ "description": "<p><font color=\"red\">As of September 29, 2016, the linked data file is not updated anymore. This page and the linked data file will be removed in the future.</font></p><p>The following data file contains statistics about agreement or disagreement among the <a href=\"about.html#directory-authority\">directory authorities</a>. Once per hour the directory authorities exchange votes with their view of the <a href=\"about.html#relay\">relays</a> in the network including attributes like <a href=\"about.html#relay-flag\">relay flags</a> or bandwidth measurements. This data file includes counts of relays by number of directory authorities assigning them a given attribute.</p>",
"data_file": "stats/disagreement.csv",
"data_column_spec": [
"<b>validafter:</b> UTC timestamp (YYYY-MM-DD HH:MM:SS) after which votes became valid, which may be used as the vote publication time.",
