[tor-commits] [metrics-web/master] Process bridge network statuses from two authorities.
karsten at torproject.org
karsten at torproject.org
Thu Sep 1 19:10:54 UTC 2016
commit d1de2a89cd96c0f9dc6615042ac47c984fbbd590
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Thu Sep 1 15:56:56 2016 +0200
Process bridge network statuses from two authorities.
Fixes #20049.
---
modules/legacy/build.xml | 2 +-
.../cron/network/ConsensusStatsFileHandler.java | 160 ++++++++++++---------
2 files changed, 96 insertions(+), 66 deletions(-)
diff --git a/modules/legacy/build.xml b/modules/legacy/build.xml
index 6cacf50..1e4049b 100644
--- a/modules/legacy/build.xml
+++ b/modules/legacy/build.xml
@@ -12,7 +12,7 @@
<include name="commons-compress-1.9.jar"/>
<include name="postgresql-jdbc3-9.2.jar"/>
<include name="commons-lang-2.6.jar"/>
- <include name="descriptor-1.2.0.jar"/>
+ <include name="descriptor-1.4.0.jar"/>
</fileset>
</path>
diff --git a/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java b/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java
index 6222859..824e1eb 100644
--- a/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java
+++ b/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java
@@ -51,15 +51,16 @@ public class ConsensusStatsFileHandler {
private File bridgeConsensusStatsRawFile;
/**
- * Number of running bridges in a given bridge status. Map keys are
- * bridge status times formatted as "yyyy-MM-dd HH:mm:ss", map values
- * are lines as read from <code>stats/bridge-consensus-stats-raw</code>.
+ * Number of running bridges in a given bridge status. Map keys are the bridge
+ * status time formatted as "yyyy-MM-dd HH:mm:ss", a comma, and the bridge
+ * authority nickname, map values are lines as read from
+ * <code>stats/bridge-consensus-stats-raw</code>.
*/
private SortedMap<String, String> bridgesRaw;
/**
* Average number of running bridges per day. Map keys are dates
- * formatted as "yyyy-MM-dd", map values are the last column as written
+ * formatted as "yyyy-MM-dd", map values are the remaining columns as written
* to <code>stats/consensus-stats</code>.
*/
private SortedMap<String, String> bridgesPerDay;
@@ -132,17 +133,26 @@ public class ConsensusStatsFileHandler {
continue;
}
String[] parts = line.split(",");
- String dateTime = parts[0];
- if (parts.length == 2) {
- this.bridgesRaw.put(dateTime, line + ",0");
- } else if (parts.length == 3) {
- this.bridgesRaw.put(dateTime, line);
- } else {
+ if (parts.length < 2 || parts.length > 4) {
this.logger.warning("Corrupt line '" + line + "' in file "
+ this.bridgeConsensusStatsRawFile.getAbsolutePath()
+ "! Aborting to read this file!");
break;
}
+ String key = parts[0] + "," + (parts.length < 4 ? "Tonga" : parts[1]);
+ String value = null;
+ if (parts.length == 2) {
+ value = key + "," + parts[1] + ",0";
+ } else if (parts.length == 3) {
+ value = key + "," + parts[1] + "," + parts[2];
+ } else if (parts.length == 4) {
+ value = key + "," + parts[2] + "," + parts[3];
+ } else {
+ /* Impossible, we already checked the range above. */
+ }
+ /* Assume that all lines without authority nickname are based on
+ * Tonga's network status, not Bifroest's. */
+ this.bridgesRaw.put(key, value);
}
br.close();
this.logger.fine("Finished reading file "
@@ -159,20 +169,21 @@ public class ConsensusStatsFileHandler {
* Adds the intermediate results of the number of running bridges in a
* given bridge status to the existing observations.
*/
- public void addBridgeConsensusResults(long publishedMillis, int running,
- int runningEc2Bridges) {
- String published = dateTimeFormat.format(publishedMillis);
- String line = published + "," + running + "," + runningEc2Bridges;
- if (!this.bridgesRaw.containsKey(published)) {
+ public void addBridgeConsensusResults(long publishedMillis,
+ String authorityNickname, int running, int runningEc2Bridges) {
+ String publishedAuthority = dateTimeFormat.format(publishedMillis) + ","
+ + authorityNickname;
+ String line = publishedAuthority + "," + running + "," + runningEc2Bridges;
+ if (!this.bridgesRaw.containsKey(publishedAuthority)) {
this.logger.finer("Adding new bridge numbers: " + line);
- this.bridgesRaw.put(published, line);
+ this.bridgesRaw.put(publishedAuthority, line);
this.bridgeResultsAdded++;
- } else if (!line.equals(this.bridgesRaw.get(published))) {
+ } else if (!line.equals(this.bridgesRaw.get(publishedAuthority))) {
this.logger.warning("The numbers of running bridges we were just "
+ "given (" + line + ") are different from what we learned "
- + "before (" + this.bridgesRaw.get(published) + ")! "
+ + "before (" + this.bridgesRaw.get(publishedAuthority) + ")! "
+ "Overwriting!");
- this.bridgesRaw.put(published, line);
+ this.bridgesRaw.put(publishedAuthority, line);
}
}
@@ -191,10 +202,24 @@ public class ConsensusStatsFileHandler {
while (descriptorFiles.hasNext()) {
DescriptorFile descriptorFile = descriptorFiles.next();
if (descriptorFile.getDescriptors() != null) {
+ String authority = null;
+ if (descriptorFile.getFileName().contains(
+ "4A0CCD2DDC7995083D73F5D667100C8A5831F16D")) {
+ authority = "Tonga";
+ } else if (descriptorFile.getFileName().contains(
+ "1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1")) {
+ authority = "Bifroest";
+ }
for (Descriptor descriptor : descriptorFile.getDescriptors()) {
if (descriptor instanceof BridgeNetworkStatus) {
+ if (authority == null) {
+ this.logger.warning("Did not recognize the bridge authority "
+ + "that generated " + descriptorFile.getFileName()
+ + ". Skipping.");
+ continue;
+ }
this.addBridgeNetworkStatus(
- (BridgeNetworkStatus) descriptor);
+ (BridgeNetworkStatus) descriptor, authority);
}
}
}
@@ -203,7 +228,8 @@ public class ConsensusStatsFileHandler {
}
}
- private void addBridgeNetworkStatus(BridgeNetworkStatus status) {
+ private void addBridgeNetworkStatus(BridgeNetworkStatus status,
+ String authority) {
int runningBridges = 0;
int runningEc2Bridges = 0;
for (NetworkStatusEntry statusEntry
@@ -215,7 +241,7 @@ public class ConsensusStatsFileHandler {
}
}
}
- this.addBridgeConsensusResults(status.getPublishedMillis(),
+ this.addBridgeConsensusResults(status.getPublishedMillis(), authority,
runningBridges, runningEc2Bridges);
}
@@ -225,49 +251,51 @@ public class ConsensusStatsFileHandler {
*/
public void writeFiles() {
- /* Go through raw observations of numbers of running bridges in bridge
- * statuses, calculate averages per day, and add these averages to
- * final results. */
- if (!this.bridgesRaw.isEmpty()) {
- String tempDate = null;
+ /* Go through raw observations and put everything into nested maps by day
+ * and bridge authority. */
+ Map<String, Map<String, int[]>> bridgesPerDayAndAuthority =
+ new HashMap<String, Map<String, int[]>>();
+ for (String bridgesRawLine : this.bridgesRaw.values()) {
+ String date = bridgesRawLine.substring(0, 10);
+ if (!bridgesPerDayAndAuthority.containsKey(date)) {
+ bridgesPerDayAndAuthority.put(date, new TreeMap<String, int[]>());
+ }
+ String[] parts = bridgesRawLine.split(",");
+ String authority = parts[1];
+ if (!bridgesPerDayAndAuthority.get(date).containsKey(authority)) {
+ bridgesPerDayAndAuthority.get(date).put(authority, new int[3]);
+ }
+ int[] bridges = bridgesPerDayAndAuthority.get(date).get(authority);
+ bridges[0] += Integer.parseInt(parts[2]);
+ bridges[1] += Integer.parseInt(parts[3]);
+ bridges[2]++;
+ }
+
+ /* Sum up average numbers of running bridges per day reported by all bridge
+ * authorities and add these averages to final results. */
+ for (Map.Entry<String, Map<String, int[]>> perDay
+ : bridgesPerDayAndAuthority.entrySet()) {
+ String date = perDay.getKey();
int brunning = 0;
int brunningEc2 = 0;
- int statuses = 0;
- Iterator<String> it = this.bridgesRaw.values().iterator();
- boolean haveWrittenFinalLine = false;
- while (it.hasNext() || !haveWrittenFinalLine) {
- String next = it.hasNext() ? it.next() : null;
- /* Finished reading a day or even all lines? */
- if (tempDate != null && (next == null
- || !next.substring(0, 10).equals(tempDate))) {
- /* Only write results if we have seen at least half of all
- * statuses. */
- if (statuses >= 24) {
- String line = "," + (brunning / statuses) + ","
- + (brunningEc2 / statuses);
- /* Are our results new? */
- if (!this.bridgesPerDay.containsKey(tempDate)) {
- this.logger.finer("Adding new average bridge numbers: "
- + tempDate + line);
- this.bridgesPerDay.put(tempDate, line);
- } else if (!line.equals(this.bridgesPerDay.get(tempDate))) {
- this.logger.finer("Replacing existing average bridge "
- + "numbers (" + this.bridgesPerDay.get(tempDate)
- + " with new numbers: " + line);
- this.bridgesPerDay.put(tempDate, line);
- }
- }
- brunning = brunningEc2 = statuses = 0;
- haveWrittenFinalLine = (next == null);
- }
- /* Sum up number of running bridges. */
- if (next != null) {
- tempDate = next.substring(0, 10);
- statuses++;
- String[] parts = next.split(",");
- brunning += Integer.parseInt(parts[1]);
- brunningEc2 += Integer.parseInt(parts[2]);
+ for (int[] perAuthority : perDay.getValue().values()) {
+ int statuses = perAuthority[2];
+ if (statuses < 12) {
+ /* Only write results if we have seen at least a dozen statuses. */
+ continue;
}
+ brunning += perAuthority[0] / statuses;
+ brunningEc2 += perAuthority[1] / statuses;
+ }
+ String line = "," + brunning + "," + brunningEc2;
+ /* Are our results new? */
+ if (!this.bridgesPerDay.containsKey(date)) {
+ this.logger.finer("Adding new average bridge numbers: " + date + line);
+ this.bridgesPerDay.put(date, line);
+ } else if (!line.equals(this.bridgesPerDay.get(date))) {
+ this.logger.finer("Replacing existing average bridge numbers ("
+ + this.bridgesPerDay.get(date) + " with new numbers: " + line);
+ this.bridgesPerDay.put(date, line);
}
}
@@ -278,9 +306,11 @@ public class ConsensusStatsFileHandler {
this.bridgeConsensusStatsRawFile.getParentFile().mkdirs();
BufferedWriter bw = new BufferedWriter(
new FileWriter(this.bridgeConsensusStatsRawFile));
- bw.append("datetime,brunning,brunningec2\n");
+ bw.append("datetime,authority,brunning,brunningec2");
+ bw.newLine();
for (String line : this.bridgesRaw.values()) {
- bw.append(line + "\n");
+ bw.append(line);
+ bw.newLine();
}
bw.close();
this.logger.fine("Finished writing file "
@@ -376,7 +406,7 @@ public class ConsensusStatsFileHandler {
+ "old: " + this.bridgesRaw.lastKey());
}
} catch (ParseException e) {
- /* Can't parse the timestamp? Whatever. */
+ logger.warning("Can't parse the timestamp? Reason: " + e);
}
}
logger.info(dumpStats.toString());
More information about the tor-commits
mailing list