[tor-commits] [onionoo/master] Use more appropriate node summary file in servlet.
karsten at torproject.org
karsten at torproject.org
Thu Jun 14 14:02:56 UTC 2012
commit 26da7450d5aa3bc0d9926d69686565e8fb95e75d
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Thu Jun 14 16:02:17 2012 +0200
Use more appropriate node summary file in servlet.
We used a JSON-formatted summary document and a .csv file to decide which
relays and bridges to return in the servlet. This is overly complex and
unnecessary. We have a fine summary file that contains all information we
need and that can be easily extended.
---
src/org/torproject/onionoo/DetailDataWriter.java | 29 ----
src/org/torproject/onionoo/Main.java | 8 +-
src/org/torproject/onionoo/Node.java | 15 ++
src/org/torproject/onionoo/ResourceServlet.java | 170 ++++++++-------------
src/org/torproject/onionoo/SummaryDataWriter.java | 93 -----------
5 files changed, 82 insertions(+), 233 deletions(-)
diff --git a/src/org/torproject/onionoo/DetailDataWriter.java b/src/org/torproject/onionoo/DetailDataWriter.java
index 1e3be72..bb4dbfa 100644
--- a/src/org/torproject/onionoo/DetailDataWriter.java
+++ b/src/org/torproject/onionoo/DetailDataWriter.java
@@ -211,8 +211,6 @@ public class DetailDataWriter {
SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
"yyyy-MM-dd HH:mm:ss");
dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- Map<String, Long> relaysByConsensusWeight =
- new HashMap<String, Long>();
for (Map.Entry<String, Node> relay : this.relays.entrySet()) {
String fingerprint = relay.getKey();
@@ -416,33 +414,6 @@ public class DetailDataWriter {
+ "broken. Ignoring.");
e.printStackTrace();
}
-
- /* Remember consensus weight to facilitate ordering of results in
- * the servlet. */
- relaysByConsensusWeight.put(fingerprint, consensusWeight);
- }
-
- /* Write consensus weights to disk to facilitate ordering of results
- * in the servlet. */
- File relaysByConsensusWeightFile =
- new File("out/relays-by-consensus-weight.csv");
- try {
- relaysByConsensusWeightFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- relaysByConsensusWeightFile));
- for (Map.Entry<String, Long> e :
- relaysByConsensusWeight.entrySet()) {
- String fingerprint = e.getKey();
- long consensusWeight = e.getValue();
- bw.write(fingerprint + "," + String.valueOf(consensusWeight)
- + "\n");
- }
- bw.close();
- } catch (IOException e) {
- System.err.println("Could not write file '"
- + relaysByConsensusWeightFile.getAbsolutePath() + "'. "
- + "Ordering by consensus_weight may now be broken. Ignoring.");
- e.printStackTrace();
}
/* Return the files that we didn't update. */
diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java
index 20367d2..edac926 100644
--- a/src/org/torproject/onionoo/Main.java
+++ b/src/org/torproject/onionoo/Main.java
@@ -15,7 +15,6 @@ public class Main {
cn.lookUpASes();
cn.readBridgeNetworkStatuses();
cn.setBridgeRunningBits();
- cn.writeRelaySearchDataFile();
printStatus("Updating detail data.");
DetailDataWriter ddw = new DetailDataWriter();
@@ -35,12 +34,7 @@ public class Main {
bdw.deleteObsoleteBandwidthFiles();
printStatus("Updating summary data.");
- SummaryDataWriter sdw = new SummaryDataWriter();
- sdw.setLastValidAfterMillis(cn.getLastValidAfterMillis());
- sdw.setLastPublishedMillis(cn.getLastPublishedMillis());
- sdw.setCurrentRelays(cn.getCurrentRelays());
- sdw.setCurrentBridges(cn.getCurrentBridges());
- sdw.writeSummaryDataFile();
+ cn.writeRelaySearchDataFile();
printStatus("Terminating.");
}
diff --git a/src/org/torproject/onionoo/Node.java b/src/org/torproject/onionoo/Node.java
index 3daa4be..b7bc126 100644
--- a/src/org/torproject/onionoo/Node.java
+++ b/src/org/torproject/onionoo/Node.java
@@ -4,10 +4,15 @@ package org.torproject.onionoo;
import java.util.SortedSet;
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+
/* Store search data of a single relay that was running in the past seven
* days. */
public class Node {
private String fingerprint;
+ private String hashedFingerprint;
private String nickname;
private String address;
private String latitude;
@@ -29,6 +34,13 @@ public class Node {
SortedSet<String> relayFlags, long consensusWeight) {
this.nickname = nickname;
this.fingerprint = fingerprint;
+ try {
+ this.hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex(
+ fingerprint.toCharArray())).toUpperCase();
+ } catch (DecoderException e) {
+ throw new IllegalArgumentException("Fingerprint '" + fingerprint
+ + "' is not a valid fingerprint.");
+ }
this.address = address;
this.lastSeenMillis = lastSeenMillis;
this.orPort = orPort;
@@ -39,6 +51,9 @@ public class Node {
public String getFingerprint() {
return this.fingerprint;
}
+ public String getHashedFingerprint() {
+ return this.hashedFingerprint;
+ }
public String getNickname() {
return this.nickname;
}
diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java
index 4f1749d..38ae248 100644
--- a/src/org/torproject/onionoo/ResourceServlet.java
+++ b/src/org/torproject/onionoo/ResourceServlet.java
@@ -7,6 +7,7 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -15,6 +16,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TimeZone;
import java.util.regex.Pattern;
import javax.servlet.ServletConfig;
@@ -23,14 +25,12 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.commons.codec.DecoderException;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-
public class ResourceServlet extends HttpServlet {
private static final long serialVersionUID = 7236658979947465319L;
+ private File summaryFile = new File("status/summary.csv");
+
private String outDirString;
public void init(ServletConfig config) throws ServletException {
@@ -41,118 +41,80 @@ public class ResourceServlet extends HttpServlet {
long summaryFileLastModified = -1L;
boolean readSummaryFile = false;
- private String relaysPublishedLine = null, bridgesPublishedLine = null;
- private List<String> relayLines = new ArrayList<String>(),
- bridgeLines = new ArrayList<String>(),
- relaysByConsensusWeight = new ArrayList<String>();
+ private String relaysPublishedString, bridgesPublishedString;
+ private List<String> relaysByConsensusWeight = new ArrayList<String>();
private Map<String, String>
relayFingerprintSummaryLines = new HashMap<String, String>(),
bridgeFingerprintSummaryLines = new HashMap<String, String>();
private void readSummaryFile() {
- File summaryFile = new File(this.outDirString + "summary.json");
if (!summaryFile.exists()) {
readSummaryFile = false;
return;
}
if (summaryFile.lastModified() > this.summaryFileLastModified) {
- this.relayLines.clear();
- this.bridgeLines.clear();
- this.relayFingerprintSummaryLines.clear();
- this.bridgeFingerprintSummaryLines.clear();
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- summaryFile));
- String line;
- while ((line = br.readLine()) != null) {
- if (line.contains("\"relays_published\":")) {
- this.relaysPublishedLine = line.startsWith("{") ? line :
- "{" + line;
- } else if (line.startsWith("\"bridges_published\":")) {
- this.bridgesPublishedLine = line;
- } else if (line.startsWith("\"relays\":")) {
- while ((line = br.readLine()) != null && !line.equals("],")) {
- this.relayLines.add(line);
- int fingerprintStart = line.indexOf("\"f\":\"");
- if (fingerprintStart > 0) {
- fingerprintStart += "\"f\":\"".length();
- String fingerprint = line.substring(fingerprintStart,
- fingerprintStart + 40);
- String hashedFingerprint = DigestUtils.shaHex(
- Hex.decodeHex(fingerprint.toCharArray())).
- toUpperCase();
- this.relayFingerprintSummaryLines.put(fingerprint, line);
- this.relayFingerprintSummaryLines.put(hashedFingerprint,
- line);
- }
- }
- } else if (line.startsWith("\"bridges\":")) {
- while ((line = br.readLine()) != null && !line.equals("]}")) {
- this.bridgeLines.add(line);
- int hashedFingerprintStart = line.indexOf("\"h\":\"");
- if (hashedFingerprintStart > 0) {
- hashedFingerprintStart += "\"h\":\"".length();
- String hashedFingerprint = line.substring(
- hashedFingerprintStart, hashedFingerprintStart + 40);
- String hashedHashedFingerprint = DigestUtils.shaHex(
- Hex.decodeHex(hashedFingerprint.toCharArray())).
- toUpperCase();
- this.bridgeFingerprintSummaryLines.put(hashedFingerprint,
- line);
- this.bridgeFingerprintSummaryLines.put(
- hashedHashedFingerprint, line);
- }
- }
- }
- }
- br.close();
- } catch (IOException e) {
- return;
- } catch (DecoderException e) {
- return;
- }
+ CurrentNodes cn = new CurrentNodes();
+ cn.readRelaySearchDataFile();
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ this.relaysPublishedString = dateTimeFormat.format(
+ cn.getLastValidAfterMillis());
+ this.bridgesPublishedString = dateTimeFormat.format(
+ cn.getLastPublishedMillis());
List<String> orderRelaysByConsensusWeight = new ArrayList<String>();
- File relaysByConsensusWeightFile =
- new File(this.outDirString + "/relays-by-consensus-weight.csv");
- if (relaysByConsensusWeightFile.exists()) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- relaysByConsensusWeightFile));
- String line;
- while ((line = br.readLine()) != null) {
- String[] parts = line.split(",");
- if (parts.length != 2) {
- return;
- }
- long consensusWeight = Long.parseLong(parts[1]);
- String fingerprint = parts[0];
- orderRelaysByConsensusWeight.add(
- String.format("%020d %s", consensusWeight, fingerprint));
- String hashedFingerprint = DigestUtils.shaHex(
- Hex.decodeHex(fingerprint.toCharArray())).
- toUpperCase();
- orderRelaysByConsensusWeight.add(
- String.format("%020d %s", consensusWeight,
- hashedFingerprint));
- }
- br.close();
- Collections.sort(orderRelaysByConsensusWeight);
- this.relaysByConsensusWeight = new ArrayList<String>();
- for (String relay : orderRelaysByConsensusWeight) {
- this.relaysByConsensusWeight.add(relay.split(" ")[1]);
- }
- } catch (IOException e) {
- return;
- } catch (NumberFormatException e) {
- return;
- } catch (DecoderException e) {
- return;
- }
+ for (Node entry : cn.getCurrentRelays().values()) {
+ String fingerprint = entry.getFingerprint().toUpperCase();
+ String hashedFingerprint = entry.getHashedFingerprint().
+ toUpperCase();
+ String line = this.formatRelaySummaryLine(entry);
+ this.relayFingerprintSummaryLines.put(fingerprint, line);
+ this.relayFingerprintSummaryLines.put(hashedFingerprint, line);
+ long consensusWeight = entry.getConsensusWeight();
+ orderRelaysByConsensusWeight.add(String.format("%020d %s",
+ consensusWeight, fingerprint));
+ orderRelaysByConsensusWeight.add(String.format("%020d %s",
+ consensusWeight, hashedFingerprint));
+ }
+ Collections.sort(orderRelaysByConsensusWeight);
+ this.relaysByConsensusWeight = new ArrayList<String>();
+ for (String relay : orderRelaysByConsensusWeight) {
+ this.relaysByConsensusWeight.add(relay.split(" ")[1]);
+ }
+ for (Node entry : cn.getCurrentBridges().values()) {
+ String hashedFingerprint = entry.getFingerprint().toUpperCase();
+ String hashedHashedFingerprint = entry.getHashedFingerprint().
+ toUpperCase();
+ String line = this.formatBridgeSummaryLine(entry);
+ this.bridgeFingerprintSummaryLines.put(hashedFingerprint, line);
+ this.bridgeFingerprintSummaryLines.put(hashedHashedFingerprint,
+ line);
}
}
this.summaryFileLastModified = summaryFile.lastModified();
this.readSummaryFile = true;
}
+ private String formatRelaySummaryLine(Node entry) {
+ String nickname = !entry.getNickname().equals("Unnamed") ?
+ entry.getNickname() : null;
+ String fingerprint = entry.getFingerprint();
+ String running = entry.getRunning() ? "true" : "false";
+ String address = entry.getAddress();
+ return String.format("\n{%s\"f\":\"%s\",\"a\":[\"%s\"],\"r\":%s}",
+ (nickname == null ? "" : "\"n\":\"" + nickname + "\","),
+ fingerprint, address, running);
+ }
+
+ private String formatBridgeSummaryLine(Node entry) {
+ String nickname = !entry.getNickname().equals("Unnamed") ?
+ entry.getNickname() : null;
+ String hashedFingerprint = entry.getFingerprint();
+ String running = entry.getRunning() ? "true" : "false";
+ return String.format("\n{%s\"h\":\"%s\",\"r\":%s}",
+ (nickname == null ? "" : "\"n\":\"" + nickname + "\","),
+ hashedFingerprint, running);
+ }
+
public long getLastModified(HttpServletRequest request) {
this.readSummaryFile();
return this.summaryFileLastModified;
@@ -541,8 +503,8 @@ public class ResourceServlet extends HttpServlet {
private void writeRelays(List<String> relays, PrintWriter pw,
String resourceType) {
- pw.print(this.relaysPublishedLine + "\n");
- pw.print("\"relays\":[");
+ pw.write("{\"relays_published\":\"" + this.relaysPublishedString
+ + "\",\n\"relays\":[");
int written = 0;
for (String line : relays) {
if (line == null) {
@@ -559,8 +521,8 @@ public class ResourceServlet extends HttpServlet {
private void writeBridges(List<String> bridges, PrintWriter pw,
String resourceType) {
- pw.print(this.bridgesPublishedLine + "\n");
- pw.print("\"bridges\":[");
+ pw.write("\"bridges_published\":\"" + this.bridgesPublishedString
+ + "\",\n\"bridges\":[");
int written = 0;
for (String line : bridges) {
if (line == null) {
diff --git a/src/org/torproject/onionoo/SummaryDataWriter.java b/src/org/torproject/onionoo/SummaryDataWriter.java
deleted file mode 100644
index 1391c02..0000000
--- a/src/org/torproject/onionoo/SummaryDataWriter.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.SortedMap;
-import java.util.TimeZone;
-
-/* Write relay and bridge summary data to disk. */
-public class SummaryDataWriter {
-
- private long lastValidAfterMillis;
- public void setLastValidAfterMillis(long lastValidAfterMillis) {
- this.lastValidAfterMillis = lastValidAfterMillis;
- }
-
- private long lastPublishedMillis;
- public void setLastPublishedMillis(long lastPublishedMillis) {
- this.lastPublishedMillis = lastPublishedMillis;
- }
-
- private SortedMap<String, Node> currentRelays;
- public void setCurrentRelays(SortedMap<String, Node> currentRelays) {
- this.currentRelays = currentRelays;
- }
-
- private SortedMap<String, Node> currentBridges;
- public void setCurrentBridges(SortedMap<String, Node> currentBridges) {
- this.currentBridges = currentBridges;
- }
-
- private File relaySearchDataFile = new File("out/summary.json");
- public void writeSummaryDataFile() {
- try {
- SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- String relaysPublishedString = dateTimeFormat.format(
- this.lastValidAfterMillis);
- String bridgesPublishedString = dateTimeFormat.format(
- this.lastPublishedMillis);
- this.relaySearchDataFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.relaySearchDataFile));
- bw.write("{\"relays_published\":\"" + relaysPublishedString
- + "\",\n\"relays\":[");
- int written = 0;
- for (Node entry : this.currentRelays.values()) {
- String nickname = !entry.getNickname().equals("Unnamed") ?
- entry.getNickname() : null;
- String fingerprint = entry.getFingerprint();
- String running = entry.getRunning() ? "true" : "false";
- String address = entry.getAddress();
- if (written++ > 0) {
- bw.write(",");
- }
- bw.write("\n{"
- + (nickname == null ? "" : "\"n\":\"" + nickname + "\",")
- + "\"f\":\"" + fingerprint + "\","
- + "\"a\":[\"" + address + "\"],"
- + "\"r\":" + running + "}");
- }
- bw.write("\n],\n\"bridges_published\":\"" + bridgesPublishedString
- + "\",\n\"bridges\":[");
- written = 0;
- for (Node entry : this.currentBridges.values()) {
- String nickname = !entry.getNickname().equals("Unnamed") ?
- entry.getNickname() : null;
- String hashedFingerprint = entry.getFingerprint();
- String running = entry.getRunning() ? "true" : "false";
- if (written++ > 0) {
- bw.write(",");
- }
- bw.write("\n{"
- + (nickname == null ? "" : "\"n\":\"" + nickname + "\",")
- + "\"h\":\"" + hashedFingerprint + "\","
- + "\"r\":" + running + "}");
- }
- bw.write("\n]}\n");
- bw.close();
- } catch (IOException e) {
- System.err.println("Could not write "
- + this.relaySearchDataFile.getAbsolutePath() + ". Exiting.");
- e.printStackTrace();
- System.exit(1);
- }
- }
-}
-
More information about the tor-commits
mailing list