commit 26da7450d5aa3bc0d9926d69686565e8fb95e75d Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Thu Jun 14 16:02:17 2012 +0200 Use more appropriate node summary file in servlet. We used a JSON-formatted summary document and a .csv file to decide which relays and bridges to return in the servlet. This is overly complex and unnecessary. We have a fine summary file that contains all information we need and that can be easily extended. --- src/org/torproject/onionoo/DetailDataWriter.java | 29 ---- src/org/torproject/onionoo/Main.java | 8 +- src/org/torproject/onionoo/Node.java | 15 ++ src/org/torproject/onionoo/ResourceServlet.java | 170 ++++++++------------- src/org/torproject/onionoo/SummaryDataWriter.java | 93 ----------- 5 files changed, 82 insertions(+), 233 deletions(-) diff --git a/src/org/torproject/onionoo/DetailDataWriter.java b/src/org/torproject/onionoo/DetailDataWriter.java index 1e3be72..bb4dbfa 100644 --- a/src/org/torproject/onionoo/DetailDataWriter.java +++ b/src/org/torproject/onionoo/DetailDataWriter.java @@ -211,8 +211,6 @@ public class DetailDataWriter { SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - Map<String, Long> relaysByConsensusWeight = - new HashMap<String, Long>(); for (Map.Entry<String, Node> relay : this.relays.entrySet()) { String fingerprint = relay.getKey(); @@ -416,33 +414,6 @@ public class DetailDataWriter { + "broken. Ignoring."); e.printStackTrace(); } - - /* Remember consensus weight to facilitate ordering of results in - * the servlet. */ - relaysByConsensusWeight.put(fingerprint, consensusWeight); - } - - /* Write consensus weights to disk to facilitate ordering of results - * in the servlet. */ - File relaysByConsensusWeightFile = - new File("out/relays-by-consensus-weight.csv"); - try { - relaysByConsensusWeightFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - relaysByConsensusWeightFile)); - for (Map.Entry<String, Long> e : - relaysByConsensusWeight.entrySet()) { - String fingerprint = e.getKey(); - long consensusWeight = e.getValue(); - bw.write(fingerprint + "," + String.valueOf(consensusWeight) - + "\n"); - } - bw.close(); - } catch (IOException e) { - System.err.println("Could not write file '" - + relaysByConsensusWeightFile.getAbsolutePath() + "'. " - + "Ordering by consensus_weight may now be broken. Ignoring."); - e.printStackTrace(); } /* Return the files that we didn't update. */ diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 20367d2..edac926 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -15,7 +15,6 @@ public class Main { cn.lookUpASes(); cn.readBridgeNetworkStatuses(); cn.setBridgeRunningBits(); - cn.writeRelaySearchDataFile(); printStatus("Updating detail data."); DetailDataWriter ddw = new DetailDataWriter(); @@ -35,12 +34,7 @@ public class Main { bdw.deleteObsoleteBandwidthFiles(); printStatus("Updating summary data."); - SummaryDataWriter sdw = new SummaryDataWriter(); - sdw.setLastValidAfterMillis(cn.getLastValidAfterMillis()); - sdw.setLastPublishedMillis(cn.getLastPublishedMillis()); - sdw.setCurrentRelays(cn.getCurrentRelays()); - sdw.setCurrentBridges(cn.getCurrentBridges()); - sdw.writeSummaryDataFile(); + cn.writeRelaySearchDataFile(); printStatus("Terminating."); } diff --git a/src/org/torproject/onionoo/Node.java b/src/org/torproject/onionoo/Node.java index 3daa4be..b7bc126 100644 --- a/src/org/torproject/onionoo/Node.java +++ b/src/org/torproject/onionoo/Node.java @@ -4,10 +4,15 @@ package org.torproject.onionoo; import java.util.SortedSet; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; + /* Store search data of a single relay that was running in the past seven * days. */ public class Node { private String fingerprint; + private String hashedFingerprint; private String nickname; private String address; private String latitude; @@ -29,6 +34,13 @@ public class Node { SortedSet<String> relayFlags, long consensusWeight) { this.nickname = nickname; this.fingerprint = fingerprint; + try { + this.hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex( + fingerprint.toCharArray())).toUpperCase(); + } catch (DecoderException e) { + throw new IllegalArgumentException("Fingerprint '" + fingerprint + + "' is not a valid fingerprint."); + } this.address = address; this.lastSeenMillis = lastSeenMillis; this.orPort = orPort; @@ -39,6 +51,9 @@ public class Node { public String getFingerprint() { return this.fingerprint; } + public String getHashedFingerprint() { + return this.hashedFingerprint; + } public String getNickname() { return this.nickname; } diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java index 4f1749d..38ae248 100644 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ b/src/org/torproject/onionoo/ResourceServlet.java @@ -7,6 +7,7 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.io.PrintWriter; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -15,6 +16,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TimeZone; import java.util.regex.Pattern; import javax.servlet.ServletConfig; @@ -23,14 +25,12 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; - public class ResourceServlet extends HttpServlet { private static final long serialVersionUID = 7236658979947465319L; + private File summaryFile = new File("status/summary.csv"); + private String outDirString; public void init(ServletConfig config) throws ServletException { @@ -41,118 +41,80 @@ public class ResourceServlet extends HttpServlet { long summaryFileLastModified = -1L; boolean readSummaryFile = false; - private String relaysPublishedLine = null, bridgesPublishedLine = null; - private List<String> relayLines = new ArrayList<String>(), - bridgeLines = new ArrayList<String>(), - relaysByConsensusWeight = new ArrayList<String>(); + private String relaysPublishedString, bridgesPublishedString; + private List<String> relaysByConsensusWeight = new ArrayList<String>(); private Map<String, String> relayFingerprintSummaryLines = new HashMap<String, String>(), bridgeFingerprintSummaryLines = new HashMap<String, String>(); private void readSummaryFile() { - File summaryFile = new File(this.outDirString + "summary.json"); if (!summaryFile.exists()) { readSummaryFile = false; return; } if (summaryFile.lastModified() > this.summaryFileLastModified) { - this.relayLines.clear(); - this.bridgeLines.clear(); - this.relayFingerprintSummaryLines.clear(); - this.bridgeFingerprintSummaryLines.clear(); - try { - BufferedReader br = new BufferedReader(new FileReader( - summaryFile)); - String line; - while ((line = br.readLine()) != null) { - if (line.contains("\"relays_published\":")) { - this.relaysPublishedLine = line.startsWith("{") ? line : - "{" + line; - } else if (line.startsWith("\"bridges_published\":")) { - this.bridgesPublishedLine = line; - } else if (line.startsWith("\"relays\":")) { - while ((line = br.readLine()) != null && !line.equals("],")) { - this.relayLines.add(line); - int fingerprintStart = line.indexOf("\"f\":\""); - if (fingerprintStart > 0) { - fingerprintStart += "\"f\":\"".length(); - String fingerprint = line.substring(fingerprintStart, - fingerprintStart + 40); - String hashedFingerprint = DigestUtils.shaHex( - Hex.decodeHex(fingerprint.toCharArray())). - toUpperCase(); - this.relayFingerprintSummaryLines.put(fingerprint, line); - this.relayFingerprintSummaryLines.put(hashedFingerprint, - line); - } - } - } else if (line.startsWith("\"bridges\":")) { - while ((line = br.readLine()) != null && !line.equals("]}")) { - this.bridgeLines.add(line); - int hashedFingerprintStart = line.indexOf("\"h\":\""); - if (hashedFingerprintStart > 0) { - hashedFingerprintStart += "\"h\":\"".length(); - String hashedFingerprint = line.substring( - hashedFingerprintStart, hashedFingerprintStart + 40); - String hashedHashedFingerprint = DigestUtils.shaHex( - Hex.decodeHex(hashedFingerprint.toCharArray())). - toUpperCase(); - this.bridgeFingerprintSummaryLines.put(hashedFingerprint, - line); - this.bridgeFingerprintSummaryLines.put( - hashedHashedFingerprint, line); - } - } - } - } - br.close(); - } catch (IOException e) { - return; - } catch (DecoderException e) { - return; - } + CurrentNodes cn = new CurrentNodes(); + cn.readRelaySearchDataFile(); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + this.relaysPublishedString = dateTimeFormat.format( + cn.getLastValidAfterMillis()); + this.bridgesPublishedString = dateTimeFormat.format( + cn.getLastPublishedMillis()); List<String> orderRelaysByConsensusWeight = new ArrayList<String>(); - File relaysByConsensusWeightFile = - new File(this.outDirString + "/relays-by-consensus-weight.csv"); - if (relaysByConsensusWeightFile.exists()) { - try { - BufferedReader br = new BufferedReader(new FileReader( - relaysByConsensusWeightFile)); - String line; - while ((line = br.readLine()) != null) { - String[] parts = line.split(","); - if (parts.length != 2) { - return; - } - long consensusWeight = Long.parseLong(parts[1]); - String fingerprint = parts[0]; - orderRelaysByConsensusWeight.add( - String.format("%020d %s", consensusWeight, fingerprint)); - String hashedFingerprint = DigestUtils.shaHex( - Hex.decodeHex(fingerprint.toCharArray())). - toUpperCase(); - orderRelaysByConsensusWeight.add( - String.format("%020d %s", consensusWeight, - hashedFingerprint)); - } - br.close(); - Collections.sort(orderRelaysByConsensusWeight); - this.relaysByConsensusWeight = new ArrayList<String>(); - for (String relay : orderRelaysByConsensusWeight) { - this.relaysByConsensusWeight.add(relay.split(" ")[1]); - } - } catch (IOException e) { - return; - } catch (NumberFormatException e) { - return; - } catch (DecoderException e) { - return; - } + for (Node entry : cn.getCurrentRelays().values()) { + String fingerprint = entry.getFingerprint().toUpperCase(); + String hashedFingerprint = entry.getHashedFingerprint(). + toUpperCase(); + String line = this.formatRelaySummaryLine(entry); + this.relayFingerprintSummaryLines.put(fingerprint, line); + this.relayFingerprintSummaryLines.put(hashedFingerprint, line); + long consensusWeight = entry.getConsensusWeight(); + orderRelaysByConsensusWeight.add(String.format("%020d %s", + consensusWeight, fingerprint)); + orderRelaysByConsensusWeight.add(String.format("%020d %s", + consensusWeight, hashedFingerprint)); + } + Collections.sort(orderRelaysByConsensusWeight); + this.relaysByConsensusWeight = new ArrayList<String>(); + for (String relay : orderRelaysByConsensusWeight) { + this.relaysByConsensusWeight.add(relay.split(" ")[1]); + } + for (Node entry : cn.getCurrentBridges().values()) { + String hashedFingerprint = entry.getFingerprint().toUpperCase(); + String hashedHashedFingerprint = entry.getHashedFingerprint(). + toUpperCase(); + String line = this.formatBridgeSummaryLine(entry); + this.bridgeFingerprintSummaryLines.put(hashedFingerprint, line); + this.bridgeFingerprintSummaryLines.put(hashedHashedFingerprint, + line); } } this.summaryFileLastModified = summaryFile.lastModified(); this.readSummaryFile = true; } + private String formatRelaySummaryLine(Node entry) { + String nickname = !entry.getNickname().equals("Unnamed") ? + entry.getNickname() : null; + String fingerprint = entry.getFingerprint(); + String running = entry.getRunning() ? "true" : "false"; + String address = entry.getAddress(); + return String.format("\n{%s\"f\":\"%s\",\"a\":[\"%s\"],\"r\":%s}", + (nickname == null ? "" : "\"n\":\"" + nickname + "\","), + fingerprint, address, running); + } + + private String formatBridgeSummaryLine(Node entry) { + String nickname = !entry.getNickname().equals("Unnamed") ? + entry.getNickname() : null; + String hashedFingerprint = entry.getFingerprint(); + String running = entry.getRunning() ? "true" : "false"; + return String.format("\n{%s\"h\":\"%s\",\"r\":%s}", + (nickname == null ? "" : "\"n\":\"" + nickname + "\","), + hashedFingerprint, running); + } + public long getLastModified(HttpServletRequest request) { this.readSummaryFile(); return this.summaryFileLastModified; @@ -541,8 +503,8 @@ public class ResourceServlet extends HttpServlet { private void writeRelays(List<String> relays, PrintWriter pw, String resourceType) { - pw.print(this.relaysPublishedLine + "\n"); - pw.print("\"relays\":["); + pw.write("{\"relays_published\":\"" + this.relaysPublishedString + + "\",\n\"relays\":["); int written = 0; for (String line : relays) { if (line == null) { @@ -559,8 +521,8 @@ public class ResourceServlet extends HttpServlet { private void writeBridges(List<String> bridges, PrintWriter pw, String resourceType) { - pw.print(this.bridgesPublishedLine + "\n"); - pw.print("\"bridges\":["); + pw.write("\"bridges_published\":\"" + this.bridgesPublishedString + + "\",\n\"bridges\":["); int written = 0; for (String line : bridges) { if (line == null) { diff --git a/src/org/torproject/onionoo/SummaryDataWriter.java b/src/org/torproject/onionoo/SummaryDataWriter.java deleted file mode 100644 index 1391c02..0000000 --- a/src/org/torproject/onionoo/SummaryDataWriter.java +++ /dev/null @@ -1,93 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.text.SimpleDateFormat; -import java.util.SortedMap; -import java.util.TimeZone; - -/* Write relay and bridge summary data to disk. */ -public class SummaryDataWriter { - - private long lastValidAfterMillis; - public void setLastValidAfterMillis(long lastValidAfterMillis) { - this.lastValidAfterMillis = lastValidAfterMillis; - } - - private long lastPublishedMillis; - public void setLastPublishedMillis(long lastPublishedMillis) { - this.lastPublishedMillis = lastPublishedMillis; - } - - private SortedMap<String, Node> currentRelays; - public void setCurrentRelays(SortedMap<String, Node> currentRelays) { - this.currentRelays = currentRelays; - } - - private SortedMap<String, Node> currentBridges; - public void setCurrentBridges(SortedMap<String, Node> currentBridges) { - this.currentBridges = currentBridges; - } - - private File relaySearchDataFile = new File("out/summary.json"); - public void writeSummaryDataFile() { - try { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String relaysPublishedString = dateTimeFormat.format( - this.lastValidAfterMillis); - String bridgesPublishedString = dateTimeFormat.format( - this.lastPublishedMillis); - this.relaySearchDataFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - this.relaySearchDataFile)); - bw.write("{\"relays_published\":\"" + relaysPublishedString - + "\",\n\"relays\":["); - int written = 0; - for (Node entry : this.currentRelays.values()) { - String nickname = !entry.getNickname().equals("Unnamed") ? - entry.getNickname() : null; - String fingerprint = entry.getFingerprint(); - String running = entry.getRunning() ? "true" : "false"; - String address = entry.getAddress(); - if (written++ > 0) { - bw.write(","); - } - bw.write("\n{" - + (nickname == null ? "" : "\"n\":\"" + nickname + "\",") - + "\"f\":\"" + fingerprint + "\"," - + "\"a\":[\"" + address + "\"]," - + "\"r\":" + running + "}"); - } - bw.write("\n],\n\"bridges_published\":\"" + bridgesPublishedString - + "\",\n\"bridges\":["); - written = 0; - for (Node entry : this.currentBridges.values()) { - String nickname = !entry.getNickname().equals("Unnamed") ? - entry.getNickname() : null; - String hashedFingerprint = entry.getFingerprint(); - String running = entry.getRunning() ? "true" : "false"; - if (written++ > 0) { - bw.write(","); - } - bw.write("\n{" - + (nickname == null ? "" : "\"n\":\"" + nickname + "\",") - + "\"h\":\"" + hashedFingerprint + "\"," - + "\"r\":" + running + "}"); - } - bw.write("\n]}\n"); - bw.close(); - } catch (IOException e) { - System.err.println("Could not write " - + this.relaySearchDataFile.getAbsolutePath() + ". Exiting."); - e.printStackTrace(); - System.exit(1); - } - } -} -