[tor-commits] [onionoo/master] Add abstractions for descriptor parsing and document storage.

commit abd10ca8c755af2e8257b4232fb07ef076f858c2 Author: Karsten Loesing karsten.loesing@gmx.net Date: Sat Jun 15 15:11:59 2013 +0200

Add abstractions for descriptor parsing and document storage.

- Add new DescriptorSource class that abstracts away details of configuring metrics-lib's DescriptorReader and handling parse histories. This moves descriptor file paths to a single place and makes it easier to test classes that process descriptors.

- Add new DocumentStore class that abstracts away storing, retrieving, listing, and deleting files produced as output of processing descriptors. Allows for easier testing of classes that store results from parsing descriptors, and prepares exchanging file system storage with a database.

- Write parse history file to disk after all new descriptors are processed, rather than have the descriptor reader do that after providing the last descriptor. Avoids edge cases where processing breaks and we don't re-process descriptors in the next execution because we think we already processed them last time.

- When overwriting files, write contents to temporary files, delete original files, and rename written temporary file to original file. Minimizes effects of concurrent executions changing the same set of files.

- Print out statistics of parsed descriptors and document store activity. Can help figuring out performance bottlenecks in the future. --- .../torproject/onionoo/BandwidthDataWriter.java | 205 ++++------ src/org/torproject/onionoo/CurrentNodes.java | 336 +++++++-------- src/org/torproject/onionoo/DescriptorSource.java | 304 ++++++++++++++ src/org/torproject/onionoo/DetailDataWriter.java | 431 ++++++++------------ src/org/torproject/onionoo/DocumentStore.java | 264 ++++++++++++ src/org/torproject/onionoo/Main.java | 62 ++- src/org/torproject/onionoo/ResourceServlet.java | 127 +++--- src/org/torproject/onionoo/WeightsDataWriter.java | 203 ++++----- .../torproject/onionoo/ResourceServletTest.java | 7 +- 9 files changed, 1183 insertions(+), 756 deletions(-)

diff --git a/src/org/torproject/onionoo/BandwidthDataWriter.java b/src/org/torproject/onionoo/BandwidthDataWriter.java index af0fc61..664c050 100644 --- a/src/org/torproject/onionoo/BandwidthDataWriter.java +++ b/src/org/torproject/onionoo/BandwidthDataWriter.java @@ -2,18 +2,12 @@ * See LICENSE for licensing information */ package org.torproject.onionoo;

-import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import java.util.Locale; +import java.util.Scanner; import java.util.SortedMap; import java.util.SortedSet; import java.util.TimeZone; @@ -21,9 +15,6 @@ import java.util.TreeMap; import java.util.TreeSet;

import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; -import org.torproject.descriptor.DescriptorReader; -import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.ExtraInfoDescriptor;

/* Write bandwidth data files to disk and delete bandwidth files of relays @@ -47,6 +38,16 @@ import org.torproject.descriptor.ExtraInfoDescriptor; * work around. */ public class BandwidthDataWriter {

+ private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + public BandwidthDataWriter(DescriptorSource descriptorSource, + DocumentStore documentStore) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + } + private SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); public BandwidthDataWriter() { @@ -63,27 +64,17 @@ public class BandwidthDataWriter { }

public void readExtraInfoDescriptors() { - DescriptorReader reader = - DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(new File("in/relay-descriptors/extra-infos")); - reader.addDirectory(new File("in/bridge-descriptors/extra-infos")); - reader.setExcludeFiles(new File("status/extrainfo-history")); - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getException() != null) { - System.out.println("Could not parse " - + descriptorFile.getFileName()); - descriptorFile.getException().printStackTrace(); - } - if (descriptorFile.getDescriptors() != null) { - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof ExtraInfoDescriptor) { - ExtraInfoDescriptor extraInfoDescriptor = - (ExtraInfoDescriptor) descriptor; - this.parseDescriptor(extraInfoDescriptor); - } - } + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + new DescriptorType[] { DescriptorType.RELAY_EXTRA_INFOS, + DescriptorType.BRIDGE_EXTRA_INFOS }, + DescriptorHistory.EXTRAINFO_HISTORY); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof ExtraInfoDescriptor) { + ExtraInfoDescriptor extraInfoDescriptor = + (ExtraInfoDescriptor) descriptor; + this.parseDescriptor(extraInfoDescriptor); } } } @@ -140,49 +131,46 @@ public class BandwidthDataWriter { private void readHistoryFromDisk(String fingerprint, SortedMap<Long, long[]> writeHistory, SortedMap<Long, long[]> readHistory) { - File historyFile = new File(String.format("status/bandwidth/%s/%s/%s", - fingerprint.substring(0, 1), fingerprint.substring(1, 2), - fingerprint)); - if (historyFile.exists()) { - try { - BufferedReader br = new BufferedReader(new FileReader( - historyFile)); - String line; - while ((line = br.readLine()) != null) { - String[] parts = line.split(" "); - if (parts.length != 6) { - System.err.println("Illegal line '" + line + "' in history " - + "file '" + historyFile.getAbsolutePath() - + "'. Skipping this line."); - continue; - } - SortedMap<Long, long[]> history = parts[0].equals("r") - ? readHistory : writeHistory; - long startMillis = this.dateTimeFormat.parse(parts[1] + " " - + parts[2]).getTime(); - long endMillis = this.dateTimeFormat.parse(parts[3] + " " - + parts[4]).getTime(); - long bandwidth = Long.parseLong(parts[5]); - long previousEndMillis = history.headMap(startMillis).isEmpty() - ? startMillis - : history.get(history.headMap(startMillis).lastKey())[1]; - long nextStartMillis = history.tailMap(startMillis).isEmpty() - ? endMillis : history.tailMap(startMillis).firstKey(); - if (previousEndMillis <= startMillis && - nextStartMillis >= endMillis) { - history.put(startMillis, new long[] { startMillis, endMillis, - bandwidth }); - } + String historyString = this.documentStore.retrieve( + DocumentType.STATUS_BANDWIDTH, fingerprint); + if (historyString == null) { + return; + } + try { + Scanner s = new Scanner(historyString); + while (s.hasNextLine()) { + String line = s.nextLine(); + String[] parts = line.split(" "); + if (parts.length != 6) { + System.err.println("Illegal line '" + line + "' in bandwidth " + + "history for fingerprint '" + fingerprint + "'. " + + "Skipping this line."); + continue; + } + SortedMap<Long, long[]> history = parts[0].equals("r") + ? readHistory : writeHistory; + long startMillis = this.dateTimeFormat.parse(parts[1] + " " + + parts[2]).getTime(); + long endMillis = this.dateTimeFormat.parse(parts[3] + " " + + parts[4]).getTime(); + long bandwidth = Long.parseLong(parts[5]); + long previousEndMillis = history.headMap(startMillis).isEmpty() + ? startMillis + : history.get(history.headMap(startMillis).lastKey())[1]; + long nextStartMillis = history.tailMap(startMillis).isEmpty() + ? endMillis : history.tailMap(startMillis).firstKey(); + if (previousEndMillis <= startMillis && + nextStartMillis >= endMillis) { + history.put(startMillis, new long[] { startMillis, endMillis, + bandwidth }); } - br.close(); - } catch (ParseException e) { - System.err.println("Could not parse timestamp while reading " - + "history file '" + historyFile.getAbsolutePath() - + "'. Skipping."); - } catch (IOException e) { - System.err.println("Could not read history file '" - + historyFile.getAbsolutePath() + "'. Skipping."); } + s.close(); + } catch (ParseException e) { + System.err.println("Could not parse timestamp while reading " + + "bandwidth history for fingerprint '" + fingerprint + "'. " + + "Skipping."); + e.printStackTrace(); } }

@@ -233,30 +221,22 @@ public class BandwidthDataWriter { private void writeHistoryToDisk(String fingerprint, SortedMap<Long, long[]> writeHistory, SortedMap<Long, long[]> readHistory) { - File historyFile = new File(String.format("status/bandwidth/%s/%s/%s", - fingerprint.substring(0, 1), fingerprint.substring(1, 2), - fingerprint)); - try { - historyFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter(historyFile)); - for (long[] v : writeHistory.values()) { - bw.write("w " + this.dateTimeFormat.format(v[0]) + " " - + this.dateTimeFormat.format(v[1]) + " " - + String.valueOf(v[2]) + "\n"); - } - for (long[] v : readHistory.values()) { - bw.write("r " + this.dateTimeFormat.format(v[0]) + " " - + this.dateTimeFormat.format(v[1]) + " " - + String.valueOf(v[2]) + "\n"); - } - bw.close(); - } catch (IOException e) { - System.err.println("Could not write history file '" - + historyFile.getAbsolutePath() + "'. Skipping."); + StringBuilder sb = new StringBuilder(); + for (long[] v : writeHistory.values()) { + sb.append("w " + this.dateTimeFormat.format(v[0]) + " " + + this.dateTimeFormat.format(v[1]) + " " + + String.valueOf(v[2]) + "\n"); + } + for (long[] v : readHistory.values()) { + sb.append("r " + this.dateTimeFormat.format(v[0]) + " " + + this.dateTimeFormat.format(v[1]) + " " + + String.valueOf(v[2]) + "\n"); } + String historyString = sb.toString(); + this.documentStore.store(historyString, DocumentType.STATUS_BANDWIDTH, + fingerprint); }

- private File bandwidthFileDirectory = new File("out/bandwidth"); private void writeBandwidthDataFileToDisk(String fingerprint, SortedMap<Long, long[]> writeHistory, SortedMap<Long, long[]> readHistory) { @@ -269,19 +249,13 @@ public class BandwidthDataWriter { } String writeHistoryString = formatHistoryString(writeHistory); String readHistoryString = formatHistoryString(readHistory); - File bandwidthFile = new File("out/bandwidth", fingerprint); - try { - bandwidthFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - bandwidthFile)); - bw.write("{"fingerprint":"" + fingerprint + "",\n" - + ""write_history":{\n" + writeHistoryString + "},\n" - + ""read_history":{\n" + readHistoryString + "}}\n"); - bw.close(); - } catch (IOException e) { - System.err.println("Could not write bandwidth data file '" - + bandwidthFile.getAbsolutePath() + "'. Skipping."); - } + StringBuilder sb = new StringBuilder(); + sb.append("{"fingerprint":"" + fingerprint + "",\n" + + ""write_history":{\n" + writeHistoryString + "},\n" + + ""read_history":{\n" + readHistoryString + "}}\n"); + String historyString = sb.toString(); + this.documentStore.store(historyString, DocumentType.OUT_BANDWIDTH, + fingerprint); }

private String[] graphNames = new String[] { @@ -402,23 +376,16 @@ public class BandwidthDataWriter { }

public void deleteObsoleteBandwidthFiles() { - SortedMap<String, File> obsoleteBandwidthFiles = - new TreeMap<String, File>(); - if (bandwidthFileDirectory.exists() && - bandwidthFileDirectory.isDirectory()) { - for (File file : bandwidthFileDirectory.listFiles()) { - if (file.getName().length() == 40) { - obsoleteBandwidthFiles.put(file.getName(), file); - } - } - } + SortedSet<String> obsoleteBandwidthFiles = this.documentStore.list( + DocumentType.OUT_BANDWIDTH); for (String fingerprint : this.currentFingerprints) { - if (obsoleteBandwidthFiles.containsKey(fingerprint)) { + if (obsoleteBandwidthFiles.contains(fingerprint)) { obsoleteBandwidthFiles.remove(fingerprint); } } - for (File bandwidthFile : obsoleteBandwidthFiles.values()) { - bandwidthFile.delete(); + for (String fingerprint : obsoleteBandwidthFiles) { + this.documentStore.remove(DocumentType.OUT_BANDWIDTH, + fingerprint); } } } diff --git a/src/org/torproject/onionoo/CurrentNodes.java b/src/org/torproject/onionoo/CurrentNodes.java index 32d6576..64db6a1 100644 --- a/src/org/torproject/onionoo/CurrentNodes.java +++ b/src/org/torproject/onionoo/CurrentNodes.java @@ -3,10 +3,8 @@ package org.torproject.onionoo;

import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.File; import java.io.FileReader; -import java.io.FileWriter; import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -15,8 +13,8 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.Map; +import java.util.Scanner; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -27,9 +25,6 @@ import java.util.regex.Pattern;

import org.torproject.descriptor.BridgeNetworkStatus; import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; -import org.torproject.descriptor.DescriptorReader; -import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus;

@@ -37,23 +32,44 @@ import org.torproject.descriptor.RelayNetworkStatusConsensus; * days. */ public class CurrentNodes {

- /* Read the internal relay search data file from disk. */ - public void readRelaySearchDataFile(File summaryFile) { - if (summaryFile.exists() && !summaryFile.isDirectory()) { - try { - BufferedReader br = new BufferedReader(new FileReader( - summaryFile)); - String line; - while ((line = br.readLine()) != null) { - this.parseSummaryFileLine(line); - } - br.close(); - } catch (IOException e) { - System.err.println("I/O error while reading " - + summaryFile.getAbsolutePath() + ": " + e.getMessage() - + ". Ignoring."); - } + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + /* Initialize an instance for the back-end that is read-only and doesn't + * support parsing new descriptor contents. */ + public CurrentNodes(DocumentStore documentStore) { + this(null, documentStore); + } + + public CurrentNodes(DescriptorSource descriptorSource, + DocumentStore documentStore) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + } + + public void readStatusSummary() { + String summaryString = this.documentStore.retrieve( + DocumentType.STATUS_SUMMARY); + this.initializeFromSummaryString(summaryString); + } + + public void readOutSummary() { + String summaryString = this.documentStore.retrieve( + DocumentType.OUT_SUMMARY); + this.initializeFromSummaryString(summaryString); + } + + private void initializeFromSummaryString(String summaryString) { + if (summaryString == null) { + return; + } + Scanner s = new Scanner(summaryString); + while (s.hasNextLine()) { + String line = s.nextLine(); + this.parseSummaryFileLine(line); } + s.close(); }

private void parseSummaryFileLine(String line) { @@ -169,137 +185,127 @@ public class CurrentNodes { } }

- /* Write the internal relay search data file to disk. */ - public void writeRelaySearchDataFile(File summaryFile, - boolean includeOldNodes) { - try { - summaryFile.getParentFile().mkdirs(); - File summaryTempFile = new File( - summaryFile.getAbsolutePath() + ".tmp"); - BufferedWriter bw = new BufferedWriter(new FileWriter( - summaryTempFile)); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - Collection<Node> relays = includeOldNodes - ? this.knownRelays.values() : this.getCurrentRelays().values(); - for (Node entry : relays) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); - String address = entry.getAddress(); - StringBuilder addressesBuilder = new StringBuilder(); - addressesBuilder.append(address + ";"); - int written = 0; - for (String orAddressAndPort : entry.getOrAddressesAndPorts()) { - addressesBuilder.append((written++ > 0 ? "+" : "") + - orAddressAndPort); - } - addressesBuilder.append(";"); - written = 0; - for (String exitAddress : entry.getExitAddresses()) { - addressesBuilder.append((written++ > 0 ? "+" : "") - + exitAddress); - } - String lastSeen = dateTimeFormat.format( - entry.getLastSeenMillis()); - String orPort = String.valueOf(entry.getOrPort()); - String dirPort = String.valueOf(entry.getDirPort()); - StringBuilder flagsBuilder = new StringBuilder(); - written = 0; - for (String relayFlag : entry.getRelayFlags()) { - flagsBuilder.append((written++ > 0 ? "," : "") + relayFlag); - } - String consensusWeight = String.valueOf( - entry.getConsensusWeight()); - String countryCode = entry.getCountryCode() != null - ? entry.getCountryCode() : "??"; - String hostName = entry.getHostName() != null - ? entry.getHostName() : "null"; - long lastRdnsLookup = entry.getLastRdnsLookup(); - String defaultPolicy = entry.getDefaultPolicy() != null - ? entry.getDefaultPolicy() : "null"; - String portList = entry.getPortList() != null - ? entry.getPortList() : "null"; - String firstSeen = dateTimeFormat.format( - entry.getFirstSeenMillis()); - String lastChangedAddresses = dateTimeFormat.format( - entry.getLastChangedOrAddress()); - String aSNumber = entry.getASNumber() != null - ? entry.getASNumber() : "null"; - bw.write("r " + nickname + " " + fingerprint + " " - + addressesBuilder.toString() + " " + lastSeen + " " - + orPort + " " + dirPort + " " + flagsBuilder.toString() + " " - + consensusWeight + " " + countryCode + " " + hostName + " " - + String.valueOf(lastRdnsLookup) + " " + defaultPolicy + " " - + portList + " " + firstSeen + " " + lastChangedAddresses - + " " + aSNumber + "\n"); - } - Collection<Node> bridges = includeOldNodes - ? this.knownBridges.values() - : this.getCurrentBridges().values(); - for (Node entry : bridges) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); - String published = dateTimeFormat.format( - entry.getLastSeenMillis()); - String address = entry.getAddress(); - StringBuilder addressesBuilder = new StringBuilder(); - addressesBuilder.append(address + ";"); - int written = 0; - for (String orAddressAndPort : entry.getOrAddressesAndPorts()) { - addressesBuilder.append((written++ > 0 ? "+" : "") + - orAddressAndPort); - } - addressesBuilder.append(";"); - String orPort = String.valueOf(entry.getOrPort()); - String dirPort = String.valueOf(entry.getDirPort()); - StringBuilder flagsBuilder = new StringBuilder(); - written = 0; - for (String relayFlag : entry.getRelayFlags()) { - flagsBuilder.append((written++ > 0 ? "," : "") + relayFlag); - } - String firstSeen = dateTimeFormat.format( - entry.getFirstSeenMillis()); - bw.write("b " + nickname + " " + fingerprint + " " - + addressesBuilder.toString() + " " + published + " " + orPort - + " " + dirPort + " " + flagsBuilder.toString() - + " -1 ?? null -1 null null " + firstSeen + " null null " - + "null\n"); - } - bw.close(); - summaryFile.delete(); - summaryTempFile.renameTo(summaryFile); - } catch (IOException e) { - System.err.println("Could not write '" - + summaryFile.getAbsolutePath() + "' to disk. Exiting."); - e.printStackTrace(); - System.exit(1); - } + public void writeStatusSummary() { + String summaryString = this.writeSummaryString(true); + this.documentStore.store(summaryString, DocumentType.STATUS_SUMMARY); + } + + public void writeOutSummary() { + String summaryString = this.writeSummaryString(false); + this.documentStore.store(summaryString, DocumentType.OUT_SUMMARY); + this.documentStore.store(String.valueOf(System.currentTimeMillis()), + DocumentType.OUT_UPDATE); + } + + /* Write internal relay search data to a string. */ + private String writeSummaryString(boolean includeOldNodes) { + StringBuilder sb = new StringBuilder(); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + Collection<Node> relays = includeOldNodes + ? this.knownRelays.values() : this.getCurrentRelays().values(); + for (Node entry : relays) { + String nickname = entry.getNickname(); + String fingerprint = entry.getFingerprint(); + String address = entry.getAddress(); + StringBuilder addressesBuilder = new StringBuilder(); + addressesBuilder.append(address + ";"); + int written = 0; + for (String orAddressAndPort : entry.getOrAddressesAndPorts()) { + addressesBuilder.append((written++ > 0 ? "+" : "") + + orAddressAndPort); + } + addressesBuilder.append(";"); + written = 0; + for (String exitAddress : entry.getExitAddresses()) { + addressesBuilder.append((written++ > 0 ? "+" : "") + + exitAddress); + } + String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); + String orPort = String.valueOf(entry.getOrPort()); + String dirPort = String.valueOf(entry.getDirPort()); + StringBuilder flagsBuilder = new StringBuilder(); + written = 0; + for (String relayFlag : entry.getRelayFlags()) { + flagsBuilder.append((written++ > 0 ? "," : "") + relayFlag); + } + String consensusWeight = String.valueOf(entry.getConsensusWeight()); + String countryCode = entry.getCountryCode() != null + ? entry.getCountryCode() : "??"; + String hostName = entry.getHostName() != null + ? entry.getHostName() : "null"; + long lastRdnsLookup = entry.getLastRdnsLookup(); + String defaultPolicy = entry.getDefaultPolicy() != null + ? entry.getDefaultPolicy() : "null"; + String portList = entry.getPortList() != null + ? entry.getPortList() : "null"; + String firstSeen = dateTimeFormat.format( + entry.getFirstSeenMillis()); + String lastChangedAddresses = dateTimeFormat.format( + entry.getLastChangedOrAddress()); + String aSNumber = entry.getASNumber() != null + ? entry.getASNumber() : "null"; + sb.append("r " + nickname + " " + fingerprint + " " + + addressesBuilder.toString() + " " + lastSeen + " " + + orPort + " " + dirPort + " " + flagsBuilder.toString() + " " + + consensusWeight + " " + countryCode + " " + hostName + " " + + String.valueOf(lastRdnsLookup) + " " + defaultPolicy + " " + + portList + " " + firstSeen + " " + lastChangedAddresses + + " " + aSNumber + "\n"); + } + Collection<Node> bridges = includeOldNodes + ? this.knownBridges.values() : this.getCurrentBridges().values(); + for (Node entry : bridges) { + String nickname = entry.getNickname(); + String fingerprint = entry.getFingerprint(); + String published = dateTimeFormat.format( + entry.getLastSeenMillis()); + String address = entry.getAddress(); + StringBuilder addressesBuilder = new StringBuilder(); + addressesBuilder.append(address + ";"); + int written = 0; + for (String orAddressAndPort : entry.getOrAddressesAndPorts()) { + addressesBuilder.append((written++ > 0 ? "+" : "") + + orAddressAndPort); + } + addressesBuilder.append(";"); + String orPort = String.valueOf(entry.getOrPort()); + String dirPort = String.valueOf(entry.getDirPort()); + StringBuilder flagsBuilder = new StringBuilder(); + written = 0; + for (String relayFlag : entry.getRelayFlags()) { + flagsBuilder.append((written++ > 0 ? "," : "") + relayFlag); + } + String firstSeen = dateTimeFormat.format( + entry.getFirstSeenMillis()); + sb.append("b " + nickname + " " + fingerprint + " " + + addressesBuilder.toString() + " " + published + " " + orPort + + " " + dirPort + " " + flagsBuilder.toString() + + " -1 ?? null -1 null null " + firstSeen + " null null " + + "null\n"); + } + return sb.toString(); }

private long lastValidAfterMillis = 0L; private long lastPublishedMillis = 0L;

public void readRelayNetworkConsensuses() { - DescriptorReader reader = - DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(new File("in/relay-descriptors/consensuses")); - reader.setExcludeFiles(new File("status/relay-consensus-history")); - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getException() != null) { - System.out.println("Could not parse " - + descriptorFile.getFileName()); - descriptorFile.getException().printStackTrace(); - } - if (descriptorFile.getDescriptors() != null) { - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof RelayNetworkStatusConsensus) { - updateRelayNetworkStatusConsensus( - (RelayNetworkStatusConsensus) descriptor); - } - } + if (this.descriptorSource == null) { + System.err.println("Not configured to read relay network " + + "consensuses."); + return; + } + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.RELAY_CONSENSUSES, + DescriptorHistory.RELAY_CONSENSUS_HISTORY); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + updateRelayNetworkStatusConsensus( + (RelayNetworkStatusConsensus) descriptor); } } } @@ -407,6 +413,11 @@ public class CurrentNodes { public void lookUpCitiesAndASes() {

/* Make sure we have all required .csv files. */ + // TODO Make paths configurable or allow passing file contents as + // strings in order to facilitate testing. + // TODO Move look-up code to new LookupService class that is + // initialized with geoip files, receives a sorted set of addresses, + // performs lookups, and returns results to CurrentNodes. File[] geoLiteCityBlocksCsvFiles = new File[] { new File("geoip/Manual-GeoLiteCity-Blocks.csv"), new File("geoip/Automatic-GeoLiteCity-Blocks.csv"), @@ -744,24 +755,19 @@ public class CurrentNodes { }

public void readBridgeNetworkStatuses() { - DescriptorReader reader = - DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(new File("in/bridge-descriptors/statuses")); - reader.setExcludeFiles(new File("status/bridge-status-history")); - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getException() != null) { - System.out.println("Could not parse " - + descriptorFile.getFileName()); - descriptorFile.getException().printStackTrace(); - } - if (descriptorFile.getDescriptors() != null) { - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof BridgeNetworkStatus) { - updateBridgeNetworkStatus((BridgeNetworkStatus) descriptor); - } - } + if (this.descriptorSource == null) { + System.err.println("Not configured to read bridge network " + + "statuses."); + return; + } + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.BRIDGE_STATUSES, + DescriptorHistory.BRIDGE_STATUS_HISTORY); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof BridgeNetworkStatus) { + updateBridgeNetworkStatus((BridgeNetworkStatus) descriptor); } } } diff --git a/src/org/torproject/onionoo/DescriptorSource.java b/src/org/torproject/onionoo/DescriptorSource.java new file mode 100644 index 0000000..5936a93 --- /dev/null +++ b/src/org/torproject/onionoo/DescriptorSource.java @@ -0,0 +1,304 @@ +package org.torproject.onionoo; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorFile; +import org.torproject.descriptor.DescriptorReader; +import org.torproject.descriptor.DescriptorSourceFactory; + +enum DescriptorType { + RELAY_CONSENSUSES, + RELAY_SERVER_DESCRIPTORS, + RELAY_EXTRA_INFOS, + BRIDGE_STATUSES, + BRIDGE_SERVER_DESCRIPTORS, + BRIDGE_EXTRA_INFOS, + BRIDGE_POOL_ASSIGNMENTS, + EXIT_LISTS, +} + +enum DescriptorHistory { + EXTRAINFO_HISTORY, + EXIT_LIST_HISTORY, + BRIDGE_POOLASSIGN_HISTORY, + WEIGHTS_RELAY_CONSENSUS_HISTORY, + RELAY_CONSENSUS_HISTORY, + BRIDGE_STATUS_HISTORY, +} + +class DescriptorQueue { + + private File inDir; + + private File statusDir; + + private DescriptorReader descriptorReader; + + private File historyFile; + + private Iterator<DescriptorFile> descriptorFiles; + + private List<Descriptor> descriptors; + + int historySizeBefore; + + int historySizeAfter; + + long returnedDescriptors = 0L; + + long returnedBytes = 0L; + + public DescriptorQueue(File inDir, File statusDir) { + this.inDir = inDir; + this.statusDir = statusDir; + this.descriptorReader = + DescriptorSourceFactory.createDescriptorReader(); + } + + public void addDirectory(DescriptorType descriptorType) { + String directoryName = null; + switch (descriptorType) { + case RELAY_CONSENSUSES: + directoryName = "relay-descriptors/consensuses"; + break; + case RELAY_SERVER_DESCRIPTORS: + directoryName = "relay-descriptors/server-descriptors"; + break; + case RELAY_EXTRA_INFOS: + directoryName = "relay-descriptors/extra-infos"; + break; + case BRIDGE_STATUSES: + directoryName = "bridge-descriptors/statuses"; + break; + case BRIDGE_SERVER_DESCRIPTORS: + directoryName = "bridge-descriptors/server-descriptors"; + break; + case BRIDGE_EXTRA_INFOS: + directoryName = "bridge-descriptors/extra-infos"; + break; + case BRIDGE_POOL_ASSIGNMENTS: + directoryName = "bridge-pool-assignments"; + break; + case EXIT_LISTS: + directoryName = "exit-lists"; + break; + default: + System.err.println("Unknown descriptor type. Not adding directory " + + "to descriptor reader."); + return; + } + this.descriptorReader.addDirectory(new File(this.inDir, + directoryName)); + } + + public void readHistoryFile(DescriptorHistory descriptorHistory) { + String historyFileName = null; + switch (descriptorHistory) { + case EXTRAINFO_HISTORY: + historyFileName = "extrainfo-history"; + break; + case EXIT_LIST_HISTORY: + historyFileName = "exit-list-history"; + break; + case BRIDGE_POOLASSIGN_HISTORY: + historyFileName = "bridge-poolassign-history"; + break; + case WEIGHTS_RELAY_CONSENSUS_HISTORY: + historyFileName = "weights-relay-consensus-history"; + break; + case RELAY_CONSENSUS_HISTORY: + historyFileName = "relay-consensus-history"; + break; + case BRIDGE_STATUS_HISTORY: + historyFileName = "bridge-status-history"; + break; + default: + System.err.println("Unknown descriptor history. Not excluding " + + "files."); + return; + } + this.historyFile = new File(this.statusDir, historyFileName); + if (this.historyFile.exists()) { + SortedMap<String, Long> excludedFiles = new TreeMap<String, Long>(); + try { + BufferedReader br = new BufferedReader(new FileReader( + this.historyFile)); + String line; + while ((line = br.readLine()) != null) { + try { + String[] parts = line.split(" ", 2); + excludedFiles.put(parts[1], Long.parseLong(parts[0])); + } catch (NumberFormatException e) { + System.err.println("Illegal line '" + line + "' in parse " + + "history. Skipping line."); + } + } + br.close(); + } catch (IOException e) { + System.err.println("Could not read history file '" + + this.historyFile.getAbsolutePath() + "'. Not excluding " + + "descriptors in this execution."); + e.printStackTrace(); + return; + } + this.historySizeBefore = excludedFiles.size(); + this.descriptorReader.setExcludedFiles(excludedFiles); + } + } + + public void writeHistoryFile() { + if (this.historyFile == null) { + return; + } + SortedMap<String, Long> excludedAndParsedFiles = + new TreeMap<String, Long>(); + excludedAndParsedFiles.putAll( + this.descriptorReader.getExcludedFiles()); + excludedAndParsedFiles.putAll(this.descriptorReader.getParsedFiles()); + this.historySizeAfter = excludedAndParsedFiles.size(); + try { + this.historyFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.historyFile)); + for (Map.Entry<String, Long> e : excludedAndParsedFiles.entrySet()) { + String absolutePath = e.getKey(); + long lastModifiedMillis = e.getValue(); + bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath + + "\n"); + } + bw.close(); + } catch (IOException e) { + System.err.println("Could not write history file '" + + this.historyFile.getAbsolutePath() + "'. Not excluding " + + "descriptors in next execution."); + return; + } + } + + public Descriptor nextDescriptor() { + Descriptor nextDescriptor = null; + if (this.descriptorFiles == null) { + this.descriptorFiles = this.descriptorReader.readDescriptors(); + } + while (this.descriptors == null && this.descriptorFiles.hasNext()) { + DescriptorFile descriptorFile = this.descriptorFiles.next(); + if (descriptorFile.getException() != null) { + System.err.println("Could not parse " + + descriptorFile.getFileName()); + descriptorFile.getException().printStackTrace(); + } + if (descriptorFile.getDescriptors() != null && + !descriptorFile.getDescriptors().isEmpty()) { + this.descriptors = descriptorFile.getDescriptors(); + } + } + if (this.descriptors != null) { + nextDescriptor = this.descriptors.remove(0); + this.returnedDescriptors++; + this.returnedBytes += nextDescriptor.getRawDescriptorBytes().length; + if (this.descriptors.isEmpty()) { + this.descriptors = null; + } + } + return nextDescriptor; + } +} + +public class DescriptorSource { + + private File inDir; + + private File statusDir; + + private List<DescriptorQueue> descriptorQueues; + + public DescriptorSource(File inDir, File statusDir) { + this.inDir = inDir; + this.statusDir = statusDir; + this.descriptorQueues = new ArrayList<DescriptorQueue>(); + } + + public DescriptorQueue getDescriptorQueue( + DescriptorType descriptorType) { + DescriptorQueue descriptorQueue = new DescriptorQueue(this.inDir, + this.statusDir); + descriptorQueue.addDirectory(descriptorType); + this.descriptorQueues.add(descriptorQueue); + return descriptorQueue; + } + + public DescriptorQueue getDescriptorQueue( + DescriptorType[] descriptorTypes, + DescriptorHistory descriptorHistory) { + DescriptorQueue descriptorQueue = new DescriptorQueue(this.inDir, + this.statusDir); + for (DescriptorType descriptorType : descriptorTypes) { + descriptorQueue.addDirectory(descriptorType); + } + descriptorQueue.readHistoryFile(descriptorHistory); + this.descriptorQueues.add(descriptorQueue); + return descriptorQueue; + } + + public DescriptorQueue getDescriptorQueue(DescriptorType descriptorType, + DescriptorHistory descriptorHistory) { + DescriptorQueue descriptorQueue = new DescriptorQueue(this.inDir, + this.statusDir); + descriptorQueue.addDirectory(descriptorType); + descriptorQueue.readHistoryFile(descriptorHistory); + this.descriptorQueues.add(descriptorQueue); + return descriptorQueue; + } + + public void writeHistoryFiles() { + for (DescriptorQueue descriptorQueue : this.descriptorQueues) { + descriptorQueue.writeHistoryFile(); + } + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + this.descriptorQueues.size() + " descriptor " + + "queues created\n"); + int historySizeBefore = 0, historySizeAfter = 0; + long descriptors = 0L, bytes = 0L; + for (DescriptorQueue descriptorQueue : descriptorQueues) { + historySizeBefore += descriptorQueue.historySizeBefore; + historySizeAfter += descriptorQueue.historySizeAfter; + descriptors += descriptorQueue.returnedDescriptors; + bytes += descriptorQueue.returnedBytes; + } + sb.append(" " + String.format("%,d", historySizeBefore) + + " descriptors excluded from this execution\n"); + sb.append(" " + String.format("%,d", descriptors) + + " descriptors provided\n"); + sb.append(" " + formatBytes(bytes) + " provided\n"); + sb.append(" " + String.format("%,d", historySizeAfter) + + " descriptors excluded from next execution\n"); + return sb.toString(); + } + + // TODO This method should go into a utility class. + private static String formatBytes(long bytes) { + if (bytes < 1024) { + return bytes + " B"; + } else { + int exp = (int) (Math.log(bytes) / Math.log(1024)); + return String.format("%.1f %siB", bytes / Math.pow(1024, exp), + "KMGTPE".charAt(exp-1)); + } + } +} + diff --git a/src/org/torproject/onionoo/DetailDataWriter.java b/src/org/torproject/onionoo/DetailDataWriter.java index 8239de9..e95a919 100644 --- a/src/org/torproject/onionoo/DetailDataWriter.java +++ b/src/org/torproject/onionoo/DetailDataWriter.java @@ -2,12 +2,6 @@ * See LICENSE for licensing information */ package org.torproject.onionoo;

-import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.text.ParseException; @@ -16,9 +10,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Scanner; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; @@ -30,9 +24,6 @@ import org.apache.commons.lang.StringEscapeUtils;

import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; -import org.torproject.descriptor.DescriptorReader; -import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.ExitList; import org.torproject.descriptor.ExitListEntry; import org.torproject.descriptor.ServerDescriptor; @@ -45,6 +36,16 @@ import org.torproject.descriptor.ServerDescriptor; * descriptor that was last referenced in a network status. */ public class DetailDataWriter {

+ private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + public DetailDataWriter(DescriptorSource descriptorSource, + DocumentStore documentStore) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + } + private SortedMap<String, Node> relays; public void setCurrentRelays(SortedMap<String, Node> currentRelays) { this.relays = currentRelays; @@ -164,37 +165,25 @@ public class DetailDataWriter { private Map<String, ServerDescriptor> relayServerDescriptors = new HashMap<String, ServerDescriptor>(); public void readRelayServerDescriptors() { - DescriptorReader reader = - DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(new File( - "in/relay-descriptors/server-descriptors")); /* Don't remember which server descriptors we already parsed. If we * parse a server descriptor now and first learn about the relay in a * later consensus, we'll never write the descriptor content anywhere. * The result would be details files containing no descriptor parts * until the relay publishes the next descriptor. */ - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getException() != null) { - System.out.println("Could not parse " - + descriptorFile.getFileName()); - descriptorFile.getException().printStackTrace(); - } - if (descriptorFile.getDescriptors() != null) { - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof ServerDescriptor) { - ServerDescriptor serverDescriptor = - (ServerDescriptor) descriptor; - String fingerprint = serverDescriptor.getFingerprint(); - if (!this.relayServerDescriptors.containsKey(fingerprint) || - this.relayServerDescriptors.get(fingerprint). - getPublishedMillis() - < serverDescriptor.getPublishedMillis()) { - this.relayServerDescriptors.put(fingerprint, - serverDescriptor); - } - } + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.RELAY_SERVER_DESCRIPTORS); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof ServerDescriptor) { + ServerDescriptor serverDescriptor = (ServerDescriptor) descriptor; + String fingerprint = serverDescriptor.getFingerprint(); + if (!this.relayServerDescriptors.containsKey(fingerprint) || + this.relayServerDescriptors.get(fingerprint). + getPublishedMillis() + < serverDescriptor.getPublishedMillis()) { + this.relayServerDescriptors.put(fingerprint, + serverDescriptor); } } } @@ -318,37 +307,25 @@ public class DetailDataWriter { private Map<String, Set<ExitListEntry>> exitListEntries = new HashMap<String, Set<ExitListEntry>>(); public void readExitLists() { - DescriptorReader reader = - DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(new File( - "in/exit-lists")); - reader.setExcludeFiles(new File("status/exit-list-history")); - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getException() != null) { - System.out.println("Could not parse " - + descriptorFile.getFileName()); - descriptorFile.getException().printStackTrace(); - } - if (descriptorFile.getDescriptors() != null) { - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof ExitList) { - ExitList exitList = (ExitList) descriptor; - for (ExitListEntry exitListEntry : - exitList.getExitListEntries()) { - if (exitListEntry.getScanMillis() < - this.now - 24L * 60L * 60L * 1000L) { - continue; - } - String fingerprint = exitListEntry.getFingerprint(); - if (!this.exitListEntries.containsKey(fingerprint)) { - this.exitListEntries.put(fingerprint, - new HashSet<ExitListEntry>()); - } - this.exitListEntries.get(fingerprint).add(exitListEntry); - } + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.EXIT_LISTS, DescriptorHistory.EXIT_LIST_HISTORY); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof ExitList) { + ExitList exitList = (ExitList) descriptor; + for (ExitListEntry exitListEntry : + exitList.getExitListEntries()) { + if (exitListEntry.getScanMillis() < + this.now - 24L * 60L * 60L * 1000L) { + continue; + } + String fingerprint = exitListEntry.getFingerprint(); + if (!this.exitListEntries.containsKey(fingerprint)) { + this.exitListEntries.put(fingerprint, + new HashSet<ExitListEntry>()); } + this.exitListEntries.get(fingerprint).add(exitListEntry); } } } @@ -357,37 +334,25 @@ public class DetailDataWriter { private Map<String, ServerDescriptor> bridgeServerDescriptors = new HashMap<String, ServerDescriptor>(); public void readBridgeServerDescriptors() { - DescriptorReader reader = - DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(new File( - "in/bridge-descriptors/server-descriptors")); /* Don't remember which server descriptors we already parsed. If we * parse a server descriptor now and first learn about the relay in a * later status, we'll never write the descriptor content anywhere. * The result would be details files containing no descriptor parts * until the bridge publishes the next descriptor. */ - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getException() != null) { - System.out.println("Could not parse " - + descriptorFile.getFileName()); - descriptorFile.getException().printStackTrace(); - } - if (descriptorFile.getDescriptors() != null) { - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof ServerDescriptor) { - ServerDescriptor serverDescriptor = - (ServerDescriptor) descriptor; - String fingerprint = serverDescriptor.getFingerprint(); - if (!this.bridgeServerDescriptors.containsKey(fingerprint) || - this.bridgeServerDescriptors.get(fingerprint). - getPublishedMillis() - < serverDescriptor.getPublishedMillis()) { - this.bridgeServerDescriptors.put(fingerprint, - serverDescriptor); - } - } + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.BRIDGE_SERVER_DESCRIPTORS); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof ServerDescriptor) { + ServerDescriptor serverDescriptor = (ServerDescriptor) descriptor; + String fingerprint = serverDescriptor.getFingerprint(); + if (!this.bridgeServerDescriptors.containsKey(fingerprint) || + this.bridgeServerDescriptors.get(fingerprint). + getPublishedMillis() + < serverDescriptor.getPublishedMillis()) { + this.bridgeServerDescriptors.put(fingerprint, + serverDescriptor); } } } @@ -396,67 +361,40 @@ public class DetailDataWriter { private Map<String, String> bridgePoolAssignments = new HashMap<String, String>(); public void readBridgePoolAssignments() { - DescriptorReader reader = - DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(new File("in/bridge-pool-assignments")); - reader.setExcludeFiles(new File("status/bridge-poolassign-history")); - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getException() != null) { - System.out.println("Could not parse " - + descriptorFile.getFileName()); - descriptorFile.getException().printStackTrace(); - } - if (descriptorFile.getDescriptors() != null) { - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof BridgePoolAssignment) { - BridgePoolAssignment bridgePoolAssignment = - (BridgePoolAssignment) descriptor; - for (Map.Entry<String, String> e : - bridgePoolAssignment.getEntries().entrySet()) { - String fingerprint = e.getKey(); - String details = e.getValue(); - this.bridgePoolAssignments.put(fingerprint, details); - } - } + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.BRIDGE_POOL_ASSIGNMENTS, + DescriptorHistory.BRIDGE_POOLASSIGN_HISTORY); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof BridgePoolAssignment) { + BridgePoolAssignment bridgePoolAssignment = + (BridgePoolAssignment) descriptor; + for (Map.Entry<String, String> e : + bridgePoolAssignment.getEntries().entrySet()) { + String fingerprint = e.getKey(); + String details = e.getValue(); + this.bridgePoolAssignments.put(fingerprint, details); } } } }

- public void writeDetailDataFiles() { - SortedMap<String, File> remainingDetailsFiles = - this.listAllDetailsFiles(); - remainingDetailsFiles = this.updateRelayDetailsFiles( - remainingDetailsFiles); - remainingDetailsFiles = this.updateBridgeDetailsFiles( - remainingDetailsFiles); + public void writeOutDetails() { + SortedSet<String> remainingDetailsFiles = new TreeSet<String>(); + remainingDetailsFiles.addAll(this.documentStore.list( + DocumentType.OUT_DETAILS)); + this.updateRelayDetailsFiles(remainingDetailsFiles); + this.updateBridgeDetailsFiles(remainingDetailsFiles); this.deleteDetailsFiles(remainingDetailsFiles); }

- private File detailsFileDirectory = new File("out/details"); - private SortedMap<String, File> listAllDetailsFiles() { - SortedMap<String, File> result = new TreeMap<String, File>(); - if (detailsFileDirectory.exists() && - detailsFileDirectory.isDirectory()) { - for (File file : detailsFileDirectory.listFiles()) { - if (file.getName().length() == 40) { - result.put(file.getName(), file); - } - } - } - return result; - } - private static String escapeJSON(String s) { return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\'", "'"); }

- private SortedMap<String, File> updateRelayDetailsFiles( - SortedMap<String, File> remainingDetailsFiles) { - SortedMap<String, File> result = - new TreeMap<String, File>(remainingDetailsFiles); + private void updateRelayDetailsFiles( + SortedSet<String> remainingDetailsFiles) { SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -466,42 +404,40 @@ public class DetailDataWriter { /* Read details file for this relay if it exists. */ String descriptorParts = null; long publishedMillis = -1L; - if (result.containsKey(fingerprint)) { - File detailsFile = result.remove(fingerprint); - try { - BufferedReader br = new BufferedReader(new FileReader( - detailsFile)); - String line; - boolean copyDescriptorParts = false; - StringBuilder sb = new StringBuilder(); - while ((line = br.readLine()) != null) { - if (line.startsWith(""desc_published":")) { - String published = line.substring( - ""desc_published":"".length(), - ""desc_published":"1970-01-01 00:00:00".length()); - publishedMillis = dateTimeFormat.parse(published).getTime(); - copyDescriptorParts = true; + if (remainingDetailsFiles.contains(fingerprint)) { + remainingDetailsFiles.remove(fingerprint); + String documentString = this.documentStore.retrieve( + DocumentType.OUT_DETAILS, fingerprint); + if (documentString != null) { + try { + boolean copyDescriptorParts = false; + StringBuilder sb = new StringBuilder(); + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + if (line.startsWith(""desc_published":")) { + String published = line.substring( + ""desc_published":"".length(), + ""desc_published":"1970-01-01 00:00:00".length()); + publishedMillis = dateTimeFormat.parse(published). + getTime(); + copyDescriptorParts = true; + } + if (copyDescriptorParts) { + sb.append(line + "\n"); + } } - if (copyDescriptorParts) { - sb.append(line + "\n"); + s.close(); + if (sb.length() > 0) { + descriptorParts = sb.toString(); } + } catch (ParseException e) { + System.err.println("Could not parse timestamp in details.json " + + "file for '" + fingerprint + "'. Ignoring."); + e.printStackTrace(); + publishedMillis = -1L; + descriptorParts = null; } - br.close(); - if (sb.length() > 0) { - descriptorParts = sb.toString(); - } - } catch (IOException e) { - System.err.println("Could not read '" - + detailsFile.getAbsolutePath() + "'. Skipping"); - e.printStackTrace(); - publishedMillis = -1L; - descriptorParts = null; - } catch (ParseException e) { - System.err.println("Could not read '" - + detailsFile.getAbsolutePath() + "'. Skipping"); - e.printStackTrace(); - publishedMillis = -1L; - descriptorParts = null; } }

@@ -708,37 +644,23 @@ public class DetailDataWriter { } sb.append("]"); } - String statusParts = sb.toString();

- /* Write details file to disk. */ - File detailsFile = new File(detailsFileDirectory, fingerprint); - try { - detailsFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - detailsFile)); - bw.write(statusParts); - if (descriptorParts != null) { - bw.write(",\n" + descriptorParts); - } else { - bw.write("\n}\n"); - } - bw.close(); - } catch (IOException e) { - System.err.println("Could not write details file '" - + detailsFile.getAbsolutePath() + "'. This file may now be " - + "broken. Ignoring."); - e.printStackTrace(); + /* Add descriptor parts. */ + if (descriptorParts != null) { + sb.append(",\n" + descriptorParts); + } else { + sb.append("\n}\n"); } - }

- /* Return the files that we didn't update. */ - return result; + /* Write details file to disk. */ + String detailsLines = sb.toString(); + this.documentStore.store(detailsLines, DocumentType.OUT_DETAILS, + fingerprint); + } }

- private SortedMap<String, File> updateBridgeDetailsFiles( - SortedMap<String, File> remainingDetailsFiles) { - SortedMap<String, File> result = - new TreeMap<String, File>(remainingDetailsFiles); + private void updateBridgeDetailsFiles( + SortedSet<String> remainingDetailsFiles) { SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); @@ -748,52 +670,51 @@ public class DetailDataWriter { /* Read details file for this bridge if it exists. */ String descriptorParts = null, bridgePoolAssignment = null; long publishedMillis = -1L; - if (result.containsKey(fingerprint)) { - File detailsFile = result.remove(fingerprint); - try { - BufferedReader br = new BufferedReader(new FileReader( - detailsFile)); - String line; + if (remainingDetailsFiles.contains(fingerprint)) { + remainingDetailsFiles.remove(fingerprint); + String documentString = this.documentStore.retrieve( + DocumentType.OUT_DETAILS, fingerprint); + if (documentString != null) { + try { boolean copyDescriptorParts = false; - StringBuilder sb = new StringBuilder(); - while ((line = br.readLine()) != null) { - if (line.startsWith(""desc_published":")) { - String published = line.substring( - ""desc_published":"".length(), - ""desc_published":"1970-01-01 00:00:00".length()); - publishedMillis = dateTimeFormat.parse(published).getTime(); - copyDescriptorParts = true; - } else if (line.startsWith(""pool_assignment":")) { - bridgePoolAssignment = line; - copyDescriptorParts = false; - } else if (line.equals("}")) { - copyDescriptorParts = false; + StringBuilder sb = new StringBuilder(); + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + if (line.startsWith(""desc_published":")) { + String published = line.substring( + ""desc_published":"".length(), + ""desc_published":"1970-01-01 00:00:00".length()); + publishedMillis = dateTimeFormat.parse(published). + getTime(); + copyDescriptorParts = true; + } else if (line.startsWith(""pool_assignment":")) { + bridgePoolAssignment = line; + copyDescriptorParts = false; + } else if (line.equals("}")) { + copyDescriptorParts = false; + } + if (copyDescriptorParts) { + sb.append(line + "\n"); + } } - if (copyDescriptorParts) { - sb.append(line + "\n"); + s.close(); + descriptorParts = sb.toString(); + if (descriptorParts.endsWith(",\n")) { + descriptorParts = descriptorParts.substring(0, + descriptorParts.length() - 2); + } else if (descriptorParts.endsWith("\n")) { + descriptorParts = descriptorParts.substring(0, + descriptorParts.length() - 1); } + } catch (ParseException e) { + System.err.println("Could not parse timestamp in " + + "details.json file for '" + fingerprint + "'. " + + "Ignoring."); + e.printStackTrace(); + publishedMillis = -1L; + descriptorParts = null; } - br.close(); - descriptorParts = sb.toString(); - if (descriptorParts.endsWith(",\n")) { - descriptorParts = descriptorParts.substring(0, - descriptorParts.length() - 2); - } else if (descriptorParts.endsWith("\n")) { - descriptorParts = descriptorParts.substring(0, - descriptorParts.length() - 1); - } - } catch (IOException e) { - System.err.println("Could not read '" - + detailsFile.getAbsolutePath() + "'. Skipping"); - e.printStackTrace(); - publishedMillis = -1L; - descriptorParts = null; - } catch (ParseException e) { - System.err.println("Could not read '" - + detailsFile.getAbsolutePath() + "'. Skipping"); - e.printStackTrace(); - publishedMillis = -1L; - descriptorParts = null; } }

@@ -871,32 +792,18 @@ public class DetailDataWriter { sb.append(",\n" + bridgePoolAssignment); } sb.append("\n}\n"); - String detailsLines = sb.toString();

/* Write details file to disk. */ - File detailsFile = new File(detailsFileDirectory, fingerprint); - try { - detailsFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - detailsFile)); - bw.write(detailsLines); - bw.close(); - } catch (IOException e) { - System.err.println("Could not write details file '" - + detailsFile.getAbsolutePath() + "'. This file may now be " - + "broken. Ignoring."); - e.printStackTrace(); - } + String detailsLines = sb.toString(); + this.documentStore.store(detailsLines, DocumentType.OUT_DETAILS, + fingerprint); } - - /* Return the files that we didn't update. */ - return result; }

private void deleteDetailsFiles( - SortedMap<String, File> remainingDetailsFiles) { - for (File detailsFile : remainingDetailsFiles.values()) { - detailsFile.delete(); + SortedSet<String> remainingDetailsFiles) { + for (String fingerprint : remainingDetailsFiles) { + this.documentStore.remove(DocumentType.OUT_DETAILS, fingerprint); } } } diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java new file mode 100644 index 0000000..c1097a0 --- /dev/null +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -0,0 +1,264 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.onionoo; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Arrays; +import java.util.SortedSet; +import java.util.Stack; +import java.util.TreeSet; + +enum DocumentType { + STATUS_SUMMARY, + STATUS_BANDWIDTH, + STATUS_WEIGHTS, + OUT_UPDATE, + OUT_SUMMARY, + OUT_DETAILS, + OUT_BANDWIDTH, + OUT_WEIGHTS; +} + +// TODO For later migration from disk to database, do the following: +// - read from database and then from disk if not found +// - write only to database, delete from disk once in database +// - move entirely to database once disk is "empty" +// TODO Also look into simple key-value stores instead of real databases. +public class DocumentStore { + + private File statusDir; + + private File outDir; + + long listOperations = 0L, listedFiles = 0L, storedFiles = 0L, + storedBytes = 0L, retrievedFiles = 0L, retrievedBytes = 0L, + removedFiles = 0L; + + public DocumentStore(File outDir) { + this.outDir = outDir; + } + + public DocumentStore(File statusDir, File outDir) { + this.statusDir = statusDir; + this.outDir = outDir; + } + + public SortedSet<String> list(DocumentType documentType) { + SortedSet<String> fingerprints = new TreeSet<String>(); + File directory = null; + String subdirectory = null; + switch (documentType) { + case STATUS_BANDWIDTH: + directory = this.statusDir; + subdirectory = "bandwidth"; + break; + case STATUS_WEIGHTS: + directory = this.statusDir; + subdirectory = "weights"; + break; + case OUT_DETAILS: + directory = this.outDir; + subdirectory = "details"; + break; + case OUT_BANDWIDTH: + directory = this.outDir; + subdirectory = "bandwidth"; + break; + case OUT_WEIGHTS: + directory = this.outDir; + break; + default: + break; + } + if (directory != null && subdirectory != null) { + Stack<File> files = new Stack<File>(); + files.add(new File(directory, subdirectory)); + while (!files.isEmpty()) { + File file = files.pop(); + if (file.isDirectory()) { + files.addAll(Arrays.asList(file.listFiles())); + } else if (file.getName().length() == 40) { + fingerprints.add(file.getName()); + } + } + } + this.listOperations++; + this.listedFiles += fingerprints.size(); + return fingerprints; + } + + public boolean store(String documentString, DocumentType documentType) { + return this.store(documentString, documentType, null); + } + + public boolean store(String documentString, DocumentType documentType, + String fingerprint) { + File documentFile = this.getDocumentFile(documentType, fingerprint); + if (documentFile == null) { + return false; + } + try { + documentFile.getParentFile().mkdirs(); + File documentTempFile = new File( + documentFile.getAbsolutePath() + ".tmp"); + BufferedWriter bw = new BufferedWriter(new FileWriter( + documentTempFile)); + bw.write(documentString); + bw.close(); + documentFile.delete(); + documentTempFile.renameTo(documentFile); + this.storedFiles++; + this.storedBytes += documentString.length(); + } catch (IOException e) { + System.err.println("Could not write file '" + + documentFile.getAbsolutePath() + "'."); + e.printStackTrace(); + return false; + } + return true; + } + + public String retrieve(DocumentType documentType) { + return this.retrieve(documentType, null); + } + + public String retrieve(DocumentType documentType, String fingerprint) { + File documentFile = this.getDocumentFile(documentType, fingerprint); + if (documentFile == null || !documentFile.exists()) { + return null; + } else if (documentFile.isDirectory()) { + System.err.println("Could not read file '" + + documentFile.getAbsolutePath() + "', because it is a " + + "directory."); + return null; + } + try { + BufferedReader br = new BufferedReader(new FileReader( + documentFile)); + StringBuilder sb = new StringBuilder(); + String line; + while ((line = br.readLine()) != null) { + sb.append(line + "\n"); + } + br.close(); + this.retrievedFiles++; + this.retrievedBytes += sb.length(); + return sb.toString(); + } catch (IOException e) { + System.err.println("Could not read file '" + + documentFile.getAbsolutePath() + "'."); + e.printStackTrace(); + return null; + } + } + + public boolean remove(DocumentType documentType) { + return this.remove(documentType, null); + } + + public boolean remove(DocumentType documentType, String fingerprint) { + File documentFile = this.getDocumentFile(documentType, fingerprint); + if (documentFile == null || !documentFile.delete()) { + System.err.println("Could not delete file '" + + documentFile.getAbsolutePath() + "'."); + return false; + } + this.removedFiles++; + return true; + } + + private File getDocumentFile(DocumentType documentType, + String fingerprint) { + File documentFile = null; + if (fingerprint == null && !( + documentType == DocumentType.STATUS_SUMMARY || + documentType == DocumentType.OUT_UPDATE|| + documentType == DocumentType.OUT_SUMMARY)) { + return null; + } + File directory = null; + String fileName = null; + switch (documentType) { + case STATUS_SUMMARY: + directory = this.statusDir; + fileName = "summary"; + break; + case STATUS_BANDWIDTH: + directory = this.statusDir; + fileName = String.format("bandwidth/%s/%s/%s", + fingerprint.substring(0, 1), fingerprint.substring(1, 2), + fingerprint); + break; + case STATUS_WEIGHTS: + directory = this.statusDir; + fileName = String.format("weights/%s/%s/%s", + fingerprint.substring(0, 1), fingerprint.substring(1, 2), + fingerprint); + break; + case OUT_UPDATE: + directory = this.outDir; + fileName = "update"; + break; + case OUT_SUMMARY: + directory = this.outDir; + fileName = "summary"; + break; + case OUT_DETAILS: + directory = this.outDir; + fileName = String.format("details/%s", fingerprint); + break; + case OUT_BANDWIDTH: + directory = this.outDir; + fileName = String.format("bandwidth/%s", fingerprint); + break; + case OUT_WEIGHTS: + directory = this.outDir; + fileName = String.format("weights/%s", fingerprint); + break; + } + if (directory != null && fileName != null) { + documentFile = new File(directory, fileName); + } + return documentFile; + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + formatDecimalNumber(listOperations) + + " list operations performed\n"); + sb.append(" " + formatDecimalNumber(listedFiles) + + " files listed\n"); + sb.append(" " + formatDecimalNumber(storedFiles) + + " files stored\n"); + sb.append(" " + formatBytes(storedBytes) + " stored\n"); + sb.append(" " + formatDecimalNumber(retrievedFiles) + + " files retrieved\n"); + sb.append(" " + formatBytes(retrievedBytes) + " retrieved\n"); + sb.append(" " + formatDecimalNumber(removedFiles) + + " files removed\n"); + return sb.toString(); + } + + //TODO This method should go into a utility class. + private static String formatDecimalNumber(long decimalNumber) { + return String.format("%,d", decimalNumber); + } + + // TODO This method should go into a utility class. + private static String formatBytes(long bytes) { + if (bytes < 1024) { + return bytes + " B"; + } else { + int exp = (int) (Math.log(bytes) / Math.log(1024)); + return String.format("%.1f %siB", bytes / Math.pow(1024, exp), + "KMGTPE".charAt(exp-1)); + } + } +} + diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index e83e4b5..f636b5e 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -9,10 +9,20 @@ import java.util.Date; public class Main { public static void main(String[] args) {

+ printStatus("Initializing descriptor source."); + DescriptorSource dso = new DescriptorSource(new File("in"), + new File("status")); + printStatusTime("Initialized descriptor source"); + + printStatus("Initializing document store."); + DocumentStore ds = new DocumentStore(new File("status"), + new File("out")); + printStatusTime("Initialized document store"); + printStatus("Updating internal node list."); - CurrentNodes cn = new CurrentNodes(); - cn.readRelaySearchDataFile(new File("status/summary")); - printStatusTime("Read status/summary"); + CurrentNodes cn = new CurrentNodes(dso, ds); + cn.readStatusSummary(); + printStatusTime("Read status summary"); cn.readRelayNetworkConsensuses(); printStatusTime("Read network status consensuses"); cn.setRelayRunningBits(); @@ -23,11 +33,12 @@ public class Main { printStatusTime("Read bridge network statuses"); cn.setBridgeRunningBits(); printStatusTime("Set bridge running bits"); - cn.writeRelaySearchDataFile(new File("status/summary"), true); - printStatusTime("Wrote status/summary"); + cn.writeStatusSummary(); + printStatusTime("Wrote status summary"); + // TODO Could write statistics here, too.

printStatus("Updating detail data."); - DetailDataWriter ddw = new DetailDataWriter(); + DetailDataWriter ddw = new DetailDataWriter(dso, ds); ddw.setCurrentRelays(cn.getCurrentRelays()); printStatusTime("Set current relays"); ddw.setCurrentBridges(cn.getCurrentBridges()); @@ -46,11 +57,12 @@ public class Main { printStatusTime("Read bridge-pool assignments"); ddw.finishReverseDomainNameLookups(); printStatusTime("Finished reverse domain name lookups"); - ddw.writeDetailDataFiles(); + ddw.writeOutDetails(); printStatusTime("Wrote detail data files"); + // TODO Could write statistics here, too.

printStatus("Updating bandwidth data."); - BandwidthDataWriter bdw = new BandwidthDataWriter(); + BandwidthDataWriter bdw = new BandwidthDataWriter(dso, ds); bdw.setCurrentRelays(cn.getCurrentRelays()); printStatusTime("Set current relays"); bdw.setCurrentBridges(cn.getCurrentBridges()); @@ -59,9 +71,10 @@ public class Main { printStatusTime("Read extra-info descriptors"); bdw.deleteObsoleteBandwidthFiles(); printStatusTime("Deleted obsolete bandwidth files"); + // TODO Could write statistics here, too.

printStatus("Updating weights data."); - WeightsDataWriter wdw = new WeightsDataWriter(); + WeightsDataWriter wdw = new WeightsDataWriter(dso, ds); wdw.setCurrentRelays(cn.getCurrentRelays()); printStatusTime("Set current relays"); wdw.readRelayServerDescriptors(); @@ -72,10 +85,22 @@ public class Main { printStatusTime("Wrote weights data files"); wdw.deleteObsoleteWeightsDataFiles(); printStatusTime("Deleted obsolete weights files"); + // TODO Could write statistics here, too.

printStatus("Updating summary data."); - cn.writeRelaySearchDataFile(new File("out/summary"), false); - printStatusTime("Wrote out/summary"); + cn.writeOutSummary(); + printStatusTime("Wrote out summary"); + // TODO Could write statistics here, too. + + printStatus("Shutting down descriptor source."); + dso.writeHistoryFiles(); + printStatusTime("Wrote parse histories"); + printStatistics(dso.getStatsString()); + printStatusTime("Shut down descriptor source"); + + printStatus("Shutting down document store."); + printStatistics(ds.getStatsString()); + printStatusTime("Shut down document store");

printStatus("Terminating."); } @@ -88,11 +113,22 @@ public class Main { printedLastStatusMessage = System.currentTimeMillis(); }

+ private static void printStatistics(String message) { + System.out.print(" Statistics:\n" + message); + } + private static void printStatusTime(String message) { long now = System.currentTimeMillis(); - System.out.println(" " + message + " (" - + (now - printedLastStatusMessage) + " millis)."); + long millis = now - printedLastStatusMessage; + System.out.println(" " + message + " (" + formatMillis(millis) + + ")."); printedLastStatusMessage = now; } + + // TODO This method should go into a utility class. + private static String formatMillis(long millis) { + return String.format("%02d:%02d.%03d minutes", + millis / (1000L * 60L), (millis / 1000L) % 60L, millis % 1000L); + } }

diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java index 75c2ac4..3ad33ae 100644 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ b/src/org/torproject/onionoo/ResourceServlet.java @@ -2,9 +2,7 @@ * See LICENSE for licensing information */ package org.torproject.onionoo;

-import java.io.BufferedReader; import java.io.File; -import java.io.FileReader; import java.io.IOException; import java.io.PrintWriter; import java.text.SimpleDateFormat; @@ -15,12 +13,11 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Scanner; import java.util.Set; import java.util.SortedMap; -import java.util.SortedSet; import java.util.TimeZone; import java.util.TreeMap; -import java.util.TreeSet; import java.util.regex.Pattern;

import javax.servlet.ServletConfig; @@ -35,7 +32,7 @@ public class ResourceServlet extends HttpServlet {

private boolean maintenanceMode = false;

- private File outDir; + private DocumentStore documentStore;

public void init(ServletConfig config) throws ServletException { super.init(config); @@ -48,7 +45,7 @@ public class ResourceServlet extends HttpServlet {

protected void init(boolean maintenanceMode, File outDir) { this.maintenanceMode = maintenanceMode; - this.outDir = outDir; + this.documentStore = new DocumentStore(outDir); if (!maintenanceMode) { this.readSummaryFile(); } @@ -66,13 +63,24 @@ public class ResourceServlet extends HttpServlet { bridgesByFirstSeenDays = null, relaysByLastSeenDays = null, bridgesByLastSeenDays = null; private void readSummaryFile() { - File summaryFile = new File(outDir, "summary"); - if (!summaryFile.exists()) { - readSummaryFile = false; + long summaryFileLastModified = -1L; + String updateString = this.documentStore.retrieve( + DocumentType.OUT_UPDATE); + if (updateString != null) { + try { + summaryFileLastModified = Long.parseLong(updateString.trim()); + } catch (NumberFormatException e) { + /* Handle below. */ + } + } + if (summaryFileLastModified < 0L) { + // TODO Does this actually solve anything? Should we instead + // switch to a variant of the maintenance mode and re-check when + // the next requests comes in that happens x seconds after this one? + this.readSummaryFile = false; return; } - if (summaryFile.lastModified() > this.summaryFileLastModified) { - long summaryFileLastModified = summaryFile.lastModified(); + if (summaryFileLastModified > this.summaryFileLastModified) { List<String> relaysByConsensusWeight = new ArrayList<String>(); Map<String, String> relayFingerprintSummaryLines = new HashMap<String, String>(), @@ -86,8 +94,11 @@ public class ResourceServlet extends HttpServlet { bridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(), relaysByLastSeenDays = new TreeMap<Integer, Set<String>>(), bridgesByLastSeenDays = new TreeMap<Integer, Set<String>>(); - CurrentNodes cn = new CurrentNodes(); - cn.readRelaySearchDataFile(summaryFile); + CurrentNodes cn = new CurrentNodes(this.documentStore); + cn.readOutSummary(); + // TODO We should be able to learn if something goes wrong when + // reading the summary file, rather than silently having an empty + // CurrentNodes instance. cn.setRelayRunningBits(); cn.setBridgeRunningBits(); SimpleDateFormat dateTimeFormat = new SimpleDateFormat( @@ -197,7 +208,7 @@ public class ResourceServlet extends HttpServlet { this.bridgesByFirstSeenDays = bridgesByFirstSeenDays; this.bridgesByLastSeenDays = bridgesByLastSeenDays; } - this.summaryFileLastModified = summaryFile.lastModified(); + this.summaryFileLastModified = summaryFileLastModified; this.readSummaryFile = true; }

@@ -910,37 +921,38 @@ public class ResourceServlet extends HttpServlet { return ""; } fingerprint = fingerprint.substring(0, 40); - File detailsFile = new File(this.outDir, "details/" + fingerprint); + String documentString = this.documentStore.retrieve( + DocumentType.OUT_DETAILS, fingerprint); StringBuilder sb = new StringBuilder(); String detailsLines = null; - if (detailsFile.exists()) { - try { - BufferedReader br = new BufferedReader(new FileReader( - detailsFile)); - String line = br.readLine(); - if (line != null) { - sb.append("{"); - while ((line = br.readLine()) != null) { - if (line.equals("}")) { - sb.append("}\n"); - break; - } else if (!line.startsWith(""desc_published":")) { - sb.append(line + "\n"); - } - } - } - br.close(); - detailsLines = sb.toString(); - if (detailsLines.length() > 1) { - detailsLines = detailsLines.substring(0, - detailsLines.length() - 1); + if (documentString != null) { + Scanner s = new Scanner(documentString); + sb.append("{"); + if (s.hasNextLine()) { + /* Skip version line. */ + s.nextLine(); + } + while (s.hasNextLine()) { + String line = s.nextLine(); + if (line.equals("}")) { + sb.append("}\n"); + break; + } else if (!line.startsWith(""desc_published":")) { + sb.append(line + "\n"); } - } catch (IOException e) { + } + s.close(); + detailsLines = sb.toString(); + if (detailsLines.length() > 1) { + detailsLines = detailsLines.substring(0, + detailsLines.length() - 1); } } if (detailsLines != null) { return detailsLines; } else { + // TODO We should probably log that we didn't find a details + // document that we expected to exist. return ""; } } @@ -957,28 +969,15 @@ public class ResourceServlet extends HttpServlet { return ""; } fingerprint = fingerprint.substring(0, 40); - File bandwidthFile = new File(this.outDir, "bandwidth/" - + fingerprint); - StringBuilder sb = new StringBuilder(); - String bandwidthLines = null; - if (bandwidthFile.exists()) { - try { - BufferedReader br = new BufferedReader(new FileReader( - bandwidthFile)); - String line; - while ((line = br.readLine()) != null) { - sb.append(line + "\n"); - } - br.close(); - bandwidthLines = sb.toString(); - } catch (IOException e) { - } - } + String bandwidthLines = this.documentStore.retrieve( + DocumentType.OUT_BANDWIDTH, fingerprint); if (bandwidthLines != null) { bandwidthLines = bandwidthLines.substring(0, bandwidthLines.length() - 1); return bandwidthLines; } else { + // TODO We should probably log that we didn't find a bandwidth + // document that we expected to exist. return ""; } } @@ -992,26 +991,14 @@ public class ResourceServlet extends HttpServlet { return ""; } fingerprint = fingerprint.substring(0, 40); - File weightsFile = new File(this.outDir, "weights/" + fingerprint); - StringBuilder sb = new StringBuilder(); - String weightsLines = null; - if (weightsFile.exists()) { - try { - BufferedReader br = new BufferedReader(new FileReader( - weightsFile)); - String line; - while ((line = br.readLine()) != null) { - sb.append(line + "\n"); - } - br.close(); - weightsLines = sb.toString(); - } catch (IOException e) { - } - } + String weightsLines = this.documentStore.retrieve( + DocumentType.OUT_WEIGHTS, fingerprint); if (weightsLines != null) { weightsLines = weightsLines.substring(0, weightsLines.length() - 1); return weightsLines; } else { + // TODO We should probably log that we didn't find a weights + // document that we expected to exist. return ""; } } diff --git a/src/org/torproject/onionoo/WeightsDataWriter.java b/src/org/torproject/onionoo/WeightsDataWriter.java index 97e37ed..de9ad42 100644 --- a/src/org/torproject/onionoo/WeightsDataWriter.java +++ b/src/org/torproject/onionoo/WeightsDataWriter.java @@ -2,22 +2,16 @@ * See LICENSE for licensing information */ package org.torproject.onionoo;

-import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Scanner; import java.util.SortedMap; import java.util.SortedSet; import java.util.TimeZone; @@ -25,15 +19,22 @@ import java.util.TreeMap; import java.util.TreeSet;

import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; -import org.torproject.descriptor.DescriptorReader; -import org.torproject.descriptor.DescriptorSourceFactory; import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.descriptor.ServerDescriptor;

public class WeightsDataWriter {

+ private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + public WeightsDataWriter(DescriptorSource descriptorSource, + DocumentStore documentStore) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + } + private SortedSet<String> currentFingerprints = new TreeSet<String>(); public void setCurrentRelays(SortedMap<String, Node> currentRelays) { this.currentFingerprints.addAll(currentRelays.keySet()); @@ -47,63 +48,41 @@ public class WeightsDataWriter { private Map<String, Integer> advertisedBandwidths = new HashMap<String, Integer>(); public void readRelayServerDescriptors() { - DescriptorReader reader = - DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(new File( - "in/relay-descriptors/server-descriptors")); - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getException() != null) { - System.out.println("Could not parse " - + descriptorFile.getFileName()); - descriptorFile.getException().printStackTrace(); - } - if (descriptorFile.getDescriptors() != null) { - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof ServerDescriptor) { - ServerDescriptor serverDescriptor = - (ServerDescriptor) descriptor; - String digest = serverDescriptor.getServerDescriptorDigest(). - toUpperCase(); - int advertisedBandwidth = Math.min(Math.min( - serverDescriptor.getBandwidthBurst(), - serverDescriptor.getBandwidthObserved()), - serverDescriptor.getBandwidthRate()); - this.advertisedBandwidths.put(digest, advertisedBandwidth); - } - } + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.RELAY_SERVER_DESCRIPTORS); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof ServerDescriptor) { + ServerDescriptor serverDescriptor = + (ServerDescriptor) descriptor; + String digest = serverDescriptor.getServerDescriptorDigest(). + toUpperCase(); + int advertisedBandwidth = Math.min(Math.min( + serverDescriptor.getBandwidthBurst(), + serverDescriptor.getBandwidthObserved()), + serverDescriptor.getBandwidthRate()); + this.advertisedBandwidths.put(digest, advertisedBandwidth); } } }

public void readRelayNetworkConsensuses() { - DescriptorReader reader = - DescriptorSourceFactory.createDescriptorReader(); - reader.addDirectory(new File("in/relay-descriptors/consensuses")); - reader.setExcludeFiles(new File( - "status/weights-relay-consensus-history")); - Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); - while (descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = descriptorFiles.next(); - if (descriptorFile.getException() != null) { - System.out.println("Could not parse " - + descriptorFile.getFileName()); - descriptorFile.getException().printStackTrace(); - } - if (descriptorFile.getDescriptors() != null) { - for (Descriptor descriptor : descriptorFile.getDescriptors()) { - if (descriptor instanceof RelayNetworkStatusConsensus) { - RelayNetworkStatusConsensus consensus = - (RelayNetworkStatusConsensus) descriptor; - long validAfterMillis = consensus.getValidAfterMillis(), - freshUntilMillis = consensus.getFreshUntilMillis(); - SortedMap<String, double[]> pathSelectionWeights = - this.calculatePathSelectionProbabilities(consensus); - this.updateWeightsHistory(validAfterMillis, freshUntilMillis, - pathSelectionWeights); - } - } + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.RELAY_CONSENSUSES, + DescriptorHistory.WEIGHTS_RELAY_CONSENSUS_HISTORY); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + RelayNetworkStatusConsensus consensus = + (RelayNetworkStatusConsensus) descriptor; + long validAfterMillis = consensus.getValidAfterMillis(), + freshUntilMillis = consensus.getFreshUntilMillis(); + SortedMap<String, double[]> pathSelectionWeights = + this.calculatePathSelectionProbabilities(consensus); + this.updateWeightsHistory(validAfterMillis, freshUntilMillis, + pathSelectionWeights); } } } @@ -286,24 +265,22 @@ public class WeightsDataWriter { return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0; } }); - File historyFile = new File(String.format("status/weights/%s/%s/%s", - fingerprint.substring(0, 1), fingerprint.substring(1, 2), - fingerprint)); - if (historyFile.exists()) { + String historyString = this.documentStore.retrieve( + DocumentType.STATUS_WEIGHTS, fingerprint); + if (historyString != null) { SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setLenient(false); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); try { - BufferedReader br = new BufferedReader(new FileReader( - historyFile)); - String line; - while ((line = br.readLine()) != null) { + Scanner s = new Scanner(historyString); + while (s.hasNextLine()) { + String line = s.nextLine(); String[] parts = line.split(" "); if (parts.length != 9) { - System.err.println("Illegal line '" + line + "' in history " - + "file '" + historyFile.getAbsolutePath() - + "'. Skipping this line."); + System.err.println("Illegal line '" + line + "' in weights " + + "history for fingerprint '" + fingerprint + "'. " + + "Skipping this line."); continue; } long validAfterMillis = dateTimeFormat.parse(parts[0] @@ -320,14 +297,12 @@ public class WeightsDataWriter { Double.parseDouble(parts[8]) }; history.put(interval, weights); } - br.close(); + s.close(); } catch (ParseException e) { System.err.println("Could not parse timestamp while reading " - + "history file '" + historyFile.getAbsolutePath() - + "'. Skipping."); - } catch (IOException e) { - System.err.println("Could not read history file '" - + historyFile.getAbsolutePath() + "'. Skipping."); + + "weights history for fingerprint '" + fingerprint + "'. " + + "Skipping."); + e.printStackTrace(); } } return history; @@ -389,33 +364,25 @@ public class WeightsDataWriter {

private void writeHistoryToDisk(String fingerprint, SortedMap<long[], double[]> history) { - File historyFile = new File(String.format("status/weights/%s/%s/%s", - fingerprint.substring(0, 1), fingerprint.substring(1, 2), - fingerprint)); - try { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - historyFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter(historyFile)); - for (Map.Entry<long[], double[]> e : history.entrySet()) { - long[] fresh = e.getKey(); - double[] weights = e.getValue(); - bw.write(dateTimeFormat.format(fresh[0]) + " " - + dateTimeFormat.format(fresh[1])); - for (double weight : weights) { - bw.write(String.format(" %.12f", weight)); - } - bw.write("\n"); + StringBuilder sb = new StringBuilder(); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + for (Map.Entry<long[], double[]> e : history.entrySet()) { + long[] fresh = e.getKey(); + double[] weights = e.getValue(); + sb.append(dateTimeFormat.format(fresh[0]) + " " + + dateTimeFormat.format(fresh[1])); + for (double weight : weights) { + sb.append(String.format(" %.12f", weight)); } - bw.close(); - } catch (IOException e) { - System.err.println("Could not write weights file '" - + historyFile.getAbsolutePath() + "'. Skipping."); + sb.append("\n"); } + String historyString = sb.toString(); + this.documentStore.store(historyString, DocumentType.STATUS_WEIGHTS, + fingerprint); }

- private File weightsFileDirectory = new File("out/weights"); public void writeWeightsDataFiles() { for (String fingerprint : this.currentFingerprints) { SortedMap<long[], double[]> history = @@ -427,17 +394,8 @@ public class WeightsDataWriter { } String historyString = this.formatHistoryString(fingerprint, history); - File weightsFile = new File(weightsFileDirectory, fingerprint); - try { - weightsFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - weightsFile)); - bw.write(historyString); - bw.close(); - } catch (IOException e) { - System.err.println("Could not write weights data file '" - + weightsFile.getAbsolutePath() + "'. Skipping."); - } + this.documentStore.store(historyString, DocumentType.OUT_WEIGHTS, + fingerprint); } }

@@ -591,23 +549,16 @@ public class WeightsDataWriter { }

public void deleteObsoleteWeightsDataFiles() { - SortedMap<String, File> obsoleteWeightsFiles = - new TreeMap<String, File>(); - if (weightsFileDirectory.exists() && - weightsFileDirectory.isDirectory()) { - for (File file : weightsFileDirectory.listFiles()) { - if (file.getName().length() == 40) { - obsoleteWeightsFiles.put(file.getName(), file); - } - } - } + SortedSet<String> obsoleteWeightsFiles; + obsoleteWeightsFiles = this.documentStore.list( + DocumentType.OUT_WEIGHTS); for (String fingerprint : this.currentFingerprints) { - if (obsoleteWeightsFiles.containsKey(fingerprint)) { + if (obsoleteWeightsFiles.contains(fingerprint)) { obsoleteWeightsFiles.remove(fingerprint); } } - for (File weightsFile : obsoleteWeightsFiles.values()) { - weightsFile.delete(); + for (String fingerprint : obsoleteWeightsFiles) { + this.documentStore.remove(DocumentType.OUT_WEIGHTS, fingerprint); } } } diff --git a/test/org/torproject/onionoo/ResourceServletTest.java b/test/org/torproject/onionoo/ResourceServletTest.java index e3e831c..5be2fba 100644 --- a/test/org/torproject/onionoo/ResourceServletTest.java +++ b/test/org/torproject/onionoo/ResourceServletTest.java @@ -155,6 +155,8 @@ public class ResourceServletTest { } }

+ /* TODO Instead of writing out/summary and out/update to a temp + * directory, we could also write our own DocumentStore instance. */ private void writeSummaryFile() throws IOException { File summaryFile = new File(this.tempOutDir, "summary"); BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); @@ -165,7 +167,10 @@ public class ResourceServletTest { bw.write(bridge + "\n"); } bw.close(); - summaryFile.setLastModified(this.lastModified); + File updateFile = new File(this.tempOutDir, "update"); + bw = new BufferedWriter(new FileWriter(updateFile)); + bw.write(String.valueOf(this.lastModified)); + bw.close(); }

private void makeRequest() throws IOException {