[tor-commits] [onionoo/master] Refactor more parts in preparation for moving to a database.

commit 4ce5a2bb8ab487176642dd0ebfd5ef1d960bc5ea Author: Karsten Loesing karsten.loesing@gmx.net Date: Thu Jun 20 18:06:42 2013 +0200

Refactor more parts in preparation for moving to a database.

- Introduce a new document type hierarchy for internal text-based status documents (*Status) and JSON-formatted output documents (*Document). Prepares moving all formatting and parsing code to DocumentStore or Document subtypes, so that *Writer classes won't have to worry about document formats anymore.

- Rename Node to NodeStatus and make it handle its own formatting and parsing. DocumentStore now returns NodeStatus objects instead of one large summary status documents containing all nodes. Prepares moving to a database design where each NodeStatus object will have its own database entry. DocumentStore still reads and writes a single summary status document, so that file formats don't change, but keeps a local NodeStatus cache in memory and writes this file to disk before exiting.

- Don't share CurrentNodes code between back-end and front-end. Move back-end-only functionality to new NodeDataWriter and front-end-only functionality to ResourceServlet.

- Rename DetailDataWriter to DetailsDataWriter for consistency with other document types. --- .../torproject/onionoo/BandwidthDataWriter.java | 35 +- src/org/torproject/onionoo/BandwidthDocument.java | 30 + src/org/torproject/onionoo/BandwidthStatus.java | 7 + src/org/torproject/onionoo/CurrentNodes.java | 562 -------------- src/org/torproject/onionoo/DescriptorSource.java | 2 + src/org/torproject/onionoo/DetailDataWriter.java | 810 ------------------- src/org/torproject/onionoo/DetailsDataWriter.java | 821 ++++++++++++++++++++ src/org/torproject/onionoo/DetailsDocument.java | 71 ++ src/org/torproject/onionoo/Document.java | 8 + src/org/torproject/onionoo/DocumentStore.java | 360 +++++++-- src/org/torproject/onionoo/LookupService.java | 1 - src/org/torproject/onionoo/Main.java | 60 +- src/org/torproject/onionoo/Node.java | 280 ------- src/org/torproject/onionoo/NodeDataWriter.java | 237 ++++++ src/org/torproject/onionoo/NodeStatus.java | 482 ++++++++++++ src/org/torproject/onionoo/ResourceServlet.java | 64 +- src/org/torproject/onionoo/SummaryDocument.java | 27 + src/org/torproject/onionoo/UpdateStatus.java | 7 + src/org/torproject/onionoo/WeightsDataWriter.java | 35 +- src/org/torproject/onionoo/WeightsDocument.java | 31 + src/org/torproject/onionoo/WeightsStatus.java | 5 + 21 files changed, 2144 insertions(+), 1791 deletions(-)

diff --git a/src/org/torproject/onionoo/BandwidthDataWriter.java b/src/org/torproject/onionoo/BandwidthDataWriter.java index 664c050..7203b6d 100644 --- a/src/org/torproject/onionoo/BandwidthDataWriter.java +++ b/src/org/torproject/onionoo/BandwidthDataWriter.java @@ -42,6 +42,8 @@ public class BandwidthDataWriter {

private DocumentStore documentStore;

+ private SortedSet<String> currentFingerprints = new TreeSet<String>(); + public BandwidthDataWriter(DescriptorSource descriptorSource, DocumentStore documentStore) { this.descriptorSource = descriptorSource; @@ -55,12 +57,9 @@ public class BandwidthDataWriter { this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); }

- private SortedSet<String> currentFingerprints = new TreeSet<String>(); - public void setCurrentRelays(SortedMap<String, Node> currentRelays) { - this.currentFingerprints.addAll(currentRelays.keySet()); - } - public void setCurrentBridges(SortedMap<String, Node> currentBridges) { - this.currentFingerprints.addAll(currentBridges.keySet()); + public void setCurrentNodes( + SortedMap<String, NodeStatus> currentNodes) { + this.currentFingerprints.addAll(currentNodes.keySet()); }

public void readExtraInfoDescriptors() { @@ -131,11 +130,12 @@ public class BandwidthDataWriter { private void readHistoryFromDisk(String fingerprint, SortedMap<Long, long[]> writeHistory, SortedMap<Long, long[]> readHistory) { - String historyString = this.documentStore.retrieve( - DocumentType.STATUS_BANDWIDTH, fingerprint); - if (historyString == null) { + BandwidthStatus bandwidthStatus = this.documentStore.retrieve( + BandwidthStatus.class, false, fingerprint); + if (bandwidthStatus == null) { return; } + String historyString = bandwidthStatus.documentString; try { Scanner s = new Scanner(historyString); while (s.hasNextLine()) { @@ -232,9 +232,9 @@ public class BandwidthDataWriter { + this.dateTimeFormat.format(v[1]) + " " + String.valueOf(v[2]) + "\n"); } - String historyString = sb.toString(); - this.documentStore.store(historyString, DocumentType.STATUS_BANDWIDTH, - fingerprint); + BandwidthStatus bandwidthStatus = new BandwidthStatus(); + bandwidthStatus.documentString = sb.toString(); + this.documentStore.store(bandwidthStatus, fingerprint); }

private void writeBandwidthDataFileToDisk(String fingerprint, @@ -253,9 +253,9 @@ public class BandwidthDataWriter { sb.append("{"fingerprint":"" + fingerprint + "",\n" + ""write_history":{\n" + writeHistoryString + "},\n" + ""read_history":{\n" + readHistoryString + "}}\n"); - String historyString = sb.toString(); - this.documentStore.store(historyString, DocumentType.OUT_BANDWIDTH, - fingerprint); + BandwidthDocument bandwidthDocument = new BandwidthDocument(); + bandwidthDocument.documentString = sb.toString(); + this.documentStore.store(bandwidthDocument, fingerprint); }

private String[] graphNames = new String[] { @@ -377,15 +377,14 @@ public class BandwidthDataWriter {

public void deleteObsoleteBandwidthFiles() { SortedSet<String> obsoleteBandwidthFiles = this.documentStore.list( - DocumentType.OUT_BANDWIDTH); + BandwidthDocument.class, false); for (String fingerprint : this.currentFingerprints) { if (obsoleteBandwidthFiles.contains(fingerprint)) { obsoleteBandwidthFiles.remove(fingerprint); } } for (String fingerprint : obsoleteBandwidthFiles) { - this.documentStore.remove(DocumentType.OUT_BANDWIDTH, - fingerprint); + this.documentStore.remove(BandwidthDocument.class, fingerprint); } } } diff --git a/src/org/torproject/onionoo/BandwidthDocument.java b/src/org/torproject/onionoo/BandwidthDocument.java new file mode 100644 index 0000000..01b87de --- /dev/null +++ b/src/org/torproject/onionoo/BandwidthDocument.java @@ -0,0 +1,30 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.util.List; +import java.util.Map; + +class BandwidthDocument extends Document { + + class BandwidthHistory { + String first; + String last; + Integer interval; + Double factor; + Integer count; + List<Integer> values; + } + + class NodeBandwidth { + String fingerprint; + Map<String, BandwidthHistory> write_history; + Map<String, BandwidthHistory> read_history; + } + + String relays_published; + List<NodeBandwidth> relays; + String bridges_published; + List<NodeBandwidth> bridges; +} + diff --git a/src/org/torproject/onionoo/BandwidthStatus.java b/src/org/torproject/onionoo/BandwidthStatus.java new file mode 100644 index 0000000..bf6f504 --- /dev/null +++ b/src/org/torproject/onionoo/BandwidthStatus.java @@ -0,0 +1,7 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +class BandwidthStatus extends Document { +} + diff --git a/src/org/torproject/onionoo/CurrentNodes.java b/src/org/torproject/onionoo/CurrentNodes.java deleted file mode 100644 index 9e27f5b..0000000 --- a/src/org/torproject/onionoo/CurrentNodes.java +++ /dev/null @@ -1,562 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.Scanner; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TimeZone; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.torproject.descriptor.BridgeNetworkStatus; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.NetworkStatusEntry; -import org.torproject.descriptor.RelayNetworkStatusConsensus; -import org.torproject.onionoo.LookupService.LookupResult; - -/* Store relays and bridges that have been running in the past seven - * days. */ -public class CurrentNodes { - - private DescriptorSource descriptorSource; - - private LookupService lookupService; - - private DocumentStore documentStore; - - /* Initialize an instance for the back-end that is read-only and doesn't - * support parsing new descriptor contents. */ - public CurrentNodes(DocumentStore documentStore) { - this(null, null, documentStore); - } - - public CurrentNodes(DescriptorSource descriptorSource, - LookupService lookupService, DocumentStore documentStore) { - this.descriptorSource = descriptorSource; - this.lookupService = lookupService; - this.documentStore = documentStore; - } - - public void readStatusSummary() { - String summaryString = this.documentStore.retrieve( - DocumentType.STATUS_SUMMARY); - this.initializeFromSummaryString(summaryString); - } - - public void readOutSummary() { - String summaryString = this.documentStore.retrieve( - DocumentType.OUT_SUMMARY); - this.initializeFromSummaryString(summaryString); - } - - private void initializeFromSummaryString(String summaryString) { - if (summaryString == null) { - return; - } - Scanner s = new Scanner(summaryString); - while (s.hasNextLine()) { - String line = s.nextLine(); - this.parseSummaryFileLine(line); - } - s.close(); - } - - private void parseSummaryFileLine(String line) { - boolean isRelay; - String nickname, fingerprint, address, countryCode = "??", - hostName = null, defaultPolicy = null, portList = null, - aSNumber = null; - SortedSet<String> orAddressesAndPorts, exitAddresses, relayFlags; - long publishedOrValidAfterMillis, consensusWeight = -1L, - lastRdnsLookup = -1L, firstSeenMillis, lastChangedAddresses; - int orPort, dirPort; - try { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - String[] parts = line.split(" "); - isRelay = parts[0].equals("r"); - if (parts.length < 9) { - System.err.println("Too few space-separated values in line '" - + line + "'. Skipping."); - return; - } - nickname = parts[1]; - fingerprint = parts[2]; - String addresses = parts[3]; - orAddressesAndPorts = new TreeSet<String>(); - exitAddresses = new TreeSet<String>(); - if (addresses.contains(";")) { - String[] addressParts = addresses.split(";", -1); - if (addressParts.length != 3) { - System.err.println("Invalid addresses entry in line '" + line - + "'. Skipping."); - return; - } - address = addressParts[0]; - if (addressParts[1].length() > 0) { - orAddressesAndPorts.addAll(Arrays.asList( - addressParts[1].split("\+"))); - } - if (addressParts[2].length() > 0) { - exitAddresses.addAll(Arrays.asList( - addressParts[2].split("\+"))); - } - } else { - address = addresses; - } - publishedOrValidAfterMillis = dateTimeFormat.parse( - parts[4] + " " + parts[5]).getTime(); - orPort = Integer.parseInt(parts[6]); - dirPort = Integer.parseInt(parts[7]); - relayFlags = new TreeSet<String>( - Arrays.asList(parts[8].split(","))); - if (parts.length > 9) { - consensusWeight = Long.parseLong(parts[9]); - } - if (parts.length > 10) { - countryCode = parts[10]; - } - if (parts.length > 12) { - hostName = parts[11].equals("null") ? null : parts[11]; - lastRdnsLookup = Long.parseLong(parts[12]); - } - if (parts.length > 14) { - if (!parts[13].equals("null")) { - defaultPolicy = parts[13]; - } - if (!parts[14].equals("null")) { - portList = parts[14]; - } - } - firstSeenMillis = publishedOrValidAfterMillis; - if (parts.length > 16) { - firstSeenMillis = dateTimeFormat.parse(parts[15] + " " - + parts[16]).getTime(); - } - lastChangedAddresses = publishedOrValidAfterMillis; - if (parts.length > 18 && !parts[17].equals("null")) { - lastChangedAddresses = dateTimeFormat.parse(parts[17] + " " - + parts[18]).getTime(); - } - if (parts.length > 19) { - aSNumber = parts[19]; - } - } catch (NumberFormatException e) { - System.err.println("Number format exception while parsing line '" - + line + "': " + e.getMessage() + ". Skipping."); - return; - } catch (ParseException e) { - System.err.println("Parse exception while parsing line '" + line - + "': " + e.getMessage() + ". Skipping."); - return; - } catch (Exception e) { - /* This catch block is only here to handle yet unknown errors. It - * should go away once we're sure what kind of errors can occur. */ - System.err.println("Unknown exception while parsing line '" + line - + "': " + e.getMessage() + ". Skipping."); - return; - } - if (isRelay) { - this.addRelay(nickname, fingerprint, address, - orAddressesAndPorts, exitAddresses, - publishedOrValidAfterMillis, orPort, dirPort, relayFlags, - consensusWeight, countryCode, hostName, lastRdnsLookup, - defaultPolicy, portList, firstSeenMillis, - lastChangedAddresses, aSNumber); - } else { - this.addBridge(nickname, fingerprint, address, - orAddressesAndPorts, exitAddresses, - publishedOrValidAfterMillis, orPort, dirPort, relayFlags, - consensusWeight, countryCode, hostName, lastRdnsLookup, - defaultPolicy, portList, firstSeenMillis, - lastChangedAddresses, aSNumber); - } - } - - public void writeStatusSummary() { - String summaryString = this.writeSummaryString(true); - this.documentStore.store(summaryString, DocumentType.STATUS_SUMMARY); - } - - public void writeOutSummary() { - String summaryString = this.writeSummaryString(false); - this.documentStore.store(summaryString, DocumentType.OUT_SUMMARY); - this.documentStore.store(String.valueOf(System.currentTimeMillis()), - DocumentType.OUT_UPDATE); - } - - /* Write internal relay search data to a string. */ - private String writeSummaryString(boolean includeOldNodes) { - StringBuilder sb = new StringBuilder(); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - Collection<Node> relays = includeOldNodes - ? this.knownRelays.values() : this.getCurrentRelays().values(); - for (Node entry : relays) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); - String address = entry.getAddress(); - StringBuilder addressesBuilder = new StringBuilder(); - addressesBuilder.append(address + ";"); - int written = 0; - for (String orAddressAndPort : entry.getOrAddressesAndPorts()) { - addressesBuilder.append((written++ > 0 ? "+" : "") + - orAddressAndPort); - } - addressesBuilder.append(";"); - written = 0; - for (String exitAddress : entry.getExitAddresses()) { - addressesBuilder.append((written++ > 0 ? "+" : "") - + exitAddress); - } - String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); - String orPort = String.valueOf(entry.getOrPort()); - String dirPort = String.valueOf(entry.getDirPort()); - StringBuilder flagsBuilder = new StringBuilder(); - written = 0; - for (String relayFlag : entry.getRelayFlags()) { - flagsBuilder.append((written++ > 0 ? "," : "") + relayFlag); - } - String consensusWeight = String.valueOf(entry.getConsensusWeight()); - String countryCode = entry.getCountryCode() != null - ? entry.getCountryCode() : "??"; - String hostName = entry.getHostName() != null - ? entry.getHostName() : "null"; - long lastRdnsLookup = entry.getLastRdnsLookup(); - String defaultPolicy = entry.getDefaultPolicy() != null - ? entry.getDefaultPolicy() : "null"; - String portList = entry.getPortList() != null - ? entry.getPortList() : "null"; - String firstSeen = dateTimeFormat.format( - entry.getFirstSeenMillis()); - String lastChangedAddresses = dateTimeFormat.format( - entry.getLastChangedOrAddress()); - String aSNumber = entry.getASNumber() != null - ? entry.getASNumber() : "null"; - sb.append("r " + nickname + " " + fingerprint + " " - + addressesBuilder.toString() + " " + lastSeen + " " - + orPort + " " + dirPort + " " + flagsBuilder.toString() + " " - + consensusWeight + " " + countryCode + " " + hostName + " " - + String.valueOf(lastRdnsLookup) + " " + defaultPolicy + " " - + portList + " " + firstSeen + " " + lastChangedAddresses - + " " + aSNumber + "\n"); - } - Collection<Node> bridges = includeOldNodes - ? this.knownBridges.values() : this.getCurrentBridges().values(); - for (Node entry : bridges) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); - String published = dateTimeFormat.format( - entry.getLastSeenMillis()); - String address = entry.getAddress(); - StringBuilder addressesBuilder = new StringBuilder(); - addressesBuilder.append(address + ";"); - int written = 0; - for (String orAddressAndPort : entry.getOrAddressesAndPorts()) { - addressesBuilder.append((written++ > 0 ? "+" : "") + - orAddressAndPort); - } - addressesBuilder.append(";"); - String orPort = String.valueOf(entry.getOrPort()); - String dirPort = String.valueOf(entry.getDirPort()); - StringBuilder flagsBuilder = new StringBuilder(); - written = 0; - for (String relayFlag : entry.getRelayFlags()) { - flagsBuilder.append((written++ > 0 ? "," : "") + relayFlag); - } - String firstSeen = dateTimeFormat.format( - entry.getFirstSeenMillis()); - sb.append("b " + nickname + " " + fingerprint + " " - + addressesBuilder.toString() + " " + published + " " + orPort - + " " + dirPort + " " + flagsBuilder.toString() - + " -1 ?? null -1 null null " + firstSeen + " null null " - + "null\n"); - } - return sb.toString(); - } - - private long lastValidAfterMillis = 0L; - private long lastPublishedMillis = 0L; - - public void readRelayNetworkConsensuses() { - if (this.descriptorSource == null) { - System.err.println("Not configured to read relay network " - + "consensuses."); - return; - } - DescriptorQueue descriptorQueue = - this.descriptorSource.getDescriptorQueue( - DescriptorType.RELAY_CONSENSUSES, - DescriptorHistory.RELAY_CONSENSUS_HISTORY); - Descriptor descriptor; - while ((descriptor = descriptorQueue.nextDescriptor()) != null) { - if (descriptor instanceof RelayNetworkStatusConsensus) { - updateRelayNetworkStatusConsensus( - (RelayNetworkStatusConsensus) descriptor); - } - } - } - - public void setRelayRunningBits() { - if (this.lastValidAfterMillis > 0L) { - for (Node entry : this.knownRelays.values()) { - entry.setRunning(entry.getLastSeenMillis() == - this.lastValidAfterMillis); - } - } - } - - SortedMap<String, Integer> lastBandwidthWeights = null; - public SortedMap<String, Integer> getLastBandwidthWeights() { - return this.lastBandwidthWeights; - } - private void updateRelayNetworkStatusConsensus( - RelayNetworkStatusConsensus consensus) { - long validAfterMillis = consensus.getValidAfterMillis(); - for (NetworkStatusEntry entry : - consensus.getStatusEntries().values()) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); - String address = entry.getAddress(); - SortedSet<String> orAddressesAndPorts = new TreeSet<String>( - entry.getOrAddresses()); - int orPort = entry.getOrPort(); - int dirPort = entry.getDirPort(); - SortedSet<String> relayFlags = entry.getFlags(); - long consensusWeight = entry.getBandwidth(); - String defaultPolicy = entry.getDefaultPolicy(); - String portList = entry.getPortList(); - this.addRelay(nickname, fingerprint, address, orAddressesAndPorts, - null, validAfterMillis, orPort, dirPort, relayFlags, - consensusWeight, null, null, -1L, defaultPolicy, portList, - validAfterMillis, validAfterMillis, null); - } - if (this.lastValidAfterMillis == validAfterMillis) { - this.lastBandwidthWeights = consensus.getBandwidthWeights(); - } - } - - public void addRelay(String nickname, String fingerprint, - String address, SortedSet<String> orAddressesAndPorts, - SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, - int dirPort, SortedSet<String> relayFlags, long consensusWeight, - String countryCode, String hostName, long lastRdnsLookup, - String defaultPolicy, String portList, long firstSeenMillis, - long lastChangedAddresses, String aSNumber) { - /* Remember addresses and OR/dir ports that the relay advertised at - * the given time. */ - SortedMap<Long, Set<String>> lastAddresses = - new TreeMap<Long, Set<String>>(Collections.reverseOrder()); - Set<String> addresses = new HashSet<String>(); - addresses.add(address + ":" + orPort); - if (dirPort > 0) { - addresses.add(address + ":" + dirPort); - } - addresses.addAll(orAddressesAndPorts); - lastAddresses.put(lastChangedAddresses, addresses); - /* See if there's already an entry for this relay. */ - if (this.knownRelays.containsKey(fingerprint)) { - Node existingEntry = this.knownRelays.get(fingerprint); - if (lastSeenMillis < existingEntry.getLastSeenMillis()) { - /* Use latest information for nickname, current addresses, etc. */ - nickname = existingEntry.getNickname(); - address = existingEntry.getAddress(); - orAddressesAndPorts = existingEntry.getOrAddressesAndPorts(); - exitAddresses = existingEntry.getExitAddresses(); - lastSeenMillis = existingEntry.getLastSeenMillis(); - orPort = existingEntry.getOrPort(); - dirPort = existingEntry.getDirPort(); - relayFlags = existingEntry.getRelayFlags(); - consensusWeight = existingEntry.getConsensusWeight(); - countryCode = existingEntry.getCountryCode(); - defaultPolicy = existingEntry.getDefaultPolicy(); - portList = existingEntry.getPortList(); - } - if (hostName == null && - existingEntry.getAddress().equals(address)) { - /* Re-use reverse DNS lookup results if available. */ - hostName = existingEntry.getHostName(); - lastRdnsLookup = existingEntry.getLastRdnsLookup(); - } - /* Update relay-history fields. */ - firstSeenMillis = Math.min(firstSeenMillis, - existingEntry.getFirstSeenMillis()); - lastAddresses.putAll(existingEntry.getLastAddresses()); - } - /* Add or update entry. */ - Node entry = new Node(nickname, fingerprint, address, - orAddressesAndPorts, exitAddresses, lastSeenMillis, orPort, - dirPort, relayFlags, consensusWeight, countryCode, hostName, - lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, - lastAddresses, aSNumber); - this.knownRelays.put(fingerprint, entry); - /* If this entry comes from a new consensus, update our global last - * valid-after time. */ - if (lastSeenMillis > this.lastValidAfterMillis) { - this.lastValidAfterMillis = lastSeenMillis; - } - } - - public void lookUpCitiesAndASes() { - SortedSet<String> addressStrings = new TreeSet<String>(); - for (Node relay : this.knownRelays.values()) { - addressStrings.add(relay.getAddress()); - } - if (addressStrings.isEmpty()) { - System.err.println("No relay IP addresses to resolve to cities or " - + "ASN."); - return; - } - SortedMap<String, LookupResult> lookupResults = - this.lookupService.lookup(addressStrings); - for (Node relay : knownRelays.values()) { - String addressString = relay.getAddress(); - if (lookupResults.containsKey(addressString)) { - LookupResult lookupResult = lookupResults.get(addressString); - relay.setCountryCode(lookupResult.countryCode); - relay.setCountryName(lookupResult.countryName); - relay.setRegionName(lookupResult.regionName); - relay.setCityName(lookupResult.cityName); - relay.setLatitude(lookupResult.latitude); - relay.setLongitude(lookupResult.longitude); - relay.setASNumber(lookupResult.aSNumber); - relay.setASName(lookupResult.aSName); - } - } - } - - public void readBridgeNetworkStatuses() { - if (this.descriptorSource == null) { - System.err.println("Not configured to read bridge network " - + "statuses."); - return; - } - DescriptorQueue descriptorQueue = - this.descriptorSource.getDescriptorQueue( - DescriptorType.BRIDGE_STATUSES, - DescriptorHistory.BRIDGE_STATUS_HISTORY); - Descriptor descriptor; - while ((descriptor = descriptorQueue.nextDescriptor()) != null) { - if (descriptor instanceof BridgeNetworkStatus) { - updateBridgeNetworkStatus((BridgeNetworkStatus) descriptor); - } - } - } - - public void setBridgeRunningBits() { - if (this.lastPublishedMillis > 0L) { - for (Node entry : this.knownBridges.values()) { - entry.setRunning(entry.getRelayFlags().contains("Running") && - entry.getLastSeenMillis() == this.lastPublishedMillis); - } - } - } - - private void updateBridgeNetworkStatus(BridgeNetworkStatus status) { - long publishedMillis = status.getPublishedMillis(); - for (NetworkStatusEntry entry : status.getStatusEntries().values()) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); - String address = entry.getAddress(); - SortedSet<String> orAddressesAndPorts = new TreeSet<String>( - entry.getOrAddresses()); - int orPort = entry.getOrPort(); - int dirPort = entry.getDirPort(); - SortedSet<String> relayFlags = entry.getFlags(); - this.addBridge(nickname, fingerprint, address, orAddressesAndPorts, - null, publishedMillis, orPort, dirPort, relayFlags, -1, "??", - null, -1L, null, null, publishedMillis, -1L, null); - } - } - - public void addBridge(String nickname, String fingerprint, - String address, SortedSet<String> orAddressesAndPorts, - SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, - int dirPort, SortedSet<String> relayFlags, long consensusWeight, - String countryCode, String hostname, long lastRdnsLookup, - String defaultPolicy, String portList, long firstSeenMillis, - long lastChangedAddresses, String aSNumber) { - /* See if there's already an entry for this bridge. */ - if (this.knownBridges.containsKey(fingerprint)) { - Node existingEntry = this.knownBridges.get(fingerprint); - if (lastSeenMillis < existingEntry.getLastSeenMillis()) { - /* Use latest information for nickname, current addresses, etc. */ - nickname = existingEntry.getNickname(); - address = existingEntry.getAddress(); - orAddressesAndPorts = existingEntry.getOrAddressesAndPorts(); - exitAddresses = existingEntry.getExitAddresses(); - lastSeenMillis = existingEntry.getLastSeenMillis(); - orPort = existingEntry.getOrPort(); - dirPort = existingEntry.getDirPort(); - relayFlags = existingEntry.getRelayFlags(); - consensusWeight = existingEntry.getConsensusWeight(); - countryCode = existingEntry.getCountryCode(); - defaultPolicy = existingEntry.getDefaultPolicy(); - portList = existingEntry.getPortList(); - aSNumber = existingEntry.getASNumber(); - } - /* Update relay-history fields. */ - firstSeenMillis = Math.min(firstSeenMillis, - existingEntry.getFirstSeenMillis()); - } - /* Add or update entry. */ - Node entry = new Node(nickname, fingerprint, address, - orAddressesAndPorts, exitAddresses, lastSeenMillis, orPort, - dirPort, relayFlags, consensusWeight, countryCode, hostname, - lastRdnsLookup, defaultPolicy, portList, firstSeenMillis, null, - aSNumber); - this.knownBridges.put(fingerprint, entry); - /* If this entry comes from a new status, update our global last - * published time. */ - if (lastSeenMillis > this.lastPublishedMillis) { - this.lastPublishedMillis = lastSeenMillis; - } - } - - private SortedMap<String, Node> knownRelays = - new TreeMap<String, Node>(); - public SortedMap<String, Node> getCurrentRelays() { - long cutoff = this.lastValidAfterMillis - - 7L * 24L * 60L * 60L * 1000L; - SortedMap<String, Node> currentRelays = new TreeMap<String, Node>(); - for (Map.Entry<String, Node> e : this.knownRelays.entrySet()) { - if (e.getValue().getLastSeenMillis() >= cutoff) { - currentRelays.put(e.getKey(), e.getValue()); - } - } - return currentRelays; - } - - private SortedMap<String, Node> knownBridges = - new TreeMap<String, Node>(); - public SortedMap<String, Node> getCurrentBridges() { - long cutoff = this.lastPublishedMillis - 7L * 24L * 60L * 60L * 1000L; - SortedMap<String, Node> currentBridges = new TreeMap<String, Node>(); - for (Map.Entry<String, Node> e : this.knownBridges.entrySet()) { - if (e.getValue().getLastSeenMillis() >= cutoff) { - currentBridges.put(e.getKey(), e.getValue()); - } - } - return currentBridges; - } - - public long getLastValidAfterMillis() { - return this.lastValidAfterMillis; - } - - public long getLastPublishedMillis() { - return this.lastPublishedMillis; - } -} - diff --git a/src/org/torproject/onionoo/DescriptorSource.java b/src/org/torproject/onionoo/DescriptorSource.java index 5936a93..23febc8 100644 --- a/src/org/torproject/onionoo/DescriptorSource.java +++ b/src/org/torproject/onionoo/DescriptorSource.java @@ -1,3 +1,5 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ package org.torproject.onionoo;

import java.io.BufferedReader; diff --git a/src/org/torproject/onionoo/DetailDataWriter.java b/src/org/torproject/onionoo/DetailDataWriter.java deleted file mode 100644 index e95a919..0000000 --- a/src/org/torproject/onionoo/DetailDataWriter.java +++ /dev/null @@ -1,810 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Scanner; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TimeZone; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.apache.commons.lang.StringEscapeUtils; - -import org.torproject.descriptor.BridgePoolAssignment; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.ExitList; -import org.torproject.descriptor.ExitListEntry; -import org.torproject.descriptor.ServerDescriptor; - -/* Write updated detail data files to disk and delete files of relays or - * bridges that fell out of the summary list. - * - * The parts of details files coming from server descriptors always come - * from the last known descriptor of a relay or bridge, not from the - * descriptor that was last referenced in a network status. */ -public class DetailDataWriter { - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - public DetailDataWriter(DescriptorSource descriptorSource, - DocumentStore documentStore) { - this.descriptorSource = descriptorSource; - this.documentStore = documentStore; - } - - private SortedMap<String, Node> relays; - public void setCurrentRelays(SortedMap<String, Node> currentRelays) { - this.relays = currentRelays; - } - - private SortedMap<String, Node> bridges; - public void setCurrentBridges(SortedMap<String, Node> currentBridges) { - this.bridges = currentBridges; - } - - private static final long RDNS_LOOKUP_MAX_REQUEST_MILLIS = 10L * 1000L; - private static final long RDNS_LOOKUP_MAX_DURATION_MILLIS = 5L * 60L - * 1000L; - private static final long RDNS_LOOKUP_MAX_AGE_MILLIS = 12L * 60L * 60L - * 1000L; - private static final int RDNS_LOOKUP_WORKERS_NUM = 5; - private Set<String> rdnsLookupJobs; - private Map<String, String> rdnsLookupResults; - private long startedRdnsLookups; - private List<RdnsLookupWorker> rdnsLookupWorkers; - public void startReverseDomainNameLookups() { - this.startedRdnsLookups = System.currentTimeMillis(); - this.rdnsLookupJobs = new HashSet<String>(); - for (Node relay : relays.values()) { - if (relay.getLastRdnsLookup() < this.startedRdnsLookups - - RDNS_LOOKUP_MAX_AGE_MILLIS) { - this.rdnsLookupJobs.add(relay.getAddress()); - } - } - this.rdnsLookupResults = new HashMap<String, String>(); - this.rdnsLookupWorkers = new ArrayList<RdnsLookupWorker>(); - for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) { - RdnsLookupWorker rdnsLookupWorker = new RdnsLookupWorker(); - this.rdnsLookupWorkers.add(rdnsLookupWorker); - rdnsLookupWorker.setDaemon(true); - rdnsLookupWorker.start(); - } - } - - public void finishReverseDomainNameLookups() { - for (RdnsLookupWorker rdnsLookupWorker : this.rdnsLookupWorkers) { - try { - rdnsLookupWorker.join(); - } catch (InterruptedException e) { - /* This is not something that we can take care of. Just leave the - * worker thread alone. */ - } - } - synchronized (this.rdnsLookupResults) { - for (Node relay : relays.values()) { - if (this.rdnsLookupResults.containsKey(relay.getAddress())) { - relay.setHostName(this.rdnsLookupResults.get( - relay.getAddress())); - relay.setLastRdnsLookup(this.startedRdnsLookups); - } - } - } - } - - private class RdnsLookupWorker extends Thread { - public void run() { - while (System.currentTimeMillis() - RDNS_LOOKUP_MAX_DURATION_MILLIS - <= startedRdnsLookups) { - String rdnsLookupJob = null; - synchronized (rdnsLookupJobs) { - for (String job : rdnsLookupJobs) { - rdnsLookupJob = job; - rdnsLookupJobs.remove(job); - break; - } - } - if (rdnsLookupJob == null) { - break; - } - RdnsLookupRequest request = new RdnsLookupRequest(this, - rdnsLookupJob); - request.setDaemon(true); - request.start(); - try { - Thread.sleep(RDNS_LOOKUP_MAX_REQUEST_MILLIS); - } catch (InterruptedException e) { - /* Getting interrupted should be the default case. */ - } - String hostName = request.getHostName(); - if (hostName != null) { - synchronized (rdnsLookupResults) { - rdnsLookupResults.put(rdnsLookupJob, hostName); - } - } - } - } - } - - private class RdnsLookupRequest extends Thread { - RdnsLookupWorker parent; - String address, hostName; - public RdnsLookupRequest(RdnsLookupWorker parent, String address) { - this.parent = parent; - this.address = address; - } - public void run() { - try { - String result = InetAddress.getByName(this.address).getHostName(); - synchronized (this) { - this.hostName = result; - } - } catch (UnknownHostException e) { - /* We'll try again the next time. */ - } - this.parent.interrupt(); - } - public synchronized String getHostName() { - return hostName; - } - } - - private Map<String, ServerDescriptor> relayServerDescriptors = - new HashMap<String, ServerDescriptor>(); - public void readRelayServerDescriptors() { - /* Don't remember which server descriptors we already parsed. If we - * parse a server descriptor now and first learn about the relay in a - * later consensus, we'll never write the descriptor content anywhere. - * The result would be details files containing no descriptor parts - * until the relay publishes the next descriptor. */ - DescriptorQueue descriptorQueue = - this.descriptorSource.getDescriptorQueue( - DescriptorType.RELAY_SERVER_DESCRIPTORS); - Descriptor descriptor; - while ((descriptor = descriptorQueue.nextDescriptor()) != null) { - if (descriptor instanceof ServerDescriptor) { - ServerDescriptor serverDescriptor = (ServerDescriptor) descriptor; - String fingerprint = serverDescriptor.getFingerprint(); - if (!this.relayServerDescriptors.containsKey(fingerprint) || - this.relayServerDescriptors.get(fingerprint). - getPublishedMillis() - < serverDescriptor.getPublishedMillis()) { - this.relayServerDescriptors.put(fingerprint, - serverDescriptor); - } - } - } - } - - public void calculatePathSelectionProbabilities( - SortedMap<String, Integer> bandwidthWeights) { - boolean consensusContainsBandwidthWeights = false; - double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0, - wmd = 0.0, wee = 0.0, wed = 0.0; - if (bandwidthWeights != null) { - SortedSet<String> weightKeys = new TreeSet<String>(Arrays.asList( - "Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(","))); - weightKeys.removeAll(bandwidthWeights.keySet()); - if (weightKeys.isEmpty()) { - consensusContainsBandwidthWeights = true; - wgg = ((double) bandwidthWeights.get("Wgg")) / 10000.0; - wgd = ((double) bandwidthWeights.get("Wgd")) / 10000.0; - wmg = ((double) bandwidthWeights.get("Wmg")) / 10000.0; - wmm = ((double) bandwidthWeights.get("Wmm")) / 10000.0; - wme = ((double) bandwidthWeights.get("Wme")) / 10000.0; - wmd = ((double) bandwidthWeights.get("Wmd")) / 10000.0; - wee = ((double) bandwidthWeights.get("Wee")) / 10000.0; - wed = ((double) bandwidthWeights.get("Wed")) / 10000.0; - } - } else { - System.err.println("Could not determine most recent Wxx parameter " - + "values, probably because we didn't parse a consensus in " - + "this execution. All relays' guard/middle/exit weights are " - + "going to be 0.0."); - } - SortedMap<String, Double> - advertisedBandwidths = new TreeMap<String, Double>(), - consensusWeights = new TreeMap<String, Double>(), - guardWeights = new TreeMap<String, Double>(), - middleWeights = new TreeMap<String, Double>(), - exitWeights = new TreeMap<String, Double>(); - double totalAdvertisedBandwidth = 0.0; - double totalConsensusWeight = 0.0; - double totalGuardWeight = 0.0; - double totalMiddleWeight = 0.0; - double totalExitWeight = 0.0; - for (Map.Entry<String, Node> e : this.relays.entrySet()) { - String fingerprint = e.getKey(); - Node relay = e.getValue(); - if (!relay.getRunning()) { - continue; - } - boolean isExit = relay.getRelayFlags().contains("Exit") && - !relay.getRelayFlags().contains("BadExit"); - boolean isGuard = relay.getRelayFlags().contains("Guard"); - if (this.relayServerDescriptors.containsKey(fingerprint)) { - ServerDescriptor serverDescriptor = - this.relayServerDescriptors.get(fingerprint); - double advertisedBandwidth = (double) Math.min(Math.min( - serverDescriptor.getBandwidthBurst(), - serverDescriptor.getBandwidthObserved()), - serverDescriptor.getBandwidthRate()); - advertisedBandwidths.put(fingerprint, advertisedBandwidth); - totalAdvertisedBandwidth += advertisedBandwidth; - } - double consensusWeight = (double) relay.getConsensusWeight(); - consensusWeights.put(fingerprint, consensusWeight); - totalConsensusWeight += consensusWeight; - if (consensusContainsBandwidthWeights) { - double guardWeight = consensusWeight, - middleWeight = consensusWeight, - exitWeight = consensusWeight; - if (isGuard && isExit) { - guardWeight *= wgd; - middleWeight *= wmd; - exitWeight *= wed; - } else if (isGuard) { - guardWeight *= wgg; - middleWeight *= wmg; - exitWeight = 0.0; - } else if (isExit) { - guardWeight = 0.0; - middleWeight *= wme; - exitWeight *= wee; - } else { - guardWeight = 0.0; - middleWeight *= wmm; - exitWeight = 0.0; - } - guardWeights.put(fingerprint, guardWeight); - middleWeights.put(fingerprint, middleWeight); - exitWeights.put(fingerprint, exitWeight); - totalGuardWeight += guardWeight; - totalMiddleWeight += middleWeight; - totalExitWeight += exitWeight; - } - } - for (Map.Entry<String, Node> e : this.relays.entrySet()) { - String fingerprint = e.getKey(); - Node relay = e.getValue(); - if (advertisedBandwidths.containsKey(fingerprint)) { - relay.setAdvertisedBandwidthFraction(advertisedBandwidths.get( - fingerprint) / totalAdvertisedBandwidth); - } - if (consensusWeights.containsKey(fingerprint)) { - relay.setConsensusWeightFraction(consensusWeights.get(fingerprint) - / totalConsensusWeight); - } - if (guardWeights.containsKey(fingerprint)) { - relay.setGuardProbability(guardWeights.get(fingerprint) - / totalGuardWeight); - } - if (middleWeights.containsKey(fingerprint)) { - relay.setMiddleProbability(middleWeights.get(fingerprint) - / totalMiddleWeight); - } - if (exitWeights.containsKey(fingerprint)) { - relay.setExitProbability(exitWeights.get(fingerprint) - / totalExitWeight); - } - } - } - - private long now = System.currentTimeMillis(); - private Map<String, Set<ExitListEntry>> exitListEntries = - new HashMap<String, Set<ExitListEntry>>(); - public void readExitLists() { - DescriptorQueue descriptorQueue = - this.descriptorSource.getDescriptorQueue( - DescriptorType.EXIT_LISTS, DescriptorHistory.EXIT_LIST_HISTORY); - Descriptor descriptor; - while ((descriptor = descriptorQueue.nextDescriptor()) != null) { - if (descriptor instanceof ExitList) { - ExitList exitList = (ExitList) descriptor; - for (ExitListEntry exitListEntry : - exitList.getExitListEntries()) { - if (exitListEntry.getScanMillis() < - this.now - 24L * 60L * 60L * 1000L) { - continue; - } - String fingerprint = exitListEntry.getFingerprint(); - if (!this.exitListEntries.containsKey(fingerprint)) { - this.exitListEntries.put(fingerprint, - new HashSet<ExitListEntry>()); - } - this.exitListEntries.get(fingerprint).add(exitListEntry); - } - } - } - } - - private Map<String, ServerDescriptor> bridgeServerDescriptors = - new HashMap<String, ServerDescriptor>(); - public void readBridgeServerDescriptors() { - /* Don't remember which server descriptors we already parsed. If we - * parse a server descriptor now and first learn about the relay in a - * later status, we'll never write the descriptor content anywhere. - * The result would be details files containing no descriptor parts - * until the bridge publishes the next descriptor. */ - DescriptorQueue descriptorQueue = - this.descriptorSource.getDescriptorQueue( - DescriptorType.BRIDGE_SERVER_DESCRIPTORS); - Descriptor descriptor; - while ((descriptor = descriptorQueue.nextDescriptor()) != null) { - if (descriptor instanceof ServerDescriptor) { - ServerDescriptor serverDescriptor = (ServerDescriptor) descriptor; - String fingerprint = serverDescriptor.getFingerprint(); - if (!this.bridgeServerDescriptors.containsKey(fingerprint) || - this.bridgeServerDescriptors.get(fingerprint). - getPublishedMillis() - < serverDescriptor.getPublishedMillis()) { - this.bridgeServerDescriptors.put(fingerprint, - serverDescriptor); - } - } - } - } - - private Map<String, String> bridgePoolAssignments = - new HashMap<String, String>(); - public void readBridgePoolAssignments() { - DescriptorQueue descriptorQueue = - this.descriptorSource.getDescriptorQueue( - DescriptorType.BRIDGE_POOL_ASSIGNMENTS, - DescriptorHistory.BRIDGE_POOLASSIGN_HISTORY); - Descriptor descriptor; - while ((descriptor = descriptorQueue.nextDescriptor()) != null) { - if (descriptor instanceof BridgePoolAssignment) { - BridgePoolAssignment bridgePoolAssignment = - (BridgePoolAssignment) descriptor; - for (Map.Entry<String, String> e : - bridgePoolAssignment.getEntries().entrySet()) { - String fingerprint = e.getKey(); - String details = e.getValue(); - this.bridgePoolAssignments.put(fingerprint, details); - } - } - } - } - - public void writeOutDetails() { - SortedSet<String> remainingDetailsFiles = new TreeSet<String>(); - remainingDetailsFiles.addAll(this.documentStore.list( - DocumentType.OUT_DETAILS)); - this.updateRelayDetailsFiles(remainingDetailsFiles); - this.updateBridgeDetailsFiles(remainingDetailsFiles); - this.deleteDetailsFiles(remainingDetailsFiles); - } - - private static String escapeJSON(String s) { - return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\'", "'"); - } - - private void updateRelayDetailsFiles( - SortedSet<String> remainingDetailsFiles) { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - for (Map.Entry<String, Node> relay : this.relays.entrySet()) { - String fingerprint = relay.getKey(); - - /* Read details file for this relay if it exists. */ - String descriptorParts = null; - long publishedMillis = -1L; - if (remainingDetailsFiles.contains(fingerprint)) { - remainingDetailsFiles.remove(fingerprint); - String documentString = this.documentStore.retrieve( - DocumentType.OUT_DETAILS, fingerprint); - if (documentString != null) { - try { - boolean copyDescriptorParts = false; - StringBuilder sb = new StringBuilder(); - Scanner s = new Scanner(documentString); - while (s.hasNextLine()) { - String line = s.nextLine(); - if (line.startsWith(""desc_published":")) { - String published = line.substring( - ""desc_published":"".length(), - ""desc_published":"1970-01-01 00:00:00".length()); - publishedMillis = dateTimeFormat.parse(published). - getTime(); - copyDescriptorParts = true; - } - if (copyDescriptorParts) { - sb.append(line + "\n"); - } - } - s.close(); - if (sb.length() > 0) { - descriptorParts = sb.toString(); - } - } catch (ParseException e) { - System.err.println("Could not parse timestamp in details.json " - + "file for '" + fingerprint + "'. Ignoring."); - e.printStackTrace(); - publishedMillis = -1L; - descriptorParts = null; - } - } - } - - /* Generate new descriptor-specific part if we have a more recent - * descriptor or if the part we read didn't contain a last_restarted - * line. */ - if (this.relayServerDescriptors.containsKey(fingerprint) && - (this.relayServerDescriptors.get(fingerprint). - getPublishedMillis() > publishedMillis)) { - ServerDescriptor descriptor = this.relayServerDescriptors.get( - fingerprint); - StringBuilder sb = new StringBuilder(); - String publishedDateTime = dateTimeFormat.format( - descriptor.getPublishedMillis()); - String lastRestartedString = dateTimeFormat.format( - descriptor.getPublishedMillis() - - descriptor.getUptime() * 1000L); - int bandwidthRate = descriptor.getBandwidthRate(); - int bandwidthBurst = descriptor.getBandwidthBurst(); - int observedBandwidth = descriptor.getBandwidthObserved(); - int advertisedBandwidth = Math.min(bandwidthRate, - Math.min(bandwidthBurst, observedBandwidth)); - sb.append(""desc_published":"" + publishedDateTime + "",\n" - + ""last_restarted":"" + lastRestartedString + "",\n" - + ""bandwidth_rate":" + bandwidthRate + ",\n" - + ""bandwidth_burst":" + bandwidthBurst + ",\n" - + ""observed_bandwidth":" + observedBandwidth + ",\n" - + ""advertised_bandwidth":" + advertisedBandwidth + ",\n" - + ""exit_policy":["); - int written = 0; - for (String exitPolicyLine : descriptor.getExitPolicyLines()) { - sb.append((written++ > 0 ? "," : "") + "\n "" + exitPolicyLine - + """); - } - sb.append("\n]"); - if (descriptor.getContact() != null) { - sb.append(",\n"contact":"" - + escapeJSON(descriptor.getContact()) + """); - } - if (descriptor.getPlatform() != null) { - sb.append(",\n"platform":"" - + escapeJSON(descriptor.getPlatform()) + """); - } - if (descriptor.getFamilyEntries() != null) { - sb.append(",\n"family":["); - written = 0; - for (String familyEntry : descriptor.getFamilyEntries()) { - sb.append((written++ > 0 ? "," : "") + "\n "" + familyEntry - + """); - } - sb.append("\n]"); - } - sb.append("\n}\n"); - descriptorParts = sb.toString(); - } - - /* Generate network-status-specific part. */ - Node entry = relay.getValue(); - String nickname = entry.getNickname(); - String address = entry.getAddress(); - List<String> orAddresses = new ArrayList<String>(); - orAddresses.add(address + ":" + entry.getOrPort()); - orAddresses.addAll(entry.getOrAddressesAndPorts()); - StringBuilder orAddressesAndPortsBuilder = new StringBuilder(); - int addressesWritten = 0; - for (String orAddress : orAddresses) { - orAddressesAndPortsBuilder.append( - (addressesWritten++ > 0 ? "," : "") + """ - + orAddress.toLowerCase() + """); - } - String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); - String firstSeen = dateTimeFormat.format( - entry.getFirstSeenMillis()); - String lastChangedOrAddress = dateTimeFormat.format( - entry.getLastChangedOrAddress()); - String running = entry.getRunning() ? "true" : "false"; - int dirPort = entry.getDirPort(); - String countryCode = entry.getCountryCode(); - String latitude = entry.getLatitude(); - String longitude = entry.getLongitude(); - String countryName = entry.getCountryName(); - String regionName = entry.getRegionName(); - String cityName = entry.getCityName(); - String aSNumber = entry.getASNumber(); - String aSName = entry.getASName(); - long consensusWeight = entry.getConsensusWeight(); - String hostName = entry.getHostName(); - double advertisedBandwidthFraction = - entry.getAdvertisedBandwidthFraction(); - double consensusWeightFraction = entry.getConsensusWeightFraction(); - double guardProbability = entry.getGuardProbability(); - double middleProbability = entry.getMiddleProbability(); - double exitProbability = entry.getExitProbability(); - String defaultPolicy = entry.getDefaultPolicy(); - String portList = entry.getPortList(); - StringBuilder sb = new StringBuilder(); - sb.append("{"version":1,\n" - + ""nickname":"" + nickname + "",\n" - + ""fingerprint":"" + fingerprint + "",\n" - + ""or_addresses":[" + orAddressesAndPortsBuilder.toString() - + "]"); - if (dirPort != 0) { - sb.append(",\n"dir_address":"" + address + ":" + dirPort - + """); - } - sb.append(",\n"last_seen":"" + lastSeen + """); - sb.append(",\n"first_seen":"" + firstSeen + """); - sb.append(",\n"last_changed_address_or_port":"" - + lastChangedOrAddress + """); - sb.append(",\n"running":" + running); - SortedSet<String> relayFlags = entry.getRelayFlags(); - if (!relayFlags.isEmpty()) { - sb.append(",\n"flags":["); - int written = 0; - for (String relayFlag : relayFlags) { - sb.append((written++ > 0 ? "," : "") + """ + relayFlag + """); - } - sb.append("]"); - } - if (countryCode != null) { - sb.append(",\n"country":"" + countryCode + """); - } - if (latitude != null) { - sb.append(",\n"latitude":" + latitude); - } - if (longitude != null) { - sb.append(",\n"longitude":" + longitude); - } - if (countryName != null) { - sb.append(",\n"country_name":"" - + escapeJSON(countryName) + """); - } - if (regionName != null) { - sb.append(",\n"region_name":"" - + escapeJSON(regionName) + """); - } - if (cityName != null) { - sb.append(",\n"city_name":"" - + escapeJSON(cityName) + """); - } - if (aSNumber != null) { - sb.append(",\n"as_number":"" - + escapeJSON(aSNumber) + """); - } - if (aSName != null) { - sb.append(",\n"as_name":"" - + escapeJSON(aSName) + """); - } - if (consensusWeight >= 0L) { - sb.append(",\n"consensus_weight":" - + String.valueOf(consensusWeight)); - } - if (hostName != null) { - sb.append(",\n"host_name":"" - + escapeJSON(hostName) + """); - } - if (advertisedBandwidthFraction >= 0.0) { - sb.append(String.format( - ",\n"advertised_bandwidth_fraction":%.9f", - advertisedBandwidthFraction)); - } - if (consensusWeightFraction >= 0.0) { - sb.append(String.format(",\n"consensus_weight_fraction":%.9f", - consensusWeightFraction)); - } - if (guardProbability >= 0.0) { - sb.append(String.format(",\n"guard_probability":%.9f", - guardProbability)); - } - if (middleProbability >= 0.0) { - sb.append(String.format(",\n"middle_probability":%.9f", - middleProbability)); - } - if (exitProbability >= 0.0) { - sb.append(String.format(",\n"exit_probability":%.9f", - exitProbability)); - } - if (defaultPolicy != null && (defaultPolicy.equals("accept") || - defaultPolicy.equals("reject")) && portList != null) { - sb.append(",\n"exit_policy_summary":{"" + defaultPolicy - + "":["); - int portsWritten = 0; - for (String portOrPortRange : portList.split(",")) { - sb.append((portsWritten++ > 0 ? "," : "") - + """ + portOrPortRange + """); - } - sb.append("]}"); - } - - /* Add exit addresses if at least one of them is distinct from the - * onion-routing addresses. */ - if (exitListEntries.containsKey(fingerprint)) { - for (ExitListEntry exitListEntry : - exitListEntries.get(fingerprint)) { - entry.addExitAddress(exitListEntry.getExitAddress()); - } - } - if (!entry.getExitAddresses().isEmpty()) { - sb.append(",\n"exit_addresses":["); - int written = 0; - for (String exitAddress : entry.getExitAddresses()) { - sb.append((written++ > 0 ? "," : "") + """ - + exitAddress.toLowerCase() + """); - } - sb.append("]"); - } - - /* Add descriptor parts. */ - if (descriptorParts != null) { - sb.append(",\n" + descriptorParts); - } else { - sb.append("\n}\n"); - } - - /* Write details file to disk. */ - String detailsLines = sb.toString(); - this.documentStore.store(detailsLines, DocumentType.OUT_DETAILS, - fingerprint); - } - } - - private void updateBridgeDetailsFiles( - SortedSet<String> remainingDetailsFiles) { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - for (Map.Entry<String, Node> bridge : this.bridges.entrySet()) { - String fingerprint = bridge.getKey(); - - /* Read details file for this bridge if it exists. */ - String descriptorParts = null, bridgePoolAssignment = null; - long publishedMillis = -1L; - if (remainingDetailsFiles.contains(fingerprint)) { - remainingDetailsFiles.remove(fingerprint); - String documentString = this.documentStore.retrieve( - DocumentType.OUT_DETAILS, fingerprint); - if (documentString != null) { - try { - boolean copyDescriptorParts = false; - StringBuilder sb = new StringBuilder(); - Scanner s = new Scanner(documentString); - while (s.hasNextLine()) { - String line = s.nextLine(); - if (line.startsWith(""desc_published":")) { - String published = line.substring( - ""desc_published":"".length(), - ""desc_published":"1970-01-01 00:00:00".length()); - publishedMillis = dateTimeFormat.parse(published). - getTime(); - copyDescriptorParts = true; - } else if (line.startsWith(""pool_assignment":")) { - bridgePoolAssignment = line; - copyDescriptorParts = false; - } else if (line.equals("}")) { - copyDescriptorParts = false; - } - if (copyDescriptorParts) { - sb.append(line + "\n"); - } - } - s.close(); - descriptorParts = sb.toString(); - if (descriptorParts.endsWith(",\n")) { - descriptorParts = descriptorParts.substring(0, - descriptorParts.length() - 2); - } else if (descriptorParts.endsWith("\n")) { - descriptorParts = descriptorParts.substring(0, - descriptorParts.length() - 1); - } - } catch (ParseException e) { - System.err.println("Could not parse timestamp in " - + "details.json file for '" + fingerprint + "'. " - + "Ignoring."); - e.printStackTrace(); - publishedMillis = -1L; - descriptorParts = null; - } - } - } - - /* Generate new descriptor-specific part if we have a more recent - * descriptor. */ - if (this.bridgeServerDescriptors.containsKey(fingerprint) && - this.bridgeServerDescriptors.get(fingerprint). - getPublishedMillis() > publishedMillis) { - ServerDescriptor descriptor = this.bridgeServerDescriptors.get( - fingerprint); - StringBuilder sb = new StringBuilder(); - String publishedDateTime = dateTimeFormat.format( - descriptor.getPublishedMillis()); - String lastRestartedString = dateTimeFormat.format( - descriptor.getPublishedMillis() - - descriptor.getUptime() * 1000L); - int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(), - Math.min(descriptor.getBandwidthBurst(), - descriptor.getBandwidthObserved())); - sb.append(""desc_published":"" + publishedDateTime + "",\n" - + ""last_restarted":"" + lastRestartedString + "",\n" - + ""advertised_bandwidth":" + advertisedBandwidth + ",\n" - + ""platform":"" + escapeJSON(descriptor.getPlatform()) - + """); - descriptorParts = sb.toString(); - } - - /* Look up bridge pool assignment. */ - if (this.bridgePoolAssignments.containsKey(fingerprint)) { - bridgePoolAssignment = ""pool_assignment":"" - + this.bridgePoolAssignments.get(fingerprint) + """; - } - - /* Generate network-status-specific part. */ - Node entry = bridge.getValue(); - String nickname = entry.getNickname(); - String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); - String firstSeen = dateTimeFormat.format( - entry.getFirstSeenMillis()); - String running = entry.getRunning() ? "true" : "false"; - String address = entry.getAddress(); - List<String> orAddresses = new ArrayList<String>(); - orAddresses.add(address + ":" + entry.getOrPort()); - orAddresses.addAll(entry.getOrAddressesAndPorts()); - StringBuilder orAddressesAndPortsBuilder = new StringBuilder(); - int addressesWritten = 0; - for (String orAddress : orAddresses) { - orAddressesAndPortsBuilder.append( - (addressesWritten++ > 0 ? "," : "") + """ - + orAddress.toLowerCase() + """); - } - StringBuilder sb = new StringBuilder(); - sb.append("{"version":1,\n" - + ""nickname":"" + nickname + "",\n" - + ""hashed_fingerprint":"" + fingerprint + "",\n" - + ""or_addresses":[" + orAddressesAndPortsBuilder.toString() - + "],\n"last_seen":"" + lastSeen + "",\n"first_seen":"" - + firstSeen + "",\n"running":" + running); - - SortedSet<String> relayFlags = entry.getRelayFlags(); - if (!relayFlags.isEmpty()) { - sb.append(",\n"flags":["); - int written = 0; - for (String relayFlag : relayFlags) { - sb.append((written++ > 0 ? "," : "") + """ + relayFlag + """); - } - sb.append("]"); - } - - /* Append descriptor and bridge pool assignment parts. */ - if (descriptorParts != null && descriptorParts.length() != 0) { - sb.append(",\n" + descriptorParts); - } - if (bridgePoolAssignment != null) { - sb.append(",\n" + bridgePoolAssignment); - } - sb.append("\n}\n"); - - /* Write details file to disk. */ - String detailsLines = sb.toString(); - this.documentStore.store(detailsLines, DocumentType.OUT_DETAILS, - fingerprint); - } - } - - private void deleteDetailsFiles( - SortedSet<String> remainingDetailsFiles) { - for (String fingerprint : remainingDetailsFiles) { - this.documentStore.remove(DocumentType.OUT_DETAILS, fingerprint); - } - } -} - diff --git a/src/org/torproject/onionoo/DetailsDataWriter.java b/src/org/torproject/onionoo/DetailsDataWriter.java new file mode 100644 index 0000000..3d49169 --- /dev/null +++ b/src/org/torproject/onionoo/DetailsDataWriter.java @@ -0,0 +1,821 @@ +/* Copyright 2011, 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Scanner; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.commons.lang.StringEscapeUtils; + +import org.torproject.descriptor.BridgePoolAssignment; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.ExitList; +import org.torproject.descriptor.ExitListEntry; +import org.torproject.descriptor.ServerDescriptor; + +/* Write updated detail data files to disk and delete files of relays or + * bridges that fell out of the summary list. + * + * The parts of details files coming from server descriptors always come + * from the last known descriptor of a relay or bridge, not from the + * descriptor that was last referenced in a network status. */ +public class DetailsDataWriter { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private SortedMap<String, NodeStatus> relays; + + private SortedMap<String, NodeStatus> bridges; + + public DetailsDataWriter(DescriptorSource descriptorSource, + DocumentStore documentStore) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + } + + public void setCurrentNodes( + SortedMap<String, NodeStatus> currentNodes) { + this.relays = new TreeMap<String, NodeStatus>(); + this.bridges = new TreeMap<String, NodeStatus>(); + for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) { + if (e.getValue().isRelay()) { + this.relays.put(e.getKey(), e.getValue()); + } else { + this.bridges.put(e.getKey(), e.getValue()); + } + } + } + + private static final long RDNS_LOOKUP_MAX_REQUEST_MILLIS = 10L * 1000L; + private static final long RDNS_LOOKUP_MAX_DURATION_MILLIS = 5L * 60L + * 1000L; + private static final long RDNS_LOOKUP_MAX_AGE_MILLIS = 12L * 60L * 60L + * 1000L; + private static final int RDNS_LOOKUP_WORKERS_NUM = 5; + private Set<String> rdnsLookupJobs; + private Map<String, String> rdnsLookupResults; + private long startedRdnsLookups; + private List<RdnsLookupWorker> rdnsLookupWorkers; + public void startReverseDomainNameLookups() { + this.startedRdnsLookups = System.currentTimeMillis(); + this.rdnsLookupJobs = new HashSet<String>(); + for (NodeStatus relay : relays.values()) { + if (relay.getLastRdnsLookup() < this.startedRdnsLookups + - RDNS_LOOKUP_MAX_AGE_MILLIS) { + this.rdnsLookupJobs.add(relay.getAddress()); + } + } + this.rdnsLookupResults = new HashMap<String, String>(); + this.rdnsLookupWorkers = new ArrayList<RdnsLookupWorker>(); + for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) { + RdnsLookupWorker rdnsLookupWorker = new RdnsLookupWorker(); + this.rdnsLookupWorkers.add(rdnsLookupWorker); + rdnsLookupWorker.setDaemon(true); + rdnsLookupWorker.start(); + } + } + + public void finishReverseDomainNameLookups() { + for (RdnsLookupWorker rdnsLookupWorker : this.rdnsLookupWorkers) { + try { + rdnsLookupWorker.join(); + } catch (InterruptedException e) { + /* This is not something that we can take care of. Just leave the + * worker thread alone. */ + } + } + synchronized (this.rdnsLookupResults) { + for (NodeStatus relay : relays.values()) { + if (this.rdnsLookupResults.containsKey(relay.getAddress())) { + relay.setHostName(this.rdnsLookupResults.get( + relay.getAddress())); + relay.setLastRdnsLookup(this.startedRdnsLookups); + } + } + } + } + + private class RdnsLookupWorker extends Thread { + public void run() { + while (System.currentTimeMillis() - RDNS_LOOKUP_MAX_DURATION_MILLIS + <= startedRdnsLookups) { + String rdnsLookupJob = null; + synchronized (rdnsLookupJobs) { + for (String job : rdnsLookupJobs) { + rdnsLookupJob = job; + rdnsLookupJobs.remove(job); + break; + } + } + if (rdnsLookupJob == null) { + break; + } + RdnsLookupRequest request = new RdnsLookupRequest(this, + rdnsLookupJob); + request.setDaemon(true); + request.start(); + try { + Thread.sleep(RDNS_LOOKUP_MAX_REQUEST_MILLIS); + } catch (InterruptedException e) { + /* Getting interrupted should be the default case. */ + } + String hostName = request.getHostName(); + if (hostName != null) { + synchronized (rdnsLookupResults) { + rdnsLookupResults.put(rdnsLookupJob, hostName); + } + } + } + } + } + + private class RdnsLookupRequest extends Thread { + RdnsLookupWorker parent; + String address, hostName; + public RdnsLookupRequest(RdnsLookupWorker parent, String address) { + this.parent = parent; + this.address = address; + } + public void run() { + try { + String result = InetAddress.getByName(this.address).getHostName(); + synchronized (this) { + this.hostName = result; + } + } catch (UnknownHostException e) { + /* We'll try again the next time. */ + } + this.parent.interrupt(); + } + public synchronized String getHostName() { + return hostName; + } + } + + private Map<String, ServerDescriptor> relayServerDescriptors = + new HashMap<String, ServerDescriptor>(); + public void readRelayServerDescriptors() { + /* Don't remember which server descriptors we already parsed. If we + * parse a server descriptor now and first learn about the relay in a + * later consensus, we'll never write the descriptor content anywhere. + * The result would be details files containing no descriptor parts + * until the relay publishes the next descriptor. */ + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.RELAY_SERVER_DESCRIPTORS); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof ServerDescriptor) { + ServerDescriptor serverDescriptor = (ServerDescriptor) descriptor; + String fingerprint = serverDescriptor.getFingerprint(); + if (!this.relayServerDescriptors.containsKey(fingerprint) || + this.relayServerDescriptors.get(fingerprint). + getPublishedMillis() + < serverDescriptor.getPublishedMillis()) { + this.relayServerDescriptors.put(fingerprint, + serverDescriptor); + } + } + } + } + + public void calculatePathSelectionProbabilities( + SortedMap<String, Integer> bandwidthWeights) { + boolean consensusContainsBandwidthWeights = false; + double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0, + wmd = 0.0, wee = 0.0, wed = 0.0; + if (bandwidthWeights != null) { + SortedSet<String> weightKeys = new TreeSet<String>(Arrays.asList( + "Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(","))); + weightKeys.removeAll(bandwidthWeights.keySet()); + if (weightKeys.isEmpty()) { + consensusContainsBandwidthWeights = true; + wgg = ((double) bandwidthWeights.get("Wgg")) / 10000.0; + wgd = ((double) bandwidthWeights.get("Wgd")) / 10000.0; + wmg = ((double) bandwidthWeights.get("Wmg")) / 10000.0; + wmm = ((double) bandwidthWeights.get("Wmm")) / 10000.0; + wme = ((double) bandwidthWeights.get("Wme")) / 10000.0; + wmd = ((double) bandwidthWeights.get("Wmd")) / 10000.0; + wee = ((double) bandwidthWeights.get("Wee")) / 10000.0; + wed = ((double) bandwidthWeights.get("Wed")) / 10000.0; + } + } else { + System.err.println("Could not determine most recent Wxx parameter " + + "values, probably because we didn't parse a consensus in " + + "this execution. All relays' guard/middle/exit weights are " + + "going to be 0.0."); + } + SortedMap<String, Double> + advertisedBandwidths = new TreeMap<String, Double>(), + consensusWeights = new TreeMap<String, Double>(), + guardWeights = new TreeMap<String, Double>(), + middleWeights = new TreeMap<String, Double>(), + exitWeights = new TreeMap<String, Double>(); + double totalAdvertisedBandwidth = 0.0; + double totalConsensusWeight = 0.0; + double totalGuardWeight = 0.0; + double totalMiddleWeight = 0.0; + double totalExitWeight = 0.0; + for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { + String fingerprint = e.getKey(); + NodeStatus relay = e.getValue(); + if (!relay.getRunning()) { + continue; + } + boolean isExit = relay.getRelayFlags().contains("Exit") && + !relay.getRelayFlags().contains("BadExit"); + boolean isGuard = relay.getRelayFlags().contains("Guard"); + if (this.relayServerDescriptors.containsKey(fingerprint)) { + ServerDescriptor serverDescriptor = + this.relayServerDescriptors.get(fingerprint); + double advertisedBandwidth = (double) Math.min(Math.min( + serverDescriptor.getBandwidthBurst(), + serverDescriptor.getBandwidthObserved()), + serverDescriptor.getBandwidthRate()); + advertisedBandwidths.put(fingerprint, advertisedBandwidth); + totalAdvertisedBandwidth += advertisedBandwidth; + } + double consensusWeight = (double) relay.getConsensusWeight(); + consensusWeights.put(fingerprint, consensusWeight); + totalConsensusWeight += consensusWeight; + if (consensusContainsBandwidthWeights) { + double guardWeight = consensusWeight, + middleWeight = consensusWeight, + exitWeight = consensusWeight; + if (isGuard && isExit) { + guardWeight *= wgd; + middleWeight *= wmd; + exitWeight *= wed; + } else if (isGuard) { + guardWeight *= wgg; + middleWeight *= wmg; + exitWeight = 0.0; + } else if (isExit) { + guardWeight = 0.0; + middleWeight *= wme; + exitWeight *= wee; + } else { + guardWeight = 0.0; + middleWeight *= wmm; + exitWeight = 0.0; + } + guardWeights.put(fingerprint, guardWeight); + middleWeights.put(fingerprint, middleWeight); + exitWeights.put(fingerprint, exitWeight); + totalGuardWeight += guardWeight; + totalMiddleWeight += middleWeight; + totalExitWeight += exitWeight; + } + } + for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { + String fingerprint = e.getKey(); + NodeStatus relay = e.getValue(); + if (advertisedBandwidths.containsKey(fingerprint)) { + relay.setAdvertisedBandwidthFraction(advertisedBandwidths.get( + fingerprint) / totalAdvertisedBandwidth); + } + if (consensusWeights.containsKey(fingerprint)) { + relay.setConsensusWeightFraction(consensusWeights.get(fingerprint) + / totalConsensusWeight); + } + if (guardWeights.containsKey(fingerprint)) { + relay.setGuardProbability(guardWeights.get(fingerprint) + / totalGuardWeight); + } + if (middleWeights.containsKey(fingerprint)) { + relay.setMiddleProbability(middleWeights.get(fingerprint) + / totalMiddleWeight); + } + if (exitWeights.containsKey(fingerprint)) { + relay.setExitProbability(exitWeights.get(fingerprint) + / totalExitWeight); + } + } + } + + private long now = System.currentTimeMillis(); + private Map<String, Set<ExitListEntry>> exitListEntries = + new HashMap<String, Set<ExitListEntry>>(); + public void readExitLists() { + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.EXIT_LISTS, DescriptorHistory.EXIT_LIST_HISTORY); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof ExitList) { + ExitList exitList = (ExitList) descriptor; + for (ExitListEntry exitListEntry : + exitList.getExitListEntries()) { + if (exitListEntry.getScanMillis() < + this.now - 24L * 60L * 60L * 1000L) { + continue; + } + String fingerprint = exitListEntry.getFingerprint(); + if (!this.exitListEntries.containsKey(fingerprint)) { + this.exitListEntries.put(fingerprint, + new HashSet<ExitListEntry>()); + } + this.exitListEntries.get(fingerprint).add(exitListEntry); + } + } + } + } + + private Map<String, ServerDescriptor> bridgeServerDescriptors = + new HashMap<String, ServerDescriptor>(); + public void readBridgeServerDescriptors() { + /* Don't remember which server descriptors we already parsed. If we + * parse a server descriptor now and first learn about the relay in a + * later status, we'll never write the descriptor content anywhere. + * The result would be details files containing no descriptor parts + * until the bridge publishes the next descriptor. */ + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.BRIDGE_SERVER_DESCRIPTORS); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof ServerDescriptor) { + ServerDescriptor serverDescriptor = (ServerDescriptor) descriptor; + String fingerprint = serverDescriptor.getFingerprint(); + if (!this.bridgeServerDescriptors.containsKey(fingerprint) || + this.bridgeServerDescriptors.get(fingerprint). + getPublishedMillis() + < serverDescriptor.getPublishedMillis()) { + this.bridgeServerDescriptors.put(fingerprint, + serverDescriptor); + } + } + } + } + + private Map<String, String> bridgePoolAssignments = + new HashMap<String, String>(); + public void readBridgePoolAssignments() { + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.BRIDGE_POOL_ASSIGNMENTS, + DescriptorHistory.BRIDGE_POOLASSIGN_HISTORY); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof BridgePoolAssignment) { + BridgePoolAssignment bridgePoolAssignment = + (BridgePoolAssignment) descriptor; + for (Map.Entry<String, String> e : + bridgePoolAssignment.getEntries().entrySet()) { + String fingerprint = e.getKey(); + String details = e.getValue(); + this.bridgePoolAssignments.put(fingerprint, details); + } + } + } + } + + public void writeOutDetails() { + SortedSet<String> remainingDetailsFiles = new TreeSet<String>(); + remainingDetailsFiles.addAll(this.documentStore.list( + DetailsDocument.class, false)); + this.updateRelayDetailsFiles(remainingDetailsFiles); + this.updateBridgeDetailsFiles(remainingDetailsFiles); + this.deleteDetailsFiles(remainingDetailsFiles); + } + + private static String escapeJSON(String s) { + return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\'", "'"); + } + + private void updateRelayDetailsFiles( + SortedSet<String> remainingDetailsFiles) { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + for (Map.Entry<String, NodeStatus> relay : this.relays.entrySet()) { + String fingerprint = relay.getKey(); + + /* Read details file for this relay if it exists. */ + String descriptorParts = null; + long publishedMillis = -1L; + if (remainingDetailsFiles.contains(fingerprint)) { + remainingDetailsFiles.remove(fingerprint); + // TODO Use parsed details document here. + DetailsDocument detailsDocument = this.documentStore.retrieve( + DetailsDocument.class, false, fingerprint); + String documentString = detailsDocument.documentString; + if (documentString != null) { + try { + boolean copyDescriptorParts = false; + StringBuilder sb = new StringBuilder(); + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + if (line.startsWith(""desc_published":")) { + String published = line.substring( + ""desc_published":"".length(), + ""desc_published":"1970-01-01 00:00:00".length()); + publishedMillis = dateTimeFormat.parse(published). + getTime(); + copyDescriptorParts = true; + } + if (copyDescriptorParts) { + sb.append(line + "\n"); + } + } + s.close(); + if (sb.length() > 0) { + descriptorParts = sb.toString(); + } + } catch (ParseException e) { + System.err.println("Could not parse timestamp in details.json " + + "file for '" + fingerprint + "'. Ignoring."); + e.printStackTrace(); + publishedMillis = -1L; + descriptorParts = null; + } + } + } + + /* Generate new descriptor-specific part if we have a more recent + * descriptor or if the part we read didn't contain a last_restarted + * line. */ + if (this.relayServerDescriptors.containsKey(fingerprint) && + (this.relayServerDescriptors.get(fingerprint). + getPublishedMillis() > publishedMillis)) { + ServerDescriptor descriptor = this.relayServerDescriptors.get( + fingerprint); + StringBuilder sb = new StringBuilder(); + String publishedDateTime = dateTimeFormat.format( + descriptor.getPublishedMillis()); + String lastRestartedString = dateTimeFormat.format( + descriptor.getPublishedMillis() + - descriptor.getUptime() * 1000L); + int bandwidthRate = descriptor.getBandwidthRate(); + int bandwidthBurst = descriptor.getBandwidthBurst(); + int observedBandwidth = descriptor.getBandwidthObserved(); + int advertisedBandwidth = Math.min(bandwidthRate, + Math.min(bandwidthBurst, observedBandwidth)); + sb.append(""desc_published":"" + publishedDateTime + "",\n" + + ""last_restarted":"" + lastRestartedString + "",\n" + + ""bandwidth_rate":" + bandwidthRate + ",\n" + + ""bandwidth_burst":" + bandwidthBurst + ",\n" + + ""observed_bandwidth":" + observedBandwidth + ",\n" + + ""advertised_bandwidth":" + advertisedBandwidth + ",\n" + + ""exit_policy":["); + int written = 0; + for (String exitPolicyLine : descriptor.getExitPolicyLines()) { + sb.append((written++ > 0 ? "," : "") + "\n "" + exitPolicyLine + + """); + } + sb.append("\n]"); + if (descriptor.getContact() != null) { + sb.append(",\n"contact":"" + + escapeJSON(descriptor.getContact()) + """); + } + if (descriptor.getPlatform() != null) { + sb.append(",\n"platform":"" + + escapeJSON(descriptor.getPlatform()) + """); + } + if (descriptor.getFamilyEntries() != null) { + sb.append(",\n"family":["); + written = 0; + for (String familyEntry : descriptor.getFamilyEntries()) { + sb.append((written++ > 0 ? "," : "") + "\n "" + familyEntry + + """); + } + sb.append("\n]"); + } + sb.append("\n}\n"); + descriptorParts = sb.toString(); + } + + /* Generate network-status-specific part. */ + NodeStatus entry = relay.getValue(); + String nickname = entry.getNickname(); + String address = entry.getAddress(); + List<String> orAddresses = new ArrayList<String>(); + orAddresses.add(address + ":" + entry.getOrPort()); + orAddresses.addAll(entry.getOrAddressesAndPorts()); + StringBuilder orAddressesAndPortsBuilder = new StringBuilder(); + int addressesWritten = 0; + for (String orAddress : orAddresses) { + orAddressesAndPortsBuilder.append( + (addressesWritten++ > 0 ? "," : "") + """ + + orAddress.toLowerCase() + """); + } + String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); + String firstSeen = dateTimeFormat.format( + entry.getFirstSeenMillis()); + String lastChangedOrAddress = dateTimeFormat.format( + entry.getLastChangedOrAddress()); + String running = entry.getRunning() ? "true" : "false"; + int dirPort = entry.getDirPort(); + String countryCode = entry.getCountryCode(); + String latitude = entry.getLatitude(); + String longitude = entry.getLongitude(); + String countryName = entry.getCountryName(); + String regionName = entry.getRegionName(); + String cityName = entry.getCityName(); + String aSNumber = entry.getASNumber(); + String aSName = entry.getASName(); + long consensusWeight = entry.getConsensusWeight(); + String hostName = entry.getHostName(); + double advertisedBandwidthFraction = + entry.getAdvertisedBandwidthFraction(); + double consensusWeightFraction = entry.getConsensusWeightFraction(); + double guardProbability = entry.getGuardProbability(); + double middleProbability = entry.getMiddleProbability(); + double exitProbability = entry.getExitProbability(); + String defaultPolicy = entry.getDefaultPolicy(); + String portList = entry.getPortList(); + StringBuilder sb = new StringBuilder(); + sb.append("{"version":1,\n" + + ""nickname":"" + nickname + "",\n" + + ""fingerprint":"" + fingerprint + "",\n" + + ""or_addresses":[" + orAddressesAndPortsBuilder.toString() + + "]"); + if (dirPort != 0) { + sb.append(",\n"dir_address":"" + address + ":" + dirPort + + """); + } + sb.append(",\n"last_seen":"" + lastSeen + """); + sb.append(",\n"first_seen":"" + firstSeen + """); + sb.append(",\n"last_changed_address_or_port":"" + + lastChangedOrAddress + """); + sb.append(",\n"running":" + running); + SortedSet<String> relayFlags = entry.getRelayFlags(); + if (!relayFlags.isEmpty()) { + sb.append(",\n"flags":["); + int written = 0; + for (String relayFlag : relayFlags) { + sb.append((written++ > 0 ? "," : "") + """ + relayFlag + """); + } + sb.append("]"); + } + if (countryCode != null) { + sb.append(",\n"country":"" + countryCode + """); + } + if (latitude != null) { + sb.append(",\n"latitude":" + latitude); + } + if (longitude != null) { + sb.append(",\n"longitude":" + longitude); + } + if (countryName != null) { + sb.append(",\n"country_name":"" + + escapeJSON(countryName) + """); + } + if (regionName != null) { + sb.append(",\n"region_name":"" + + escapeJSON(regionName) + """); + } + if (cityName != null) { + sb.append(",\n"city_name":"" + + escapeJSON(cityName) + """); + } + if (aSNumber != null) { + sb.append(",\n"as_number":"" + + escapeJSON(aSNumber) + """); + } + if (aSName != null) { + sb.append(",\n"as_name":"" + + escapeJSON(aSName) + """); + } + if (consensusWeight >= 0L) { + sb.append(",\n"consensus_weight":" + + String.valueOf(consensusWeight)); + } + if (hostName != null) { + sb.append(",\n"host_name":"" + + escapeJSON(hostName) + """); + } + if (advertisedBandwidthFraction >= 0.0) { + sb.append(String.format( + ",\n"advertised_bandwidth_fraction":%.9f", + advertisedBandwidthFraction)); + } + if (consensusWeightFraction >= 0.0) { + sb.append(String.format(",\n"consensus_weight_fraction":%.9f", + consensusWeightFraction)); + } + if (guardProbability >= 0.0) { + sb.append(String.format(",\n"guard_probability":%.9f", + guardProbability)); + } + if (middleProbability >= 0.0) { + sb.append(String.format(",\n"middle_probability":%.9f", + middleProbability)); + } + if (exitProbability >= 0.0) { + sb.append(String.format(",\n"exit_probability":%.9f", + exitProbability)); + } + if (defaultPolicy != null && (defaultPolicy.equals("accept") || + defaultPolicy.equals("reject")) && portList != null) { + sb.append(",\n"exit_policy_summary":{"" + defaultPolicy + + "":["); + int portsWritten = 0; + for (String portOrPortRange : portList.split(",")) { + sb.append((portsWritten++ > 0 ? "," : "") + + """ + portOrPortRange + """); + } + sb.append("]}"); + } + + /* Add exit addresses if at least one of them is distinct from the + * onion-routing addresses. */ + if (exitListEntries.containsKey(fingerprint)) { + for (ExitListEntry exitListEntry : + exitListEntries.get(fingerprint)) { + entry.addExitAddress(exitListEntry.getExitAddress()); + } + } + if (!entry.getExitAddresses().isEmpty()) { + sb.append(",\n"exit_addresses":["); + int written = 0; + for (String exitAddress : entry.getExitAddresses()) { + sb.append((written++ > 0 ? "," : "") + """ + + exitAddress.toLowerCase() + """); + } + sb.append("]"); + } + + /* Add descriptor parts. */ + if (descriptorParts != null) { + sb.append(",\n" + descriptorParts); + } else { + sb.append("\n}\n"); + } + + /* Write details file to disk. */ + DetailsDocument detailsDocument = new DetailsDocument(); + detailsDocument.documentString = sb.toString(); + this.documentStore.store(detailsDocument, fingerprint); + } + } + + private void updateBridgeDetailsFiles( + SortedSet<String> remainingDetailsFiles) { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + for (Map.Entry<String, NodeStatus> bridge : this.bridges.entrySet()) { + String fingerprint = bridge.getKey(); + + /* Read details file for this bridge if it exists. */ + String descriptorParts = null, bridgePoolAssignment = null; + long publishedMillis = -1L; + if (remainingDetailsFiles.contains(fingerprint)) { + remainingDetailsFiles.remove(fingerprint); + // TODO Use parsed details document here. + DetailsDocument detailsDocument = this.documentStore.retrieve( + DetailsDocument.class, false, fingerprint); + String documentString = detailsDocument.documentString; + if (documentString != null) { + try { + boolean copyDescriptorParts = false; + StringBuilder sb = new StringBuilder(); + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + if (line.startsWith(""desc_published":")) { + String published = line.substring( + ""desc_published":"".length(), + ""desc_published":"1970-01-01 00:00:00".length()); + publishedMillis = dateTimeFormat.parse(published). + getTime(); + copyDescriptorParts = true; + } else if (line.startsWith(""pool_assignment":")) { + bridgePoolAssignment = line; + copyDescriptorParts = false; + } else if (line.equals("}")) { + copyDescriptorParts = false; + } + if (copyDescriptorParts) { + sb.append(line + "\n"); + } + } + s.close(); + descriptorParts = sb.toString(); + if (descriptorParts.endsWith(",\n")) { + descriptorParts = descriptorParts.substring(0, + descriptorParts.length() - 2); + } else if (descriptorParts.endsWith("\n")) { + descriptorParts = descriptorParts.substring(0, + descriptorParts.length() - 1); + } + } catch (ParseException e) { + System.err.println("Could not parse timestamp in " + + "details.json file for '" + fingerprint + "'. " + + "Ignoring."); + e.printStackTrace(); + publishedMillis = -1L; + descriptorParts = null; + } + } + } + + /* Generate new descriptor-specific part if we have a more recent + * descriptor. */ + if (this.bridgeServerDescriptors.containsKey(fingerprint) && + this.bridgeServerDescriptors.get(fingerprint). + getPublishedMillis() > publishedMillis) { + ServerDescriptor descriptor = this.bridgeServerDescriptors.get( + fingerprint); + StringBuilder sb = new StringBuilder(); + String publishedDateTime = dateTimeFormat.format( + descriptor.getPublishedMillis()); + String lastRestartedString = dateTimeFormat.format( + descriptor.getPublishedMillis() + - descriptor.getUptime() * 1000L); + int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(), + Math.min(descriptor.getBandwidthBurst(), + descriptor.getBandwidthObserved())); + sb.append(""desc_published":"" + publishedDateTime + "",\n" + + ""last_restarted":"" + lastRestartedString + "",\n" + + ""advertised_bandwidth":" + advertisedBandwidth + ",\n" + + ""platform":"" + escapeJSON(descriptor.getPlatform()) + + """); + descriptorParts = sb.toString(); + } + + /* Look up bridge pool assignment. */ + if (this.bridgePoolAssignments.containsKey(fingerprint)) { + bridgePoolAssignment = ""pool_assignment":"" + + this.bridgePoolAssignments.get(fingerprint) + """; + } + + /* Generate network-status-specific part. */ + NodeStatus entry = bridge.getValue(); + String nickname = entry.getNickname(); + String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); + String firstSeen = dateTimeFormat.format( + entry.getFirstSeenMillis()); + String running = entry.getRunning() ? "true" : "false"; + String address = entry.getAddress(); + List<String> orAddresses = new ArrayList<String>(); + orAddresses.add(address + ":" + entry.getOrPort()); + orAddresses.addAll(entry.getOrAddressesAndPorts()); + StringBuilder orAddressesAndPortsBuilder = new StringBuilder(); + int addressesWritten = 0; + for (String orAddress : orAddresses) { + orAddressesAndPortsBuilder.append( + (addressesWritten++ > 0 ? "," : "") + """ + + orAddress.toLowerCase() + """); + } + StringBuilder sb = new StringBuilder(); + sb.append("{"version":1,\n" + + ""nickname":"" + nickname + "",\n" + + ""hashed_fingerprint":"" + fingerprint + "",\n" + + ""or_addresses":[" + orAddressesAndPortsBuilder.toString() + + "],\n"last_seen":"" + lastSeen + "",\n"first_seen":"" + + firstSeen + "",\n"running":" + running); + + SortedSet<String> relayFlags = entry.getRelayFlags(); + if (!relayFlags.isEmpty()) { + sb.append(",\n"flags":["); + int written = 0; + for (String relayFlag : relayFlags) { + sb.append((written++ > 0 ? "," : "") + """ + relayFlag + """); + } + sb.append("]"); + } + + /* Append descriptor and bridge pool assignment parts. */ + if (descriptorParts != null && descriptorParts.length() != 0) { + sb.append(",\n" + descriptorParts); + } + if (bridgePoolAssignment != null) { + sb.append(",\n" + bridgePoolAssignment); + } + sb.append("\n}\n"); + + /* Write details file to disk. */ + DetailsDocument detailsDocument = new DetailsDocument(); + detailsDocument.documentString = sb.toString(); + this.documentStore.store(detailsDocument, fingerprint); + } + } + + private void deleteDetailsFiles( + SortedSet<String> remainingDetailsFiles) { + for (String fingerprint : remainingDetailsFiles) { + this.documentStore.remove(DetailsDocument.class, fingerprint); + } + } +} + diff --git a/src/org/torproject/onionoo/DetailsDocument.java b/src/org/torproject/onionoo/DetailsDocument.java new file mode 100644 index 0000000..9b81bbd --- /dev/null +++ b/src/org/torproject/onionoo/DetailsDocument.java @@ -0,0 +1,71 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.util.List; + +class DetailsDocument extends Document { + + class ExitPolicySummary { + List<String> reject; + List<String> accept; + } + + class RelayDetails { + String nickname; + String fingerprint; + List<String> or_addresses; + List<String> exit_addresses; + String dir_address; + String last_seen; + String last_changed_address_or_port; + String first_seen; + Boolean running; + List<String> flags; + String country; + String country_name; + String region_name; + String city_name; + Double latitude; + Double longitude; + String as_number; + String as_name; + Double consensus_weight; + String host_name; + String last_restarted; + Integer bandwidth_rate; + Integer bandwidth_burst; + Integer observed_bandwidth; + Integer advertised_bandwidth; + List<String> exit_policy; + ExitPolicySummary exit_policy_summary; + String contact; + String platform; + List<String> family; + Double advertised_bandwidth_fraction; + Double consensus_weight_fraction; + Double guard_probability; + Double middle_probability; + Double exit_probability; + } + + class BridgeDetails { + String nickname; + String hashed_fingerprint; + List<String> or_addresses; + String last_seen; + String first_seen; + Boolean running; + List<String> flags; + String last_restarted; + Integer advertised_bandwidth; + String platform; + String pool_assignment; + } + + String relays_published; + List<RelayDetails> relays; + String bridges_published; + List<BridgeDetails> bridges; +} + diff --git a/src/org/torproject/onionoo/Document.java b/src/org/torproject/onionoo/Document.java new file mode 100644 index 0000000..9730d66 --- /dev/null +++ b/src/org/torproject/onionoo/Document.java @@ -0,0 +1,8 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +abstract class Document { + transient String documentString; +} + diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index c1097a0..829f35f 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -1,6 +1,5 @@ /* Copyright 2013 The Tor Project * See LICENSE for licensing information */ - package org.torproject.onionoo;

import java.io.BufferedReader; @@ -10,20 +9,15 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.util.Arrays; +import java.util.Map; +import java.util.SortedMap; import java.util.SortedSet; import java.util.Stack; +import java.util.TreeMap; import java.util.TreeSet;

-enum DocumentType { - STATUS_SUMMARY, - STATUS_BANDWIDTH, - STATUS_WEIGHTS, - OUT_UPDATE, - OUT_SUMMARY, - OUT_DETAILS, - OUT_BANDWIDTH, - OUT_WEIGHTS; -} +import com.google.gson.Gson; +import com.google.gson.JsonParseException;

// TODO For later migration from disk to database, do the following: // - read from database and then from disk if not found @@ -36,10 +30,19 @@ public class DocumentStore {

private File outDir;

+ boolean listedArchivedNodeStatuses = false, + listedCurrentNodeStatuses = false; + long listOperations = 0L, listedFiles = 0L, storedFiles = 0L, storedBytes = 0L, retrievedFiles = 0L, retrievedBytes = 0L, removedFiles = 0L;

+ /* Node statuses are cached in memory, as opposed to all other document + * types. This cache is initialized when listing NodeStatus documents, + * either including or excluding archived node statuses. Later retrieve + * operations depend on which NodeStatus documents were listed. */ + private SortedMap<String, NodeStatus> cachedNodeStatuses; + public DocumentStore(File outDir) { this.outDir = outDir; } @@ -49,32 +52,75 @@ public class DocumentStore { this.outDir = outDir; }

- public SortedSet<String> list(DocumentType documentType) { + public <T extends Document> SortedSet<String> list( + Class<T> documentType, boolean includeArchive) { + if (documentType.equals(NodeStatus.class)) { + return this.listNodeStatuses(includeArchive); + } else { + return this.listDocumentFiles(documentType); + } + } + + private SortedSet<String> listNodeStatuses(boolean includeArchive) { + SortedMap<String, NodeStatus> parsedNodeStatuses = + new TreeMap<String, NodeStatus>(); + File directory = includeArchive ? this.statusDir : this.outDir; + if (directory != null) { + File summaryFile = new File(directory, "summary"); + if (summaryFile.exists()) { + try { + BufferedReader br = new BufferedReader(new FileReader( + summaryFile)); + String line; + while ((line = br.readLine()) != null) { + if (line.length() == 0) { + continue; + } + NodeStatus node = NodeStatus.fromString(line); + if (node != null) { + parsedNodeStatuses.put(node.getFingerprint(), node); + } + } + br.close(); + this.listedFiles += parsedNodeStatuses.size(); + this.listOperations++; + } catch (IOException e) { + System.err.println("Could not read file '" + + summaryFile.getAbsolutePath() + "'."); + e.printStackTrace(); + return null; + } + } + } + if (includeArchive) { + this.listedArchivedNodeStatuses = true; + } else { + this.listedCurrentNodeStatuses = true; + } + this.cachedNodeStatuses = parsedNodeStatuses; + return new TreeSet<String>(this.cachedNodeStatuses.keySet()); + } + + private <T extends Document> SortedSet<String> listDocumentFiles( + Class<T> documentType) { SortedSet<String> fingerprints = new TreeSet<String>(); File directory = null; String subdirectory = null; - switch (documentType) { - case STATUS_BANDWIDTH: + if (documentType.equals(BandwidthStatus.class)) { directory = this.statusDir; subdirectory = "bandwidth"; - break; - case STATUS_WEIGHTS: + } else if (documentType.equals(WeightsStatus.class)) { directory = this.statusDir; subdirectory = "weights"; - break; - case OUT_DETAILS: + } else if (documentType.equals(DetailsDocument.class)) { directory = this.outDir; subdirectory = "details"; - break; - case OUT_BANDWIDTH: + } else if (documentType.equals(BandwidthDocument.class)) { directory = this.outDir; subdirectory = "bandwidth"; - break; - case OUT_WEIGHTS: + } else if (documentType.equals(WeightsDocument.class)) { directory = this.outDir; - break; - default: - break; + subdirectory = "weights"; } if (directory != null && subdirectory != null) { Stack<File> files = new Stack<File>(); @@ -93,16 +139,45 @@ public class DocumentStore { return fingerprints; }

- public boolean store(String documentString, DocumentType documentType) { - return this.store(documentString, documentType, null); + public <T extends Document> boolean store(T document) { + return this.store(document, null); }

- public boolean store(String documentString, DocumentType documentType, + public <T extends Document> boolean store(T document, String fingerprint) { - File documentFile = this.getDocumentFile(documentType, fingerprint); + if (document instanceof NodeStatus) { + return this.storeNodeStatus((NodeStatus) document, fingerprint); + } else { + return this.storeDocumentFile(document, fingerprint); + } + } + + private <T extends Document> boolean storeNodeStatus( + NodeStatus nodeStatus, String fingerprint) { + this.cachedNodeStatuses.put(fingerprint, nodeStatus); + return true; + } + + private <T extends Document> boolean storeDocumentFile(T document, + String fingerprint) { + File documentFile = this.getDocumentFile(document.getClass(), + fingerprint); if (documentFile == null) { return false; } + String documentString; + if (document.documentString != null) { + documentString = document.documentString; + } else if (document instanceof DetailsDocument || + document instanceof BandwidthDocument || + document instanceof WeightsDocument) { + Gson gson = new Gson(); + documentString = gson.toJson(this); + } else { + System.err.println("Serializing is not supported for type " + + document.getClass().getName() + "."); + return false; + } try { documentFile.getParentFile().mkdirs(); File documentTempFile = new File( @@ -124,11 +199,26 @@ public class DocumentStore { return true; }

- public String retrieve(DocumentType documentType) { - return this.retrieve(documentType, null); + public <T extends Document> T retrieve(Class<T> documentType, + boolean parse) { + return this.retrieve(documentType, parse, null); + } + + public <T extends Document> T retrieve(Class<T> documentType, + boolean parse, String fingerprint) { + if (documentType.equals(NodeStatus.class)) { + return documentType.cast(this.retrieveNodeStatus(fingerprint)); + } else { + return this.retrieveDocumentFile(documentType, parse, fingerprint); + } + } + + private NodeStatus retrieveNodeStatus(String fingerprint) { + return this.cachedNodeStatuses.get(fingerprint); }

- public String retrieve(DocumentType documentType, String fingerprint) { + private <T extends Document> T retrieveDocumentFile( + Class<T> documentType, boolean parse, String fingerprint) { File documentFile = this.getDocumentFile(documentType, fingerprint); if (documentFile == null || !documentFile.exists()) { return null; @@ -138,6 +228,7 @@ public class DocumentStore { + "directory."); return null; } + String documentString = null; try { BufferedReader br = new BufferedReader(new FileReader( documentFile)); @@ -149,20 +240,82 @@ public class DocumentStore { br.close(); this.retrievedFiles++; this.retrievedBytes += sb.length(); - return sb.toString(); + documentString = sb.toString(); } catch (IOException e) { System.err.println("Could not read file '" + documentFile.getAbsolutePath() + "'."); e.printStackTrace(); return null; } + T result = null; + if (!parse) { + return this.retrieveUnparsedDocumentFile(documentType, + documentString); + } else if (documentType.equals(DetailsDocument.class) || + documentType.equals(BandwidthDocument.class) || + documentType.equals(WeightsDocument.class)) { + return this.retrieveParsedDocumentFile(documentType, + documentString); + } else { + System.err.println("Parsing is not supported for type " + + documentType.getName() + "."); + } + return result; + } + + private <T extends Document> T retrieveParsedDocumentFile( + Class<T> documentType, String documentString) { + T result = null; + Gson gson = new Gson(); + try { + result = gson.fromJson(documentString, documentType); + result.documentString = documentString; + } catch (JsonParseException e) { + /* Handle below. */ + e.printStackTrace(); + } + if (result == null) { + System.err.println("Could not initialize parsed document of type " + + documentType.getName() + "."); + } + return result; + } + + private <T extends Document> T retrieveUnparsedDocumentFile( + Class<T> documentType, String documentString) { + T result = null; + try { + result = documentType.newInstance(); + result.documentString = documentString; + } catch (InstantiationException e) { + /* Handle below. */ + e.printStackTrace(); + } catch (IllegalAccessException e) { + /* Handle below. */ + e.printStackTrace(); + } + if (result == null) { + System.err.println("Could not initialize unparsed document of type " + + documentType.getName() + "."); + } + return result; }

- public boolean remove(DocumentType documentType) { + public <T extends Document> boolean remove(Class<T> documentType) { return this.remove(documentType, null); }

- public boolean remove(DocumentType documentType, String fingerprint) { + public <T extends Document> boolean remove(Class<T> documentType, + String fingerprint) { + if (documentType.equals(NodeStatus.class)) { + return this.cachedNodeStatuses.remove(fingerprint) != null; + } else { + return this.removeDocumentFile(documentType, fingerprint); + } + } + + private <T extends Document> boolean removeDocumentFile( + Class<T> documentType, String fingerprint) { File documentFile = this.getDocumentFile(documentType, fingerprint); if (documentFile == null || !documentFile.delete()) { System.err.println("Could not delete file '" @@ -173,54 +326,46 @@ public class DocumentStore { return true; }

- private File getDocumentFile(DocumentType documentType, + private <T extends Document> File getDocumentFile(Class<T> documentType, String fingerprint) { File documentFile = null; - if (fingerprint == null && !( - documentType == DocumentType.STATUS_SUMMARY || - documentType == DocumentType.OUT_UPDATE|| - documentType == DocumentType.OUT_SUMMARY)) { + if (fingerprint == null && + !documentType.equals(UpdateStatus.class)) { + // TODO Instead of using the update file workaround, add new method + // lastModified(Class<T> documentType) that serves a similar + // purpose. Once that's implemented, make fingerprint mandatory for + // all methods. return null; } File directory = null; String fileName = null; - switch (documentType) { - case STATUS_SUMMARY: - directory = this.statusDir; - fileName = "summary"; - break; - case STATUS_BANDWIDTH: + if (documentType.equals(BandwidthStatus.class)) { directory = this.statusDir; fileName = String.format("bandwidth/%s/%s/%s", fingerprint.substring(0, 1), fingerprint.substring(1, 2), fingerprint); - break; - case STATUS_WEIGHTS: + } else if (documentType.equals(WeightsStatus.class)) { directory = this.statusDir; fileName = String.format("weights/%s/%s/%s", fingerprint.substring(0, 1), fingerprint.substring(1, 2), fingerprint); - break; - case OUT_UPDATE: + } else if (documentType.equals(UpdateStatus.class)) { directory = this.outDir; fileName = "update"; - break; - case OUT_SUMMARY: - directory = this.outDir; - fileName = "summary"; - break; - case OUT_DETAILS: + // TODO Taken out, because storing/retrieving summary documents is not + // supported yet. + //} else if (documentType.equals(SummaryDocument.class)) { + // directory = this.outDir; + // fileName = "summary"; + } else if (documentType.equals(DetailsDocument.class)) { directory = this.outDir; fileName = String.format("details/%s", fingerprint); - break; - case OUT_BANDWIDTH: + } else if (documentType.equals(BandwidthDocument.class)) { directory = this.outDir; fileName = String.format("bandwidth/%s", fingerprint); - break; - case OUT_WEIGHTS: + } else if (documentType.equals(WeightsDocument.class)) { directory = this.outDir; fileName = String.format("weights/%s", fingerprint); - break; } if (directory != null && fileName != null) { documentFile = new File(directory, fileName); @@ -228,6 +373,99 @@ public class DocumentStore { return documentFile; }

+ public void flushDocumentCache() { + if (this.listedArchivedNodeStatuses) { + this.writeNodeStatuses(false); + this.writeNodeStatuses(true); + this.writeUpdateStatus(); + } else if (this.listedCurrentNodeStatuses) { + this.writeNodeStatuses(false); + this.writeUpdateStatus(); + } + } + + private void writeNodeStatuses(boolean includeArchive) { + File directory = includeArchive ? this.statusDir : this.outDir; + if (directory == null) { + return; + } + File summaryFile = new File(directory, "summary"); + SortedMap<String, NodeStatus> + cachedRelays = new TreeMap<String, NodeStatus>(), + cachedBridges = new TreeMap<String, NodeStatus>(); + long cutoff = 0L; + if (!includeArchive) { + long maxLastSeenMillis = 0L; + for (NodeStatus node : this.cachedNodeStatuses.values()) { + if (node.getLastSeenMillis() > maxLastSeenMillis) { + maxLastSeenMillis = node.getLastSeenMillis(); + } + } + cutoff = maxLastSeenMillis - 7L * 24L * 60L * 60L * 1000L; + } + for (Map.Entry<String, NodeStatus> e : + this.cachedNodeStatuses.entrySet()) { + if (e.getValue().getLastSeenMillis() < cutoff) { + continue; + } + if (e.getValue().isRelay()) { + cachedRelays.put(e.getKey(), e.getValue()); + } else { + cachedBridges.put(e.getKey(), e.getValue()); + } + } + StringBuilder sb = new StringBuilder(); + for (NodeStatus relay : cachedRelays.values()) { + String line = relay.toString(); + if (line != null) { + sb.append(line + "\n"); + } else { + System.err.println("Could not serialize relay node status '" + + relay.getFingerprint() + "'"); + } + } + for (NodeStatus bridge : cachedBridges.values()) { + String line = bridge.toString(); + if (line != null) { + sb.append(line + "\n"); + } else { + System.err.println("Could not serialize bridge node status '" + + bridge.getFingerprint() + "'"); + } + } + String documentString = sb.toString(); + try { + BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); + bw.write(documentString); + bw.close(); + this.storedFiles++; + this.storedBytes += documentString.length(); + } catch (IOException e) { + System.err.println("Could not write file '" + + summaryFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } + } + + private void writeUpdateStatus() { + if (this.outDir == null) { + return; + } + File updateFile = new File(this.outDir, "update"); + String documentString = String.valueOf(System.currentTimeMillis()); + try { + BufferedWriter bw = new BufferedWriter(new FileWriter(updateFile)); + bw.write(documentString); + bw.close(); + this.storedFiles++; + this.storedBytes += documentString.length(); + } catch (IOException e) { + System.err.println("Could not write file '" + + updateFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } + } + public String getStatsString() { StringBuilder sb = new StringBuilder(); sb.append(" " + formatDecimalNumber(listOperations) diff --git a/src/org/torproject/onionoo/LookupService.java b/src/org/torproject/onionoo/LookupService.java index bf3131e..4071e26 100644 --- a/src/org/torproject/onionoo/LookupService.java +++ b/src/org/torproject/onionoo/LookupService.java @@ -1,6 +1,5 @@ /* Copyright 2013 The Tor Project * See LICENSE for licensing information */ - package org.torproject.onionoo;

import java.io.BufferedReader; diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 3086054..2232d81 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -4,6 +4,7 @@ package org.torproject.onionoo;

import java.io.File; import java.util.Date; +import java.util.SortedMap;

/* Update search data and status data files. */ public class Main { @@ -24,34 +25,37 @@ public class Main { printStatusTime("Initialized Geoip lookup service");

printStatus("Updating internal node list."); - CurrentNodes cn = new CurrentNodes(dso, ls, ds); - cn.readStatusSummary(); + NodeDataWriter ndw = new NodeDataWriter(dso, ls, ds); + ndw.readStatusSummary(); printStatusTime("Read status summary"); - cn.readRelayNetworkConsensuses(); + ndw.readRelayNetworkConsensuses(); printStatusTime("Read network status consensuses"); - cn.setRelayRunningBits(); - printStatusTime("Set relay running bits"); - cn.lookUpCitiesAndASes(); + ndw.lookUpCitiesAndASes(); printStatusTime("Looked up cities and ASes"); - cn.readBridgeNetworkStatuses(); + ndw.readBridgeNetworkStatuses(); printStatusTime("Read bridge network statuses"); - cn.setBridgeRunningBits(); - printStatusTime("Set bridge running bits"); - cn.writeStatusSummary(); + ndw.setRunningBits(); + printStatusTime("Set running bits"); + ndw.writeStatusSummary(); printStatusTime("Wrote status summary"); + SortedMap<String, NodeStatus> currentNodes = ndw.getCurrentNodes(); + SortedMap<String, Integer> lastBandwidthWeights = + ndw.getLastBandwidthWeights(); // TODO Could write statistics here, too.

printStatus("Updating detail data."); - DetailDataWriter ddw = new DetailDataWriter(dso, ds); - ddw.setCurrentRelays(cn.getCurrentRelays()); - printStatusTime("Set current relays"); - ddw.setCurrentBridges(cn.getCurrentBridges()); - printStatusTime("Set current bridges"); + DetailsDataWriter ddw = new DetailsDataWriter(dso, ds); + // TODO Instead of using ndw's currentNodes and lastBandwidthWeights, + // parse statuses once again, keeping separate parse history. Allows + // us to run ndw and ddw in parallel in the future. Alternatively, + // merge ndw and ddw, because they're doing similar things anyway. + ddw.setCurrentNodes(currentNodes); + printStatusTime("Set current node fingerprints"); ddw.startReverseDomainNameLookups(); printStatusTime("Started reverse domain name lookups"); ddw.readRelayServerDescriptors(); printStatusTime("Read relay server descriptors"); - ddw.calculatePathSelectionProbabilities(cn.getLastBandwidthWeights()); + ddw.calculatePathSelectionProbabilities(lastBandwidthWeights); printStatusTime("Calculated path selection probabilities"); ddw.readExitLists(); printStatusTime("Read exit lists"); @@ -67,35 +71,43 @@ public class Main {

printStatus("Updating bandwidth data."); BandwidthDataWriter bdw = new BandwidthDataWriter(dso, ds); - bdw.setCurrentRelays(cn.getCurrentRelays()); - printStatusTime("Set current relays"); - bdw.setCurrentBridges(cn.getCurrentBridges()); - printStatusTime("Set current bridges"); + bdw.setCurrentNodes(currentNodes); + printStatusTime("Set current node fingerprints"); bdw.readExtraInfoDescriptors(); printStatusTime("Read extra-info descriptors"); + // TODO Evaluate overhead of not deleting obsolete bandwidth files. + // An advantage would be that we don't need ndw's currentNodes + // anymore, which allows us to run ndw and bdw in parallel in the + // future. bdw.deleteObsoleteBandwidthFiles(); printStatusTime("Deleted obsolete bandwidth files"); // TODO Could write statistics here, too.

printStatus("Updating weights data."); WeightsDataWriter wdw = new WeightsDataWriter(dso, ds); - wdw.setCurrentRelays(cn.getCurrentRelays()); - printStatusTime("Set current relays"); + wdw.setCurrentNodes(currentNodes); + printStatusTime("Set current node fingerprints"); wdw.readRelayServerDescriptors(); printStatusTime("Read relay server descriptors"); wdw.readRelayNetworkConsensuses(); printStatusTime("Read relay network consensuses"); wdw.writeWeightsDataFiles(); printStatusTime("Wrote weights data files"); + // TODO Evaluate overhead of not deleting obsolete weights files. An + // advantage would be that we don't need ndw's currentNodes anymore, + // which allows us to run ndw and wdw in parallel in the future. wdw.deleteObsoleteWeightsDataFiles(); printStatusTime("Deleted obsolete weights files"); // TODO Could write statistics here, too.

printStatus("Updating summary data."); - cn.writeOutSummary(); + ndw.writeOutSummary(); printStatusTime("Wrote out summary"); // TODO Could write statistics here, too.

+ // TODO "Shut down" lookup service and write statistics about number + // of (successfully) looked up addresses. + printStatus("Shutting down descriptor source."); dso.writeHistoryFiles(); printStatusTime("Wrote parse histories"); @@ -103,6 +115,8 @@ public class Main { printStatusTime("Shut down descriptor source");

printStatus("Shutting down document store."); + ds.flushDocumentCache(); + printStatusTime("Flushed document cache"); printStatistics(ds.getStatsString()); printStatusTime("Shut down document store");

diff --git a/src/org/torproject/onionoo/Node.java b/src/org/torproject/onionoo/Node.java deleted file mode 100644 index 35d81c3..0000000 --- a/src/org/torproject/onionoo/Node.java +++ /dev/null @@ -1,280 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Map; -import java.util.Set; -import java.util.SortedSet; -import java.util.SortedMap; -import java.util.TreeSet; -import java.util.TreeMap; - -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; - -/* Store search data of a single relay that was running in the past seven - * days. */ -public class Node { - private String fingerprint; - private String hashedFingerprint; - private String nickname; - private String address; - private SortedSet<String> orAddresses; - private SortedSet<String> orAddressesAndPorts; - private SortedSet<String> exitAddresses; - private String latitude; - private String longitude; - private String countryCode; - private String countryName; - private String regionName; - private String cityName; - private String aSName; - private String aSNumber; - private long firstSeenMillis; - private long lastSeenMillis; - private int orPort; - private int dirPort; - private SortedSet<String> relayFlags; - private long consensusWeight; - private boolean running; - private String hostName; - private long lastRdnsLookup = -1L; - private double advertisedBandwidthFraction = -1.0; - private double consensusWeightFraction = -1.0; - private double guardProbability = -1.0; - private double middleProbability = -1.0; - private double exitProbability = -1.0; - private String defaultPolicy; - private String portList; - private SortedMap<Long, Set<String>> lastAddresses; - public Node(String nickname, String fingerprint, String address, - SortedSet<String> orAddressesAndPorts, - SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, - int dirPort, SortedSet<String> relayFlags, long consensusWeight, - String countryCode, String hostName, long lastRdnsLookup, - String defaultPolicy, String portList, long firstSeenMillis, - SortedMap<Long, Set<String>> lastAddresses, String aSNumber) { - this.nickname = nickname; - this.fingerprint = fingerprint; - try { - this.hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex( - fingerprint.toCharArray())).toUpperCase(); - } catch (DecoderException e) { - throw new IllegalArgumentException("Fingerprint '" + fingerprint - + "' is not a valid fingerprint."); - } - this.address = address; - this.exitAddresses = new TreeSet<String>(); - if (exitAddresses != null) { - this.exitAddresses.addAll(exitAddresses); - } - this.exitAddresses.remove(this.address); - this.orAddresses = new TreeSet<String>(); - this.orAddressesAndPorts = new TreeSet<String>(); - if (orAddressesAndPorts != null) { - for (String orAddressAndPort : orAddressesAndPorts) { - this.addOrAddressAndPort(orAddressAndPort); - } - } - this.lastSeenMillis = lastSeenMillis; - this.orPort = orPort; - this.dirPort = dirPort; - this.relayFlags = relayFlags; - this.consensusWeight = consensusWeight; - this.countryCode = countryCode; - this.hostName = hostName; - this.lastRdnsLookup = lastRdnsLookup; - this.defaultPolicy = defaultPolicy; - this.portList = portList; - this.firstSeenMillis = firstSeenMillis; - this.lastAddresses = lastAddresses; - this.aSNumber = aSNumber; - } - public String getFingerprint() { - return this.fingerprint; - } - public String getHashedFingerprint() { - return this.hashedFingerprint; - } - public String getNickname() { - return this.nickname; - } - public String getAddress() { - return this.address; - } - public SortedSet<String> getOrAddresses() { - return new TreeSet<String>(this.orAddresses); - } - public void addOrAddressAndPort(String orAddressAndPort) { - if (!orAddressAndPort.contains(":")) { - System.err.println("Illegal OR address:port '" + orAddressAndPort - + "'. Exiting."); - System.exit(1); - } else if (orAddressAndPort.length() > 0) { - String orAddress = orAddressAndPort.substring(0, - orAddressAndPort.lastIndexOf(":")); - if (this.exitAddresses.contains(orAddress)) { - this.exitAddresses.remove(orAddress); - } - this.orAddresses.add(orAddress); - this.orAddressesAndPorts.add(orAddressAndPort); - } - } - public SortedSet<String> getOrAddressesAndPorts() { - return new TreeSet<String>(this.orAddressesAndPorts); - } - public void addExitAddress(String exitAddress) { - if (exitAddress.length() > 0 && !this.address.equals(exitAddress) && - !this.orAddresses.contains(exitAddress)) { - this.exitAddresses.add(exitAddress); - } - } - public SortedSet<String> getExitAddresses() { - return new TreeSet<String>(this.exitAddresses); - } - public void setLatitude(String latitude) { - this.latitude = latitude; - } - public String getLatitude() { - return this.latitude; - } - public void setLongitude(String longitude) { - this.longitude = longitude; - } - public String getLongitude() { - return this.longitude; - } - public void setCountryCode(String countryCode) { - this.countryCode = countryCode; - } - public String getCountryCode() { - return this.countryCode; - } - public void setCountryName(String countryName) { - this.countryName = countryName; - } - public String getCountryName() { - return this.countryName; - } - public void setRegionName(String regionName) { - this.regionName = regionName; - } - public String getRegionName() { - return this.regionName; - } - public void setCityName(String cityName) { - this.cityName = cityName; - } - public String getCityName() { - return this.cityName; - } - public void setASNumber(String aSNumber) { - this.aSNumber = aSNumber; - } - public String getASNumber() { - return this.aSNumber; - } - public void setASName(String aSName) { - this.aSName = aSName; - } - public String getASName() { - return this.aSName; - } - public long getFirstSeenMillis() { - return this.firstSeenMillis; - } - public long getLastSeenMillis() { - return this.lastSeenMillis; - } - public int getOrPort() { - return this.orPort; - } - public int getDirPort() { - return this.dirPort; - } - public SortedSet<String> getRelayFlags() { - return this.relayFlags; - } - public long getConsensusWeight() { - return this.consensusWeight; - } - public void setRunning(boolean running) { - this.running = running; - } - public boolean getRunning() { - return this.running; - } - public void setHostName(String hostName) { - this.hostName = hostName; - } - public String getHostName() { - return this.hostName; - } - public void setLastRdnsLookup(long lastRdnsLookup) { - this.lastRdnsLookup = lastRdnsLookup; - } - public long getLastRdnsLookup() { - return this.lastRdnsLookup; - } - public void setAdvertisedBandwidthFraction( - double advertisedBandwidthFraction) { - this.advertisedBandwidthFraction = advertisedBandwidthFraction; - } - public double getAdvertisedBandwidthFraction() { - return this.advertisedBandwidthFraction; - } - public void setConsensusWeightFraction(double consensusWeightFraction) { - this.consensusWeightFraction = consensusWeightFraction; - } - public double getConsensusWeightFraction() { - return this.consensusWeightFraction; - } - public void setGuardProbability(double guardProbability) { - this.guardProbability = guardProbability; - } - public double getGuardProbability() { - return this.guardProbability; - } - public void setMiddleProbability(double middleProbability) { - this.middleProbability = middleProbability; - } - public double getMiddleProbability() { - return this.middleProbability; - } - public void setExitProbability(double exitProbability) { - this.exitProbability = exitProbability; - } - public double getExitProbability() { - return this.exitProbability; - } - public String getDefaultPolicy() { - return this.defaultPolicy; - } - public String getPortList() { - return this.portList; - } - public SortedMap<Long, Set<String>> getLastAddresses() { - return this.lastAddresses == null ? null : - new TreeMap<Long, Set<String>>(this.lastAddresses); - } - public long getLastChangedOrAddress() { - long lastChangedAddressesMillis = -1L; - if (this.lastAddresses != null) { - Set<String> lastAddresses = null; - for (Map.Entry<Long, Set<String>> e : this.lastAddresses.entrySet()) { - if (lastAddresses != null) { - for (String address : e.getValue()) { - if (!lastAddresses.contains(address)) { - return lastChangedAddressesMillis; - } - } - } - lastChangedAddressesMillis = e.getKey(); - lastAddresses = e.getValue(); - } - } - return lastChangedAddressesMillis; - } -} - diff --git a/src/org/torproject/onionoo/NodeDataWriter.java b/src/org/torproject/onionoo/NodeDataWriter.java new file mode 100644 index 0000000..9cfbb21 --- /dev/null +++ b/src/org/torproject/onionoo/NodeDataWriter.java @@ -0,0 +1,237 @@ +/* Copyright 2011, 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.torproject.descriptor.BridgeNetworkStatus; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.onionoo.LookupService.LookupResult; + +/* Store relays and bridges that have been running in the past seven + * days. */ +public class NodeDataWriter { + + private DescriptorSource descriptorSource; + + private LookupService lookupService; + + private DocumentStore documentStore; + + private SortedMap<String, NodeStatus> knownNodes = + new TreeMap<String, NodeStatus>(); + + private long relaysLastValidAfterMillis = -1L; + + private long bridgesLastPublishedMillis = -1L; + + private SortedMap<String, Integer> lastBandwidthWeights = null; + + public NodeDataWriter(DescriptorSource descriptorSource, + LookupService lookupService, DocumentStore documentStore) { + this.descriptorSource = descriptorSource; + this.lookupService = lookupService; + this.documentStore = documentStore; + } + + public void readStatusSummary() { + SortedSet<String> fingerprints = this.documentStore.list( + NodeStatus.class, true); + for (String fingerprint : fingerprints) { + NodeStatus node = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (node.isRelay()) { + this.relaysLastValidAfterMillis = Math.max( + this.relaysLastValidAfterMillis, node.getLastSeenMillis()); + } else { + this.bridgesLastPublishedMillis = Math.max( + this.bridgesLastPublishedMillis, node.getLastSeenMillis()); + } + this.knownNodes.put(fingerprint, node); + } + } + + public void readRelayNetworkConsensuses() { + if (this.descriptorSource == null) { + System.err.println("Not configured to read relay network " + + "consensuses."); + return; + } + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.RELAY_CONSENSUSES, + DescriptorHistory.RELAY_CONSENSUS_HISTORY); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + updateRelayNetworkStatusConsensus( + (RelayNetworkStatusConsensus) descriptor); + } + } + } + + public void setRunningBits() { + for (NodeStatus node : this.knownNodes.values()) { + if (node.isRelay() && + node.getLastSeenMillis() == this.relaysLastValidAfterMillis) { + node.setRunning(true); + } + if (!node.isRelay() && + node.getLastSeenMillis() == this.bridgesLastPublishedMillis) { + node.setRunning(true); + } + } + } + + private void updateRelayNetworkStatusConsensus( + RelayNetworkStatusConsensus consensus) { + long validAfterMillis = consensus.getValidAfterMillis(); + if (validAfterMillis > this.relaysLastValidAfterMillis) { + this.relaysLastValidAfterMillis = validAfterMillis; + } + for (NetworkStatusEntry entry : + consensus.getStatusEntries().values()) { + String nickname = entry.getNickname(); + String fingerprint = entry.getFingerprint(); + String address = entry.getAddress(); + SortedSet<String> orAddressesAndPorts = new TreeSet<String>( + entry.getOrAddresses()); + int orPort = entry.getOrPort(); + int dirPort = entry.getDirPort(); + SortedSet<String> relayFlags = entry.getFlags(); + long consensusWeight = entry.getBandwidth(); + String defaultPolicy = entry.getDefaultPolicy(); + String portList = entry.getPortList(); + NodeStatus newNodeStatus = new NodeStatus(true, nickname, + fingerprint, address, orAddressesAndPorts, null, + validAfterMillis, orPort, dirPort, relayFlags, consensusWeight, + null, null, -1L, defaultPolicy, portList, validAfterMillis, + validAfterMillis, null); + if (this.knownNodes.containsKey(fingerprint)) { + this.knownNodes.get(fingerprint).update(newNodeStatus); + } else { + this.knownNodes.put(fingerprint, newNodeStatus); + } + } + if (this.relaysLastValidAfterMillis == validAfterMillis) { + this.lastBandwidthWeights = consensus.getBandwidthWeights(); + } + } + + public void lookUpCitiesAndASes() { + SortedSet<String> addressStrings = new TreeSet<String>(); + for (NodeStatus node : this.knownNodes.values()) { + if (node.isRelay()) { + addressStrings.add(node.getAddress()); + } + } + if (addressStrings.isEmpty()) { + System.err.println("No relay IP addresses to resolve to cities or " + + "ASN."); + return; + } + SortedMap<String, LookupResult> lookupResults = + this.lookupService.lookup(addressStrings); + for (NodeStatus node : knownNodes.values()) { + if (!node.isRelay()) { + continue; + } + String addressString = node.getAddress(); + if (lookupResults.containsKey(addressString)) { + LookupResult lookupResult = lookupResults.get(addressString); + node.setCountryCode(lookupResult.countryCode); + node.setCountryName(lookupResult.countryName); + node.setRegionName(lookupResult.regionName); + node.setCityName(lookupResult.cityName); + node.setLatitude(lookupResult.latitude); + node.setLongitude(lookupResult.longitude); + node.setASNumber(lookupResult.aSNumber); + node.setASName(lookupResult.aSName); + } + } + } + + public void readBridgeNetworkStatuses() { + if (this.descriptorSource == null) { + System.err.println("Not configured to read bridge network " + + "statuses."); + return; + } + DescriptorQueue descriptorQueue = + this.descriptorSource.getDescriptorQueue( + DescriptorType.BRIDGE_STATUSES, + DescriptorHistory.BRIDGE_STATUS_HISTORY); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + if (descriptor instanceof BridgeNetworkStatus) { + updateBridgeNetworkStatus((BridgeNetworkStatus) descriptor); + } + } + } + + private void updateBridgeNetworkStatus(BridgeNetworkStatus status) { + long publishedMillis = status.getPublishedMillis(); + if (publishedMillis > this.bridgesLastPublishedMillis) { + this.bridgesLastPublishedMillis = publishedMillis; + } + for (NetworkStatusEntry entry : status.getStatusEntries().values()) { + String nickname = entry.getNickname(); + String fingerprint = entry.getFingerprint(); + String address = entry.getAddress(); + SortedSet<String> orAddressesAndPorts = new TreeSet<String>( + entry.getOrAddresses()); + int orPort = entry.getOrPort(); + int dirPort = entry.getDirPort(); + SortedSet<String> relayFlags = entry.getFlags(); + NodeStatus newNodeStatus = new NodeStatus(false, nickname, + fingerprint, address, orAddressesAndPorts, null, + publishedMillis, orPort, dirPort, relayFlags, -1L, "??", null, + -1L, null, null, publishedMillis, -1L, null); + if (this.knownNodes.containsKey(fingerprint)) { + this.knownNodes.get(fingerprint).update(newNodeStatus); + } else { + this.knownNodes.put(fingerprint, newNodeStatus); + } + } + } + + public void writeStatusSummary() { + this.writeSummary(true); + } + + public void writeOutSummary() { + this.writeSummary(false); + } + + private void writeSummary(boolean includeArchive) { + SortedMap<String, NodeStatus> nodes = includeArchive + ? this.knownNodes : this.getCurrentNodes(); + for (Map.Entry<String, NodeStatus> e : nodes.entrySet()) { + this.documentStore.store(e.getValue(), e.getKey()); + } + } + + public SortedMap<String, NodeStatus> getCurrentNodes() { + long cutoff = Math.max(this.relaysLastValidAfterMillis, + this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L; + SortedMap<String, NodeStatus> currentNodes = + new TreeMap<String, NodeStatus>(); + for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { + if (e.getValue().getLastSeenMillis() >= cutoff) { + currentNodes.put(e.getKey(), e.getValue()); + } + } + return currentNodes; + } + + public SortedMap<String, Integer> getLastBandwidthWeights() { + return this.lastBandwidthWeights; + } +} + diff --git a/src/org/torproject/onionoo/NodeStatus.java b/src/org/torproject/onionoo/NodeStatus.java new file mode 100644 index 0000000..81e5c7f --- /dev/null +++ b/src/org/torproject/onionoo/NodeStatus.java @@ -0,0 +1,482 @@ +/* Copyright 2011, 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.SortedMap; +import java.util.TimeZone; +import java.util.TreeSet; +import java.util.TreeMap; + +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; + +/* Store search data of a single relay that was running in the past seven + * days. */ +public class NodeStatus extends Document { + private boolean isRelay; + private String fingerprint; + private String hashedFingerprint; + private String nickname; + private String address; + private SortedSet<String> orAddresses; + private SortedSet<String> orAddressesAndPorts; + private SortedSet<String> exitAddresses; + private String latitude; + private String longitude; + private String countryCode; + private String countryName; + private String regionName; + private String cityName; + private String aSName; + private String aSNumber; + private long firstSeenMillis; + private long lastSeenMillis; + private int orPort; + private int dirPort; + private SortedSet<String> relayFlags; + private long consensusWeight; + private boolean running; + private String hostName; + private long lastRdnsLookup = -1L; + private double advertisedBandwidthFraction = -1.0; + private double consensusWeightFraction = -1.0; + private double guardProbability = -1.0; + private double middleProbability = -1.0; + private double exitProbability = -1.0; + private String defaultPolicy; + private String portList; + private SortedMap<Long, Set<String>> lastAddresses; + public NodeStatus(boolean isRelay, String nickname, String fingerprint, + String address, SortedSet<String> orAddressesAndPorts, + SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, + int dirPort, SortedSet<String> relayFlags, long consensusWeight, + String countryCode, String hostName, long lastRdnsLookup, + String defaultPolicy, String portList, long firstSeenMillis, + long lastChangedAddresses, String aSNumber) { + this.isRelay = isRelay; + this.nickname = nickname; + this.fingerprint = fingerprint; + try { + this.hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex( + this.fingerprint.toCharArray())).toUpperCase(); + } catch (DecoderException e) { + throw new IllegalArgumentException("Fingerprint '" + fingerprint + + "' is not a valid fingerprint."); + } + this.address = address; + this.exitAddresses = new TreeSet<String>(); + if (exitAddresses != null) { + this.exitAddresses.addAll(exitAddresses); + } + this.exitAddresses.remove(this.address); + this.orAddresses = new TreeSet<String>(); + this.orAddressesAndPorts = new TreeSet<String>(); + if (orAddressesAndPorts != null) { + for (String orAddressAndPort : orAddressesAndPorts) { + this.addOrAddressAndPort(orAddressAndPort); + } + } + this.lastSeenMillis = lastSeenMillis; + this.orPort = orPort; + this.dirPort = dirPort; + this.relayFlags = relayFlags; + this.consensusWeight = consensusWeight; + this.countryCode = countryCode; + this.hostName = hostName; + this.lastRdnsLookup = lastRdnsLookup; + this.defaultPolicy = defaultPolicy; + this.portList = portList; + this.firstSeenMillis = firstSeenMillis; + this.lastAddresses = + new TreeMap<Long, Set<String>>(Collections.reverseOrder()); + Set<String> addresses = new HashSet<String>(); + addresses.add(address + ":" + orPort); + if (dirPort > 0) { + addresses.add(address + ":" + dirPort); + } + addresses.addAll(orAddressesAndPorts); + this.lastAddresses.put(lastChangedAddresses, addresses); + this.aSNumber = aSNumber; + } + + public static NodeStatus fromString(String documentString) { + boolean isRelay = false; + String nickname = null, fingerprint = null, address = null, + countryCode = null, hostName = null, defaultPolicy = null, + portList = null, aSNumber = null; + SortedSet<String> orAddressesAndPorts = null, exitAddresses = null, + relayFlags = null; + long lastSeenMillis = -1L, consensusWeight = -1L, + lastRdnsLookup = -1L, firstSeenMillis = -1L, + lastChangedAddresses = -1L; + int orPort = -1, dirPort = -1; + try { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String[] parts = documentString.trim().split(" "); + isRelay = parts[0].equals("r"); + if (parts.length < 9) { + System.err.println("Too few space-separated values in line '" + + documentString.trim() + "'. Skipping."); + return null; + } + nickname = parts[1]; + fingerprint = parts[2]; + orAddressesAndPorts = new TreeSet<String>(); + exitAddresses = new TreeSet<String>(); + String addresses = parts[3]; + if (addresses.contains(";")) { + String[] addressParts = addresses.split(";", -1); + if (addressParts.length != 3) { + System.err.println("Invalid addresses entry in line '" + + documentString.trim() + "'. Skipping."); + return null; + } + address = addressParts[0]; + if (addressParts[1].length() > 0) { + orAddressesAndPorts.addAll(Arrays.asList( + addressParts[1].split("\+"))); + } + if (addressParts[2].length() > 0) { + exitAddresses.addAll(Arrays.asList( + addressParts[2].split("\+"))); + } + } else { + address = addresses; + } + lastSeenMillis = dateTimeFormat.parse(parts[4] + " " + parts[5]). + getTime(); + orPort = Integer.parseInt(parts[6]); + dirPort = Integer.parseInt(parts[7]); + relayFlags = new TreeSet<String>( + Arrays.asList(parts[8].split(","))); + if (parts.length > 9) { + consensusWeight = Long.parseLong(parts[9]); + } + if (parts.length > 10) { + countryCode = parts[10]; + } + if (parts.length > 12) { + hostName = parts[11].equals("null") ? null : parts[11]; + lastRdnsLookup = Long.parseLong(parts[12]); + } + if (parts.length > 14) { + if (!parts[13].equals("null")) { + defaultPolicy = parts[13]; + } + if (!parts[14].equals("null")) { + portList = parts[14]; + } + } + firstSeenMillis = lastSeenMillis; + if (parts.length > 16) { + firstSeenMillis = dateTimeFormat.parse(parts[15] + " " + + parts[16]).getTime(); + } + lastChangedAddresses = lastSeenMillis; + if (parts.length > 18 && !parts[17].equals("null")) { + lastChangedAddresses = dateTimeFormat.parse(parts[17] + " " + + parts[18]).getTime(); + } + if (parts.length > 19) { + aSNumber = parts[19]; + } + } catch (NumberFormatException e) { + System.err.println("Number format exception while parsing node " + + "status line '" + documentString + "': " + e.getMessage() + + ". Skipping."); + return null; + } catch (ParseException e) { + System.err.println("Parse exception while parsing node status " + + "line '" + documentString + "': " + e.getMessage() + ". " + + "Skipping."); + return null; + } catch (Exception e) { + /* This catch block is only here to handle yet unknown errors. It + * should go away once we're sure what kind of errors can occur. */ + System.err.println("Unknown exception while parsing node status " + + "line '" + documentString + "': " + e.getMessage() + ". " + + "Skipping."); + return null; + } + NodeStatus newNodeStatus = new NodeStatus(isRelay, nickname, + fingerprint, address, orAddressesAndPorts, exitAddresses, + lastSeenMillis, orPort, dirPort, relayFlags, consensusWeight, + countryCode, hostName, lastRdnsLookup, defaultPolicy, portList, + firstSeenMillis, lastChangedAddresses, aSNumber); + return newNodeStatus; + } + + public void update(NodeStatus newNodeStatus) { + if (newNodeStatus.lastSeenMillis > this.lastSeenMillis) { + this.nickname = newNodeStatus.nickname; + this.address = newNodeStatus.address; + this.orAddressesAndPorts = newNodeStatus.orAddressesAndPorts; + this.exitAddresses = newNodeStatus.exitAddresses; + this.lastSeenMillis = newNodeStatus.lastSeenMillis; + this.orPort = newNodeStatus.orPort; + this.dirPort = newNodeStatus.dirPort; + this.relayFlags = newNodeStatus.relayFlags; + this.consensusWeight = newNodeStatus.consensusWeight; + this.countryCode = newNodeStatus.countryCode; + this.defaultPolicy = newNodeStatus.defaultPolicy; + this.portList = newNodeStatus.portList; + this.aSNumber = newNodeStatus.aSNumber; + } + if (this.isRelay && newNodeStatus.isRelay) { + this.lastAddresses.putAll(newNodeStatus.lastAddresses); + } + this.firstSeenMillis = Math.min(newNodeStatus.firstSeenMillis, + this.getFirstSeenMillis()); + } + + public String toString() { + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + StringBuilder sb = new StringBuilder(); + sb.append(this.isRelay ? "r" : "b"); + sb.append(" " + this.nickname); + sb.append(" " + this.fingerprint); + sb.append(" " + this.address + ";"); + int written = 0; + for (String orAddressAndPort : this.orAddressesAndPorts) { + sb.append((written++ > 0 ? "+" : "") + orAddressAndPort); + } + sb.append(";"); + if (this.isRelay) { + written = 0; + for (String exitAddress : this.exitAddresses) { + sb.append((written++ > 0 ? "+" : "") + + exitAddress); + } + } + sb.append(" " + dateTimeFormat.format(this.lastSeenMillis)); + sb.append(" " + this.orPort); + sb.append(" " + this.dirPort + " "); + written = 0; + for (String relayFlag : this.relayFlags) { + sb.append((written++ > 0 ? "," : "") + relayFlag); + } + if (this.isRelay) { + sb.append(" " + String.valueOf(this.consensusWeight)); + sb.append(" " + (this.countryCode != null ? this.countryCode : "??")); + sb.append(" " + (this.hostName != null ? this.hostName : "null")); + sb.append(" " + String.valueOf(this.lastRdnsLookup)); + sb.append(" " + (this.defaultPolicy != null ? this.defaultPolicy + : "null")); + sb.append(" " + (this.portList != null ? this.portList : "null")); + } else { + sb.append(" -1 ?? null -1 null null"); + } + sb.append(" " + dateTimeFormat.format(this.firstSeenMillis)); + if (this.isRelay) { + sb.append(" " + dateTimeFormat.format( + this.getLastChangedOrAddress())); + sb.append(" " + (this.aSNumber != null ? this.aSNumber : "null")); + } else { + sb.append(" null null null"); + } + return sb.toString(); + } + + public boolean isRelay() { + return this.isRelay; + } + public String getFingerprint() { + return this.fingerprint; + } + public String getHashedFingerprint() { + return this.hashedFingerprint; + } + public String getNickname() { + return this.nickname; + } + public String getAddress() { + return this.address; + } + public SortedSet<String> getOrAddresses() { + return new TreeSet<String>(this.orAddresses); + } + public void addOrAddressAndPort(String orAddressAndPort) { + if (!orAddressAndPort.contains(":")) { + System.err.println("Illegal OR address:port '" + orAddressAndPort + + "'. Exiting."); + System.exit(1); + } else if (orAddressAndPort.length() > 0) { + String orAddress = orAddressAndPort.substring(0, + orAddressAndPort.lastIndexOf(":")); + if (this.exitAddresses.contains(orAddress)) { + this.exitAddresses.remove(orAddress); + } + this.orAddresses.add(orAddress); + this.orAddressesAndPorts.add(orAddressAndPort); + } + } + public SortedSet<String> getOrAddressesAndPorts() { + return new TreeSet<String>(this.orAddressesAndPorts); + } + public void addExitAddress(String exitAddress) { + if (exitAddress.length() > 0 && !this.address.equals(exitAddress) && + !this.orAddresses.contains(exitAddress)) { + this.exitAddresses.add(exitAddress); + } + } + public SortedSet<String> getExitAddresses() { + return new TreeSet<String>(this.exitAddresses); + } + public void setLatitude(String latitude) { + this.latitude = latitude; + } + public String getLatitude() { + return this.latitude; + } + public void setLongitude(String longitude) { + this.longitude = longitude; + } + public String getLongitude() { + return this.longitude; + } + public void setCountryCode(String countryCode) { + this.countryCode = countryCode; + } + public String getCountryCode() { + return this.countryCode; + } + public void setCountryName(String countryName) { + this.countryName = countryName; + } + public String getCountryName() { + return this.countryName; + } + public void setRegionName(String regionName) { + this.regionName = regionName; + } + public String getRegionName() { + return this.regionName; + } + public void setCityName(String cityName) { + this.cityName = cityName; + } + public String getCityName() { + return this.cityName; + } + public void setASNumber(String aSNumber) { + this.aSNumber = aSNumber; + } + public String getASNumber() { + return this.aSNumber; + } + public void setASName(String aSName) { + this.aSName = aSName; + } + public String getASName() { + return this.aSName; + } + public long getFirstSeenMillis() { + return this.firstSeenMillis; + } + public long getLastSeenMillis() { + return this.lastSeenMillis; + } + public int getOrPort() { + return this.orPort; + } + public int getDirPort() { + return this.dirPort; + } + public SortedSet<String> getRelayFlags() { + return this.relayFlags; + } + public long getConsensusWeight() { + return this.consensusWeight; + } + public void setRunning(boolean running) { + this.running = running; + } + public boolean getRunning() { + return this.running; + } + public void setHostName(String hostName) { + this.hostName = hostName; + } + public String getHostName() { + return this.hostName; + } + public void setLastRdnsLookup(long lastRdnsLookup) { + this.lastRdnsLookup = lastRdnsLookup; + } + public long getLastRdnsLookup() { + return this.lastRdnsLookup; + } + public void setAdvertisedBandwidthFraction( + double advertisedBandwidthFraction) { + this.advertisedBandwidthFraction = advertisedBandwidthFraction; + } + public double getAdvertisedBandwidthFraction() { + return this.advertisedBandwidthFraction; + } + public void setConsensusWeightFraction(double consensusWeightFraction) { + this.consensusWeightFraction = consensusWeightFraction; + } + public double getConsensusWeightFraction() { + return this.consensusWeightFraction; + } + public void setGuardProbability(double guardProbability) { + this.guardProbability = guardProbability; + } + public double getGuardProbability() { + return this.guardProbability; + } + public void setMiddleProbability(double middleProbability) { + this.middleProbability = middleProbability; + } + public double getMiddleProbability() { + return this.middleProbability; + } + public void setExitProbability(double exitProbability) { + this.exitProbability = exitProbability; + } + public double getExitProbability() { + return this.exitProbability; + } + public String getDefaultPolicy() { + return this.defaultPolicy; + } + public String getPortList() { + return this.portList; + } + public SortedMap<Long, Set<String>> getLastAddresses() { + return this.lastAddresses == null ? null : + new TreeMap<Long, Set<String>>(this.lastAddresses); + } + public long getLastChangedOrAddress() { + long lastChangedAddressesMillis = -1L; + if (this.lastAddresses != null) { + Set<String> lastAddresses = null; + for (Map.Entry<Long, Set<String>> e : this.lastAddresses.entrySet()) { + if (lastAddresses != null) { + for (String address : e.getValue()) { + if (!lastAddresses.contains(address)) { + return lastChangedAddressesMillis; + } + } + } + lastChangedAddressesMillis = e.getKey(); + lastAddresses = e.getValue(); + } + } + return lastChangedAddressesMillis; + } +} + diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java index 3ad33ae..996b870 100644 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ b/src/org/torproject/onionoo/ResourceServlet.java @@ -16,6 +16,7 @@ import java.util.Map; import java.util.Scanner; import java.util.Set; import java.util.SortedMap; +import java.util.SortedSet; import java.util.TimeZone; import java.util.TreeMap; import java.util.regex.Pattern; @@ -64,9 +65,10 @@ public class ResourceServlet extends HttpServlet { bridgesByLastSeenDays = null; private void readSummaryFile() { long summaryFileLastModified = -1L; - String updateString = this.documentStore.retrieve( - DocumentType.OUT_UPDATE); - if (updateString != null) { + UpdateStatus updateStatus = this.documentStore.retrieve( + UpdateStatus.class, false); + if (updateStatus != null && updateStatus.documentString != null) { + String updateString = updateStatus.documentString; try { summaryFileLastModified = Long.parseLong(updateString.trim()); } catch (NumberFormatException e) { @@ -94,25 +96,42 @@ public class ResourceServlet extends HttpServlet { bridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(), relaysByLastSeenDays = new TreeMap<Integer, Set<String>>(), bridgesByLastSeenDays = new TreeMap<Integer, Set<String>>(); - CurrentNodes cn = new CurrentNodes(this.documentStore); - cn.readOutSummary(); + long relaysLastValidAfterMillis = -1L, + bridgesLastPublishedMillis = -1L; + Set<NodeStatus> currentRelays = new HashSet<NodeStatus>(), + currentBridges = new HashSet<NodeStatus>(); + SortedSet<String> fingerprints = this.documentStore.list( + NodeStatus.class, false); // TODO We should be able to learn if something goes wrong when // reading the summary file, rather than silently having an empty - // CurrentNodes instance. - cn.setRelayRunningBits(); - cn.setBridgeRunningBits(); + // list of fingerprints. + for (String fingerprint : fingerprints) { + NodeStatus node = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (node.isRelay()) { + relaysLastValidAfterMillis = Math.max( + relaysLastValidAfterMillis, node.getLastSeenMillis()); + currentRelays.add(node); + } else { + bridgesLastPublishedMillis = Math.max( + bridgesLastPublishedMillis, node.getLastSeenMillis()); + currentBridges.add(node); + } + } SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); this.relaysPublishedString = dateTimeFormat.format( - cn.getLastValidAfterMillis()); + relaysLastValidAfterMillis); this.bridgesPublishedString = dateTimeFormat.format( - cn.getLastPublishedMillis()); + bridgesLastPublishedMillis); List<String> orderRelaysByConsensusWeight = new ArrayList<String>(); - for (Node entry : cn.getCurrentRelays().values()) { + for (NodeStatus entry : currentRelays) { String fingerprint = entry.getFingerprint().toUpperCase(); String hashedFingerprint = entry.getHashedFingerprint(). toUpperCase(); + entry.setRunning(entry.getLastSeenMillis() == + relaysLastValidAfterMillis); String line = this.formatRelaySummaryLine(entry); relayFingerprintSummaryLines.put(fingerprint, line); relayFingerprintSummaryLines.put(hashedFingerprint, line); @@ -169,10 +188,12 @@ public class ResourceServlet extends HttpServlet { for (String relay : orderRelaysByConsensusWeight) { relaysByConsensusWeight.add(relay.split(" ")[1]); } - for (Node entry : cn.getCurrentBridges().values()) { + for (NodeStatus entry : currentBridges) { String hashedFingerprint = entry.getFingerprint().toUpperCase(); String hashedHashedFingerprint = entry.getHashedFingerprint(). toUpperCase(); + entry.setRunning(entry.getRelayFlags().contains("Running") && + entry.getLastSeenMillis() == bridgesLastPublishedMillis); String line = this.formatBridgeSummaryLine(entry); bridgeFingerprintSummaryLines.put(hashedFingerprint, line); bridgeFingerprintSummaryLines.put(hashedHashedFingerprint, line); @@ -212,7 +233,7 @@ public class ResourceServlet extends HttpServlet { this.readSummaryFile = true; }

- private String formatRelaySummaryLine(Node entry) { + private String formatRelaySummaryLine(NodeStatus entry) { String nickname = !entry.getNickname().equals("Unnamed") ? entry.getNickname() : null; String fingerprint = entry.getFingerprint(); @@ -238,7 +259,7 @@ public class ResourceServlet extends HttpServlet { fingerprint, addressesBuilder.toString(), running); }

- private String formatBridgeSummaryLine(Node entry) { + private String formatBridgeSummaryLine(NodeStatus entry) { String nickname = !entry.getNickname().equals("Unnamed") ? entry.getNickname() : null; String hashedFingerprint = entry.getFingerprint(); @@ -921,8 +942,9 @@ public class ResourceServlet extends HttpServlet { return ""; } fingerprint = fingerprint.substring(0, 40); - String documentString = this.documentStore.retrieve( - DocumentType.OUT_DETAILS, fingerprint); + DetailsDocument detailsDocument = this.documentStore.retrieve( + DetailsDocument.class, false, fingerprint); + String documentString = detailsDocument.documentString; StringBuilder sb = new StringBuilder(); String detailsLines = null; if (documentString != null) { @@ -969,8 +991,9 @@ public class ResourceServlet extends HttpServlet { return ""; } fingerprint = fingerprint.substring(0, 40); - String bandwidthLines = this.documentStore.retrieve( - DocumentType.OUT_BANDWIDTH, fingerprint); + BandwidthDocument bandwidthDocument = this.documentStore.retrieve( + BandwidthDocument.class, false, fingerprint); + String bandwidthLines = bandwidthDocument.documentString; if (bandwidthLines != null) { bandwidthLines = bandwidthLines.substring(0, bandwidthLines.length() - 1); @@ -991,8 +1014,9 @@ public class ResourceServlet extends HttpServlet { return ""; } fingerprint = fingerprint.substring(0, 40); - String weightsLines = this.documentStore.retrieve( - DocumentType.OUT_WEIGHTS, fingerprint); + WeightsDocument weightsDocument = this.documentStore.retrieve( + WeightsDocument.class, false, fingerprint); + String weightsLines = weightsDocument.documentString; if (weightsLines != null) { weightsLines = weightsLines.substring(0, weightsLines.length() - 1); return weightsLines; diff --git a/src/org/torproject/onionoo/SummaryDocument.java b/src/org/torproject/onionoo/SummaryDocument.java new file mode 100644 index 0000000..e0aadb5 --- /dev/null +++ b/src/org/torproject/onionoo/SummaryDocument.java @@ -0,0 +1,27 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.util.List; + +class SummaryDocument extends Document { + + class RelaySummary { + String n; + String f; + String[] a; + Boolean r; + } + + class BridgeSummary { + String n; + String h; + Boolean r; + } + + String relays_published; + List<RelaySummary> relays; + String bridges_published; + List<BridgeSummary> bridges; +} + diff --git a/src/org/torproject/onionoo/UpdateStatus.java b/src/org/torproject/onionoo/UpdateStatus.java new file mode 100644 index 0000000..bd3648d --- /dev/null +++ b/src/org/torproject/onionoo/UpdateStatus.java @@ -0,0 +1,7 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +class UpdateStatus extends Document { +} + diff --git a/src/org/torproject/onionoo/WeightsDataWriter.java b/src/org/torproject/onionoo/WeightsDataWriter.java index de9ad42..317b8e2 100644 --- a/src/org/torproject/onionoo/WeightsDataWriter.java +++ b/src/org/torproject/onionoo/WeightsDataWriter.java @@ -29,15 +29,17 @@ public class WeightsDataWriter {

private DocumentStore documentStore;

+ private SortedSet<String> currentFingerprints = new TreeSet<String>(); + public WeightsDataWriter(DescriptorSource descriptorSource, DocumentStore documentStore) { this.descriptorSource = descriptorSource; this.documentStore = documentStore; }

- private SortedSet<String> currentFingerprints = new TreeSet<String>(); - public void setCurrentRelays(SortedMap<String, Node> currentRelays) { - this.currentFingerprints.addAll(currentRelays.keySet()); + public void setCurrentNodes( + SortedMap<String, NodeStatus> currentNodes) { + this.currentFingerprints.addAll(currentNodes.keySet()); }

/* Read advertised bandwidths of all server descriptors in @@ -265,9 +267,10 @@ public class WeightsDataWriter { return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0; } }); - String historyString = this.documentStore.retrieve( - DocumentType.STATUS_WEIGHTS, fingerprint); - if (historyString != null) { + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, false, fingerprint); + if (weightsStatus != null) { + String historyString = weightsStatus.documentString; SimpleDateFormat dateTimeFormat = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss"); dateTimeFormat.setLenient(false); @@ -378,9 +381,9 @@ public class WeightsDataWriter { } sb.append("\n"); } - String historyString = sb.toString(); - this.documentStore.store(historyString, DocumentType.STATUS_WEIGHTS, - fingerprint); + WeightsStatus weightsStatus = new WeightsStatus(); + weightsStatus.documentString = sb.toString(); + this.documentStore.store(weightsStatus, fingerprint); }

public void writeWeightsDataFiles() { @@ -392,10 +395,10 @@ public class WeightsDataWriter { /* Don't write weights data file to disk. */ continue; } - String historyString = this.formatHistoryString(fingerprint, - history); - this.documentStore.store(historyString, DocumentType.OUT_WEIGHTS, - fingerprint); + WeightsDocument weightsDocument = new WeightsDocument(); + weightsDocument.documentString = this.formatHistoryString( + fingerprint, history); + this.documentStore.store(weightsDocument, fingerprint); } }

@@ -550,15 +553,15 @@ public class WeightsDataWriter {

public void deleteObsoleteWeightsDataFiles() { SortedSet<String> obsoleteWeightsFiles; - obsoleteWeightsFiles = this.documentStore.list( - DocumentType.OUT_WEIGHTS); + obsoleteWeightsFiles = this.documentStore.list(WeightsDocument.class, + false); for (String fingerprint : this.currentFingerprints) { if (obsoleteWeightsFiles.contains(fingerprint)) { obsoleteWeightsFiles.remove(fingerprint); } } for (String fingerprint : obsoleteWeightsFiles) { - this.documentStore.remove(DocumentType.OUT_WEIGHTS, fingerprint); + this.documentStore.remove(WeightsDocument.class, fingerprint); } } } diff --git a/src/org/torproject/onionoo/WeightsDocument.java b/src/org/torproject/onionoo/WeightsDocument.java new file mode 100644 index 0000000..6739333 --- /dev/null +++ b/src/org/torproject/onionoo/WeightsDocument.java @@ -0,0 +1,31 @@ +package org.torproject.onionoo; + +import java.util.List; +import java.util.Map; + +class WeightsDocument extends Document { + + class WeightsHistory { + String first; + String last; + Integer interval; + Double factor; + Integer count; + List<Integer> values; + } + + class NodeWeights { + String fingerprint; + Map<String, WeightsHistory> advertised_bandwidth_fraction; + Map<String, WeightsHistory> consensus_weight_fraction; + Map<String, WeightsHistory> guard_probability; + Map<String, WeightsHistory> middle_probability; + Map<String, WeightsHistory> exit_probability; + } + + String relays_published; + List<NodeWeights> relays; + String bridges_published; + List<NodeWeights> bridges; +} + diff --git a/src/org/torproject/onionoo/WeightsStatus.java b/src/org/torproject/onionoo/WeightsStatus.java new file mode 100644 index 0000000..f8f78ad --- /dev/null +++ b/src/org/torproject/onionoo/WeightsStatus.java @@ -0,0 +1,5 @@ +package org.torproject.onionoo; + +class WeightsStatus extends Document { +} +