commit a96e55ce18e3e4b89eae5bc6e318db6217460192 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Wed Oct 29 22:39:06 2014 +0100 Clarify responsibilities of node and details statuses. This big refactoring patch clarifies responsibilities of node and details statuses. These responsibilities have been watered down by past feature implementations which made it hard to add future enhancements. This patch builds upon the earlier separation of node status parts into summary documents. It does not change the role of details documents. This patch is an important requirement for separating the update and the write step of the hourly updater. New responsibilities of node and details statuses are: - Node statuses hold selected information about relays and bridges that must be retrieved efficiently in the update process. They are kept in memory and not written to disk until the update process has completed. They are also the sole basis for writing summary documents that are used in the server package to build the node index. - Details statuses hold detailed information about relays and bridges and are written only once at the end of the update process. They are not kept in memory. They are the sole basis for writing details documents. Changes in more detail: - NodeStatus now only contains attributes that are made persistent. Previously, some attributes, like lookup results, were set by NodeDetailsStatusUpdater and later read by DetailsDocumentWriter, but that approach will break as soon as the two classes may be executed in distinct runs. - NodeStatus now has a constructor with fewer arguments, and attributes now have setters. NodeStatus also doesn't have the confusing update() method anymore. - DetailsStatus contains the removed attributes from NodeStatus which are made persistent and used by DetailsDocumentWriter to create and write DetailsDocument instances. - NodeDetailsStatusUpdater tries harder not to retrieve and update DetailsStatus documents in order to reduce disk I/O operations. DetailsStatus documents are only retrieved and possibly written when parsing server descriptors and at the end of the update process. - NodeDetailsStatusUpdater handles bridge pool assignments in the same way as exit lists: by storing updates in memory during the parse step and updating DetailsStatus instances as part of the update step. - DetailsDocumentWriter only uses DetailsStatus documents and no NodeStatus documents to write new DetailsDocument files. --- .../torproject/onionoo/docs/DetailsDocument.java | 23 +- .../org/torproject/onionoo/docs/DetailsStatus.java | 273 +++++++++- .../org/torproject/onionoo/docs/NodeStatus.java | 565 ++++++++----------- .../onionoo/updater/NodeDetailsStatusUpdater.java | 571 ++++++++++++-------- .../onionoo/writer/DetailsDocumentWriter.java | 270 ++++----- .../onionoo/writer/SummaryDocumentWriter.java | 21 +- 6 files changed, 978 insertions(+), 745 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java index 455097a..86abf9f 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsDocument.java @@ -2,8 +2,10 @@ * See LICENSE for licensing information */ package org.torproject.onionoo.docs; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.SortedSet; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; @@ -61,10 +63,11 @@ public class DetailsDocument extends Document { private List<String> exit_addresses; public void setExitAddresses(List<String> exitAddresses) { - this.exit_addresses = exitAddresses; + this.exit_addresses = !exitAddresses.isEmpty() ? exitAddresses : null; } public List<String> getExitAddresses() { - return this.exit_addresses; + return this.exit_addresses == null ? new ArrayList<String>() + : this.exit_addresses; } private String dir_address; @@ -109,11 +112,11 @@ public class DetailsDocument extends Document { return this.running; } - private List<String> flags; - public void setFlags(List<String> flags) { + private SortedSet<String> flags; + public void setFlags(SortedSet<String> flags) { this.flags = flags; } - public List<String> getFlags() { + public SortedSet<String> getFlags() { return this.flags; } @@ -198,11 +201,13 @@ public class DetailsDocument extends Document { } private String last_restarted; - public void setLastRestarted(long lastRestarted) { - this.last_restarted = DateTimeHelper.format(lastRestarted); + public void setLastRestarted(Long lastRestarted) { + this.last_restarted = (lastRestarted == null ? null : + DateTimeHelper.format(lastRestarted)); } - public long getLastRestarted() { - return DateTimeHelper.parse(this.last_restarted); + public Long getLastRestarted() { + return this.last_restarted == null ? null : + DateTimeHelper.parse(this.last_restarted); } private Integer bandwidth_rate; diff --git a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java index 42c835f..967e493 100644 --- a/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/DetailsStatus.java @@ -4,6 +4,8 @@ package org.torproject.onionoo.docs; import java.util.List; import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; import org.apache.commons.lang3.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; @@ -27,20 +29,24 @@ public class DetailsStatus extends Document { new String[] { "'" }, new String[] { "\\'" })); } + /* From most recently published server descriptor: */ + private String desc_published; - public void setDescPublished(long descPublished) { + public void setDescPublished(Long descPublished) { this.desc_published = DateTimeHelper.format(descPublished); } - public long getDescPublished() { - return DateTimeHelper.parse(this.desc_published); + public Long getDescPublished() { + return this.desc_published == null ? null : + DateTimeHelper.parse(this.desc_published); } private String last_restarted; - public void setLastRestarted(long lastRestarted) { + public void setLastRestarted(Long lastRestarted) { this.last_restarted = DateTimeHelper.format(lastRestarted); } - public long getLastRestarted() { - return DateTimeHelper.parse(this.last_restarted); + public Long getLastRestarted() { + return this.last_restarted == null ? null : + DateTimeHelper.parse(this.last_restarted); } private Integer bandwidth_rate; @@ -124,6 +130,149 @@ public class DetailsStatus extends Document { return this.hibernating; } + /* From network status entries: */ + + private boolean is_relay; + public void setRelay(boolean isRelay) { + this.is_relay = isRelay; + } + public boolean isRelay() { + return this.is_relay; + } + + private boolean running; + public void setRunning(boolean isRunning) { + this.running = isRunning; + } + public boolean isRunning() { + return this.running; + } + + private String nickname; + public void setNickname(String nickname) { + this.nickname = nickname; + } + public String getNickname() { + return this.nickname; + } + + private String address; + public void setAddress(String address) { + this.address = address; + } + public String getAddress() { + return this.address; + } + + private SortedSet<String> or_addresses_and_ports; + public void setOrAddressesAndPorts( + SortedSet<String> orAddressesAndPorts) { + this.or_addresses_and_ports = orAddressesAndPorts; + } + public SortedSet<String> getOrAddressesAndPorts() { + return this.or_addresses_and_ports == null ? new TreeSet<String>() : + this.or_addresses_and_ports; + } + public SortedSet<String> getOrAddresses() { + SortedSet<String> orAddresses = new TreeSet<String>(); + if (this.address != null) { + orAddresses.add(this.address); + } + if (this.or_addresses_and_ports != null) { + for (String orAddressAndPort : this.or_addresses_and_ports) { + if (orAddressAndPort.contains(":")) { + String orAddress = orAddressAndPort.substring(0, + orAddressAndPort.lastIndexOf(':')); + orAddresses.add(orAddress); + } + } + } + return orAddresses; + } + + private long first_seen_millis; + public void setFirstSeenMillis(long firstSeenMillis) { + this.first_seen_millis = firstSeenMillis; + } + public long getFirstSeenMillis() { + return this.first_seen_millis; + } + + private long last_seen_millis; + public void setLastSeenMillis(long lastSeenMillis) { + this.last_seen_millis = lastSeenMillis; + } + public long getLastSeenMillis() { + return this.last_seen_millis; + } + + private int or_port; + public void setOrPort(int orPort) { + this.or_port = orPort; + } + public int getOrPort() { + return this.or_port; + } + + private int dir_port; + public void setDirPort(int dirPort) { + this.dir_port = dirPort; + } + public int getDirPort() { + return this.dir_port; + } + + private SortedSet<String> relay_flags; + public void setRelayFlags(SortedSet<String> relayFlags) { + this.relay_flags = relayFlags; + } + public SortedSet<String> getRelayFlags() { + return this.relay_flags; + } + + private long consensus_weight; + public void setConsensusWeight(long consensusWeight) { + this.consensus_weight = consensusWeight; + } + public long getConsensusWeight() { + return this.consensus_weight; + } + + private String default_policy; + public void setDefaultPolicy(String defaultPolicy) { + this.default_policy = defaultPolicy; + } + public String getDefaultPolicy() { + return this.default_policy; + } + + private String port_list; + public void setPortList(String portList) { + this.port_list = portList; + } + public String getPortList() { + return this.port_list; + } + + private long last_changed_or_address_or_port; + public void setLastChangedOrAddressOrPort( + long lastChangedOrAddressOrPort) { + this.last_changed_or_address_or_port = lastChangedOrAddressOrPort; + } + public long getLastChangedOrAddressOrPort() { + return this.last_changed_or_address_or_port; + } + + private Boolean recommended_version; + public void setRecommendedVersion(Boolean recommendedVersion) { + this.recommended_version = recommendedVersion; + } + public Boolean getRecommendedVersion() { + return this.recommended_version; + } + + /* From bridge pool assignments: */ + private String pool_assignment; public void setPoolAssignment(String poolAssignment) { this.pool_assignment = poolAssignment; @@ -132,6 +281,8 @@ public class DetailsStatus extends Document { return this.pool_assignment; } + /* From exit lists: */ + private Map<String, Long> exit_addresses; public void setExitAddresses(Map<String, Long> exitAddresses) { this.exit_addresses = exitAddresses; @@ -139,4 +290,114 @@ public class DetailsStatus extends Document { public Map<String, Long> getExitAddresses() { return this.exit_addresses; } + + /* Calculated path-selection probabilities: */ + + private Float consensus_weight_fraction; + public void setConsensusWeightFraction(Float consensusWeightFraction) { + this.consensus_weight_fraction = consensusWeightFraction; + } + public Float getConsensusWeightFraction() { + return this.consensus_weight_fraction; + } + + private Float guard_probability; + public void setGuardProbability(Float guardProbability) { + this.guard_probability = guardProbability; + } + public Float getGuardProbability() { + return this.guard_probability; + } + + private Float middle_probability; + public void setMiddleProbability(Float middleProbability) { + this.middle_probability = middleProbability; + } + public Float getMiddleProbability() { + return this.middle_probability; + } + + private Float exit_probability; + public void setExitProbability(Float exitProbability) { + this.exit_probability = exitProbability; + } + public Float getExitProbability() { + return this.exit_probability; + } + + /* GeoIP lookup results: */ + + private Float latitude; + public void setLatitude(Float latitude) { + this.latitude = latitude; + } + public Float getLatitude() { + return this.latitude; + } + + private Float longitude; + public void setLongitude(Float longitude) { + this.longitude = longitude; + } + public Float getLongitude() { + return this.longitude; + } + + private String country_code; + public void setCountryCode(String countryCode) { + this.country_code = countryCode; + } + public String getCountryCode() { + return this.country_code; + } + + private String country_name; + public void setCountryName(String countryName) { + this.country_name = countryName; + } + public String getCountryName() { + return this.country_name; + } + + private String region_name; + public void setRegionName(String regionName) { + this.region_name = regionName; + } + public String getRegionName() { + return this.region_name; + } + + private String city_name; + public void setCityName(String cityName) { + this.city_name = cityName; + } + public String getCityName() { + return this.city_name; + } + + private String as_name; + public void setASName(String aSName) { + this.as_name = aSName; + } + public String getASName() { + return this.as_name; + } + + private String as_number; + public void setASNumber(String aSNumber) { + this.as_number = aSNumber; + } + public String getASNumber() { + return this.as_number; + } + + /* Reverse DNS lookup result: */ + + private String host_name; + public void setHostName(String hostName) { + this.host_name = hostName; + } + public String getHostName() { + return this.host_name; + } } diff --git a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java index ced6cbe..e3dcdaf 100644 --- a/src/main/java/org/torproject/onionoo/docs/NodeStatus.java +++ b/src/main/java/org/torproject/onionoo/docs/NodeStatus.java @@ -18,144 +18,148 @@ import java.util.TreeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/* Store search data of a single relay that was running in the past seven - * days. */ public class NodeStatus extends Document { private final static Logger log = LoggerFactory.getLogger( NodeStatus.class); - private boolean isRelay; - public boolean isRelay() { - return this.isRelay; - } - - private String fingerprint; - public String getFingerprint() { - return this.fingerprint; - } - - private String nickname; - public String getNickname() { - return this.nickname; - } - - private String address; - public String getAddress() { - return this.address; - } + /* From most recently published server descriptor: */ - private SortedSet<String> orAddresses; - private SortedSet<String> orAddressesAndPorts; - public SortedSet<String> getOrAddresses() { - return new TreeSet<String>(this.orAddresses); - } - public void addOrAddressAndPort(String orAddressAndPort) { - if (orAddressAndPort.contains(":") && orAddressAndPort.length() > 0) { - String orAddress = orAddressAndPort.substring(0, - orAddressAndPort.lastIndexOf(':')); - if (this.exitAddresses.contains(orAddress)) { - this.exitAddresses.remove(orAddress); + private String contact; + public void setContact(String contact) { + if (contact == null) { + this.contact = null; + } else { + StringBuilder sb = new StringBuilder(); + for (char c : contact.toLowerCase().toCharArray()) { + if (c >= 32 && c < 127) { + sb.append(c); + } else { + sb.append(" "); + } } - this.orAddresses.add(orAddress); - this.orAddressesAndPorts.add(orAddressAndPort); + this.contact = sb.toString(); } } - public SortedSet<String> getOrAddressesAndPorts() { - return new TreeSet<String>(this.orAddressesAndPorts); + public String getContact() { + return this.contact; } - private SortedSet<String> exitAddresses; - public void addExitAddress(String exitAddress) { - if (exitAddress.length() > 0 && !this.address.equals(exitAddress) && - !this.orAddresses.contains(exitAddress)) { - this.exitAddresses.add(exitAddress); - } + private String[] familyFingerprints; + public void setFamilyFingerprints( + SortedSet<String> familyFingerprints) { + this.familyFingerprints = collectionToStringArray(familyFingerprints); } - public SortedSet<String> getExitAddresses() { - return new TreeSet<String>(this.exitAddresses); + public SortedSet<String> getFamilyFingerprints() { + return stringArrayToSortedSet(this.familyFingerprints); } - private Float latitude; - public void setLatitude(Float latitude) { - this.latitude = latitude; + private static String[] collectionToStringArray( + Collection<String> collection) { + String[] stringArray = null; + if (collection != null && !collection.isEmpty()) { + stringArray = new String[collection.size()]; + int i = 0; + for (String string : collection) { + stringArray[i++] = string; + } + } + return stringArray; } - public Float getLatitude() { - return this.latitude; + private SortedSet<String> stringArrayToSortedSet(String[] stringArray) { + SortedSet<String> sortedSet = new TreeSet<String>(); + if (stringArray != null) { + sortedSet.addAll(Arrays.asList(stringArray)); + } + return sortedSet; } - private Float longitude; - public void setLongitude(Float longitude) { - this.longitude = longitude; - } - public Float getLongitude() { - return this.longitude; - } + /* From network status entries: */ - private String countryCode; - public void setCountryCode(String countryCode) { - this.countryCode = countryCode; + private boolean isRelay; + public void setRelay(boolean isRelay) { + this.isRelay = isRelay; } - public String getCountryCode() { - return this.countryCode; + public boolean isRelay() { + return this.isRelay; } - private String countryName; - public void setCountryName(String countryName) { - this.countryName = countryName; - } - public String getCountryName() { - return this.countryName; + private final String fingerprint; + public String getFingerprint() { + return this.fingerprint; } - private String regionName; - public void setRegionName(String regionName) { - this.regionName = regionName; + private String nickname; + public void setNickname(String nickname) { + this.nickname = nickname; } - public String getRegionName() { - return this.regionName; + public String getNickname() { + return this.nickname; } - private String cityName; - public void setCityName(String cityName) { - this.cityName = cityName; + private String address; + public void setAddress(String address) { + this.address = address; } - public String getCityName() { - return this.cityName; + public String getAddress() { + return this.address; } - private String aSName; - public void setASName(String aSName) { - this.aSName = aSName; - } - public String getASName() { - return this.aSName; + private SortedSet<String> orAddressesAndPorts; + public void setOrAddressesAndPorts( + SortedSet<String> orAddressesAndPorts) { + this.orAddressesAndPorts = orAddressesAndPorts; } - - private String aSNumber; - public void setASNumber(String aSNumber) { - this.aSNumber = aSNumber; + public SortedSet<String> getOrAddressesAndPorts() { + return this.orAddressesAndPorts == null ? new TreeSet<String>() : + this.orAddressesAndPorts; } - public String getASNumber() { - return this.aSNumber; + public SortedSet<String> getOrAddresses() { + SortedSet<String> orAddresses = new TreeSet<String>(); + if (this.address != null) { + orAddresses.add(this.address); + } + if (this.orAddressesAndPorts != null) { + for (String orAddressAndPort : this.orAddressesAndPorts) { + if (orAddressAndPort.contains(":") && + orAddressAndPort.length() > 0) { + String orAddress = orAddressAndPort.substring(0, + orAddressAndPort.lastIndexOf(':')); + orAddresses.add(orAddress); + } + } + } + return orAddresses; } private long firstSeenMillis; + public void setFirstSeenMillis(long firstSeenMillis) { + this.firstSeenMillis = firstSeenMillis; + } public long getFirstSeenMillis() { return this.firstSeenMillis; } private long lastSeenMillis; + public void setLastSeenMillis(long lastSeenMillis) { + this.lastSeenMillis = lastSeenMillis; + } public long getLastSeenMillis() { return this.lastSeenMillis; } private int orPort; + public void setOrPort(int orPort) { + this.orPort = orPort; + } public int getOrPort() { return this.orPort; } private int dirPort; + public void setDirPort(int dirPort) { + this.dirPort = dirPort; + } public int getDirPort() { return this.dirPort; } @@ -189,86 +193,54 @@ public class NodeStatus extends Document { } private long consensusWeight; + public void setConsensusWeight(long consensusWeight) { + this.consensusWeight = consensusWeight; + } public long getConsensusWeight() { return this.consensusWeight; } - private boolean running; - public void setRunning(boolean running) { - this.running = running; - } - public boolean getRunning() { - return this.running; - } - - private String hostName; - public void setHostName(String hostName) { - this.hostName = hostName; - } - public String getHostName() { - return this.hostName; - } - - private long lastRdnsLookup = -1L; - public void setLastRdnsLookup(long lastRdnsLookup) { - this.lastRdnsLookup = lastRdnsLookup; - } - public long getLastRdnsLookup() { - return this.lastRdnsLookup; - } - - private double consensusWeightFraction = -1.0; - public void setConsensusWeightFraction(double consensusWeightFraction) { - this.consensusWeightFraction = consensusWeightFraction; - } - public double getConsensusWeightFraction() { - return this.consensusWeightFraction; - } - - private double guardProbability = -1.0; - public void setGuardProbability(double guardProbability) { - this.guardProbability = guardProbability; - } - public double getGuardProbability() { - return this.guardProbability; - } - - private double middleProbability = -1.0; - public void setMiddleProbability(double middleProbability) { - this.middleProbability = middleProbability; - } - public double getMiddleProbability() { - return this.middleProbability; - } - - private double exitProbability = -1.0; - public void setExitProbability(double exitProbability) { - this.exitProbability = exitProbability; - } - public double getExitProbability() { - return this.exitProbability; - } - private String defaultPolicy; + public void setDefaultPolicy(String defaultPolicy) { + this.defaultPolicy = defaultPolicy; + } public String getDefaultPolicy() { return this.defaultPolicy; } private String portList; + public void setPortList(String portList) { + this.portList = portList; + } public String getPortList() { return this.portList; } - private SortedMap<Long, Set<String>> lastAddresses; + private SortedMap<Long, Set<String>> lastAddresses = + new TreeMap<Long, Set<String>>(Collections.reverseOrder()); public SortedMap<Long, Set<String>> getLastAddresses() { - return this.lastAddresses == null ? null : - new TreeMap<Long, Set<String>>(this.lastAddresses); + return new TreeMap<Long, Set<String>>(this.lastAddresses); + } + public void addLastAddresses(long lastSeenMillis, String address, + int orPort, int dirPort, SortedSet<String> orAddressesAndPorts) { + Set<String> addressesAndPorts = new HashSet<String>(); + addressesAndPorts.add(address + ":" + orPort); + if (dirPort > 0) { + addressesAndPorts.add(address + ":" + dirPort); + } + addressesAndPorts.addAll(orAddressesAndPorts); + if (this.lastAddresses.containsKey(lastSeenMillis)) { + this.lastAddresses.get(lastSeenMillis).addAll(addressesAndPorts); + } else { + this.lastAddresses.put(lastSeenMillis, addressesAndPorts); + } } - public long getLastChangedOrAddress() { + public long getLastChangedOrAddressOrPort() { long lastChangedAddressesMillis = -1L; if (this.lastAddresses != null) { Set<String> lastAddresses = null; - for (Map.Entry<Long, Set<String>> e : this.lastAddresses.entrySet()) { + for (Map.Entry<Long, Set<String>> e : + this.lastAddresses.entrySet()) { if (lastAddresses != null) { for (String address : e.getValue()) { if (!lastAddresses.contains(address)) { @@ -283,136 +255,73 @@ public class NodeStatus extends Document { return lastChangedAddressesMillis; } - private String contact; - public void setContact(String contact) { - if (contact == null) { - this.contact = null; - } else { - StringBuilder sb = new StringBuilder(); - for (char c : contact.toLowerCase().toCharArray()) { - if (c >= 32 && c < 127) { - sb.append(c); - } else { - sb.append(" "); - } - } - this.contact = sb.toString(); - } - } - public String getContact() { - return this.contact; - } - private Boolean recommendedVersion; + public void setRecommendedVersion(Boolean recommendedVersion) { + this.recommendedVersion = recommendedVersion; + } public Boolean getRecommendedVersion() { return this.recommendedVersion; } - private String[] familyFingerprints; - public void setFamilyFingerprints( - SortedSet<String> familyFingerprints) { - this.familyFingerprints = collectionToStringArray(familyFingerprints); + /* From exit lists: */ + + private SortedSet<String> exitAddresses; + public void setExitAddresses(SortedSet<String> exitAddresses) { + this.exitAddresses = exitAddresses; } - public SortedSet<String> getFamilyFingerprints() { - return stringArrayToSortedSet(this.familyFingerprints); + public SortedSet<String> getExitAddresses() { + return new TreeSet<String>(this.exitAddresses); } - private static String[] collectionToStringArray( - Collection<String> collection) { - String[] stringArray = null; - if (collection != null && !collection.isEmpty()) { - stringArray = new String[collection.size()]; - int i = 0; - for (String string : collection) { - stringArray[i++] = string; - } - } - return stringArray; + /* GeoIP lookup results: */ + + private String countryCode; + public void setCountryCode(String countryCode) { + this.countryCode = countryCode; } - private SortedSet<String> stringArrayToSortedSet(String[] stringArray) { - SortedSet<String> sortedSet = new TreeSet<String>(); - if (stringArray != null) { - sortedSet.addAll(Arrays.asList(stringArray)); - } - return sortedSet; + public String getCountryCode() { + return this.countryCode; } - public NodeStatus(boolean isRelay, String nickname, String fingerprint, - String address, SortedSet<String> orAddressesAndPorts, - SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, - int dirPort, SortedSet<String> relayFlags, long consensusWeight, - String countryCode, String hostName, long lastRdnsLookup, - String defaultPolicy, String portList, long firstSeenMillis, - long lastChangedAddresses, String aSNumber, String contact, - Boolean recommendedVersion, SortedSet<String> familyFingerprints) { - this.isRelay = isRelay; - this.nickname = nickname; - this.fingerprint = fingerprint; - this.address = address; - this.exitAddresses = new TreeSet<String>(); - if (exitAddresses != null) { - this.exitAddresses.addAll(exitAddresses); - } - this.exitAddresses.remove(this.address); - this.orAddresses = new TreeSet<String>(); - this.orAddressesAndPorts = new TreeSet<String>(); - if (orAddressesAndPorts != null) { - for (String orAddressAndPort : orAddressesAndPorts) { - this.addOrAddressAndPort(orAddressAndPort); - } - } - this.lastSeenMillis = lastSeenMillis; - this.orPort = orPort; - this.dirPort = dirPort; - this.setRelayFlags(relayFlags); - this.consensusWeight = consensusWeight; - this.countryCode = countryCode; - this.hostName = hostName; - this.lastRdnsLookup = lastRdnsLookup; - this.defaultPolicy = defaultPolicy; - this.portList = portList; - this.firstSeenMillis = firstSeenMillis; - this.lastAddresses = - new TreeMap<Long, Set<String>>(Collections.reverseOrder()); - Set<String> addresses = new HashSet<String>(); - addresses.add(address + ":" + orPort); - if (dirPort > 0) { - addresses.add(address + ":" + dirPort); - } - addresses.addAll(orAddressesAndPorts); - this.lastAddresses.put(lastChangedAddresses, addresses); + private String aSNumber; + public void setASNumber(String aSNumber) { this.aSNumber = aSNumber; - this.contact = contact; - this.recommendedVersion = recommendedVersion; - this.setFamilyFingerprints(familyFingerprints); + } + public String getASNumber() { + return this.aSNumber; + } + + /* Reverse DNS lookup result */ + + private long lastRdnsLookup = -1L; + public void setLastRdnsLookup(long lastRdnsLookup) { + this.lastRdnsLookup = lastRdnsLookup; + } + public long getLastRdnsLookup() { + return this.lastRdnsLookup; + } + + /* Constructor and (de-)serialization methods: */ + + public NodeStatus(String fingerprint) { + this.fingerprint = fingerprint; } public static NodeStatus fromString(String documentString) { - boolean isRelay = false; - String nickname = null, fingerprint = null, address = null, - countryCode = null, hostName = null, defaultPolicy = null, - portList = null, aSNumber = null, contact = null; - SortedSet<String> orAddressesAndPorts = null, exitAddresses = null, - relayFlags = null, familyFingerprints = null; - long lastSeenMillis = -1L, consensusWeight = -1L, - lastRdnsLookup = -1L, firstSeenMillis = -1L, - lastChangedAddresses = -1L; - int orPort = -1, dirPort = -1; - Boolean recommendedVersion = null; try { - String separator = documentString.contains("\t") ? "\t" : " "; - String[] parts = documentString.trim().split(separator); - isRelay = parts[0].equals("r"); - if (parts.length < 9) { + String[] parts = documentString.trim().split("\t"); + if (parts.length < 23) { log.error("Too few space-separated values in line '" + documentString.trim() + "'. Skipping."); return null; } - nickname = parts[1]; - fingerprint = parts[2]; - orAddressesAndPorts = new TreeSet<String>(); - exitAddresses = new TreeSet<String>(); - String addresses = parts[3]; + String fingerprint = parts[2]; + NodeStatus nodeStatus = new NodeStatus(fingerprint); + nodeStatus.setRelay(parts[0].equals("r")); + nodeStatus.setNickname(parts[1]); + SortedSet<String> orAddressesAndPorts = new TreeSet<String>(); + SortedSet<String> exitAddresses = new TreeSet<String>(); + String addresses = parts[3], address = null; if (addresses.contains(";")) { String[] addressParts = addresses.split(";", -1); if (addressParts.length != 3) { @@ -432,48 +341,51 @@ public class NodeStatus extends Document { } else { address = addresses; } - lastSeenMillis = DateTimeHelper.parse(parts[4] + " " + parts[5]); + nodeStatus.setAddress(address); + nodeStatus.setOrAddressesAndPorts(orAddressesAndPorts); + nodeStatus.setExitAddresses(exitAddresses); + long lastSeenMillis = DateTimeHelper.parse(parts[4] + " " + + parts[5]); if (lastSeenMillis < 0L) { log.error("Parse exception while parsing node status " + "line '" + documentString + "'. Skipping."); return null; } - orPort = Integer.parseInt(parts[6]); - dirPort = Integer.parseInt(parts[7]); - relayFlags = new TreeSet<String>(); - if (parts[8].length() > 0) { - relayFlags.addAll(Arrays.asList(parts[8].split(","))); - } - if (parts.length > 9) { - consensusWeight = Long.parseLong(parts[9]); + nodeStatus.setLastSeenMillis(lastSeenMillis); + int orPort = Integer.parseInt(parts[6]), + dirPort = Integer.parseInt(parts[7]); + nodeStatus.setOrPort(orPort); + nodeStatus.setDirPort(dirPort); + nodeStatus.setRelayFlags(new TreeSet<String>( + Arrays.asList(parts[8].split(",")))); + nodeStatus.setConsensusWeight(Long.parseLong(parts[9])); + nodeStatus.setCountryCode(parts[10]); + if (!parts[11].equals("")) { + /* This is a (possibly surprising) hack that is part of moving the + * host name field from node status to details status. As part of + * that move we ignore all previously looked up host names trigger + * a new lookup by setting the last lookup time to 1969-12-31 + * 23:59:59.999. This hack may be removed after it has been run + * at least once. */ + parts[12] = "-1"; } - if (parts.length > 10) { - countryCode = parts[10]; + nodeStatus.setLastRdnsLookup(Long.parseLong(parts[12])); + if (!parts[13].equals("null")) { + nodeStatus.setDefaultPolicy(parts[13]); } - if (parts.length > 12) { - hostName = parts[11].equals("null") ? null : parts[11]; - lastRdnsLookup = Long.parseLong(parts[12]); + if (!parts[14].equals("null")) { + nodeStatus.setPortList(parts[14]); } - if (parts.length > 14) { - if (!parts[13].equals("null")) { - defaultPolicy = parts[13]; - } - if (!parts[14].equals("null")) { - portList = parts[14]; - } - } - firstSeenMillis = lastSeenMillis; - if (parts.length > 16) { - firstSeenMillis = DateTimeHelper.parse(parts[15] + " " - + parts[16]); - if (firstSeenMillis < 0L) { - log.error("Parse exception while parsing node status " - + "line '" + documentString + "'. Skipping."); - return null; - } + long firstSeenMillis = lastSeenMillis; + firstSeenMillis = DateTimeHelper.parse(parts[15] + " " + parts[16]); + if (firstSeenMillis < 0L) { + log.error("Parse exception while parsing node status " + + "line '" + documentString + "'. Skipping."); + return null; } - lastChangedAddresses = lastSeenMillis; - if (parts.length > 18 && !parts[17].equals("null")) { + nodeStatus.setFirstSeenMillis(firstSeenMillis); + long lastChangedAddresses = lastSeenMillis; + if (!parts[17].equals("null")) { lastChangedAddresses = DateTimeHelper.parse(parts[17] + " " + parts[18]); if (lastChangedAddresses < 0L) { @@ -482,20 +394,18 @@ public class NodeStatus extends Document { return null; } } - if (parts.length > 19) { - aSNumber = parts[19]; - } - if (parts.length > 20) { - contact = parts[20]; - } - if (parts.length > 21) { - recommendedVersion = parts[21].equals("null") ? null : - parts[21].equals("true"); + nodeStatus.addLastAddresses(lastChangedAddresses, address, orPort, + dirPort, orAddressesAndPorts); + nodeStatus.setASNumber(parts[19]); + nodeStatus.setContact(parts[20]); + if (!parts[21].equals("null")) { + nodeStatus.setRecommendedVersion(parts[21].equals("true")); } - if (parts.length > 22 && !parts[22].equals("null")) { - familyFingerprints = new TreeSet<String>(Arrays.asList( - parts[22].split(";"))); + if (!parts[22].equals("null")) { + nodeStatus.setFamilyFingerprints(new TreeSet<String>( + Arrays.asList(parts[22].split(";")))); } + return nodeStatus; } catch (NumberFormatException e) { log.error("Number format exception while parsing node " + "status line '" + documentString + "': " + e.getMessage() @@ -509,36 +419,6 @@ public class NodeStatus extends Document { + "Skipping."); return null; } - NodeStatus newNodeStatus = new NodeStatus(isRelay, nickname, - fingerprint, address, orAddressesAndPorts, exitAddresses, - lastSeenMillis, orPort, dirPort, relayFlags, consensusWeight, - countryCode, hostName, lastRdnsLookup, defaultPolicy, portList, - firstSeenMillis, lastChangedAddresses, aSNumber, contact, - recommendedVersion, familyFingerprints); - return newNodeStatus; - } - - public void update(NodeStatus newNodeStatus) { - if (newNodeStatus.lastSeenMillis > this.lastSeenMillis) { - this.nickname = newNodeStatus.nickname; - this.address = newNodeStatus.address; - this.orAddressesAndPorts = newNodeStatus.orAddressesAndPorts; - this.lastSeenMillis = newNodeStatus.lastSeenMillis; - this.orPort = newNodeStatus.orPort; - this.dirPort = newNodeStatus.dirPort; - this.relayFlags = newNodeStatus.relayFlags; - this.consensusWeight = newNodeStatus.consensusWeight; - this.countryCode = newNodeStatus.countryCode; - this.defaultPolicy = newNodeStatus.defaultPolicy; - this.portList = newNodeStatus.portList; - this.aSNumber = newNodeStatus.aSNumber; - this.recommendedVersion = newNodeStatus.recommendedVersion; - } - if (this.isRelay && newNodeStatus.isRelay) { - this.lastAddresses.putAll(newNodeStatus.lastAddresses); - } - this.firstSeenMillis = Math.min(newNodeStatus.firstSeenMillis, - this.firstSeenMillis); } public String toString() { @@ -548,15 +428,16 @@ public class NodeStatus extends Document { sb.append("\t" + this.fingerprint); sb.append("\t" + this.address + ";"); int written = 0; - for (String orAddressAndPort : this.orAddressesAndPorts) { - sb.append((written++ > 0 ? "+" : "") + orAddressAndPort); + if (this.orAddressesAndPorts != null) { + for (String orAddressAndPort : this.orAddressesAndPorts) { + sb.append((written++ > 0 ? "+" : "") + orAddressAndPort); + } } sb.append(";"); if (this.isRelay) { written = 0; for (String exitAddress : this.exitAddresses) { - sb.append((written++ > 0 ? "+" : "") - + exitAddress); + sb.append((written++ > 0 ? "+" : "") + exitAddress); } } sb.append("\t" + DateTimeHelper.format(this.lastSeenMillis, @@ -571,19 +452,19 @@ public class NodeStatus extends Document { sb.append("\t" + String.valueOf(this.consensusWeight)); sb.append("\t" + (this.countryCode != null ? this.countryCode : "??")); - sb.append("\t" + (this.hostName != null ? this.hostName : "null")); + sb.append("\t"); /* formerly used for storing host names */ sb.append("\t" + String.valueOf(this.lastRdnsLookup)); sb.append("\t" + (this.defaultPolicy != null ? this.defaultPolicy : "null")); sb.append("\t" + (this.portList != null ? this.portList : "null")); } else { - sb.append("\t-1\t??\tnull\t-1\tnull\tnull"); + sb.append("\t-1\t??\t\t-1\tnull\tnull"); } sb.append("\t" + DateTimeHelper.format(this.firstSeenMillis, DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); if (this.isRelay) { sb.append("\t" + DateTimeHelper.format( - this.getLastChangedOrAddress(), + this.getLastChangedOrAddressOrPort(), DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); sb.append("\t" + (this.aSNumber != null ? this.aSNumber : "null")); } else { diff --git a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java index 9963a74..0ce17e4 100644 --- a/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java +++ b/src/main/java/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -31,6 +31,40 @@ import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.util.FormattingUtils; import org.torproject.onionoo.util.TimeFactory; +/* + * Status updater for both node and details statuses. + * + * Node statuses hold selected information about relays and bridges that + * must be retrieved efficiently in the update process. They are kept in + * memory and not written to disk until the update process has completed. + * They are also the sole basis for writing summary documents that are + * used in the server package to build the node index. + * + * Details statuses hold detailed information about relays and bridges and + * are written at the end of the update process. They are not kept in + * memory. They are the sole basis for writing details documents. + * + * The complete update process as implemented in this class is as follows: + * + * 1. Parse descriptors and either write their contents to details + * statuses (which requires retrieving them and storing them back to + * disk, which is why this is only done for detailed information that + * don't fit into memory) and/or local data structures in memory + * (which may include node statuses). + * 2. Read all known node statuses from disk and merge their contents + * with the node statuses from parsing descriptors. Node statuses + * are not loaded from disk before the parse step in order to save + * memory for parsed descriptors. + * 3. Perform reverse DNS lookups, Look up relay IP addresses in a + * GeoIP database, and calculate path selection probabilities. + * Update node statuses accordingly. + * 4. Retrieve details statuses corresponding to nodes that have been + * changed since the start of the update process, possibly update the + * node statuses with contents from newly parsed descriptors, update + * details statuses with results from lookup operations and new path + * selection probabilities, and store details statuses and node + * statuses back to disk. + */ public class NodeDetailsStatusUpdater implements DescriptorListener, StatusUpdater { @@ -50,10 +84,6 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, private SortedMap<String, NodeStatus> knownNodes = new TreeMap<String, NodeStatus>(); - private SortedMap<String, NodeStatus> relays; - - private SortedMap<String, NodeStatus> bridges; - private long relaysLastValidAfterMillis = -1L; private long bridgesLastPublishedMillis = -1L; @@ -88,6 +118,10 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, DescriptorType.EXIT_LISTS); } + /* Step 1: parse descriptors. */ + + private SortedSet<String> updatedNodes = new TreeSet<String>(); + public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof ServerDescriptor && relay) { this.processRelayServerDescriptor((ServerDescriptor) descriptor); @@ -112,8 +146,9 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, DetailsStatus.class, true, fingerprint); if (detailsStatus == null) { detailsStatus = new DetailsStatus(); - } else if (descriptor.getPublishedMillis() <= - detailsStatus.getDescPublished()) { + } else if (detailsStatus.getDescPublished() != null && + detailsStatus.getDescPublished() >= + descriptor.getPublishedMillis()) { return; } long lastRestartedMillis = descriptor.getPublishedMillis() @@ -145,9 +180,8 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, portsOrPortRanges); detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary); } - if (descriptor.isHibernating()) { - detailsStatus.setHibernating(true); - } + detailsStatus.setHibernating(descriptor.isHibernating() ? true : + null); this.documentStore.store(detailsStatus, fingerprint); } @@ -157,8 +191,8 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, private void processExitList(ExitList exitList) { for (ExitListEntry exitListEntry : exitList.getExitListEntries()) { String fingerprint = exitListEntry.getFingerprint(); - if (exitListEntry.getScanMillis() < - this.now - DateTimeHelper.ONE_DAY) { + long scanMillis = exitListEntry.getScanMillis(); + if (scanMillis < this.now - DateTimeHelper.ONE_DAY) { continue; } if (!this.exitListEntries.containsKey(fingerprint)) { @@ -166,7 +200,6 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, new HashMap<String, Long>()); } String exitAddress = exitListEntry.getExitAddress(); - long scanMillis = exitListEntry.getScanMillis(); if (!this.exitListEntries.get(fingerprint).containsKey(exitAddress) || this.exitListEntries.get(fingerprint).get(exitAddress) < scanMillis) { @@ -190,31 +223,40 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, recommendedVersions.add("Tor " + recommendedVersion); } } - for (NetworkStatusEntry entry : - consensus.getStatusEntries().values()) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); + for (Map.Entry<String, NetworkStatusEntry> e : + consensus.getStatusEntries().entrySet()) { + String fingerprint = e.getKey(); + NetworkStatusEntry entry = e.getValue(); + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + if (nodeStatus == null) { + nodeStatus = new NodeStatus(fingerprint); + this.knownNodes.put(fingerprint, nodeStatus); + } String address = entry.getAddress(); + int orPort = entry.getOrPort(), dirPort = entry.getDirPort(); SortedSet<String> orAddressesAndPorts = new TreeSet<String>( entry.getOrAddresses()); - int orPort = entry.getOrPort(); - int dirPort = entry.getDirPort(); - SortedSet<String> relayFlags = entry.getFlags(); - long consensusWeight = entry.getBandwidth(); - String defaultPolicy = entry.getDefaultPolicy(); - String portList = entry.getPortList(); - Boolean recommendedVersion = (recommendedVersions == null || - entry.getVersion() == null) ? null : - recommendedVersions.contains(entry.getVersion()); - NodeStatus newNodeStatus = new NodeStatus(true, nickname, - fingerprint, address, orAddressesAndPorts, null, - validAfterMillis, orPort, dirPort, relayFlags, consensusWeight, - null, null, -1L, defaultPolicy, portList, validAfterMillis, - validAfterMillis, null, null, recommendedVersion, null); - if (this.knownNodes.containsKey(fingerprint)) { - this.knownNodes.get(fingerprint).update(newNodeStatus); - } else { - this.knownNodes.put(fingerprint, newNodeStatus); + nodeStatus.addLastAddresses(validAfterMillis, address, orPort, + dirPort, orAddressesAndPorts); + if (nodeStatus.getFirstSeenMillis() == 0L || + validAfterMillis < nodeStatus.getFirstSeenMillis()) { + nodeStatus.setFirstSeenMillis(validAfterMillis); + } + if (validAfterMillis > nodeStatus.getLastSeenMillis()) { + nodeStatus.setLastSeenMillis(validAfterMillis); + nodeStatus.setRelay(true); + nodeStatus.setNickname(entry.getNickname()); + nodeStatus.setAddress(address); + nodeStatus.setOrAddressesAndPorts(orAddressesAndPorts); + nodeStatus.setOrPort(orPort); + nodeStatus.setDirPort(dirPort); + nodeStatus.setRelayFlags(entry.getFlags()); + nodeStatus.setConsensusWeight(entry.getBandwidth()); + nodeStatus.setDefaultPolicy(entry.getDefaultPolicy()); + nodeStatus.setPortList(entry.getPortList()); + nodeStatus.setRecommendedVersion((recommendedVersions == null || + entry.getVersion() == null) ? null : + recommendedVersions.contains(entry.getVersion())); } } this.relayConsensusesProcessed++; @@ -244,24 +286,15 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, detailsStatus.setAdvertisedBandwidth(advertisedBandwidth); detailsStatus.setPlatform(descriptor.getPlatform()); this.documentStore.store(detailsStatus, fingerprint); + this.updatedNodes.add(fingerprint); } + private Map<String, String> bridgePoolAssignments = + new HashMap<String, String>(); + private void processBridgePoolAssignment( BridgePoolAssignment bridgePoolAssignment) { - for (Map.Entry<String, String> e : - bridgePoolAssignment.getEntries().entrySet()) { - String fingerprint = e.getKey(); - String details = e.getValue(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus == null) { - detailsStatus = new DetailsStatus(); - } else if (details.equals(detailsStatus.getPoolAssignment())) { - continue; - } - detailsStatus.setPoolAssignment(details); - this.documentStore.store(detailsStatus, fingerprint); - } + this.bridgePoolAssignments.putAll(bridgePoolAssignment.getEntries()); } private void processBridgeNetworkStatus(BridgeNetworkStatus status) { @@ -269,108 +302,185 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, if (publishedMillis > this.bridgesLastPublishedMillis) { this.bridgesLastPublishedMillis = publishedMillis; } - for (NetworkStatusEntry entry : status.getStatusEntries().values()) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); - String address = entry.getAddress(); - SortedSet<String> orAddressesAndPorts = new TreeSet<String>( - entry.getOrAddresses()); - int orPort = entry.getOrPort(); - int dirPort = entry.getDirPort(); - SortedSet<String> relayFlags = entry.getFlags(); - NodeStatus newNodeStatus = new NodeStatus(false, nickname, - fingerprint, address, orAddressesAndPorts, null, - publishedMillis, orPort, dirPort, relayFlags, -1L, "??", null, - -1L, null, null, publishedMillis, -1L, null, null, null, null); - if (this.knownNodes.containsKey(fingerprint)) { - this.knownNodes.get(fingerprint).update(newNodeStatus); - } else { - this.knownNodes.put(fingerprint, newNodeStatus); + for (Map.Entry<String, NetworkStatusEntry> e : + status.getStatusEntries().entrySet()) { + String fingerprint = e.getKey(); + NetworkStatusEntry entry = e.getValue(); + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + if (nodeStatus == null) { + nodeStatus = new NodeStatus(fingerprint); + this.knownNodes.put(fingerprint, nodeStatus); + } + if (nodeStatus.getFirstSeenMillis() == 0L || + publishedMillis < nodeStatus.getFirstSeenMillis()) { + nodeStatus.setFirstSeenMillis(publishedMillis); + } + if (publishedMillis > nodeStatus.getLastSeenMillis()) { + nodeStatus.setRelay(false); + nodeStatus.setNickname(entry.getNickname()); + nodeStatus.setAddress(entry.getAddress()); + nodeStatus.setOrAddressesAndPorts(new TreeSet<String>( + entry.getOrAddresses())); + nodeStatus.setOrPort(entry.getOrPort()); + nodeStatus.setDirPort(entry.getDirPort()); + nodeStatus.setRelayFlags(entry.getFlags()); + nodeStatus.setLastSeenMillis(publishedMillis); } } this.bridgeStatusesProcessed++; } public void updateStatuses() { - this.readStatusSummary(); - log.info("Read status summary"); - this.setCurrentNodes(); - log.info("Set current node fingerprints"); + this.readNodeStatuses(); + log.info("Read node statuses"); this.startReverseDomainNameLookups(); log.info("Started reverse domain name lookups"); this.lookUpCitiesAndASes(); log.info("Looked up cities and ASes"); - this.setDescriptorPartsOfNodeStatus(); - log.info("Set descriptor parts of node statuses."); this.calculatePathSelectionProbabilities(); log.info("Calculated path selection probabilities"); this.finishReverseDomainNameLookups(); log.info("Finished reverse domain name lookups"); - this.writeStatusSummary(); - log.info("Wrote status summary"); - this.updateDetailsStatuses(); - log.info("Updated exit addresses in details statuses"); + this.updateNodeDetailsStatuses(); + log.info("Updated node and details statuses"); } - private void readStatusSummary() { - SortedSet<String> fingerprints = this.documentStore.list( + /* Step 2: read node statuses from disk. */ + + private SortedSet<String> currentRelays = new TreeSet<String>(), + runningRelays = new TreeSet<String>(); + + private void readNodeStatuses() { + SortedSet<String> previouslyKnownNodes = this.documentStore.list( NodeStatus.class); - for (String fingerprint : fingerprints) { - NodeStatus node = this.documentStore.retrieve(NodeStatus.class, - true, fingerprint); - if (node.isRelay()) { - this.relaysLastValidAfterMillis = Math.max( - this.relaysLastValidAfterMillis, node.getLastSeenMillis()); - } else { - this.bridgesLastPublishedMillis = Math.max( - this.bridgesLastPublishedMillis, node.getLastSeenMillis()); - } - if (this.knownNodes.containsKey(fingerprint)) { - this.knownNodes.get(fingerprint).update(node); - } else { - this.knownNodes.put(fingerprint, node); + long previousRelaysLastValidAfterMillis = -1L, + previousBridgesLastValidAfterMillis = -1L; + for (String fingerprint : previouslyKnownNodes) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, true, fingerprint); + if (nodeStatus.isRelay() && nodeStatus.getLastSeenMillis() > + previousRelaysLastValidAfterMillis) { + previousRelaysLastValidAfterMillis = + nodeStatus.getLastSeenMillis(); + } else if (!nodeStatus.isRelay() && nodeStatus.getLastSeenMillis() > + previousBridgesLastValidAfterMillis) { + previousBridgesLastValidAfterMillis = + nodeStatus.getLastSeenMillis(); } } - } - - private void setCurrentNodes() { + if (previousRelaysLastValidAfterMillis > + this.relaysLastValidAfterMillis) { + this.relaysLastValidAfterMillis = + previousRelaysLastValidAfterMillis; + } + if (previousBridgesLastValidAfterMillis > + this.bridgesLastPublishedMillis) { + this.bridgesLastPublishedMillis = + previousBridgesLastValidAfterMillis; + } long cutoff = Math.max(this.relaysLastValidAfterMillis, - this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L; - SortedMap<String, NodeStatus> currentNodes = - new TreeMap<String, NodeStatus>(); + this.bridgesLastPublishedMillis) - DateTimeHelper.ONE_WEEK; for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { - if (e.getValue().getLastSeenMillis() >= cutoff) { - currentNodes.put(e.getKey(), e.getValue()); + String fingerprint = e.getKey(); + NodeStatus nodeStatus = e.getValue(); + this.updatedNodes.add(fingerprint); + if (nodeStatus.isRelay() && + nodeStatus.getLastSeenMillis() >= cutoff) { + this.currentRelays.add(fingerprint); + if (nodeStatus.getLastSeenMillis() == + this.relaysLastValidAfterMillis) { + this.runningRelays.add(fingerprint); + } } } - this.relays = new TreeMap<String, NodeStatus>(); - this.bridges = new TreeMap<String, NodeStatus>(); - for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) { - if (e.getValue().isRelay()) { - this.relays.put(e.getKey(), e.getValue()); + for (String fingerprint : previouslyKnownNodes) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, true, fingerprint); + NodeStatus updatedNodeStatus = null; + if (this.knownNodes.containsKey(fingerprint)) { + updatedNodeStatus = this.knownNodes.get(fingerprint); + String address = nodeStatus.getAddress(); + int orPort = nodeStatus.getOrPort(), + dirPort = nodeStatus.getDirPort(); + SortedSet<String> orAddressesAndPorts = + nodeStatus.getOrAddressesAndPorts(); + updatedNodeStatus.addLastAddresses( + nodeStatus.getLastChangedOrAddressOrPort(), address, orPort, + dirPort, orAddressesAndPorts); + if (nodeStatus.getLastSeenMillis() > + updatedNodeStatus.getLastSeenMillis()) { + updatedNodeStatus.setNickname(nodeStatus.getNickname()); + updatedNodeStatus.setAddress(address); + updatedNodeStatus.setOrAddressesAndPorts(orAddressesAndPorts); + updatedNodeStatus.setLastSeenMillis( + nodeStatus.getLastSeenMillis()); + updatedNodeStatus.setOrPort(orPort); + updatedNodeStatus.setDirPort(dirPort); + updatedNodeStatus.setRelayFlags(nodeStatus.getRelayFlags()); + updatedNodeStatus.setConsensusWeight( + nodeStatus.getConsensusWeight()); + updatedNodeStatus.setCountryCode(nodeStatus.getCountryCode()); + updatedNodeStatus.setDefaultPolicy( + nodeStatus.getDefaultPolicy()); + updatedNodeStatus.setPortList(nodeStatus.getPortList()); + updatedNodeStatus.setASNumber(nodeStatus.getASNumber()); + updatedNodeStatus.setRecommendedVersion( + nodeStatus.getRecommendedVersion()); + } + if (nodeStatus.getFirstSeenMillis() < + updatedNodeStatus.getFirstSeenMillis()) { + updatedNodeStatus.setFirstSeenMillis( + nodeStatus.getFirstSeenMillis()); + } + updatedNodeStatus.setLastRdnsLookup( + nodeStatus.getLastRdnsLookup()); } else { - this.bridges.put(e.getKey(), e.getValue()); + updatedNodeStatus = nodeStatus; + this.knownNodes.put(fingerprint, nodeStatus); + if (nodeStatus.getLastSeenMillis() == (nodeStatus.isRelay() ? + previousRelaysLastValidAfterMillis : + previousBridgesLastValidAfterMillis)) { + /* This relay or bridge was previously running, but we didn't + * parse any descriptors with its fingerprint. Make sure to + * update its details status file later on, so it has the + * correct running bit. */ + this.updatedNodes.add(fingerprint); + } + } + if (updatedNodeStatus.isRelay() && + updatedNodeStatus.getLastSeenMillis() >= cutoff) { + this.currentRelays.add(fingerprint); + if (updatedNodeStatus.getLastSeenMillis() == + this.relaysLastValidAfterMillis) { + this.runningRelays.add(fingerprint); + } } } } + /* Step 3: perform lookups and calculate path selection + * probabilities. */ + private void startReverseDomainNameLookups() { Map<String, Long> addressLastLookupTimes = new HashMap<String, Long>(); - for (NodeStatus relay : relays.values()) { - addressLastLookupTimes.put(relay.getAddress(), - relay.getLastRdnsLookup()); + for (String fingerprint : this.currentRelays) { + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + addressLastLookupTimes.put(nodeStatus.getAddress(), + nodeStatus.getLastRdnsLookup()); } this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes); this.reverseDomainNameResolver.startReverseDomainNameLookups(); } + private SortedMap<String, LookupResult> geoIpLookupResults = + new TreeMap<String, LookupResult>(); + private void lookUpCitiesAndASes() { SortedSet<String> addressStrings = new TreeSet<String>(); - for (NodeStatus node : this.knownNodes.values()) { - if (node.isRelay()) { - addressStrings.add(node.getAddress()); - } + for (String fingerprint : this.currentRelays) { + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + addressStrings.add(nodeStatus.getAddress()); } if (addressStrings.isEmpty()) { log.error("No relay IP addresses to resolve to cities or " @@ -379,67 +489,22 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, } SortedMap<String, LookupResult> lookupResults = this.lookupService.lookup(addressStrings); - for (NodeStatus node : knownNodes.values()) { - if (!node.isRelay()) { - continue; - } - String addressString = node.getAddress(); - if (lookupResults.containsKey(addressString)) { - LookupResult lookupResult = lookupResults.get(addressString); - node.setCountryCode(lookupResult.getCountryCode()); - node.setCountryName(lookupResult.getCountryName()); - node.setRegionName(lookupResult.getRegionName()); - node.setCityName(lookupResult.getCityName()); - node.setLatitude(lookupResult.getLatitude()); - node.setLongitude(lookupResult.getLongitude()); - node.setASNumber(lookupResult.getAsNumber()); - node.setASName(lookupResult.getAsName()); + for (String fingerprint : this.currentRelays) { + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + LookupResult lookupResult = lookupResults.get( + nodeStatus.getAddress()); + if (lookupResult != null) { + this.geoIpLookupResults.put(fingerprint, lookupResult); + this.updatedNodes.add(fingerprint); } } } - private void setDescriptorPartsOfNodeStatus() { - for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus node = e.getValue(); - if (node.isRelay()) { - if (node.getRelayFlags().contains("Running") && - node.getLastSeenMillis() == this.relaysLastValidAfterMillis) { - node.setRunning(true); - } - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus != null) { - node.setContact(detailsStatus.getContact()); - if (detailsStatus.getExitAddresses() != null) { - for (Map.Entry<String, Long> ea : - detailsStatus.getExitAddresses().entrySet()) { - if (ea.getValue() >= this.now - DateTimeHelper.ONE_DAY) { - node.addExitAddress(ea.getKey()); - } - } - } - if (detailsStatus.getFamily() != null && - !detailsStatus.getFamily().isEmpty()) { - SortedSet<String> familyFingerprints = new TreeSet<String>(); - for (String familyMember : detailsStatus.getFamily()) { - if (familyMember.startsWith("$") && - familyMember.length() == 41) { - familyFingerprints.add(familyMember.substring(1)); - } - } - if (!familyFingerprints.isEmpty()) { - node.setFamilyFingerprints(familyFingerprints); - } - } - } - } - if (!node.isRelay() && node.getRelayFlags().contains("Running") && - node.getLastSeenMillis() == this.bridgesLastPublishedMillis) { - node.setRunning(true); - } - } - } + private SortedMap<String, Float> + consensusWeightFractions = new TreeMap<String, Float>(), + guardProbabilities = new TreeMap<String, Float>(), + middleProbabilities = new TreeMap<String, Float>(), + exitProbabilities = new TreeMap<String, Float>(); private void calculatePathSelectionProbabilities() { boolean consensusContainsBandwidthWeights = false; @@ -475,16 +540,12 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, double totalGuardWeight = 0.0; double totalMiddleWeight = 0.0; double totalExitWeight = 0.0; - for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus relay = e.getValue(); - if (!relay.getRunning()) { - continue; - } - boolean isExit = relay.getRelayFlags().contains("Exit") && - !relay.getRelayFlags().contains("BadExit"); - boolean isGuard = relay.getRelayFlags().contains("Guard"); - double consensusWeight = (double) relay.getConsensusWeight(); + for (String fingerprint : this.runningRelays) { + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + boolean isExit = nodeStatus.getRelayFlags().contains("Exit") && + !nodeStatus.getRelayFlags().contains("BadExit"); + boolean isGuard = nodeStatus.getRelayFlags().contains("Guard"); + double consensusWeight = (double) nodeStatus.getConsensusWeight(); consensusWeights.put(fingerprint, consensusWeight); totalConsensusWeight += consensusWeight; if (consensusContainsBandwidthWeights) { @@ -516,57 +577,67 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, totalExitWeight += exitWeight; } } - for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus relay = e.getValue(); + for (String fingerprint : this.runningRelays) { if (consensusWeights.containsKey(fingerprint)) { - relay.setConsensusWeightFraction(consensusWeights.get(fingerprint) - / totalConsensusWeight); + this.consensusWeightFractions.put(fingerprint, (float) + (consensusWeights.get(fingerprint) / totalConsensusWeight)); + this.updatedNodes.add(fingerprint); } if (guardWeights.containsKey(fingerprint)) { - relay.setGuardProbability(guardWeights.get(fingerprint) - / totalGuardWeight); + this.guardProbabilities.put(fingerprint, (float) + (guardWeights.get(fingerprint) / totalGuardWeight)); + this.updatedNodes.add(fingerprint); } if (middleWeights.containsKey(fingerprint)) { - relay.setMiddleProbability(middleWeights.get(fingerprint) - / totalMiddleWeight); + this.middleProbabilities.put(fingerprint, (float) + (middleWeights.get(fingerprint) / totalMiddleWeight)); + this.updatedNodes.add(fingerprint); } if (exitWeights.containsKey(fingerprint)) { - relay.setExitProbability(exitWeights.get(fingerprint) - / totalExitWeight); + this.exitProbabilities.put(fingerprint, (float) + (exitWeights.get(fingerprint) / totalExitWeight)); + this.updatedNodes.add(fingerprint); } } } + private long startedRdnsLookups = -1L; + + private SortedMap<String, String> rdnsLookupResults = + new TreeMap<String, String>(); + private void finishReverseDomainNameLookups() { this.reverseDomainNameResolver.finishReverseDomainNameLookups(); + this.startedRdnsLookups = + this.reverseDomainNameResolver.getLookupStartMillis(); Map<String, String> lookupResults = this.reverseDomainNameResolver.getLookupResults(); - long startedRdnsLookups = - this.reverseDomainNameResolver.getLookupStartMillis(); - for (NodeStatus relay : relays.values()) { - if (lookupResults.containsKey(relay.getAddress())) { - relay.setHostName(lookupResults.get(relay.getAddress())); - relay.setLastRdnsLookup(startedRdnsLookups); + for (String fingerprint : this.currentRelays) { + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + String hostName = lookupResults.get(nodeStatus.getAddress()); + if (hostName != null) { + this.rdnsLookupResults.put(fingerprint, hostName); + this.updatedNodes.add(fingerprint); } } } - private void writeStatusSummary() { - for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { - this.documentStore.store(e.getValue(), e.getKey()); - } - } + /* Step 4: update details statuses and then node statuses. */ - private void updateDetailsStatuses() { - SortedSet<String> fingerprints = new TreeSet<String>(); - fingerprints.addAll(this.exitListEntries.keySet()); - for (String fingerprint : fingerprints) { + private void updateNodeDetailsStatuses() { + for (String fingerprint : this.updatedNodes) { + NodeStatus nodeStatus = this.knownNodes.get(fingerprint); + if (nodeStatus == null) { + nodeStatus = new NodeStatus(fingerprint); + } DetailsStatus detailsStatus = this.documentStore.retrieve( DetailsStatus.class, true, fingerprint); if (detailsStatus == null) { detailsStatus = new DetailsStatus(); } + + nodeStatus.setContact(detailsStatus.getContact()); + Map<String, Long> exitAddresses = new HashMap<String, Long>(); if (detailsStatus.getExitAddresses() != null) { for (Map.Entry<String, Long> e : @@ -579,20 +650,94 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, if (this.exitListEntries.containsKey(fingerprint)) { for (Map.Entry<String, Long> e : this.exitListEntries.get(fingerprint).entrySet()) { - if (!exitAddresses.containsKey(e.getKey()) || - exitAddresses.get(e.getKey()) < e.getValue()) { - exitAddresses.put(e.getKey(), e.getValue()); + String exitAddress = e.getKey(); + long scanMillis = e.getValue(); + if (!exitAddresses.containsKey(exitAddress) || + exitAddresses.get(exitAddress) < scanMillis) { + exitAddresses.put(exitAddress, scanMillis); } } } - if (this.knownNodes.containsKey(fingerprint)) { - for (String orAddress : - this.knownNodes.get(fingerprint).getOrAddresses()) { - this.exitListEntries.remove(orAddress); + detailsStatus.setExitAddresses(exitAddresses); + SortedSet<String> exitAddressesWithoutOrAddresses = + new TreeSet<String>(exitAddresses.keySet()); + exitAddressesWithoutOrAddresses.removeAll( + nodeStatus.getOrAddresses()); + nodeStatus.setExitAddresses(exitAddressesWithoutOrAddresses); + + if (detailsStatus.getFamily() != null && + !detailsStatus.getFamily().isEmpty()) { + SortedSet<String> familyFingerprints = new TreeSet<String>(); + for (String familyMember : detailsStatus.getFamily()) { + if (familyMember.startsWith("$") && + familyMember.length() == 41) { + familyFingerprints.add(familyMember.substring(1)); + } + } + if (!familyFingerprints.isEmpty()) { + nodeStatus.setFamilyFingerprints(familyFingerprints); } } - detailsStatus.setExitAddresses(exitAddresses); + + if (this.geoIpLookupResults.containsKey(fingerprint)) { + LookupResult lookupResult = this.geoIpLookupResults.get( + fingerprint); + detailsStatus.setCountryCode(lookupResult.getCountryCode()); + detailsStatus.setCountryName(lookupResult.getCountryName()); + detailsStatus.setRegionName(lookupResult.getRegionName()); + detailsStatus.setCityName(lookupResult.getCityName()); + detailsStatus.setLatitude(lookupResult.getLatitude()); + detailsStatus.setLongitude(lookupResult.getLongitude()); + detailsStatus.setASNumber(lookupResult.getAsNumber()); + detailsStatus.setASName(lookupResult.getAsName()); + nodeStatus.setCountryCode(lookupResult.getCountryCode()); + nodeStatus.setASNumber(lookupResult.getAsNumber()); + } + + detailsStatus.setConsensusWeightFraction( + this.consensusWeightFractions.get(fingerprint)); + detailsStatus.setGuardProbability( + this.guardProbabilities.get(fingerprint)); + detailsStatus.setMiddleProbability( + this.middleProbabilities.get(fingerprint)); + detailsStatus.setExitProbability( + this.exitProbabilities.get(fingerprint)); + + if (this.rdnsLookupResults.containsKey(fingerprint)) { + String hostName = this.rdnsLookupResults.get(fingerprint); + detailsStatus.setHostName(hostName); + nodeStatus.setLastRdnsLookup(this.startedRdnsLookups); + } + + if (this.bridgePoolAssignments.containsKey(fingerprint)) { + detailsStatus.setPoolAssignment( + this.bridgePoolAssignments.get(fingerprint)); + } + + detailsStatus.setRelay(nodeStatus.isRelay()); + detailsStatus.setRunning(nodeStatus.getLastSeenMillis() == + (nodeStatus.isRelay() + ? this.relaysLastValidAfterMillis + : this.bridgesLastPublishedMillis)); + detailsStatus.setNickname(nodeStatus.getNickname()); + detailsStatus.setAddress(nodeStatus.getAddress()); + detailsStatus.setOrAddressesAndPorts( + nodeStatus.getOrAddressesAndPorts()); + detailsStatus.setFirstSeenMillis(nodeStatus.getFirstSeenMillis()); + detailsStatus.setLastSeenMillis(nodeStatus.getLastSeenMillis()); + detailsStatus.setOrPort(nodeStatus.getOrPort()); + detailsStatus.setDirPort(nodeStatus.getDirPort()); + detailsStatus.setRelayFlags(nodeStatus.getRelayFlags()); + detailsStatus.setConsensusWeight(nodeStatus.getConsensusWeight()); + detailsStatus.setDefaultPolicy(nodeStatus.getDefaultPolicy()); + detailsStatus.setPortList(nodeStatus.getPortList()); + detailsStatus.setRecommendedVersion( + nodeStatus.getRecommendedVersion()); + detailsStatus.setLastChangedOrAddressOrPort( + nodeStatus.getLastChangedOrAddressOrPort()); + this.documentStore.store(detailsStatus, fingerprint); + this.documentStore.store(nodeStatus, fingerprint); } } diff --git a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java index 0692070..7f464b6 100644 --- a/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -10,14 +10,11 @@ import java.util.TreeSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.torproject.onionoo.docs.DateTimeHelper; import org.torproject.onionoo.docs.DetailsDocument; import org.torproject.onionoo.docs.DetailsStatus; import org.torproject.onionoo.docs.DocumentStore; import org.torproject.onionoo.docs.DocumentStoreFactory; -import org.torproject.onionoo.docs.NodeStatus; import org.torproject.onionoo.docs.UpdateStatus; -import org.torproject.onionoo.util.TimeFactory; public class DetailsDocumentWriter implements DocumentWriter { @@ -26,200 +23,135 @@ public class DetailsDocumentWriter implements DocumentWriter { private DocumentStore documentStore; - private long now; - public DetailsDocumentWriter() { this.documentStore = DocumentStoreFactory.getDocumentStore(); - this.now = TimeFactory.getTime().currentTimeMillis(); } - private SortedSet<String> newRelays = new TreeSet<String>(), - newBridges = new TreeSet<String>(); - public void writeDocuments() { UpdateStatus updateStatus = this.documentStore.retrieve( UpdateStatus.class, true); long updatedMillis = updateStatus != null ? updateStatus.getUpdatedMillis() : 0L; - SortedSet<String> updatedNodeStatuses = this.documentStore.list( - NodeStatus.class, updatedMillis); - for (String fingerprint : updatedNodeStatuses) { - NodeStatus nodeStatus = this.documentStore.retrieve( - NodeStatus.class, true, fingerprint); - if (nodeStatus.isRelay()) { - newRelays.add(fingerprint); - } else { - newBridges.add(fingerprint); - } - } SortedSet<String> updatedDetailsStatuses = this.documentStore.list( DetailsStatus.class, updatedMillis); for (String fingerprint : updatedDetailsStatuses) { - NodeStatus nodeStatus = this.documentStore.retrieve( - NodeStatus.class, true, fingerprint); - if (nodeStatus == null) { - continue; - } else if (nodeStatus.isRelay()) { - newRelays.add(fingerprint); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + if (detailsStatus.isRelay()) { + this.updateRelayDetailsFile(fingerprint, detailsStatus); } else { - newBridges.add(fingerprint); + this.updateBridgeDetailsFile(fingerprint, detailsStatus); } } - this.updateRelayDetailsFiles(); - this.updateBridgeDetailsFiles(); log.info("Wrote details document files"); } - private void updateRelayDetailsFiles() { - for (String fingerprint : this.newRelays) { - - /* Generate network-status-specific part. */ - NodeStatus entry = this.documentStore.retrieve(NodeStatus.class, - true, fingerprint); - if (entry == null) { - continue; - } - DetailsDocument detailsDocument = new DetailsDocument(); - detailsDocument.setNickname(entry.getNickname()); - detailsDocument.setFingerprint(fingerprint); - List<String> orAddresses = new ArrayList<String>(); - orAddresses.add(entry.getAddress() + ":" + entry.getOrPort()); - for (String orAddress : entry.getOrAddressesAndPorts()) { - orAddresses.add(orAddress.toLowerCase()); - } - detailsDocument.setOrAddresses(orAddresses); - if (entry.getDirPort() != 0) { - detailsDocument.setDirAddress(entry.getAddress() + ":" - + entry.getDirPort()); - } - detailsDocument.setLastSeen(entry.getLastSeenMillis()); - detailsDocument.setFirstSeen(entry.getFirstSeenMillis()); - detailsDocument.setLastChangedAddressOrPort( - entry.getLastChangedOrAddress()); - detailsDocument.setRunning(entry.getRunning()); - if (!entry.getRelayFlags().isEmpty()) { - detailsDocument.setFlags(new ArrayList<String>( - entry.getRelayFlags())); - } - detailsDocument.setCountry(entry.getCountryCode()); - detailsDocument.setLatitude(entry.getLatitude()); - detailsDocument.setLongitude(entry.getLongitude()); - detailsDocument.setCountryName(entry.getCountryName()); - detailsDocument.setRegionName(entry.getRegionName()); - detailsDocument.setCityName(entry.getCityName()); - detailsDocument.setAsNumber(entry.getASNumber()); - detailsDocument.setAsName(entry.getASName()); - detailsDocument.setConsensusWeight(entry.getConsensusWeight()); - detailsDocument.setHostName(entry.getHostName()); + private void updateRelayDetailsFile(String fingerprint, + DetailsStatus detailsStatus) { + DetailsDocument detailsDocument = new DetailsDocument(); + detailsDocument.setNickname(detailsStatus.getNickname()); + detailsDocument.setFingerprint(fingerprint); + List<String> orAddresses = new ArrayList<String>(); + orAddresses.add(detailsStatus.getAddress() + ":" + + detailsStatus.getOrPort()); + for (String orAddress : detailsStatus.getOrAddressesAndPorts()) { + orAddresses.add(orAddress.toLowerCase()); + } + detailsDocument.setOrAddresses(orAddresses); + if (detailsStatus.getDirPort() != 0) { + detailsDocument.setDirAddress(detailsStatus.getAddress() + ":" + + detailsStatus.getDirPort()); + } + detailsDocument.setLastSeen(detailsStatus.getLastSeenMillis()); + detailsDocument.setFirstSeen(detailsStatus.getFirstSeenMillis()); + detailsDocument.setLastChangedAddressOrPort( + detailsStatus.getLastChangedOrAddressOrPort()); + detailsDocument.setRunning(detailsStatus.isRunning()); + detailsDocument.setFlags(detailsStatus.getRelayFlags()); + detailsDocument.setConsensusWeight( + detailsStatus.getConsensusWeight()); + detailsDocument.setHostName(detailsStatus.getHostName()); + String defaultPolicy = detailsStatus.getDefaultPolicy(); + String portList = detailsStatus.getPortList(); + if (defaultPolicy != null && (defaultPolicy.equals("accept") || + defaultPolicy.equals("reject")) && portList != null) { + Map<String, List<String>> exitPolicySummary = + new HashMap<String, List<String>>(); + List<String> portsOrPortRanges = Arrays.asList(portList.split(",")); + exitPolicySummary.put(defaultPolicy, portsOrPortRanges); + detailsDocument.setExitPolicySummary(exitPolicySummary); + } + detailsDocument.setRecommendedVersion( + detailsStatus.getRecommendedVersion()); + detailsDocument.setCountry(detailsStatus.getCountryCode()); + detailsDocument.setLatitude(detailsStatus.getLatitude()); + detailsDocument.setLongitude(detailsStatus.getLongitude()); + detailsDocument.setCountryName(detailsStatus.getCountryName()); + detailsDocument.setRegionName(detailsStatus.getRegionName()); + detailsDocument.setCityName(detailsStatus.getCityName()); + detailsDocument.setAsNumber(detailsStatus.getASNumber()); + detailsDocument.setAsName(detailsStatus.getASName()); + if (detailsStatus.isRunning()) { detailsDocument.setConsensusWeightFraction( - (float) entry.getConsensusWeightFraction()); + detailsStatus.getConsensusWeightFraction()); detailsDocument.setGuardProbability( - (float) entry.getGuardProbability()); + detailsStatus.getGuardProbability()); detailsDocument.setMiddleProbability( - (float) entry.getMiddleProbability()); + detailsStatus.getMiddleProbability()); detailsDocument.setExitProbability( - (float) entry.getExitProbability()); - String defaultPolicy = entry.getDefaultPolicy(); - String portList = entry.getPortList(); - if (defaultPolicy != null && (defaultPolicy.equals("accept") || - defaultPolicy.equals("reject")) && portList != null) { - Map<String, List<String>> exitPolicySummary = - new HashMap<String, List<String>>(); - List<String> portsOrPortRanges = Arrays.asList( - portList.split(",")); - exitPolicySummary.put(defaultPolicy, portsOrPortRanges); - detailsDocument.setExitPolicySummary(exitPolicySummary); - } - detailsDocument.setRecommendedVersion( - entry.getRecommendedVersion()); - - /* Append descriptor-specific part and exit addresses from details - * status file. */ - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus != null) { - detailsDocument.setLastRestarted( - detailsStatus.getLastRestarted()); - detailsDocument.setBandwidthRate( - detailsStatus.getBandwidthRate()); - detailsDocument.setBandwidthBurst( - detailsStatus.getBandwidthBurst()); - detailsDocument.setObservedBandwidth( - detailsStatus.getObservedBandwidth()); - detailsDocument.setAdvertisedBandwidth( - detailsStatus.getAdvertisedBandwidth()); - detailsDocument.setExitPolicy(detailsStatus.getExitPolicy()); - detailsDocument.setContact(detailsStatus.getContact()); - detailsDocument.setPlatform(detailsStatus.getPlatform()); - detailsDocument.setFamily(detailsStatus.getFamily()); - detailsDocument.setExitPolicyV6Summary( - detailsStatus.getExitPolicyV6Summary()); - detailsDocument.setHibernating(detailsStatus.getHibernating()); - if (detailsStatus.getExitAddresses() != null) { - SortedSet<String> exitAddresses = new TreeSet<String>(); - for (Map.Entry<String, Long> e : - detailsStatus.getExitAddresses().entrySet()) { - String exitAddress = e.getKey().toLowerCase(); - long scanMillis = e.getValue(); - if (!entry.getAddress().equals(exitAddress) && - !entry.getOrAddresses().contains(exitAddress) && - scanMillis >= this.now - DateTimeHelper.ONE_DAY) { - exitAddresses.add(exitAddress); - } - } - if (!exitAddresses.isEmpty()) { - detailsDocument.setExitAddresses(new ArrayList<String>( - exitAddresses)); - } - } - } - - /* Write details file to disk. */ - this.documentStore.store(detailsDocument, fingerprint); + detailsStatus.getExitProbability()); + } + detailsDocument.setLastRestarted(detailsStatus.getLastRestarted()); + detailsDocument.setBandwidthRate(detailsStatus.getBandwidthRate()); + detailsDocument.setBandwidthBurst(detailsStatus.getBandwidthBurst()); + detailsDocument.setObservedBandwidth( + detailsStatus.getObservedBandwidth()); + detailsDocument.setAdvertisedBandwidth( + detailsStatus.getAdvertisedBandwidth()); + detailsDocument.setExitPolicy(detailsStatus.getExitPolicy()); + detailsDocument.setContact(detailsStatus.getContact()); + detailsDocument.setPlatform(detailsStatus.getPlatform()); + detailsDocument.setFamily(detailsStatus.getFamily()); + detailsDocument.setExitPolicyV6Summary( + detailsStatus.getExitPolicyV6Summary()); + detailsDocument.setHibernating(detailsStatus.getHibernating()); + if (detailsStatus.getExitAddresses() != null) { + SortedSet<String> exitAddressesWithoutOrAddresses = + new TreeSet<String>(detailsStatus.getExitAddresses().keySet()); + exitAddressesWithoutOrAddresses.removeAll( + detailsStatus.getOrAddresses()); + detailsDocument.setExitAddresses(new ArrayList<String>( + exitAddressesWithoutOrAddresses)); } + this.documentStore.store(detailsDocument, fingerprint); } - private void updateBridgeDetailsFiles() { - for (String fingerprint : this.newBridges) { - - /* Generate network-status-specific part. */ - NodeStatus entry = this.documentStore.retrieve(NodeStatus.class, - true, fingerprint); - if (entry == null) { - continue; - } - DetailsDocument detailsDocument = new DetailsDocument(); - detailsDocument.setNickname(entry.getNickname()); - detailsDocument.setHashedFingerprint(fingerprint); - String address = entry.getAddress(); - List<String> orAddresses = new ArrayList<String>(); - orAddresses.add(address + ":" + entry.getOrPort()); - for (String orAddress : entry.getOrAddressesAndPorts()) { + private void updateBridgeDetailsFile(String fingerprint, + DetailsStatus detailsStatus) { + DetailsDocument detailsDocument = new DetailsDocument(); + detailsDocument.setNickname(detailsStatus.getNickname()); + detailsDocument.setHashedFingerprint(fingerprint); + String address = detailsStatus.getAddress(); + List<String> orAddresses = new ArrayList<String>(); + orAddresses.add(address + ":" + detailsStatus.getOrPort()); + SortedSet<String> orAddressesAndPorts = + detailsStatus.getOrAddressesAndPorts(); + if (orAddressesAndPorts != null) { + for (String orAddress : orAddressesAndPorts) { orAddresses.add(orAddress.toLowerCase()); } - detailsDocument.setOrAddresses(orAddresses); - detailsDocument.setLastSeen(entry.getLastSeenMillis()); - detailsDocument.setFirstSeen(entry.getFirstSeenMillis()); - detailsDocument.setRunning(entry.getRunning()); - detailsDocument.setFlags(new ArrayList<String>( - entry.getRelayFlags())); - - /* Append descriptor-specific part from details status file. */ - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus != null) { - detailsDocument.setLastRestarted( - detailsStatus.getLastRestarted()); - detailsDocument.setAdvertisedBandwidth( - detailsStatus.getAdvertisedBandwidth()); - detailsDocument.setPlatform(detailsStatus.getPlatform()); - detailsDocument.setPoolAssignment( - detailsStatus.getPoolAssignment()); - } - - /* Write details file to disk. */ - this.documentStore.store(detailsDocument, fingerprint); } + detailsDocument.setOrAddresses(orAddresses); + detailsDocument.setLastSeen(detailsStatus.getLastSeenMillis()); + detailsDocument.setFirstSeen(detailsStatus.getFirstSeenMillis()); + detailsDocument.setRunning(detailsStatus.isRunning()); + detailsDocument.setFlags(detailsStatus.getRelayFlags()); + detailsDocument.setLastRestarted(detailsStatus.getLastRestarted()); + detailsDocument.setAdvertisedBandwidth( + detailsStatus.getAdvertisedBandwidth()); + detailsDocument.setPlatform(detailsStatus.getPlatform()); + detailsDocument.setPoolAssignment(detailsStatus.getPoolAssignment()); + this.documentStore.store(detailsDocument, fingerprint); } public String getStatsString() { diff --git a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java index d7686cb..6406a28 100644 --- a/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java +++ b/src/main/java/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -29,16 +29,23 @@ public class SummaryDocumentWriter implements DocumentWriter { private int writtenDocuments = 0, deletedDocuments = 0; public void writeDocuments() { - long maxLastSeenMillis = 0L; + long relaysLastValidAfterMillis = -1L, + bridgesLastPublishedMillis = -1L; for (String fingerprint : this.documentStore.list(NodeStatus.class)) { NodeStatus nodeStatus = this.documentStore.retrieve( NodeStatus.class, true, fingerprint); - if (nodeStatus != null && - nodeStatus.getLastSeenMillis() > maxLastSeenMillis) { - maxLastSeenMillis = nodeStatus.getLastSeenMillis(); + if (nodeStatus != null) { + if (nodeStatus.isRelay()) { + relaysLastValidAfterMillis = Math.max( + relaysLastValidAfterMillis, nodeStatus.getLastSeenMillis()); + } else { + bridgesLastPublishedMillis = Math.max( + bridgesLastPublishedMillis, nodeStatus.getLastSeenMillis()); + } } } - long cutoff = maxLastSeenMillis - DateTimeHelper.ONE_WEEK; + long cutoff = Math.max(relaysLastValidAfterMillis, + bridgesLastPublishedMillis) - DateTimeHelper.ONE_WEEK; for (String fingerprint : this.documentStore.list(NodeStatus.class)) { NodeStatus nodeStatus = this.documentStore.retrieve( NodeStatus.class, @@ -68,8 +75,10 @@ public class SummaryDocumentWriter implements DocumentWriter { } } long lastSeenMillis = nodeStatus.getLastSeenMillis(); - boolean running = nodeStatus.getRunning(); SortedSet<String> relayFlags = nodeStatus.getRelayFlags(); + boolean running = relayFlags.contains("Running") && (isRelay ? + lastSeenMillis == relaysLastValidAfterMillis : + lastSeenMillis == bridgesLastPublishedMillis); long consensusWeight = nodeStatus.getConsensusWeight(); String countryCode = nodeStatus.getCountryCode(); long firstSeenMillis = nodeStatus.getFirstSeenMillis();