April 2014 - tor-commits - lists.torproject.org

[onionoo/master] Add fingerprint listener to learn which documents need updating.
by karsten＠torproject.org 11 Apr '14

11 Apr '14

commit 64fecd018ad2373635c94fa167523ef476a349ef Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Mar 12 20:16:34 2014 +0100 Add fingerprint listener to learn which documents need updating. Our data writers register at the descriptor source as descriptor listeners to be notified when new descriptors are parsed. They use these descriptors to update their internal status files and to know which output documents need to be rewritten. We now add a second type of listener, the fingerprint listener, which can be used to learn relay or bridge fingerprints from new descriptors. Knowing the fingerprints is sufficient to rewrite output documents, because their content is only based on internal status files. This will enable us to later split data writers into status updaters and document writers. --- .../torproject/onionoo/BandwidthDataWriter.java | 21 ++++- src/org/torproject/onionoo/ClientsDataWriter.java | 24 +++++- src/org/torproject/onionoo/DescriptorSource.java | 82 +++++++++++++++++++- src/org/torproject/onionoo/NodeDataWriter.java | 37 +++++++-- src/org/torproject/onionoo/UptimeDataWriter.java | 31 ++++++-- src/org/torproject/onionoo/WeightsDataWriter.java | 23 +++++- 6 files changed, 191 insertions(+), 27 deletions(-) diff --git a/src/org/torproject/onionoo/BandwidthDataWriter.java b/src/org/torproject/onionoo/BandwidthDataWriter.java index 9f2f97e..67dc427 100644 --- a/src/org/torproject/onionoo/BandwidthDataWriter.java +++ b/src/org/torproject/onionoo/BandwidthDataWriter.java @@ -9,6 +9,7 @@ import java.util.List; import java.util.Locale; import java.util.Scanner; import java.util.SortedMap; +import java.util.SortedSet; import java.util.TimeZone; import java.util.TreeMap; @@ -35,7 +36,7 @@ import org.torproject.descriptor.ExtraInfoDescriptor; * 3 days as of publishing the document, but that's something clients can * work around. */ public class BandwidthDataWriter implements DataWriter, - DescriptorListener { + DescriptorListener, FingerprintListener { private DescriptorSource descriptorSource; @@ -54,21 +55,35 @@ public class BandwidthDataWriter implements DataWriter, this.dateTimeFormat.setLenient(false); this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); this.registerDescriptorListeners(); + this.registerFingerprintListeners(); } private void registerDescriptorListeners() { - this.descriptorSource.registerListener(this, + this.descriptorSource.registerDescriptorListener(this, DescriptorType.RELAY_EXTRA_INFOS); - this.descriptorSource.registerListener(this, + this.descriptorSource.registerDescriptorListener(this, DescriptorType.BRIDGE_EXTRA_INFOS); } + private void registerFingerprintListeners() { + /* TODO Not used yet. + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_EXTRA_INFOS); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_EXTRA_INFOS);*/ + } + public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof ExtraInfoDescriptor) { this.parseDescriptor((ExtraInfoDescriptor) descriptor); } } + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + /* TODO Not used yet. */ + } + public void updateStatuses() { /* Status files are already updated while processing descriptors. */ } diff --git a/src/org/torproject/onionoo/ClientsDataWriter.java b/src/org/torproject/onionoo/ClientsDataWriter.java index b956f59..b045bfa 100644 --- a/src/org/torproject/onionoo/ClientsDataWriter.java +++ b/src/org/torproject/onionoo/ClientsDataWriter.java @@ -51,7 +51,8 @@ import org.torproject.descriptor.ExtraInfoDescriptor; * "transports":{"obfs2":0.4581}, * "versions":{"v4":1.0000}} */ -public class ClientsDataWriter implements DataWriter, DescriptorListener { +public class ClientsDataWriter implements DataWriter, DescriptorListener, + FingerprintListener { private static class ResponseHistory implements Comparable<ResponseHistory> { @@ -206,10 +207,16 @@ public class ClientsDataWriter implements DataWriter, DescriptorListener { this.documentStore = documentStore; this.now = time.currentTimeMillis(); this.registerDescriptorListeners(); + this.registerFingerprintListeners(); } private void registerDescriptorListeners() { - this.descriptorSource.registerListener(this, + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_EXTRA_INFOS); + } + + private void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, DescriptorType.BRIDGE_EXTRA_INFOS); } @@ -402,8 +409,17 @@ public class ClientsDataWriter implements DataWriter, DescriptorListener { this.documentStore.store(clientsStatus, hashedFingerprint); } + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + if (!relay) { + this.updateDocuments.addAll(fingerprints); + } + } + + private SortedSet<String> updateDocuments = new TreeSet<String>(); + public void updateDocuments() { - for (String hashedFingerprint : this.newResponses.keySet()) { + for (String hashedFingerprint : this.updateDocuments) { SortedSet<ResponseHistory> history = this.readHistory(hashedFingerprint); ClientsDocument clientsDocument = new ClientsDocument(); @@ -631,7 +647,7 @@ public class ClientsDataWriter implements DataWriter, DescriptorListener { + Logger.formatDecimalNumber(this.newResponses.size()) + " client status files updated\n"); sb.append(" " - + Logger.formatDecimalNumber(this.newResponses.size()) + + Logger.formatDecimalNumber(this.updateDocuments.size()) + " client document files updated\n"); return sb.toString(); } diff --git a/src/org/torproject/onionoo/DescriptorSource.java b/src/org/torproject/onionoo/DescriptorSource.java index bdf9be8..c72e8ce 100644 --- a/src/org/torproject/onionoo/DescriptorSource.java +++ b/src/org/torproject/onionoo/DescriptorSource.java @@ -16,12 +16,21 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; +import java.util.SortedSet; import java.util.TreeMap; +import java.util.TreeSet; +import org.torproject.descriptor.BridgeNetworkStatus; +import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorFile; import org.torproject.descriptor.DescriptorReader; import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.ExitList; +import org.torproject.descriptor.ExitListEntry; +import org.torproject.descriptor.ExtraInfoDescriptor; +import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.ServerDescriptor; enum DescriptorType { RELAY_CONSENSUSES, @@ -38,6 +47,11 @@ interface DescriptorListener { abstract void processDescriptor(Descriptor descriptor, boolean relay); } +interface FingerprintListener { + abstract void processFingerprints(SortedSet<String> fingerprints, + boolean relay); +} + enum DescriptorHistory { RELAY_CONSENSUS_HISTORY, RELAY_SERVER_HISTORY, @@ -253,6 +267,8 @@ public class DescriptorSource { this.descriptorQueues = new ArrayList<DescriptorQueue>(); this.descriptorListeners = new HashMap<DescriptorType, Set<DescriptorListener>>(); + this.fingerprintListeners = + new HashMap<DescriptorType, Set<FingerprintListener>>(); } private DescriptorQueue getDescriptorQueue( @@ -271,7 +287,10 @@ public class DescriptorSource { private Map<DescriptorType, Set<DescriptorListener>> descriptorListeners; - public void registerListener(DescriptorListener listener, + private Map<DescriptorType, Set<FingerprintListener>> + fingerprintListeners; + + public void registerDescriptorListener(DescriptorListener listener, DescriptorType descriptorType) { if (!this.descriptorListeners.containsKey(descriptorType)) { this.descriptorListeners.put(descriptorType, @@ -280,6 +299,15 @@ public class DescriptorSource { this.descriptorListeners.get(descriptorType).add(listener); } + public void registerFingerprintListener(FingerprintListener listener, + DescriptorType descriptorType) { + if (!this.fingerprintListeners.containsKey(descriptorType)) { + this.fingerprintListeners.put(descriptorType, + new HashSet<FingerprintListener>()); + } + this.fingerprintListeners.get(descriptorType).add(listener); + } + public void readRelayNetworkConsensuses() { this.readDescriptors(DescriptorType.RELAY_CONSENSUSES, DescriptorHistory.RELAY_CONSENSUS_HISTORY, true); @@ -322,11 +350,14 @@ public class DescriptorSource { private void readDescriptors(DescriptorType descriptorType, DescriptorHistory descriptorHistory, boolean relay) { - Set<DescriptorListener> descriptorListeners = - this.descriptorListeners.get(descriptorType); - if (descriptorListeners == null || descriptorListeners.isEmpty()) { + if (!this.descriptorListeners.containsKey(descriptorType) && + !this.fingerprintListeners.containsKey(descriptorType)) { return; } + Set<DescriptorListener> descriptorListeners = + this.descriptorListeners.get(descriptorType); + Set<FingerprintListener> fingerprintListeners = + this.fingerprintListeners.get(descriptorType); DescriptorQueue descriptorQueue = this.getDescriptorQueue( descriptorType, descriptorHistory); Descriptor descriptor; @@ -334,6 +365,49 @@ public class DescriptorSource { for (DescriptorListener descriptorListener : descriptorListeners) { descriptorListener.processDescriptor(descriptor, relay); } + SortedSet<String> fingerprints = new TreeSet<String>(); + if (descriptorType == DescriptorType.RELAY_CONSENSUSES && + descriptor instanceof RelayNetworkStatusConsensus) { + fingerprints.addAll(((RelayNetworkStatusConsensus) descriptor). + getStatusEntries().keySet()); + } else if (descriptorType + == DescriptorType.RELAY_SERVER_DESCRIPTORS && + descriptor instanceof ServerDescriptor) { + fingerprints.add(((ServerDescriptor) descriptor). + getFingerprint()); + } else if (descriptorType == DescriptorType.RELAY_EXTRA_INFOS && + descriptor instanceof ExtraInfoDescriptor) { + fingerprints.add(((ExtraInfoDescriptor) descriptor). + getFingerprint()); + } else if (descriptorType == DescriptorType.EXIT_LISTS && + descriptor instanceof ExitList) { + for (ExitListEntry entry : + ((ExitList) descriptor).getExitListEntries()) { + fingerprints.add(entry.getFingerprint()); + } + } else if (descriptorType == DescriptorType.BRIDGE_STATUSES && + descriptor instanceof BridgeNetworkStatus) { + fingerprints.addAll(((BridgeNetworkStatus) descriptor). + getStatusEntries().keySet()); + } else if (descriptorType == + DescriptorType.BRIDGE_SERVER_DESCRIPTORS && + descriptor instanceof ServerDescriptor) { + fingerprints.add(((ServerDescriptor) descriptor). + getFingerprint()); + } else if (descriptorType == DescriptorType.BRIDGE_EXTRA_INFOS && + descriptor instanceof ExtraInfoDescriptor) { + fingerprints.add(((ExtraInfoDescriptor) descriptor). + getFingerprint()); + } else if (descriptorType == + DescriptorType.BRIDGE_POOL_ASSIGNMENTS && + descriptor instanceof BridgePoolAssignment) { + fingerprints.addAll(((BridgePoolAssignment) descriptor). + getEntries().keySet()); + } + for (FingerprintListener fingerprintListener : + fingerprintListeners) { + fingerprintListener.processFingerprints(fingerprints, relay); + } } } diff --git a/src/org/torproject/onionoo/NodeDataWriter.java b/src/org/torproject/onionoo/NodeDataWriter.java index cc9addb..b59500a 100644 --- a/src/org/torproject/onionoo/NodeDataWriter.java +++ b/src/org/torproject/onionoo/NodeDataWriter.java @@ -33,7 +33,8 @@ import org.torproject.onionoo.LookupService.LookupResult; * The parts of details files coming from server descriptors always come * from the last known descriptor of a relay or bridge, not from the * descriptor that was last referenced in a network status. */ -public class NodeDataWriter implements DataWriter, DescriptorListener { +public class NodeDataWriter implements DataWriter, DescriptorListener, + FingerprintListener { private DescriptorSource descriptorSource; @@ -70,23 +71,40 @@ public class NodeDataWriter implements DataWriter, DescriptorListener { this.documentStore = documentStore; this.now = time.currentTimeMillis(); this.registerDescriptorListeners(); + this.registerFingerprintListeners(); } private void registerDescriptorListeners() { - this.descriptorSource.registerListener(this, + this.descriptorSource.registerDescriptorListener(this, DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerListener(this, + this.descriptorSource.registerDescriptorListener(this, DescriptorType.RELAY_SERVER_DESCRIPTORS); - this.descriptorSource.registerListener(this, + this.descriptorSource.registerDescriptorListener(this, DescriptorType.BRIDGE_STATUSES); - this.descriptorSource.registerListener(this, + this.descriptorSource.registerDescriptorListener(this, DescriptorType.BRIDGE_SERVER_DESCRIPTORS); - this.descriptorSource.registerListener(this, + this.descriptorSource.registerDescriptorListener(this, DescriptorType.BRIDGE_POOL_ASSIGNMENTS); - this.descriptorSource.registerListener(this, + this.descriptorSource.registerDescriptorListener(this, DescriptorType.EXIT_LISTS); } + private void registerFingerprintListeners() { + /* TODO Not used yet. + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_SERVER_DESCRIPTORS); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_STATUSES); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_SERVER_DESCRIPTORS); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_POOL_ASSIGNMENTS); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.EXIT_LISTS);*/ + } + public void processDescriptor(Descriptor descriptor, boolean relay) { if (descriptor instanceof RelayNetworkStatusConsensus) { this.updateRelayNetworkStatusConsensus( @@ -104,6 +122,11 @@ public class NodeDataWriter implements DataWriter, DescriptorListener { } } + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + /* TODO Not used yet. */ + } + public void updateStatuses() { this.readStatusSummary(); Logger.printStatusTime("Read status summary"); diff --git a/src/org/torproject/onionoo/UptimeDataWriter.java b/src/org/torproject/onionoo/UptimeDataWriter.java index 7a8b3af..e2b1bee 100644 --- a/src/org/torproject/onionoo/UptimeDataWriter.java +++ b/src/org/torproject/onionoo/UptimeDataWriter.java @@ -20,7 +20,8 @@ import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; -public class UptimeDataWriter implements DataWriter, DescriptorListener { +public class UptimeDataWriter implements DataWriter, DescriptorListener, + FingerprintListener { private DescriptorSource descriptorSource; @@ -34,12 +35,20 @@ public class UptimeDataWriter implements DataWriter, DescriptorListener { this.documentStore = documentStore; this.now = time.currentTimeMillis(); this.registerDescriptorListeners(); + this.registerFingerprintListeners(); } private void registerDescriptorListeners() { - this.descriptorSource.registerListener(this, + this.descriptorSource.registerDescriptorListener(this, DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerListener(this, + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_STATUSES); + } + + public void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerFingerprintListener(this, DescriptorType.BRIDGE_STATUSES); } @@ -284,6 +293,18 @@ public class UptimeDataWriter implements DataWriter, DescriptorListener { } } + private SortedSet<String> newRelayFingerprints = new TreeSet<String>(), + newBridgeFingerprints = new TreeSet<String>(); + + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + if (relay) { + this.newRelayFingerprints.addAll(fingerprints); + } else { + this.newBridgeFingerprints.addAll(fingerprints); + } + } + public void updateDocuments() { SortedSet<UptimeHistory> knownRelayStatuses = new TreeSet<UptimeHistory>(), @@ -296,10 +317,10 @@ public class UptimeDataWriter implements DataWriter, DescriptorListener { knownBridgeStatuses.add(status); } } - for (String fingerprint : this.newRunningRelays.keySet()) { + for (String fingerprint : this.newRelayFingerprints) { this.updateDocument(true, fingerprint, knownRelayStatuses); } - for (String fingerprint : this.newRunningBridges.keySet()) { + for (String fingerprint : this.newBridgeFingerprints) { this.updateDocument(false, fingerprint, knownBridgeStatuses); } Logger.printStatusTime("Wrote uptime document files"); diff --git a/src/org/torproject/onionoo/WeightsDataWriter.java b/src/org/torproject/onionoo/WeightsDataWriter.java index 855e3e9..21c3272 100644 --- a/src/org/torproject/onionoo/WeightsDataWriter.java +++ b/src/org/torproject/onionoo/WeightsDataWriter.java @@ -25,7 +25,8 @@ import org.torproject.descriptor.NetworkStatusEntry; import org.torproject.descriptor.RelayNetworkStatusConsensus; import org.torproject.descriptor.ServerDescriptor; -public class WeightsDataWriter implements DataWriter, DescriptorListener { +public class WeightsDataWriter implements DataWriter, DescriptorListener, + FingerprintListener { private DescriptorSource descriptorSource; @@ -39,12 +40,20 @@ public class WeightsDataWriter implements DataWriter, DescriptorListener { this.documentStore = documentStore; this.now = time.currentTimeMillis(); this.registerDescriptorListeners(); + this.registerFingerprintListeners(); } private void registerDescriptorListeners() { - this.descriptorSource.registerListener(this, + this.descriptorSource.registerDescriptorListener(this, DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerListener(this, + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_SERVER_DESCRIPTORS); + } + + private void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerFingerprintListener(this, DescriptorType.RELAY_SERVER_DESCRIPTORS); } @@ -289,7 +298,6 @@ public class WeightsDataWriter implements DataWriter, DescriptorListener { history.put(interval, weights); history = this.compressHistory(history); this.writeHistoryToDisk(fingerprint, history); - this.updateWeightsDocuments.add(fingerprint); this.updateWeightsStatuses.remove(fingerprint); } } @@ -456,6 +464,13 @@ public class WeightsDataWriter implements DataWriter, DescriptorListener { this.documentStore.store(weightsStatus, fingerprint); } + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + if (relay) { + this.updateWeightsDocuments.addAll(fingerprints); + } + } + private void writeWeightsDataFiles() { for (String fingerprint : this.updateWeightsDocuments) { SortedMap<long[], double[]> history =

1 0

[onionoo/master] Move (de-)serialization code into ClientsStatus.
by karsten＠torproject.org 11 Apr '14

11 Apr '14

commit bf01739df763e75030aa8731bf91ca7add4f8b83 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Apr 8 23:57:54 2014 +0200 Move (de-)serialization code into ClientsStatus. --- src/org/torproject/onionoo/ClientsDataWriter.java | 237 +++------------------ src/org/torproject/onionoo/ClientsStatus.java | 192 +++++++++++++++++ src/org/torproject/onionoo/DocumentStore.java | 6 +- 3 files changed, 230 insertions(+), 205 deletions(-) diff --git a/src/org/torproject/onionoo/ClientsDataWriter.java b/src/org/torproject/onionoo/ClientsDataWriter.java index 2ccdff7..7662f54 100644 --- a/src/org/torproject/onionoo/ClientsDataWriter.java +++ b/src/org/torproject/onionoo/ClientsDataWriter.java @@ -2,13 +2,11 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Scanner; import java.util.SortedMap; import java.util.SortedSet; import java.util.TimeZone; @@ -54,147 +52,6 @@ import org.torproject.descriptor.ExtraInfoDescriptor; public class ClientsDataWriter implements DescriptorListener, StatusUpdater, FingerprintListener, DocumentWriter { - private static class ResponseHistory - implements Comparable<ResponseHistory> { - private long startMillis; - private long endMillis; - private double totalResponses; - private SortedMap<String, Double> responsesByCountry; - private SortedMap<String, Double> responsesByTransport; - private SortedMap<String, Double> responsesByVersion; - private ResponseHistory(long startMillis, long endMillis, - double totalResponses, - SortedMap<String, Double> responsesByCountry, - SortedMap<String, Double> responsesByTransport, - SortedMap<String, Double> responsesByVersion) { - this.startMillis = startMillis; - this.endMillis = endMillis; - this.totalResponses = totalResponses; - this.responsesByCountry = responsesByCountry; - this.responsesByTransport = responsesByTransport; - this.responsesByVersion = responsesByVersion; - } - public static ResponseHistory fromString( - String responseHistoryString) { - String[] parts = responseHistoryString.split(" ", 8); - if (parts.length != 8) { - return null; - } - long startMillis = -1L, endMillis = -1L; - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - try { - startMillis = dateTimeFormat.parse(parts[0] + " " + parts[1]). - getTime(); - endMillis = dateTimeFormat.parse(parts[2] + " " + parts[3]). - getTime(); - } catch (ParseException e) { - return null; - } - if (startMillis >= endMillis) { - return null; - } - double totalResponses = 0.0; - try { - totalResponses = Double.parseDouble(parts[4]); - } catch (NumberFormatException e) { - return null; - } - SortedMap<String, Double> responsesByCountry = - parseResponses(parts[5]); - SortedMap<String, Double> responsesByTransport = - parseResponses(parts[6]); - SortedMap<String, Double> responsesByVersion = - parseResponses(parts[7]); - if (responsesByCountry == null || responsesByTransport == null || - responsesByVersion == null) { - return null; - } - return new ResponseHistory(startMillis, endMillis, totalResponses, - responsesByCountry, responsesByTransport, responsesByVersion); - } - private static SortedMap<String, Double> parseResponses( - String responsesString) { - SortedMap<String, Double> responses = new TreeMap<String, Double>(); - if (responsesString.length() > 0) { - for (String pair : responsesString.split(",")) { - String[] keyValue = pair.split("="); - if (keyValue.length != 2) { - return null; - } - double value = 0.0; - try { - value = Double.parseDouble(keyValue[1]); - } catch (NumberFormatException e) { - return null; - } - responses.put(keyValue[0], value); - } - } - return responses; - } - public String toString() { - StringBuilder sb = new StringBuilder(); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - sb.append(dateTimeFormat.format(startMillis)); - sb.append(" " + dateTimeFormat.format(endMillis)); - sb.append(" " + String.format("%.3f", this.totalResponses)); - this.appendResponses(sb, this.responsesByCountry); - this.appendResponses(sb, this.responsesByTransport); - this.appendResponses(sb, this.responsesByVersion); - return sb.toString(); - } - private void appendResponses(StringBuilder sb, - SortedMap<String, Double> responses) { - sb.append(" "); - int written = 0; - for (Map.Entry<String, Double> e : responses.entrySet()) { - sb.append((written++ > 0 ? "," : "") + e.getKey() + "=" - + String.format("%.3f", e.getValue())); - } - } - public void addResponses(ResponseHistory other) { - this.totalResponses += other.totalResponses; - this.addResponsesByCategory(this.responsesByCountry, - other.responsesByCountry); - this.addResponsesByCategory(this.responsesByTransport, - other.responsesByTransport); - this.addResponsesByCategory(this.responsesByVersion, - other.responsesByVersion); - if (this.startMillis > other.startMillis) { - this.startMillis = other.startMillis; - } - if (this.endMillis < other.endMillis) { - this.endMillis = other.endMillis; - } - } - private void addResponsesByCategory( - SortedMap<String, Double> thisResponses, - SortedMap<String, Double> otherResponses) { - for (Map.Entry<String, Double> e : otherResponses.entrySet()) { - if (thisResponses.containsKey(e.getKey())) { - thisResponses.put(e.getKey(), thisResponses.get(e.getKey()) - + e.getValue()); - } else { - thisResponses.put(e.getKey(), e.getValue()); - } - } - } - public int compareTo(ResponseHistory other) { - return this.startMillis < other.startMillis ? -1 : - this.startMillis > other.startMillis ? 1 : 0; - } - public boolean equals(Object other) { - return other instanceof ResponseHistory && - this.startMillis == ((ResponseHistory) other).startMillis; - } - } - private DescriptorSource descriptorSource; private DocumentStore documentStore; @@ -230,8 +87,8 @@ public class ClientsDataWriter implements DescriptorListener, private static final long ONE_HOUR_MILLIS = 60L * 60L * 1000L, ONE_DAY_MILLIS = 24L * ONE_HOUR_MILLIS; - private SortedMap<String, SortedSet<ResponseHistory>> newResponses = - new TreeMap<String, SortedSet<ResponseHistory>>(); + private SortedMap<String, SortedSet<ClientsHistory>> newResponses = + new TreeMap<String, SortedSet<ClientsHistory>>(); private void processBridgeExtraInfoDescriptor( ExtraInfoDescriptor descriptor) { @@ -271,12 +128,12 @@ public class ClientsDataWriter implements DescriptorListener, SortedMap<String, Double> responsesByVersion = this.weightResponsesWithUniqueIps(totalResponses, descriptor.getBridgeIpVersions(), ""); - ResponseHistory newResponseHistory = new ResponseHistory( + ClientsHistory newResponseHistory = new ClientsHistory( startMillis, endMillis, totalResponses, responsesByCountry, responsesByTransport, responsesByVersion); if (!this.newResponses.containsKey(hashedFingerprint)) { this.newResponses.put(hashedFingerprint, - new TreeSet<ResponseHistory>()); + new TreeSet<ClientsHistory>()); } this.newResponses.get(hashedFingerprint).add( newResponseHistory); @@ -309,44 +166,25 @@ public class ClientsDataWriter implements DescriptorListener, } public void updateStatuses() { - for (Map.Entry<String, SortedSet<ResponseHistory>> e : + for (Map.Entry<String, SortedSet<ClientsHistory>> e : this.newResponses.entrySet()) { String hashedFingerprint = e.getKey(); - SortedSet<ResponseHistory> history = - this.readHistory(hashedFingerprint); - this.addToHistory(history, e.getValue()); - history = this.compressHistory(history); - this.writeHistory(hashedFingerprint, history); - } - Logger.printStatusTime("Updated clients status files"); - } - - private SortedSet<ResponseHistory> readHistory( - String hashedFingerprint) { - SortedSet<ResponseHistory> history = new TreeSet<ResponseHistory>(); - ClientsStatus clientsStatus = this.documentStore.retrieve( - ClientsStatus.class, false, hashedFingerprint); - if (clientsStatus != null) { - Scanner s = new Scanner(clientsStatus.documentString); - while (s.hasNextLine()) { - String line = s.nextLine(); - ResponseHistory parsedLine = ResponseHistory.fromString(line); - if (parsedLine != null) { - history.add(parsedLine); - } else { - System.err.println("Could not parse clients history line '" - + line + "' for fingerprint '" + hashedFingerprint - + "'. Skipping."); - } + ClientsStatus clientsStatus = this.documentStore.retrieve( + ClientsStatus.class, true, hashedFingerprint); + if (clientsStatus == null) { + clientsStatus = new ClientsStatus(); } - s.close(); + this.addToHistory(clientsStatus, e.getValue()); + this.compressHistory(clientsStatus); + this.documentStore.store(clientsStatus, hashedFingerprint); } - return history; + Logger.printStatusTime("Updated clients status files"); } - private void addToHistory(SortedSet<ResponseHistory> history, - SortedSet<ResponseHistory> newIntervals) { - for (ResponseHistory interval : newIntervals) { + private void addToHistory(ClientsStatus clientsStatus, + SortedSet<ClientsHistory> newIntervals) { + SortedSet<ClientsHistory> history = clientsStatus.history; + for (ClientsHistory interval : newIntervals) { if ((history.headSet(interval).isEmpty() || history.headSet(interval).last().endMillis <= interval.startMillis) && @@ -358,15 +196,15 @@ public class ClientsDataWriter implements DescriptorListener, } } - private SortedSet<ResponseHistory> compressHistory( - SortedSet<ResponseHistory> history) { - SortedSet<ResponseHistory> compressedHistory = - new TreeSet<ResponseHistory>(); - ResponseHistory lastResponses = null; + private void compressHistory(ClientsStatus clientsStatus) { + SortedSet<ClientsHistory> history = clientsStatus.history; + SortedSet<ClientsHistory> compressedHistory = + new TreeSet<ClientsHistory>(); + ClientsHistory lastResponses = null; SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM"); dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); String lastMonthString = "1970-01"; - for (ResponseHistory responses : history) { + for (ClientsHistory responses : history) { long intervalLengthMillis; if (this.now - responses.endMillis <= 92L * 24L * 60L * 60L * 1000L) { @@ -395,18 +233,7 @@ public class ClientsDataWriter implements DescriptorListener, if (lastResponses != null) { compressedHistory.add(lastResponses); } - return compressedHistory; - } - - private void writeHistory(String hashedFingerprint, - SortedSet<ResponseHistory> history) { - StringBuilder sb = new StringBuilder(); - for (ResponseHistory responses : history) { - sb.append(responses.toString() + "\n"); - } - ClientsStatus clientsStatus = new ClientsStatus(); - clientsStatus.documentString = sb.toString(); - this.documentStore.store(clientsStatus, hashedFingerprint); + clientsStatus.history = compressedHistory; } public void processFingerprints(SortedSet<String> fingerprints, @@ -420,8 +247,12 @@ public class ClientsDataWriter implements DescriptorListener, public void writeDocuments() { for (String hashedFingerprint : this.updateDocuments) { - SortedSet<ResponseHistory> history = - this.readHistory(hashedFingerprint); + ClientsStatus clientsStatus = this.documentStore.retrieve( + ClientsStatus.class, true, hashedFingerprint); + if (clientsStatus == null) { + continue; + } + SortedSet<ClientsHistory> history = clientsStatus.history; ClientsDocument clientsDocument = new ClientsDocument(); clientsDocument.documentString = this.formatHistoryString( hashedFingerprint, history); @@ -452,7 +283,7 @@ public class ClientsDataWriter implements DescriptorListener, 10L * 24L * 60L * 60L * 1000L }; private String formatHistoryString(String hashedFingerprint, - SortedSet<ResponseHistory> history) { + SortedSet<ClientsHistory> history) { StringBuilder sb = new StringBuilder(); sb.append("{\"fingerprint\":\"" + hashedFingerprint + "\""); sb.append(",\n\"average_clients\":{"); @@ -471,7 +302,7 @@ public class ClientsDataWriter implements DescriptorListener, } private String formatTimeline(int graphIntervalIndex, - SortedSet<ResponseHistory> history) { + SortedSet<ClientsHistory> history) { String graphName = this.graphNames[graphIntervalIndex]; long graphInterval = this.graphIntervals[graphIntervalIndex]; long dataPointInterval = @@ -485,7 +316,7 @@ public class ClientsDataWriter implements DescriptorListener, totalResponsesByCountry = new TreeMap<String, Double>(), totalResponsesByTransport = new TreeMap<String, Double>(), totalResponsesByVersion = new TreeMap<String, Double>(); - for (ResponseHistory hist : history) { + for (ClientsHistory hist : history) { if (hist.endMillis < intervalStartMillis) { continue; } @@ -636,7 +467,7 @@ public class ClientsDataWriter implements DescriptorListener, public String getStatsString() { int newIntervals = 0; - for (SortedSet<ResponseHistory> hist : this.newResponses.values()) { + for (SortedSet<ClientsHistory> hist : this.newResponses.values()) { newIntervals += hist.size(); } StringBuilder sb = new StringBuilder(); diff --git a/src/org/torproject/onionoo/ClientsStatus.java b/src/org/torproject/onionoo/ClientsStatus.java index 65fb341..e315fc3 100644 --- a/src/org/torproject/onionoo/ClientsStatus.java +++ b/src/org/torproject/onionoo/ClientsStatus.java @@ -1,5 +1,197 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ package org.torproject.onionoo; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Map; +import java.util.Scanner; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeMap; +import java.util.TreeSet; + +class ClientsHistory implements Comparable<ClientsHistory> { + + long startMillis; + + long endMillis; + + double totalResponses; + + SortedMap<String, Double> responsesByCountry; + + SortedMap<String, Double> responsesByTransport; + + SortedMap<String, Double> responsesByVersion; + + ClientsHistory(long startMillis, long endMillis, + double totalResponses, + SortedMap<String, Double> responsesByCountry, + SortedMap<String, Double> responsesByTransport, + SortedMap<String, Double> responsesByVersion) { + this.startMillis = startMillis; + this.endMillis = endMillis; + this.totalResponses = totalResponses; + this.responsesByCountry = responsesByCountry; + this.responsesByTransport = responsesByTransport; + this.responsesByVersion = responsesByVersion; + } + + public static ClientsHistory fromString( + String responseHistoryString) { + String[] parts = responseHistoryString.split(" ", 8); + if (parts.length != 8) { + return null; + } + long startMillis = -1L, endMillis = -1L; + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + try { + startMillis = dateTimeFormat.parse(parts[0] + " " + parts[1]). + getTime(); + endMillis = dateTimeFormat.parse(parts[2] + " " + parts[3]). + getTime(); + } catch (ParseException e) { + return null; + } + if (startMillis >= endMillis) { + return null; + } + double totalResponses = 0.0; + try { + totalResponses = Double.parseDouble(parts[4]); + } catch (NumberFormatException e) { + return null; + } + SortedMap<String, Double> responsesByCountry = + parseResponses(parts[5]); + SortedMap<String, Double> responsesByTransport = + parseResponses(parts[6]); + SortedMap<String, Double> responsesByVersion = + parseResponses(parts[7]); + if (responsesByCountry == null || responsesByTransport == null || + responsesByVersion == null) { + return null; + } + return new ClientsHistory(startMillis, endMillis, totalResponses, + responsesByCountry, responsesByTransport, responsesByVersion); + } + + private static SortedMap<String, Double> parseResponses( + String responsesString) { + SortedMap<String, Double> responses = new TreeMap<String, Double>(); + if (responsesString.length() > 0) { + for (String pair : responsesString.split(",")) { + String[] keyValue = pair.split("="); + if (keyValue.length != 2) { + return null; + } + double value = 0.0; + try { + value = Double.parseDouble(keyValue[1]); + } catch (NumberFormatException e) { + return null; + } + responses.put(keyValue[0], value); + } + } + return responses; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setLenient(false); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + sb.append(dateTimeFormat.format(startMillis)); + sb.append(" " + dateTimeFormat.format(endMillis)); + sb.append(" " + String.format("%.3f", this.totalResponses)); + this.appendResponses(sb, this.responsesByCountry); + this.appendResponses(sb, this.responsesByTransport); + this.appendResponses(sb, this.responsesByVersion); + return sb.toString(); + } + + private void appendResponses(StringBuilder sb, + SortedMap<String, Double> responses) { + sb.append(" "); + int written = 0; + for (Map.Entry<String, Double> e : responses.entrySet()) { + sb.append((written++ > 0 ? "," : "") + e.getKey() + "=" + + String.format("%.3f", e.getValue())); + } + } + + public void addResponses(ClientsHistory other) { + this.totalResponses += other.totalResponses; + this.addResponsesByCategory(this.responsesByCountry, + other.responsesByCountry); + this.addResponsesByCategory(this.responsesByTransport, + other.responsesByTransport); + this.addResponsesByCategory(this.responsesByVersion, + other.responsesByVersion); + if (this.startMillis > other.startMillis) { + this.startMillis = other.startMillis; + } + if (this.endMillis < other.endMillis) { + this.endMillis = other.endMillis; + } + } + + private void addResponsesByCategory( + SortedMap<String, Double> thisResponses, + SortedMap<String, Double> otherResponses) { + for (Map.Entry<String, Double> e : otherResponses.entrySet()) { + if (thisResponses.containsKey(e.getKey())) { + thisResponses.put(e.getKey(), thisResponses.get(e.getKey()) + + e.getValue()); + } else { + thisResponses.put(e.getKey(), e.getValue()); + } + } + } + + public int compareTo(ClientsHistory other) { + return this.startMillis < other.startMillis ? -1 : + this.startMillis > other.startMillis ? 1 : 0; + } + + public boolean equals(Object other) { + return other instanceof ClientsHistory && + this.startMillis == ((ClientsHistory) other).startMillis; + } +} + class ClientsStatus extends Document { + + SortedSet<ClientsHistory> history = new TreeSet<ClientsHistory>(); + + public void fromDocumentString(String documentString) { + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + ClientsHistory parsedLine = ClientsHistory.fromString(line); + if (parsedLine != null) { + this.history.add(parsedLine); + } else { + System.err.println("Could not parse clients history line '" + + line + "'. Skipping."); + } + } + s.close(); + } + + public String toDocumentString() { + StringBuilder sb = new StringBuilder(); + for (ClientsHistory interval : this.history) { + sb.append(interval.toString() + "\n"); + } + return sb.toString(); + } } diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index f786d01..387f34c 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -196,7 +196,8 @@ public class DocumentStore { document instanceof UptimeDocument) { Gson gson = new Gson(); documentString = gson.toJson(this); - } else if (document instanceof UptimeStatus) { + } else if (document instanceof ClientsStatus || + document instanceof UptimeStatus) { documentString = document.toDocumentString(); } else { System.err.println("Serializing is not supported for type " @@ -288,7 +289,8 @@ public class DocumentStore { documentType.equals(UptimeDocument.class)) { return this.retrieveParsedDocumentFile(documentType, documentString); - } else if (documentType.equals(UptimeStatus.class)) { + } else if (documentType.equals(ClientsStatus.class) || + documentType.equals(UptimeStatus.class)) { return this.retrieveParsedStatusFile(documentType, documentString); } else { System.err.println("Parsing is not supported for type "

1 0

[onionoo/master] Split uptime data writer into two classes.
by karsten＠torproject.org 11 Apr '14

11 Apr '14

commit c2117e85a31bf38bb01bb7eeaff34b210786e386 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Apr 8 20:42:18 2014 +0200 Split uptime data writer into two classes. --- src/org/torproject/onionoo/Main.java | 20 +- src/org/torproject/onionoo/UptimeDataWriter.java | 469 -------------------- .../torproject/onionoo/UptimeDocumentWriter.java | 299 +++++++++++++ .../torproject/onionoo/UptimeStatusUpdater.java | 200 +++++++++ 4 files changed, 514 insertions(+), 474 deletions(-) diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index abae5cf..1a5d083 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -38,9 +38,12 @@ public class Main { Logger.printStatusTime("Initialized weights data writer"); ClientsDataWriter cdw = new ClientsDataWriter(dso, ds, t); Logger.printStatusTime("Initialized clients data writer"); - UptimeDataWriter udw = new UptimeDataWriter(dso, ds, t); - Logger.printStatusTime("Initialized uptime data writer"); - StatusUpdater[] sus = new StatusUpdater[] { ndw, bdw, wdw, cdw, udw }; + UptimeStatusUpdater usu = new UptimeStatusUpdater(dso, ds); + Logger.printStatusTime("Initialized uptime status updater"); + StatusUpdater[] sus = new StatusUpdater[] { ndw, bdw, wdw, cdw, usu }; + + UptimeDocumentWriter udw = new UptimeDocumentWriter(dso, ds, t); + Logger.printStatusTime("Initialized uptime document writer"); DocumentWriter[] dws = new DocumentWriter[] { ndw, bdw, wdw, cdw, udw }; @@ -86,8 +89,15 @@ public class Main { statsString); } } - /* TODO Print status updater statistics once all data writers have - * been separated. */ + /* TODO Print status updater statistics for *all* status updaters once + * all data writers have been separated. */ + for (DocumentWriter dw : new DocumentWriter[] { udw }) { + String statsString = dw.getStatsString(); + if (statsString != null) { + Logger.printStatistics(dw.getClass().getSimpleName(), + statsString); + } + } Logger.printStatistics("Descriptor source", dso.getStatsString()); Logger.printStatistics("Document store", ds.getStatsString()); Logger.printStatistics("GeoIP lookup service", ls.getStatsString()); diff --git a/src/org/torproject/onionoo/UptimeDataWriter.java b/src/org/torproject/onionoo/UptimeDataWriter.java deleted file mode 100644 index a2b2d29..0000000 --- a/src/org/torproject/onionoo/UptimeDataWriter.java +++ /dev/null @@ -1,469 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TimeZone; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.torproject.descriptor.BridgeNetworkStatus; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.NetworkStatusEntry; -import org.torproject.descriptor.RelayNetworkStatusConsensus; - -public class UptimeDataWriter implements DescriptorListener, - StatusUpdater, FingerprintListener, DocumentWriter { - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - private long now; - - public UptimeDataWriter(DescriptorSource descriptorSource, - DocumentStore documentStore, Time time) { - this.descriptorSource = descriptorSource; - this.documentStore = documentStore; - this.now = time.currentTimeMillis(); - this.registerDescriptorListeners(); - this.registerFingerprintListeners(); - } - - private void registerDescriptorListeners() { - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_STATUSES); - } - - public void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_STATUSES); - } - - public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof RelayNetworkStatusConsensus) { - this.processRelayNetworkStatusConsensus( - (RelayNetworkStatusConsensus) descriptor); - } else if (descriptor instanceof BridgeNetworkStatus) { - this.processBridgeNetworkStatus( - (BridgeNetworkStatus) descriptor); - } - } - - private SortedSet<Long> newRelayStatuses = new TreeSet<Long>(), - newBridgeStatuses = new TreeSet<Long>(); - private SortedMap<String, SortedSet<Long>> - newRunningRelays = new TreeMap<String, SortedSet<Long>>(), - newRunningBridges = new TreeMap<String, SortedSet<Long>>(); - - private static final long ONE_HOUR_MILLIS = 60L * 60L * 1000L; - - private void processRelayNetworkStatusConsensus( - RelayNetworkStatusConsensus consensus) { - SortedSet<String> fingerprints = new TreeSet<String>(); - for (NetworkStatusEntry entry : - consensus.getStatusEntries().values()) { - if (entry.getFlags().contains("Running")) { - fingerprints.add(entry.getFingerprint()); - } - } - if (!fingerprints.isEmpty()) { - long dateHourMillis = (consensus.getValidAfterMillis() - / ONE_HOUR_MILLIS) * ONE_HOUR_MILLIS; - for (String fingerprint : fingerprints) { - if (!this.newRunningRelays.containsKey(fingerprint)) { - this.newRunningRelays.put(fingerprint, new TreeSet<Long>()); - } - this.newRunningRelays.get(fingerprint).add(dateHourMillis); - } - this.newRelayStatuses.add(dateHourMillis); - } - } - - private void processBridgeNetworkStatus(BridgeNetworkStatus status) { - SortedSet<String> fingerprints = new TreeSet<String>(); - for (NetworkStatusEntry entry : - status.getStatusEntries().values()) { - if (entry.getFlags().contains("Running")) { - fingerprints.add(entry.getFingerprint()); - } - } - if (!fingerprints.isEmpty()) { - long dateHourMillis = (status.getPublishedMillis() - / ONE_HOUR_MILLIS) * ONE_HOUR_MILLIS; - for (String fingerprint : fingerprints) { - if (!this.newRunningBridges.containsKey(fingerprint)) { - this.newRunningBridges.put(fingerprint, new TreeSet<Long>()); - } - this.newRunningBridges.get(fingerprint).add(dateHourMillis); - } - this.newBridgeStatuses.add(dateHourMillis); - } - } - - public void updateStatuses() { - for (Map.Entry<String, SortedSet<Long>> e : - this.newRunningRelays.entrySet()) { - this.updateStatus(true, e.getKey(), e.getValue()); - } - this.updateStatus(true, null, this.newRelayStatuses); - for (Map.Entry<String, SortedSet<Long>> e : - this.newRunningBridges.entrySet()) { - this.updateStatus(false, e.getKey(), e.getValue()); - } - this.updateStatus(false, null, this.newBridgeStatuses); - Logger.printStatusTime("Updated uptime status files"); - } - - private void updateStatus(boolean relay, String fingerprint, - SortedSet<Long> newUptimeHours) { - UptimeStatus uptimeStatus = this.readHistory(fingerprint); - if (uptimeStatus == null) { - uptimeStatus = new UptimeStatus(); - } - this.addToHistory(uptimeStatus, relay, newUptimeHours); - this.compressHistory(uptimeStatus); - this.writeHistory(fingerprint, uptimeStatus); - } - - private UptimeStatus readHistory(String fingerprint) { - return fingerprint == null ? - documentStore.retrieve(UptimeStatus.class, true) : - documentStore.retrieve(UptimeStatus.class, true, fingerprint); - } - - private void addToHistory(UptimeStatus uptimeStatus, boolean relay, - SortedSet<Long> newIntervals) { - SortedSet<UptimeHistory> history = uptimeStatus.history; - for (long startMillis : newIntervals) { - UptimeHistory interval = new UptimeHistory(relay, startMillis, 1); - if (!history.headSet(interval).isEmpty()) { - UptimeHistory prev = history.headSet(interval).last(); - if (prev.relay == interval.relay && - prev.startMillis + ONE_HOUR_MILLIS * prev.uptimeHours > - interval.startMillis) { - continue; - } - } - if (!history.tailSet(interval).isEmpty()) { - UptimeHistory next = history.tailSet(interval).first(); - if (next.relay == interval.relay && - next.startMillis < interval.startMillis + ONE_HOUR_MILLIS) { - continue; - } - } - history.add(interval); - } - } - - private void compressHistory(UptimeStatus uptimeStatus) { - SortedSet<UptimeHistory> history = uptimeStatus.history; - SortedSet<UptimeHistory> compressedHistory = - new TreeSet<UptimeHistory>(); - UptimeHistory lastInterval = null; - for (UptimeHistory interval : history) { - if (lastInterval != null && - lastInterval.startMillis + ONE_HOUR_MILLIS - * lastInterval.uptimeHours == interval.startMillis && - lastInterval.relay == interval.relay) { - lastInterval.addUptime(interval); - } else { - if (lastInterval != null) { - compressedHistory.add(lastInterval); - } - lastInterval = interval; - } - } - if (lastInterval != null) { - compressedHistory.add(lastInterval); - } - uptimeStatus.history = compressedHistory; - } - - private void writeHistory(String fingerprint, - UptimeStatus uptimeStatus) { - if (fingerprint == null) { - this.documentStore.store(uptimeStatus); - } else { - this.documentStore.store(uptimeStatus, fingerprint); - } - } - - private SortedSet<String> newRelayFingerprints = new TreeSet<String>(), - newBridgeFingerprints = new TreeSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (relay) { - this.newRelayFingerprints.addAll(fingerprints); - } else { - this.newBridgeFingerprints.addAll(fingerprints); - } - } - - public void writeDocuments() { - SortedSet<UptimeHistory> - knownRelayStatuses = new TreeSet<UptimeHistory>(), - knownBridgeStatuses = new TreeSet<UptimeHistory>(); - UptimeStatus uptimeStatus = this.documentStore.retrieve( - UptimeStatus.class, true); - if (uptimeStatus == null) { - return; - } - SortedSet<UptimeHistory> knownStatuses = uptimeStatus.history; - for (UptimeHistory status : knownStatuses) { - if (status.relay) { - knownRelayStatuses.add(status); - } else { - knownBridgeStatuses.add(status); - } - } - for (String fingerprint : this.newRelayFingerprints) { - this.updateDocument(true, fingerprint, knownRelayStatuses); - } - for (String fingerprint : this.newBridgeFingerprints) { - this.updateDocument(false, fingerprint, knownBridgeStatuses); - } - Logger.printStatusTime("Wrote uptime document files"); - } - - private void updateDocument(boolean relay, String fingerprint, - SortedSet<UptimeHistory> knownStatuses) { - UptimeStatus uptimeStatus = this.documentStore.retrieve( - UptimeStatus.class, true, fingerprint); - if (uptimeStatus != null) { - SortedSet<UptimeHistory> history = uptimeStatus.history; - UptimeDocument uptimeDocument = new UptimeDocument(); - uptimeDocument.documentString = this.formatHistoryString(relay, - fingerprint, history, knownStatuses); - this.documentStore.store(uptimeDocument, fingerprint); - } - } - - private String[] graphNames = new String[] { - "1_week", - "1_month", - "3_months", - "1_year", - "5_years" }; - - private long[] graphIntervals = new long[] { - 7L * 24L * 60L * 60L * 1000L, - 31L * 24L * 60L * 60L * 1000L, - 92L * 24L * 60L * 60L * 1000L, - 366L * 24L * 60L * 60L * 1000L, - 5L * 366L * 24L * 60L * 60L * 1000L }; - - private long[] dataPointIntervals = new long[] { - 60L * 60L * 1000L, - 4L * 60L * 60L * 1000L, - 12L * 60L * 60L * 1000L, - 2L * 24L * 60L * 60L * 1000L, - 10L * 24L * 60L * 60L * 1000L }; - - private String formatHistoryString(boolean relay, String fingerprint, - SortedSet<UptimeHistory> history, - SortedSet<UptimeHistory> knownStatuses) { - StringBuilder sb = new StringBuilder(); - sb.append("{\"fingerprint\":\"" + fingerprint + "\""); - sb.append(",\n\"uptime\":{"); - int graphIntervalsWritten = 0; - for (int graphIntervalIndex = 0; graphIntervalIndex < - this.graphIntervals.length; graphIntervalIndex++) { - String timeline = this.formatTimeline(graphIntervalIndex, relay, - history, knownStatuses); - if (timeline != null) { - sb.append((graphIntervalsWritten++ > 0 ? "," : "") + "\n" - + timeline); - } - } - sb.append("}"); - sb.append("\n}\n"); - return sb.toString(); - } - - private String formatTimeline(int graphIntervalIndex, boolean relay, - SortedSet<UptimeHistory> history, - SortedSet<UptimeHistory> knownStatuses) { - String graphName = this.graphNames[graphIntervalIndex]; - long graphInterval = this.graphIntervals[graphIntervalIndex]; - long dataPointInterval = - this.dataPointIntervals[graphIntervalIndex]; - int dataPointIntervalHours = (int) (dataPointInterval - / ONE_HOUR_MILLIS); - List<Integer> statusDataPoints = new ArrayList<Integer>(); - long intervalStartMillis = ((this.now - graphInterval) - / dataPointInterval) * dataPointInterval; - int statusHours = 0; - for (UptimeHistory hist : knownStatuses) { - if (hist.relay != relay) { - continue; - } - long histEndMillis = hist.startMillis + ONE_HOUR_MILLIS - * hist.uptimeHours; - if (histEndMillis < intervalStartMillis) { - continue; - } - while (hist.startMillis >= intervalStartMillis - + dataPointInterval) { - statusDataPoints.add(statusHours * 5 > dataPointIntervalHours - ? statusHours : -1); - statusHours = 0; - intervalStartMillis += dataPointInterval; - } - while (histEndMillis >= intervalStartMillis + dataPointInterval) { - statusHours += (int) ((intervalStartMillis + dataPointInterval - - Math.max(hist.startMillis, intervalStartMillis)) - / ONE_HOUR_MILLIS); - statusDataPoints.add(statusHours * 5 > dataPointIntervalHours - ? statusHours : -1); - statusHours = 0; - intervalStartMillis += dataPointInterval; - } - statusHours += (int) ((histEndMillis - Math.max(hist.startMillis, - intervalStartMillis)) / ONE_HOUR_MILLIS); - } - statusDataPoints.add(statusHours * 5 > dataPointIntervalHours - ? statusHours : -1); - List<Integer> uptimeDataPoints = new ArrayList<Integer>(); - intervalStartMillis = ((this.now - graphInterval) - / dataPointInterval) * dataPointInterval; - int uptimeHours = 0; - long firstStatusStartMillis = -1L; - for (UptimeHistory hist : history) { - if (hist.relay != relay) { - continue; - } - if (firstStatusStartMillis < 0L) { - firstStatusStartMillis = hist.startMillis; - } - long histEndMillis = hist.startMillis + ONE_HOUR_MILLIS - * hist.uptimeHours; - if (histEndMillis < intervalStartMillis) { - continue; - } - while (hist.startMillis >= intervalStartMillis - + dataPointInterval) { - if (firstStatusStartMillis < intervalStartMillis - + dataPointInterval) { - uptimeDataPoints.add(uptimeHours); - } else { - uptimeDataPoints.add(-1); - } - uptimeHours = 0; - intervalStartMillis += dataPointInterval; - } - while (histEndMillis >= intervalStartMillis + dataPointInterval) { - uptimeHours += (int) ((intervalStartMillis + dataPointInterval - - Math.max(hist.startMillis, intervalStartMillis)) - / ONE_HOUR_MILLIS); - uptimeDataPoints.add(uptimeHours); - uptimeHours = 0; - intervalStartMillis += dataPointInterval; - } - uptimeHours += (int) ((histEndMillis - Math.max(hist.startMillis, - intervalStartMillis)) / ONE_HOUR_MILLIS); - } - uptimeDataPoints.add(uptimeHours); - List<Double> dataPoints = new ArrayList<Double>(); - for (int dataPointIndex = 0; dataPointIndex < statusDataPoints.size(); - dataPointIndex++) { - if (dataPointIndex >= uptimeDataPoints.size()) { - dataPoints.add(0.0); - } else if (uptimeDataPoints.get(dataPointIndex) >= 0 && - statusDataPoints.get(dataPointIndex) > 0) { - dataPoints.add(((double) uptimeDataPoints.get(dataPointIndex)) - / ((double) statusDataPoints.get(dataPointIndex))); - } else { - dataPoints.add(-1.0); - } - } - int firstNonNullIndex = -1, lastNonNullIndex = -1; - for (int dataPointIndex = 0; dataPointIndex < dataPoints.size(); - dataPointIndex++) { - double dataPoint = dataPoints.get(dataPointIndex); - if (dataPoint >= 0.0) { - if (firstNonNullIndex < 0) { - firstNonNullIndex = dataPointIndex; - } - lastNonNullIndex = dataPointIndex; - } - } - if (firstNonNullIndex < 0) { - return null; - } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) - * dataPointInterval + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis >= - this.now - graphIntervals[graphIntervalIndex - 1]) { - /* Skip uptime history object, because it doesn't contain - * anything new that wasn't already contained in the last - * uptime history object(s). */ - return null; - } - long lastDataPointMillis = firstDataPointMillis - + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; - double factor = 1.0 / 999.0; - int count = lastNonNullIndex - firstNonNullIndex + 1; - StringBuilder sb = new StringBuilder(); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - sb.append("\"" + graphName + "\":{" - + "\"first\":\"" + dateTimeFormat.format(firstDataPointMillis) - + "\",\"last\":\"" + dateTimeFormat.format(lastDataPointMillis) - + "\",\"interval\":" + String.valueOf(dataPointInterval / 1000L) - + ",\"factor\":" + String.format(Locale.US, "%.9f", factor) - + ",\"count\":" + String.valueOf(count) + ",\"values\":["); - int dataPointsWritten = 0, previousNonNullIndex = -2; - boolean foundTwoAdjacentDataPoints = false; - for (int dataPointIndex = firstNonNullIndex; dataPointIndex <= - lastNonNullIndex; dataPointIndex++) { - double dataPoint = dataPoints.get(dataPointIndex); - if (dataPoint >= 0.0) { - if (dataPointIndex - previousNonNullIndex == 1) { - foundTwoAdjacentDataPoints = true; - } - previousNonNullIndex = dataPointIndex; - } - sb.append((dataPointsWritten++ > 0 ? "," : "") - + (dataPoint < -0.5 ? "null" : - String.valueOf((long) (dataPoint * 999.0)))); - } - sb.append("]}"); - if (foundTwoAdjacentDataPoints) { - return sb.toString(); - } else { - return null; - } - } - - public String getStatsString() { - StringBuilder sb = new StringBuilder(); - sb.append(" " + Logger.formatDecimalNumber( - this.newRelayStatuses.size()) + " hours of relay uptimes " - + "processed\n"); - sb.append(" " + Logger.formatDecimalNumber( - this.newBridgeStatuses.size()) + " hours of bridge uptimes " - + "processed\n"); - sb.append(" " + Logger.formatDecimalNumber( - this.newRunningRelays.size() + this.newRunningBridges.size()) - + " uptime status files updated\n"); - sb.append(" " + Logger.formatDecimalNumber( - this.newRunningRelays.size() + this.newRunningBridges.size()) - + " uptime document files updated\n"); - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/UptimeDocumentWriter.java b/src/org/torproject/onionoo/UptimeDocumentWriter.java new file mode 100644 index 0000000..5b03153 --- /dev/null +++ b/src/org/torproject/onionoo/UptimeDocumentWriter.java @@ -0,0 +1,299 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeSet; + +public class UptimeDocumentWriter implements FingerprintListener, + DocumentWriter { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public UptimeDocumentWriter(DescriptorSource descriptorSource, + DocumentStore documentStore, Time time) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + this.now = time.currentTimeMillis(); + this.registerFingerprintListeners(); + } + + public void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_STATUSES); + } + + private static final long ONE_HOUR_MILLIS = 60L * 60L * 1000L; + + private SortedSet<String> newRelayFingerprints = new TreeSet<String>(), + newBridgeFingerprints = new TreeSet<String>(); + + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + if (relay) { + this.newRelayFingerprints.addAll(fingerprints); + } else { + this.newBridgeFingerprints.addAll(fingerprints); + } + } + + public void writeDocuments() { + SortedSet<UptimeHistory> + knownRelayStatuses = new TreeSet<UptimeHistory>(), + knownBridgeStatuses = new TreeSet<UptimeHistory>(); + UptimeStatus uptimeStatus = this.documentStore.retrieve( + UptimeStatus.class, true); + if (uptimeStatus == null) { + return; + } + SortedSet<UptimeHistory> knownStatuses = uptimeStatus.history; + for (UptimeHistory status : knownStatuses) { + if (status.relay) { + knownRelayStatuses.add(status); + } else { + knownBridgeStatuses.add(status); + } + } + for (String fingerprint : this.newRelayFingerprints) { + this.updateDocument(true, fingerprint, knownRelayStatuses); + } + for (String fingerprint : this.newBridgeFingerprints) { + this.updateDocument(false, fingerprint, knownBridgeStatuses); + } + Logger.printStatusTime("Wrote uptime document files"); + } + + private int writtenDocuments = 0; + + private void updateDocument(boolean relay, String fingerprint, + SortedSet<UptimeHistory> knownStatuses) { + UptimeStatus uptimeStatus = this.documentStore.retrieve( + UptimeStatus.class, true, fingerprint); + if (uptimeStatus != null) { + SortedSet<UptimeHistory> history = uptimeStatus.history; + UptimeDocument uptimeDocument = new UptimeDocument(); + uptimeDocument.documentString = this.formatHistoryString(relay, + fingerprint, history, knownStatuses); + this.documentStore.store(uptimeDocument, fingerprint); + this.writtenDocuments++; + } + } + + private String[] graphNames = new String[] { + "1_week", + "1_month", + "3_months", + "1_year", + "5_years" }; + + private long[] graphIntervals = new long[] { + 7L * 24L * 60L * 60L * 1000L, + 31L * 24L * 60L * 60L * 1000L, + 92L * 24L * 60L * 60L * 1000L, + 366L * 24L * 60L * 60L * 1000L, + 5L * 366L * 24L * 60L * 60L * 1000L }; + + private long[] dataPointIntervals = new long[] { + 60L * 60L * 1000L, + 4L * 60L * 60L * 1000L, + 12L * 60L * 60L * 1000L, + 2L * 24L * 60L * 60L * 1000L, + 10L * 24L * 60L * 60L * 1000L }; + + private String formatHistoryString(boolean relay, String fingerprint, + SortedSet<UptimeHistory> history, + SortedSet<UptimeHistory> knownStatuses) { + StringBuilder sb = new StringBuilder(); + sb.append("{\"fingerprint\":\"" + fingerprint + "\""); + sb.append(",\n\"uptime\":{"); + int graphIntervalsWritten = 0; + for (int graphIntervalIndex = 0; graphIntervalIndex < + this.graphIntervals.length; graphIntervalIndex++) { + String timeline = this.formatTimeline(graphIntervalIndex, relay, + history, knownStatuses); + if (timeline != null) { + sb.append((graphIntervalsWritten++ > 0 ? "," : "") + "\n" + + timeline); + } + } + sb.append("}"); + sb.append("\n}\n"); + return sb.toString(); + } + + private String formatTimeline(int graphIntervalIndex, boolean relay, + SortedSet<UptimeHistory> history, + SortedSet<UptimeHistory> knownStatuses) { + String graphName = this.graphNames[graphIntervalIndex]; + long graphInterval = this.graphIntervals[graphIntervalIndex]; + long dataPointInterval = + this.dataPointIntervals[graphIntervalIndex]; + int dataPointIntervalHours = (int) (dataPointInterval + / ONE_HOUR_MILLIS); + List<Integer> statusDataPoints = new ArrayList<Integer>(); + long intervalStartMillis = ((this.now - graphInterval) + / dataPointInterval) * dataPointInterval; + int statusHours = 0; + for (UptimeHistory hist : knownStatuses) { + if (hist.relay != relay) { + continue; + } + long histEndMillis = hist.startMillis + ONE_HOUR_MILLIS + * hist.uptimeHours; + if (histEndMillis < intervalStartMillis) { + continue; + } + while (hist.startMillis >= intervalStartMillis + + dataPointInterval) { + statusDataPoints.add(statusHours * 5 > dataPointIntervalHours + ? statusHours : -1); + statusHours = 0; + intervalStartMillis += dataPointInterval; + } + while (histEndMillis >= intervalStartMillis + dataPointInterval) { + statusHours += (int) ((intervalStartMillis + dataPointInterval + - Math.max(hist.startMillis, intervalStartMillis)) + / ONE_HOUR_MILLIS); + statusDataPoints.add(statusHours * 5 > dataPointIntervalHours + ? statusHours : -1); + statusHours = 0; + intervalStartMillis += dataPointInterval; + } + statusHours += (int) ((histEndMillis - Math.max(hist.startMillis, + intervalStartMillis)) / ONE_HOUR_MILLIS); + } + statusDataPoints.add(statusHours * 5 > dataPointIntervalHours + ? statusHours : -1); + List<Integer> uptimeDataPoints = new ArrayList<Integer>(); + intervalStartMillis = ((this.now - graphInterval) + / dataPointInterval) * dataPointInterval; + int uptimeHours = 0; + long firstStatusStartMillis = -1L; + for (UptimeHistory hist : history) { + if (hist.relay != relay) { + continue; + } + if (firstStatusStartMillis < 0L) { + firstStatusStartMillis = hist.startMillis; + } + long histEndMillis = hist.startMillis + ONE_HOUR_MILLIS + * hist.uptimeHours; + if (histEndMillis < intervalStartMillis) { + continue; + } + while (hist.startMillis >= intervalStartMillis + + dataPointInterval) { + if (firstStatusStartMillis < intervalStartMillis + + dataPointInterval) { + uptimeDataPoints.add(uptimeHours); + } else { + uptimeDataPoints.add(-1); + } + uptimeHours = 0; + intervalStartMillis += dataPointInterval; + } + while (histEndMillis >= intervalStartMillis + dataPointInterval) { + uptimeHours += (int) ((intervalStartMillis + dataPointInterval + - Math.max(hist.startMillis, intervalStartMillis)) + / ONE_HOUR_MILLIS); + uptimeDataPoints.add(uptimeHours); + uptimeHours = 0; + intervalStartMillis += dataPointInterval; + } + uptimeHours += (int) ((histEndMillis - Math.max(hist.startMillis, + intervalStartMillis)) / ONE_HOUR_MILLIS); + } + uptimeDataPoints.add(uptimeHours); + List<Double> dataPoints = new ArrayList<Double>(); + for (int dataPointIndex = 0; dataPointIndex < statusDataPoints.size(); + dataPointIndex++) { + if (dataPointIndex >= uptimeDataPoints.size()) { + dataPoints.add(0.0); + } else if (uptimeDataPoints.get(dataPointIndex) >= 0 && + statusDataPoints.get(dataPointIndex) > 0) { + dataPoints.add(((double) uptimeDataPoints.get(dataPointIndex)) + / ((double) statusDataPoints.get(dataPointIndex))); + } else { + dataPoints.add(-1.0); + } + } + int firstNonNullIndex = -1, lastNonNullIndex = -1; + for (int dataPointIndex = 0; dataPointIndex < dataPoints.size(); + dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (firstNonNullIndex < 0) { + firstNonNullIndex = dataPointIndex; + } + lastNonNullIndex = dataPointIndex; + } + } + if (firstNonNullIndex < 0) { + return null; + } + long firstDataPointMillis = (((this.now - graphInterval) + / dataPointInterval) + firstNonNullIndex) + * dataPointInterval + dataPointInterval / 2L; + if (graphIntervalIndex > 0 && firstDataPointMillis >= + this.now - graphIntervals[graphIntervalIndex - 1]) { + /* Skip uptime history object, because it doesn't contain + * anything new that wasn't already contained in the last + * uptime history object(s). */ + return null; + } + long lastDataPointMillis = firstDataPointMillis + + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; + double factor = 1.0 / 999.0; + int count = lastNonNullIndex - firstNonNullIndex + 1; + StringBuilder sb = new StringBuilder(); + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + "yyyy-MM-dd HH:mm:ss"); + dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + sb.append("\"" + graphName + "\":{" + + "\"first\":\"" + dateTimeFormat.format(firstDataPointMillis) + + "\",\"last\":\"" + dateTimeFormat.format(lastDataPointMillis) + + "\",\"interval\":" + String.valueOf(dataPointInterval / 1000L) + + ",\"factor\":" + String.format(Locale.US, "%.9f", factor) + + ",\"count\":" + String.valueOf(count) + ",\"values\":["); + int dataPointsWritten = 0, previousNonNullIndex = -2; + boolean foundTwoAdjacentDataPoints = false; + for (int dataPointIndex = firstNonNullIndex; dataPointIndex <= + lastNonNullIndex; dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (dataPointIndex - previousNonNullIndex == 1) { + foundTwoAdjacentDataPoints = true; + } + previousNonNullIndex = dataPointIndex; + } + sb.append((dataPointsWritten++ > 0 ? "," : "") + + (dataPoint < -0.5 ? "null" : + String.valueOf((long) (dataPoint * 999.0)))); + } + sb.append("]}"); + if (foundTwoAdjacentDataPoints) { + return sb.toString(); + } else { + return null; + } + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments) + + " uptime document files written\n"); + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/UptimeStatusUpdater.java b/src/org/torproject/onionoo/UptimeStatusUpdater.java new file mode 100644 index 0000000..30ab703 --- /dev/null +++ b/src/org/torproject/onionoo/UptimeStatusUpdater.java @@ -0,0 +1,200 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.torproject.descriptor.BridgeNetworkStatus; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusConsensus; + +public class UptimeStatusUpdater implements DescriptorListener, + StatusUpdater { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + public UptimeStatusUpdater(DescriptorSource descriptorSource, + DocumentStore documentStore) { + this.descriptorSource = descriptorSource; + this.documentStore = documentStore; + this.registerDescriptorListeners(); + } + + private void registerDescriptorListeners() { + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_STATUSES); + } + + public void processDescriptor(Descriptor descriptor, boolean relay) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + this.processRelayNetworkStatusConsensus( + (RelayNetworkStatusConsensus) descriptor); + } else if (descriptor instanceof BridgeNetworkStatus) { + this.processBridgeNetworkStatus( + (BridgeNetworkStatus) descriptor); + } + } + + private SortedSet<Long> newRelayStatuses = new TreeSet<Long>(), + newBridgeStatuses = new TreeSet<Long>(); + private SortedMap<String, SortedSet<Long>> + newRunningRelays = new TreeMap<String, SortedSet<Long>>(), + newRunningBridges = new TreeMap<String, SortedSet<Long>>(); + + private static final long ONE_HOUR_MILLIS = 60L * 60L * 1000L; + + private void processRelayNetworkStatusConsensus( + RelayNetworkStatusConsensus consensus) { + SortedSet<String> fingerprints = new TreeSet<String>(); + for (NetworkStatusEntry entry : + consensus.getStatusEntries().values()) { + if (entry.getFlags().contains("Running")) { + fingerprints.add(entry.getFingerprint()); + } + } + if (!fingerprints.isEmpty()) { + long dateHourMillis = (consensus.getValidAfterMillis() + / ONE_HOUR_MILLIS) * ONE_HOUR_MILLIS; + for (String fingerprint : fingerprints) { + if (!this.newRunningRelays.containsKey(fingerprint)) { + this.newRunningRelays.put(fingerprint, new TreeSet<Long>()); + } + this.newRunningRelays.get(fingerprint).add(dateHourMillis); + } + this.newRelayStatuses.add(dateHourMillis); + } + } + + private void processBridgeNetworkStatus(BridgeNetworkStatus status) { + SortedSet<String> fingerprints = new TreeSet<String>(); + for (NetworkStatusEntry entry : + status.getStatusEntries().values()) { + if (entry.getFlags().contains("Running")) { + fingerprints.add(entry.getFingerprint()); + } + } + if (!fingerprints.isEmpty()) { + long dateHourMillis = (status.getPublishedMillis() + / ONE_HOUR_MILLIS) * ONE_HOUR_MILLIS; + for (String fingerprint : fingerprints) { + if (!this.newRunningBridges.containsKey(fingerprint)) { + this.newRunningBridges.put(fingerprint, new TreeSet<Long>()); + } + this.newRunningBridges.get(fingerprint).add(dateHourMillis); + } + this.newBridgeStatuses.add(dateHourMillis); + } + } + + public void updateStatuses() { + for (Map.Entry<String, SortedSet<Long>> e : + this.newRunningRelays.entrySet()) { + this.updateStatus(true, e.getKey(), e.getValue()); + } + this.updateStatus(true, null, this.newRelayStatuses); + for (Map.Entry<String, SortedSet<Long>> e : + this.newRunningBridges.entrySet()) { + this.updateStatus(false, e.getKey(), e.getValue()); + } + this.updateStatus(false, null, this.newBridgeStatuses); + Logger.printStatusTime("Updated uptime status files"); + } + + private void updateStatus(boolean relay, String fingerprint, + SortedSet<Long> newUptimeHours) { + UptimeStatus uptimeStatus = this.readHistory(fingerprint); + if (uptimeStatus == null) { + uptimeStatus = new UptimeStatus(); + } + this.addToHistory(uptimeStatus, relay, newUptimeHours); + this.compressHistory(uptimeStatus); + this.writeHistory(fingerprint, uptimeStatus); + } + + private UptimeStatus readHistory(String fingerprint) { + return fingerprint == null ? + documentStore.retrieve(UptimeStatus.class, true) : + documentStore.retrieve(UptimeStatus.class, true, fingerprint); + } + + private void addToHistory(UptimeStatus uptimeStatus, boolean relay, + SortedSet<Long> newIntervals) { + SortedSet<UptimeHistory> history = uptimeStatus.history; + for (long startMillis : newIntervals) { + UptimeHistory interval = new UptimeHistory(relay, startMillis, 1); + if (!history.headSet(interval).isEmpty()) { + UptimeHistory prev = history.headSet(interval).last(); + if (prev.relay == interval.relay && + prev.startMillis + ONE_HOUR_MILLIS * prev.uptimeHours > + interval.startMillis) { + continue; + } + } + if (!history.tailSet(interval).isEmpty()) { + UptimeHistory next = history.tailSet(interval).first(); + if (next.relay == interval.relay && + next.startMillis < interval.startMillis + ONE_HOUR_MILLIS) { + continue; + } + } + history.add(interval); + } + } + + private void compressHistory(UptimeStatus uptimeStatus) { + SortedSet<UptimeHistory> history = uptimeStatus.history; + SortedSet<UptimeHistory> compressedHistory = + new TreeSet<UptimeHistory>(); + UptimeHistory lastInterval = null; + for (UptimeHistory interval : history) { + if (lastInterval != null && + lastInterval.startMillis + ONE_HOUR_MILLIS + * lastInterval.uptimeHours == interval.startMillis && + lastInterval.relay == interval.relay) { + lastInterval.addUptime(interval); + } else { + if (lastInterval != null) { + compressedHistory.add(lastInterval); + } + lastInterval = interval; + } + } + if (lastInterval != null) { + compressedHistory.add(lastInterval); + } + uptimeStatus.history = compressedHistory; + } + + private void writeHistory(String fingerprint, + UptimeStatus uptimeStatus) { + if (fingerprint == null) { + this.documentStore.store(uptimeStatus); + } else { + this.documentStore.store(uptimeStatus, fingerprint); + } + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber( + this.newRelayStatuses.size()) + " hours of relay uptimes " + + "processed\n"); + sb.append(" " + Logger.formatDecimalNumber( + this.newBridgeStatuses.size()) + " hours of bridge uptimes " + + "processed\n"); + sb.append(" " + Logger.formatDecimalNumber( + this.newRunningRelays.size() + this.newRunningBridges.size()) + + " uptime status files updated\n"); + return sb.toString(); + } +} +

1 0

[onionoo/master] Move (de-)serialization code into UptimeStatus.
by karsten＠torproject.org 11 Apr '14

11 Apr '14

commit 9a5d33d7b2f76959ce162e83e831f7b8953ec892 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Apr 8 23:57:10 2014 +0200 Move (de-)serialization code into UptimeStatus. --- src/org/torproject/onionoo/Document.java | 10 ++ src/org/torproject/onionoo/DocumentStore.java | 24 ++++ src/org/torproject/onionoo/UptimeDataWriter.java | 151 +++++----------------- src/org/torproject/onionoo/UptimeStatus.java | 116 +++++++++++++++++ 4 files changed, 183 insertions(+), 118 deletions(-) diff --git a/src/org/torproject/onionoo/Document.java b/src/org/torproject/onionoo/Document.java index 9730d66..f0fac2c 100644 --- a/src/org/torproject/onionoo/Document.java +++ b/src/org/torproject/onionoo/Document.java @@ -3,6 +3,16 @@ package org.torproject.onionoo; abstract class Document { + transient String documentString; + + public void fromDocumentString(String documentString) { + /* Subclasses may override this method to parse documentString. */ + } + + public String toDocumentString() { + /* Subclasses may override this method to write documentString. */ + return null; + } } diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index 519b4f2..f786d01 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -196,6 +196,8 @@ public class DocumentStore { document instanceof UptimeDocument) { Gson gson = new Gson(); documentString = gson.toJson(this); + } else if (document instanceof UptimeStatus) { + documentString = document.toDocumentString(); } else { System.err.println("Serializing is not supported for type " + document.getClass().getName() + "."); @@ -286,6 +288,8 @@ public class DocumentStore { documentType.equals(UptimeDocument.class)) { return this.retrieveParsedDocumentFile(documentType, documentString); + } else if (documentType.equals(UptimeStatus.class)) { + return this.retrieveParsedStatusFile(documentType, documentString); } else { System.err.println("Parsing is not supported for type " + documentType.getName() + "."); @@ -293,6 +297,26 @@ public class DocumentStore { return result; } + private <T extends Document> T retrieveParsedStatusFile( + Class<T> documentType, String documentString) { + T result = null; + try { + result = documentType.newInstance(); + result.fromDocumentString(documentString); + } catch (InstantiationException e) { + /* Handle below. */ + e.printStackTrace(); + } catch (IllegalAccessException e) { + /* Handle below. */ + e.printStackTrace(); + } + if (result == null) { + System.err.println("Could not initialize parsed status file of " + + "type " + documentType.getName() + "."); + } + return result; + } + private <T extends Document> T retrieveParsedDocumentFile( Class<T> documentType, String documentString) { T result = null; diff --git a/src/org/torproject/onionoo/UptimeDataWriter.java b/src/org/torproject/onionoo/UptimeDataWriter.java index 4692b18..a2b2d29 100644 --- a/src/org/torproject/onionoo/UptimeDataWriter.java +++ b/src/org/torproject/onionoo/UptimeDataWriter.java @@ -2,13 +2,11 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Scanner; import java.util.SortedMap; import java.util.SortedSet; import java.util.TimeZone; @@ -127,112 +125,26 @@ public class UptimeDataWriter implements DescriptorListener, Logger.printStatusTime("Updated uptime status files"); } - private static class UptimeHistory - implements Comparable<UptimeHistory> { - private boolean relay; - private long startMillis; - private int uptimeHours; - private UptimeHistory(boolean relay, long startMillis, - int uptimeHours) { - this.relay = relay; - this.startMillis = startMillis; - this.uptimeHours = uptimeHours; - } - public static UptimeHistory fromString(String uptimeHistoryString) { - String[] parts = uptimeHistoryString.split(" ", 3); - if (parts.length != 3) { - return null; - } - boolean relay = false; - if (parts[0].equals("r")) { - relay = true; - } else if (!parts[0].equals("b")) { - return null; - } - long startMillis = -1L; - SimpleDateFormat dateHourFormat = new SimpleDateFormat( - "yyyy-MM-dd-HH"); - dateHourFormat.setLenient(false); - dateHourFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - try { - startMillis = dateHourFormat.parse(parts[1]).getTime(); - } catch (ParseException e) { - return null; - } - int uptimeHours = -1; - try { - uptimeHours = Integer.parseInt(parts[2]); - } catch (NumberFormatException e) { - return null; - } - return new UptimeHistory(relay, startMillis, uptimeHours); - } - public String toString() { - StringBuilder sb = new StringBuilder(); - SimpleDateFormat dateHourFormat = new SimpleDateFormat( - "yyyy-MM-dd-HH"); - dateHourFormat.setLenient(false); - dateHourFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - sb.append(this.relay ? "r" : "b"); - sb.append(" " + dateHourFormat.format(this.startMillis)); - sb.append(" " + String.format("%d", this.uptimeHours)); - return sb.toString(); - } - public void addUptime(UptimeHistory other) { - this.uptimeHours += other.uptimeHours; - if (this.startMillis > other.startMillis) { - this.startMillis = other.startMillis; - } - } - public int compareTo(UptimeHistory other) { - if (this.relay && !other.relay) { - return -1; - } else if (!this.relay && other.relay) { - return 1; - } - return this.startMillis < other.startMillis ? -1 : - this.startMillis > other.startMillis ? 1 : 0; - } - public boolean equals(Object other) { - return other instanceof UptimeHistory && - this.relay == ((UptimeHistory) other).relay && - this.startMillis == ((UptimeHistory) other).startMillis; - } - } - private void updateStatus(boolean relay, String fingerprint, SortedSet<Long> newUptimeHours) { - SortedSet<UptimeHistory> history = this.readHistory(fingerprint); - this.addToHistory(history, relay, newUptimeHours); - history = this.compressHistory(history); - this.writeHistory(fingerprint, history); + UptimeStatus uptimeStatus = this.readHistory(fingerprint); + if (uptimeStatus == null) { + uptimeStatus = new UptimeStatus(); + } + this.addToHistory(uptimeStatus, relay, newUptimeHours); + this.compressHistory(uptimeStatus); + this.writeHistory(fingerprint, uptimeStatus); } - private SortedSet<UptimeHistory> readHistory(String fingerprint) { - SortedSet<UptimeHistory> history = new TreeSet<UptimeHistory>(); - UptimeStatus uptimeStatus = fingerprint == null ? - documentStore.retrieve(UptimeStatus.class, false) : - documentStore.retrieve(UptimeStatus.class, false, fingerprint); - if (uptimeStatus != null) { - Scanner s = new Scanner(uptimeStatus.documentString); - while (s.hasNextLine()) { - String line = s.nextLine(); - UptimeHistory parsedLine = UptimeHistory.fromString(line); - if (parsedLine != null) { - history.add(parsedLine); - } else { - System.err.println("Could not parse uptime history line '" - + line + "' for fingerprint '" + fingerprint - + "'. Skipping."); - } - } - s.close(); - } - return history; + private UptimeStatus readHistory(String fingerprint) { + return fingerprint == null ? + documentStore.retrieve(UptimeStatus.class, true) : + documentStore.retrieve(UptimeStatus.class, true, fingerprint); } - private void addToHistory(SortedSet<UptimeHistory> history, - boolean relay, SortedSet<Long> newIntervals) { + private void addToHistory(UptimeStatus uptimeStatus, boolean relay, + SortedSet<Long> newIntervals) { + SortedSet<UptimeHistory> history = uptimeStatus.history; for (long startMillis : newIntervals) { UptimeHistory interval = new UptimeHistory(relay, startMillis, 1); if (!history.headSet(interval).isEmpty()) { @@ -254,8 +166,8 @@ public class UptimeDataWriter implements DescriptorListener, } } - private SortedSet<UptimeHistory> compressHistory( - SortedSet<UptimeHistory> history) { + private void compressHistory(UptimeStatus uptimeStatus) { + SortedSet<UptimeHistory> history = uptimeStatus.history; SortedSet<UptimeHistory> compressedHistory = new TreeSet<UptimeHistory>(); UptimeHistory lastInterval = null; @@ -275,17 +187,11 @@ public class UptimeDataWriter implements DescriptorListener, if (lastInterval != null) { compressedHistory.add(lastInterval); } - return compressedHistory; + uptimeStatus.history = compressedHistory; } private void writeHistory(String fingerprint, - SortedSet<UptimeHistory> history) { - StringBuilder sb = new StringBuilder(); - for (UptimeHistory interval : history) { - sb.append(interval.toString() + "\n"); - } - UptimeStatus uptimeStatus = new UptimeStatus(); - uptimeStatus.documentString = sb.toString(); + UptimeStatus uptimeStatus) { if (fingerprint == null) { this.documentStore.store(uptimeStatus); } else { @@ -309,7 +215,12 @@ public class UptimeDataWriter implements DescriptorListener, SortedSet<UptimeHistory> knownRelayStatuses = new TreeSet<UptimeHistory>(), knownBridgeStatuses = new TreeSet<UptimeHistory>(); - SortedSet<UptimeHistory> knownStatuses = this.readHistory(null); + UptimeStatus uptimeStatus = this.documentStore.retrieve( + UptimeStatus.class, true); + if (uptimeStatus == null) { + return; + } + SortedSet<UptimeHistory> knownStatuses = uptimeStatus.history; for (UptimeHistory status : knownStatuses) { if (status.relay) { knownRelayStatuses.add(status); @@ -328,11 +239,15 @@ public class UptimeDataWriter implements DescriptorListener, private void updateDocument(boolean relay, String fingerprint, SortedSet<UptimeHistory> knownStatuses) { - SortedSet<UptimeHistory> history = this.readHistory(fingerprint); - UptimeDocument uptimeDocument = new UptimeDocument(); - uptimeDocument.documentString = this.formatHistoryString(relay, - fingerprint, history, knownStatuses); - this.documentStore.store(uptimeDocument, fingerprint); + UptimeStatus uptimeStatus = this.documentStore.retrieve( + UptimeStatus.class, true, fingerprint); + if (uptimeStatus != null) { + SortedSet<UptimeHistory> history = uptimeStatus.history; + UptimeDocument uptimeDocument = new UptimeDocument(); + uptimeDocument.documentString = this.formatHistoryString(relay, + fingerprint, history, knownStatuses); + this.documentStore.store(uptimeDocument, fingerprint); + } } private String[] graphNames = new String[] { diff --git a/src/org/torproject/onionoo/UptimeStatus.java b/src/org/torproject/onionoo/UptimeStatus.java index d16c8fc..fedf1a1 100644 --- a/src/org/torproject/onionoo/UptimeStatus.java +++ b/src/org/torproject/onionoo/UptimeStatus.java @@ -1,5 +1,121 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ package org.torproject.onionoo; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Scanner; +import java.util.SortedSet; +import java.util.TimeZone; +import java.util.TreeSet; + +class UptimeHistory + implements Comparable<UptimeHistory> { + + boolean relay; + + long startMillis; + + int uptimeHours; + + UptimeHistory(boolean relay, long startMillis, + int uptimeHours) { + this.relay = relay; + this.startMillis = startMillis; + this.uptimeHours = uptimeHours; + } + + public static UptimeHistory fromString(String uptimeHistoryString) { + String[] parts = uptimeHistoryString.split(" ", 3); + if (parts.length != 3) { + return null; + } + boolean relay = false; + if (parts[0].equals("r")) { + relay = true; + } else if (!parts[0].equals("b")) { + return null; + } + long startMillis = -1L; + SimpleDateFormat dateHourFormat = new SimpleDateFormat( + "yyyy-MM-dd-HH"); + dateHourFormat.setLenient(false); + dateHourFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + try { + startMillis = dateHourFormat.parse(parts[1]).getTime(); + } catch (ParseException e) { + return null; + } + int uptimeHours = -1; + try { + uptimeHours = Integer.parseInt(parts[2]); + } catch (NumberFormatException e) { + return null; + } + return new UptimeHistory(relay, startMillis, uptimeHours); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + SimpleDateFormat dateHourFormat = new SimpleDateFormat( + "yyyy-MM-dd-HH"); + dateHourFormat.setLenient(false); + dateHourFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + sb.append(this.relay ? "r" : "b"); + sb.append(" " + dateHourFormat.format(this.startMillis)); + sb.append(" " + String.format("%d", this.uptimeHours)); + return sb.toString(); + } + + public void addUptime(UptimeHistory other) { + this.uptimeHours += other.uptimeHours; + if (this.startMillis > other.startMillis) { + this.startMillis = other.startMillis; + } + } + + public int compareTo(UptimeHistory other) { + if (this.relay && !other.relay) { + return -1; + } else if (!this.relay && other.relay) { + return 1; + } + return this.startMillis < other.startMillis ? -1 : + this.startMillis > other.startMillis ? 1 : 0; + } + + public boolean equals(Object other) { + return other instanceof UptimeHistory && + this.relay == ((UptimeHistory) other).relay && + this.startMillis == ((UptimeHistory) other).startMillis; + } +} + class UptimeStatus extends Document { + + SortedSet<UptimeHistory> history = new TreeSet<UptimeHistory>(); + + public void fromDocumentString(String documentString) { + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + UptimeHistory parsedLine = UptimeHistory.fromString(line); + if (parsedLine != null) { + this.history.add(parsedLine); + } else { + System.err.println("Could not parse uptime history line '" + + line + "'. Skipping."); + } + } + s.close(); + } + + public String toDocumentString() { + StringBuilder sb = new StringBuilder(); + for (UptimeHistory interval : this.history) { + sb.append(interval.toString() + "\n"); + } + return sb.toString(); + } }

1 0

[translation/torbutton-browserproperties_completed] Update translations for torbutton-browserproperties_completed
by translation＠torproject.org 11 Apr '14

11 Apr '14

commit a944bb7de67b137b866f9e4f8785d3698cbfa40c Author: Translation commit bot <translation(a)torproject.org> Date: Fri Apr 11 05:45:44 2014 +0000 Update translations for torbutton-browserproperties_completed --- tr/browser.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tr/browser.properties b/tr/browser.properties index 55550a3..c30bc36 100644 --- a/tr/browser.properties +++ b/tr/browser.properties @@ -1,5 +1,5 @@ # Default home page -browser.startup.homepage=hakkında:tor +browser.startup.homepage=about:tor # Spell checker dictionary spellchecker.dictionary=en_US

1 0

[translation/torbutton-browserproperties] Update translations for torbutton-browserproperties
by translation＠torproject.org 11 Apr '14

11 Apr '14

commit d9c9f19f879c454bbbe8daac52738ee20d386534 Author: Translation commit bot <translation(a)torproject.org> Date: Fri Apr 11 05:45:41 2014 +0000 Update translations for torbutton-browserproperties --- tr/browser.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tr/browser.properties b/tr/browser.properties index 55550a3..c30bc36 100644 --- a/tr/browser.properties +++ b/tr/browser.properties @@ -1,5 +1,5 @@ # Default home page -browser.startup.homepage=hakkında:tor +browser.startup.homepage=about:tor # Spell checker dictionary spellchecker.dictionary=en_US

1 0

[tor-browser-bundle/master] Bump Torbutton version for Turkish homepage fix.
by mikeperry＠torproject.org 11 Apr '14

11 Apr '14

commit 362bca4ed06cd0b12301492e2df8f69e9f6258e6 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Thu Apr 10 22:43:08 2014 -0700 Bump Torbutton version for Turkish homepage fix. --- Bundle-Data/Docs/ChangeLog.txt | 2 +- gitian/versions.alpha | 2 +- gitian/versions.beta | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Bundle-Data/Docs/ChangeLog.txt b/Bundle-Data/Docs/ChangeLog.txt index 194ce0f..4b72c7f 100644 --- a/Bundle-Data/Docs/ChangeLog.txt +++ b/Bundle-Data/Docs/ChangeLog.txt @@ -5,7 +5,7 @@ Tor Browser Bundle 3.6-beta-2 -- Apr 8 2014 * Bug 9387 testing: Disable JS JIT, type inference, asmjs, and ion. * Update fte transport to 0.2.12 * Update NoScript to 2.6.8.19 - * Update Torbutton to 1.6.8.0 + * Update Torbutton to 1.6.8.1 * Bug 11242: Fix improper "update needed" message after in-place upgrade. * Bug 10398: Ease translation of about:tor page elements * Update Tor Launcher to 0.2.5.3 diff --git a/gitian/versions.alpha b/gitian/versions.alpha index 2bdbb2d..f53fbd8 100755 --- a/gitian/versions.alpha +++ b/gitian/versions.alpha @@ -9,7 +9,7 @@ FIREFOX_VERSION=24.4.0esr TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1-build1 TOR_TAG=tor-0.2.5.2-alpha TORLAUNCHER_TAG=0.2.5.1a -TORBUTTON_TAG=1.6.7.0 +TORBUTTON_TAG=1.6.8.1 HTTPSE_TAG=3.4.5 NSIS_TAG=v0.1 ZLIB_TAG=v1.2.8 diff --git a/gitian/versions.beta b/gitian/versions.beta index ca59070..adb9818 100755 --- a/gitian/versions.beta +++ b/gitian/versions.beta @@ -9,7 +9,7 @@ FIREFOX_VERSION=24.4.0esr TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-1-build2 TOR_TAG=tor-0.2.4.21 TORLAUNCHER_TAG=0.2.5.3 -TORBUTTON_TAG=1.6.8.0 +TORBUTTON_TAG=1.6.8.1 HTTPSE_TAG=3.4.5 NSIS_TAG=v0.1 ZLIB_TAG=v1.2.8

1 0

[torbutton/master] Dear Turkish Translators: Don't translate the homepage URL, yo.
by mikeperry＠torproject.org 11 Apr '14

11 Apr '14

commit 5393b528462d265b87972bc1279d024b0efb9eb8 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Thu Apr 10 22:35:39 2014 -0700 Dear Turkish Translators: Don't translate the homepage URL, yo. (Or the search engine names, but we'll deal with that later). I fixed this in transifex too, but don't want to wait for transifex to get pulled again. --- src/chrome/locale/tr/browser.properties | 2 +- src/install.rdf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/chrome/locale/tr/browser.properties b/src/chrome/locale/tr/browser.properties index 55550a3..c30bc36 100644 --- a/src/chrome/locale/tr/browser.properties +++ b/src/chrome/locale/tr/browser.properties @@ -1,5 +1,5 @@ # Default home page -browser.startup.homepage=hakkında:tor +browser.startup.homepage=about:tor # Spell checker dictionary spellchecker.dictionary=en_US diff --git a/src/install.rdf b/src/install.rdf index 370cad8..87db08f 100644 --- a/src/install.rdf +++ b/src/install.rdf @@ -6,7 +6,7 @@ <em:name>Torbutton</em:name> <em:creator>Mike Perry</em:creator> <em:id>torbutton(a)torproject.org</em:id> - <em:version>1.6.8.0</em:version> + <em:version>1.6.8.1</em:version> <em:homepageURL>https://www.torproject.org/projects/torbrowser.html.en</em:homepageURL> <em:optionsURL>chrome://torbutton/content/preferences.xul</em:optionsURL> <em:iconURL>chrome://torbutton/skin/tor.png</em:iconURL>

1 0

r26702: {website} add more hints about running a mirror (website/trunk/docs/en)
by Andrew Lewman 11 Apr '14

11 Apr '14

Author: phobos Date: 2014-04-11 00:52:48 +0000 (Fri, 11 Apr 2014) New Revision: 26702 Modified: website/trunk/docs/en/running-a-mirror.wml Log: add more hints about running a mirror Modified: website/trunk/docs/en/running-a-mirror.wml =================================================================== --- website/trunk/docs/en/running-a-mirror.wml 2014-04-11 00:49:45 UTC (rev 26701) +++ website/trunk/docs/en/running-a-mirror.wml 2014-04-11 00:52:48 UTC (rev 26702) @@ -124,9 +124,13 @@ href="https://svn.torproject.org/svn/website/trunk/include/tor-mirrors.csv">this file</a> to the mirrors list. Your mirror will then be added manually if it passes availability testing and your provided information is - confirmed. Do not run your mirror behind a content delivery network + confirmed. Some general pointers on mirrors are: + <ol><li> Try not run your mirror behind a content delivery network (such as Akamai, Cloudflare, Fastly, etc), as most of them block - access from countries where the mirror is needed the most. + access from countries where the mirror is needed the most.</li> + <li> Try not to redirect http to https. Many places in the world + cannot use https due to firewalling.</li> + </ol> </p> </div>

1 0

r26701: {website} add a new mirror (website/trunk/include)
by Andrew Lewman 11 Apr '14

11 Apr '14

Author: phobos Date: 2014-04-11 00:49:45 +0000 (Fri, 11 Apr 2014) New Revision: 26701 Modified: website/trunk/include/tor-mirrors.csv Log: add a new mirror Modified: website/trunk/include/tor-mirrors.csv =================================================================== --- website/trunk/include/tor-mirrors.csv 2014-04-09 19:45:27 UTC (rev 26700) +++ website/trunk/include/tor-mirrors.csv 2014-04-11 00:49:45 UTC (rev 26701) @@ -88,3 +88,4 @@ Tor Fan, Tor Supporter, DE, Germany, DE, TRUE, FALSE, NO, http://tor-mirror.snurn.de, , , , http://tor-mirror.snurn.de/dist/, , , , Thu Mar 27 13:13:23 2014 EFF, EFF, US, United States, US, TRUE, FALSE, NO, https://tor.eff.org, https://tor.eff.org, , , https://tor.eff.org/dist/, https://tor.eff.org/dist/, , , Tue Apr 8 23:51:05 2014 Tor Fan, Tor Supporter, GR, Greece, GR, TRUE, TRUE, NO, https://tor.void.gr, https://tor.void.gr, , , https://tor.void.gr/dist/, https://tor.void.gr/dist/, , , Tue Apr 8 23:51:05 2014 +Ich Eben,Tor Supporter,DE,Germany,DE,TRUE,TRUE,No,http://reichster.de/mirrors/torproject…,,,,

1 0