April 2014 - tor-commits - lists.torproject.org

[onionoo/master] Fix minor bug in writing uptime documents.
by karsten＠torproject.org 14 Apr '14

14 Apr '14

commit 56695bcbaa409644b2036c508f4d0cf80796241a Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Apr 12 18:41:29 2014 +0200 Fix minor bug in writing uptime documents. When a relay first showed up in the middle of a, say, 4-hour interval, we discounted its uptime in that interval by the number of hours that it wasn't online. We should ignore those hours. --- .../torproject/onionoo/UptimeDocumentWriter.java | 79 +++++++++++--------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/src/org/torproject/onionoo/UptimeDocumentWriter.java b/src/org/torproject/onionoo/UptimeDocumentWriter.java index 60ca311..14c800d 100644 --- a/src/org/torproject/onionoo/UptimeDocumentWriter.java +++ b/src/org/torproject/onionoo/UptimeDocumentWriter.java @@ -129,14 +129,18 @@ public class UptimeDocumentWriter implements FingerprintListener, this.dataPointIntervals[graphIntervalIndex]; int dataPointIntervalHours = (int) (dataPointInterval / DateTimeHelper.ONE_HOUR); - List<Integer> statusDataPoints = new ArrayList<Integer>(); + List<Integer> uptimeDataPoints = new ArrayList<Integer>(); long intervalStartMillis = ((this.now - graphInterval) / dataPointInterval) * dataPointInterval; - int statusHours = 0; - for (UptimeHistory hist : knownStatuses) { + int uptimeHours = 0; + long firstStatusStartMillis = -1L; + for (UptimeHistory hist : history) { if (hist.isRelay() != relay) { continue; } + if (firstStatusStartMillis < 0L) { + firstStatusStartMillis = hist.getStartMillis(); + } long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR * hist.getUptimeHours(); if (histEndMillis < intervalStartMillis) { @@ -144,38 +148,36 @@ public class UptimeDocumentWriter implements FingerprintListener, } while (hist.getStartMillis() >= intervalStartMillis + dataPointInterval) { - statusDataPoints.add(statusHours * 5 > dataPointIntervalHours - ? statusHours : -1); - statusHours = 0; + if (firstStatusStartMillis < intervalStartMillis + + dataPointInterval) { + uptimeDataPoints.add(uptimeHours); + } else { + uptimeDataPoints.add(-1); + } + uptimeHours = 0; intervalStartMillis += dataPointInterval; } while (histEndMillis >= intervalStartMillis + dataPointInterval) { - statusHours += (int) ((intervalStartMillis + dataPointInterval + uptimeHours += (int) ((intervalStartMillis + dataPointInterval - Math.max(hist.getStartMillis(), intervalStartMillis)) / DateTimeHelper.ONE_HOUR); - statusDataPoints.add(statusHours * 5 > dataPointIntervalHours - ? statusHours : -1); - statusHours = 0; + uptimeDataPoints.add(uptimeHours); + uptimeHours = 0; intervalStartMillis += dataPointInterval; } - statusHours += (int) ((histEndMillis - Math.max( + uptimeHours += (int) ((histEndMillis - Math.max( hist.getStartMillis(), intervalStartMillis)) / DateTimeHelper.ONE_HOUR); } - statusDataPoints.add(statusHours * 5 > dataPointIntervalHours - ? statusHours : -1); - List<Integer> uptimeDataPoints = new ArrayList<Integer>(); + uptimeDataPoints.add(uptimeHours); + List<Integer> statusDataPoints = new ArrayList<Integer>(); intervalStartMillis = ((this.now - graphInterval) / dataPointInterval) * dataPointInterval; - int uptimeHours = 0; - long firstStatusStartMillis = -1L; - for (UptimeHistory hist : history) { + int statusHours = -1; + for (UptimeHistory hist : knownStatuses) { if (hist.isRelay() != relay) { continue; } - if (firstStatusStartMillis < 0L) { - firstStatusStartMillis = hist.getStartMillis(); - } long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR * hist.getUptimeHours(); if (histEndMillis < intervalStartMillis) { @@ -183,28 +185,35 @@ public class UptimeDocumentWriter implements FingerprintListener, } while (hist.getStartMillis() >= intervalStartMillis + dataPointInterval) { - if (firstStatusStartMillis < intervalStartMillis - + dataPointInterval) { - uptimeDataPoints.add(uptimeHours); - } else { - uptimeDataPoints.add(-1); - } - uptimeHours = 0; + statusDataPoints.add(statusHours * 5 > dataPointIntervalHours + ? statusHours : -1); + statusHours = -1; intervalStartMillis += dataPointInterval; } while (histEndMillis >= intervalStartMillis + dataPointInterval) { - uptimeHours += (int) ((intervalStartMillis + dataPointInterval - - Math.max(hist.getStartMillis(), intervalStartMillis)) + if (statusHours < 0) { + statusHours = 0; + } + statusHours += (int) ((intervalStartMillis + dataPointInterval + - Math.max(Math.max(hist.getStartMillis(), + firstStatusStartMillis), intervalStartMillis)) / DateTimeHelper.ONE_HOUR); - uptimeDataPoints.add(uptimeHours); - uptimeHours = 0; + statusDataPoints.add(statusHours * 5 > dataPointIntervalHours + ? statusHours : -1); + statusHours = -1; intervalStartMillis += dataPointInterval; } - uptimeHours += (int) ((histEndMillis - Math.max( - hist.getStartMillis(), intervalStartMillis)) - / DateTimeHelper.ONE_HOUR); + if (statusHours < 0) { + statusHours = 0; + } + statusHours += (int) ((histEndMillis - Math.max(Math.max( + hist.getStartMillis(), firstStatusStartMillis), + intervalStartMillis)) / DateTimeHelper.ONE_HOUR); + } + if (statusHours > 0) { + statusDataPoints.add(statusHours * 5 > dataPointIntervalHours + ? statusHours : -1); } - uptimeDataPoints.add(uptimeHours); List<Double> dataPoints = new ArrayList<Double>(); for (int dataPointIndex = 0; dataPointIndex < statusDataPoints.size(); dataPointIndex++) {

1 0

[onionoo/master] Add tests for UpdateStatusUpdater.
by karsten＠torproject.org 14 Apr '14

14 Apr '14

commit 7ad85bd18b82973a6a323fe776a0dd657a7f8a15 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sun Mar 23 17:09:38 2014 +0100 Add tests for UpdateStatusUpdater. --- .../torproject/onionoo/ClientsStatusUpdater.java | 1 - src/org/torproject/onionoo/DescriptorSource.java | 49 +++--- src/org/torproject/onionoo/Main.java | 19 +-- .../torproject/onionoo/UptimeStatusUpdater.java | 1 - test/org/torproject/onionoo/DummyBridgeStatus.java | 43 +++++ test/org/torproject/onionoo/DummyConsensus.java | 114 +++++++++++++ .../torproject/onionoo/DummyDescriptorSource.java | 133 +++++++++++++++ .../org/torproject/onionoo/DummyDocumentStore.java | 6 + test/org/torproject/onionoo/DummyStatusEntry.java | 92 ++++++++++ .../onionoo/UptimeStatusUpdaterTest.java | 176 ++++++++++++++++++++ 10 files changed, 594 insertions(+), 40 deletions(-) diff --git a/src/org/torproject/onionoo/ClientsStatusUpdater.java b/src/org/torproject/onionoo/ClientsStatusUpdater.java index ed4390e..4fe2b2d 100644 --- a/src/org/torproject/onionoo/ClientsStatusUpdater.java +++ b/src/org/torproject/onionoo/ClientsStatusUpdater.java @@ -149,7 +149,6 @@ public class ClientsStatusUpdater implements DescriptorListener, this.compressHistory(clientsStatus); this.documentStore.store(clientsStatus, hashedFingerprint); } - Logger.printStatusTime("Updated clients status files"); } private void addToHistory(ClientsStatus clientsStatus, diff --git a/src/org/torproject/onionoo/DescriptorSource.java b/src/org/torproject/onionoo/DescriptorSource.java index f5adf56..f4ff2fc 100644 --- a/src/org/torproject/onionoo/DescriptorSource.java +++ b/src/org/torproject/onionoo/DescriptorSource.java @@ -318,42 +318,21 @@ public class DescriptorSource { this.fingerprintListeners.get(descriptorType).add(listener); } - public void readRelayNetworkConsensuses() { + public void readDescriptors() { this.readDescriptors(DescriptorType.RELAY_CONSENSUSES, DescriptorHistory.RELAY_CONSENSUS_HISTORY, true); - } - - public void readRelayServerDescriptors() { this.readDescriptors(DescriptorType.RELAY_SERVER_DESCRIPTORS, DescriptorHistory.RELAY_SERVER_HISTORY, true); - } - - public void readRelayExtraInfos() { this.readDescriptors(DescriptorType.RELAY_EXTRA_INFOS, DescriptorHistory.RELAY_EXTRAINFO_HISTORY, true); - } - - public void readExitLists() { this.readDescriptors(DescriptorType.EXIT_LISTS, DescriptorHistory.EXIT_LIST_HISTORY, true); - } - - public void readBridgeNetworkStatuses() { this.readDescriptors(DescriptorType.BRIDGE_STATUSES, DescriptorHistory.BRIDGE_STATUS_HISTORY, false); - } - - public void readBridgeServerDescriptors() { this.readDescriptors(DescriptorType.BRIDGE_SERVER_DESCRIPTORS, DescriptorHistory.BRIDGE_SERVER_HISTORY, false); - } - - public void readBridgeExtraInfos() { this.readDescriptors(DescriptorType.BRIDGE_EXTRA_INFOS, DescriptorHistory.BRIDGE_EXTRAINFO_HISTORY, false); - } - - public void readBridgePoolAssignments() { this.readDescriptors(DescriptorType.BRIDGE_POOL_ASSIGNMENTS, DescriptorHistory.BRIDGE_POOLASSIGN_HISTORY, false); } @@ -419,6 +398,32 @@ public class DescriptorSource { fingerprintListener.processFingerprints(fingerprints, relay); } } + switch (descriptorType) { + case RELAY_CONSENSUSES: + Logger.printStatusTime("Read relay network consensuses"); + break; + case RELAY_SERVER_DESCRIPTORS: + Logger.printStatusTime("Read relay server descriptors"); + break; + case RELAY_EXTRA_INFOS: + Logger.printStatusTime("Read relay extra-info descriptors"); + break; + case EXIT_LISTS: + Logger.printStatusTime("Read exit lists"); + break; + case BRIDGE_STATUSES: + Logger.printStatusTime("Read bridge network statuses"); + break; + case BRIDGE_SERVER_DESCRIPTORS: + Logger.printStatusTime("Read bridge server descriptors"); + break; + case BRIDGE_EXTRA_INFOS: + Logger.printStatusTime("Read bridge extra-info descriptors"); + break; + case BRIDGE_POOL_ASSIGNMENTS: + Logger.printStatusTime("Read bridge-pool assignments"); + break; + } } public void writeHistoryFiles() { diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 141e44b..cceef92 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -59,26 +59,13 @@ public class Main { udw }; Logger.printStatus("Reading descriptors."); - dso.readRelayNetworkConsensuses(); - Logger.printStatusTime("Read relay network consensuses"); - dso.readRelayServerDescriptors(); - Logger.printStatusTime("Read relay server descriptors"); - dso.readRelayExtraInfos(); - Logger.printStatusTime("Read relay extra-info descriptors"); - dso.readExitLists(); - Logger.printStatusTime("Read exit lists"); - dso.readBridgeNetworkStatuses(); - Logger.printStatusTime("Read bridge network statuses"); - dso.readBridgeServerDescriptors(); - Logger.printStatusTime("Read bridge server descriptors"); - dso.readBridgeExtraInfos(); - Logger.printStatusTime("Read bridge extra-info descriptors"); - dso.readBridgePoolAssignments(); - Logger.printStatusTime("Read bridge-pool assignments"); + dso.readDescriptors(); Logger.printStatus("Updating internal status files."); for (StatusUpdater su : sus) { su.updateStatuses(); + Logger.printStatusTime(su.getClass().getSimpleName() + + " updated status files"); } Logger.printStatus("Updating document files."); diff --git a/src/org/torproject/onionoo/UptimeStatusUpdater.java b/src/org/torproject/onionoo/UptimeStatusUpdater.java index 6dc0b94..eccc2f2 100644 --- a/src/org/torproject/onionoo/UptimeStatusUpdater.java +++ b/src/org/torproject/onionoo/UptimeStatusUpdater.java @@ -100,7 +100,6 @@ public class UptimeStatusUpdater implements DescriptorListener, this.updateStatus(false, e.getKey(), e.getValue()); } this.updateStatus(false, null, this.newBridgeStatuses); - Logger.printStatusTime("Updated uptime status files"); } private void updateStatus(boolean relay, String fingerprint, diff --git a/test/org/torproject/onionoo/DummyBridgeStatus.java b/test/org/torproject/onionoo/DummyBridgeStatus.java new file mode 100644 index 0000000..35a9036 --- /dev/null +++ b/test/org/torproject/onionoo/DummyBridgeStatus.java @@ -0,0 +1,43 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.util.List; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.torproject.descriptor.BridgeNetworkStatus; +import org.torproject.descriptor.NetworkStatusEntry; + +public class DummyBridgeStatus implements BridgeNetworkStatus { + + public byte[] getRawDescriptorBytes() { + return null; + } + + public List<String> getAnnotations() { + return null; + } + + public List<String> getUnrecognizedLines() { + return null; + } + + private long publishedMillis; + public void setPublishedMillis(long publishedMillis) { + this.publishedMillis = publishedMillis; + } + public long getPublishedMillis() { + return this.publishedMillis; + } + + private SortedMap<String, NetworkStatusEntry> statusEntries = + new TreeMap<String, NetworkStatusEntry>(); + public void addStatusEntry(NetworkStatusEntry statusEntry) { + this.statusEntries.put(statusEntry.getFingerprint(), statusEntry); + } + public SortedMap<String, NetworkStatusEntry> getStatusEntries() { + return this.statusEntries; + } +} + diff --git a/test/org/torproject/onionoo/DummyConsensus.java b/test/org/torproject/onionoo/DummyConsensus.java new file mode 100644 index 0000000..3fa0fdd --- /dev/null +++ b/test/org/torproject/onionoo/DummyConsensus.java @@ -0,0 +1,114 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.util.List; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; + +import org.torproject.descriptor.DirSourceEntry; +import org.torproject.descriptor.DirectorySignature; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusConsensus; + +public class DummyConsensus implements RelayNetworkStatusConsensus { + + public byte[] getRawDescriptorBytes() { + return null; + } + + public List<String> getAnnotations() { + return null; + } + + public List<String> getUnrecognizedLines() { + return null; + } + + public int getNetworkStatusVersion() { + return 0; + } + + public String getConsensusFlavor() { + return null; + } + + public int getConsensusMethod() { + return 0; + } + + private long validAfterMillis; + public void setValidAfterMillis(long validAfterMillis) { + this.validAfterMillis = validAfterMillis; + } + public long getValidAfterMillis() { + return this.validAfterMillis; + } + + public long getFreshUntilMillis() { + return 0; + } + + public long getValidUntilMillis() { + return 0; + } + + public long getVoteSeconds() { + return 0; + } + + public long getDistSeconds() { + return 0; + } + + public List<String> getRecommendedServerVersions() { + return null; + } + + public List<String> getRecommendedClientVersions() { + return null; + } + + public SortedSet<String> getKnownFlags() { + return null; + } + + public SortedMap<String, Integer> getConsensusParams() { + return null; + } + + public SortedMap<String, DirSourceEntry> getDirSourceEntries() { + return null; + } + + private SortedMap<String, NetworkStatusEntry> statusEntries = + new TreeMap<String, NetworkStatusEntry>(); + public void addStatusEntry(NetworkStatusEntry statusEntry) { + this.statusEntries.put(statusEntry.getFingerprint(), statusEntry); + } + public SortedMap<String, NetworkStatusEntry> getStatusEntries() { + return this.statusEntries; + } + + public boolean containsStatusEntry(String fingerprint) { + return false; + } + + public NetworkStatusEntry getStatusEntry(String fingerprint) { + return null; + } + + public SortedMap<String, DirectorySignature> getDirectorySignatures() { + return null; + } + + public SortedMap<String, Integer> getBandwidthWeights() { + return null; + } + + public String getConsensusDigest() { + return null; + } +} + diff --git a/test/org/torproject/onionoo/DummyDescriptorSource.java b/test/org/torproject/onionoo/DummyDescriptorSource.java new file mode 100644 index 0000000..06ec499 --- /dev/null +++ b/test/org/torproject/onionoo/DummyDescriptorSource.java @@ -0,0 +1,133 @@ +package org.torproject.onionoo; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.torproject.descriptor.Descriptor; + +public class DummyDescriptorSource extends DescriptorSource { + + private Map<DescriptorType, Set<Descriptor>> descriptors = + new HashMap<DescriptorType, Set<Descriptor>>(); + + public void provideDescriptors(DescriptorType descriptorType, + Collection<Descriptor> descriptors) { + for (Descriptor descriptor : descriptors) { + this.addDescriptor(descriptorType, descriptor); + } + } + + public void addDescriptor(DescriptorType descriptorType, + Descriptor descriptor) { + this.getDescriptorsByType(descriptorType).add(descriptor); + } + + private Set<Descriptor> getDescriptorsByType( + DescriptorType descriptorType) { + if (!this.descriptors.containsKey(descriptorType)) { + this.descriptors.put(descriptorType, new HashSet<Descriptor>()); + } + return this.descriptors.get(descriptorType); + } + + private Map<DescriptorType, SortedSet<String>> fingerprints = + new HashMap<DescriptorType, SortedSet<String>>(); + + public void addFingerprints(DescriptorType descriptorType, + Collection<String> fingerprints) { + this.getFingerprintsByType(descriptorType).addAll(fingerprints); + } + + public void addFingerprint(DescriptorType descriptorType, + String fingerprint) { + this.getFingerprintsByType(descriptorType).add(fingerprint); + } + + private SortedSet<String> getFingerprintsByType( + DescriptorType descriptorType) { + if (!this.fingerprints.containsKey(descriptorType)) { + this.fingerprints.put(descriptorType, new TreeSet<String>()); + } + return this.fingerprints.get(descriptorType); + } + + private Map<DescriptorType, Set<DescriptorListener>> + descriptorListeners = new HashMap<DescriptorType, + Set<DescriptorListener>>(); + + public void registerDescriptorListener(DescriptorListener listener, + DescriptorType descriptorType) { + if (!this.descriptorListeners.containsKey(descriptorType)) { + this.descriptorListeners.put(descriptorType, + new HashSet<DescriptorListener>()); + } + this.descriptorListeners.get(descriptorType).add(listener); + } + + private Map<DescriptorType, Set<FingerprintListener>> + fingerprintListeners = new HashMap<DescriptorType, + Set<FingerprintListener>>(); + + public void registerFingerprintListener(FingerprintListener listener, + DescriptorType descriptorType) { + if (!this.fingerprintListeners.containsKey(descriptorType)) { + this.fingerprintListeners.put(descriptorType, + new HashSet<FingerprintListener>()); + } + this.fingerprintListeners.get(descriptorType).add(listener); + } + + public void readDescriptors() { + Set<DescriptorType> descriptorTypes = new HashSet<DescriptorType>(); + descriptorTypes.addAll(this.descriptorListeners.keySet()); + descriptorTypes.addAll(this.fingerprintListeners.keySet()); + for (DescriptorType descriptorType : descriptorTypes) { + boolean relay; + switch (descriptorType) { + case RELAY_CONSENSUSES: + case RELAY_SERVER_DESCRIPTORS: + case RELAY_EXTRA_INFOS: + case EXIT_LISTS: + relay = true; + break; + case BRIDGE_STATUSES: + case BRIDGE_SERVER_DESCRIPTORS: + case BRIDGE_EXTRA_INFOS: + case BRIDGE_POOL_ASSIGNMENTS: + default: + relay = false; + break; + } + if (this.descriptors.containsKey(descriptorType) && + this.descriptorListeners.containsKey(descriptorType)) { + Set<DescriptorListener> listeners = + this.descriptorListeners.get(descriptorType); + for (Descriptor descriptor : + this.getDescriptorsByType(descriptorType)) { + for (DescriptorListener listener : listeners) { + listener.processDescriptor(descriptor, relay); + } + } + } + if (this.fingerprints.containsKey(descriptorType) && + this.fingerprintListeners.containsKey(descriptorType)) { + Set<FingerprintListener> listeners = + this.fingerprintListeners.get(descriptorType); + for (FingerprintListener listener : listeners) { + listener.processFingerprints(this.getFingerprintsByType( + descriptorType), relay); + } + } + } + } + + public void writeHistoryFiles() { + /* Nothing to do here. */ + } +} + diff --git a/test/org/torproject/onionoo/DummyDocumentStore.java b/test/org/torproject/onionoo/DummyDocumentStore.java index 05723d0..6969b8f 100644 --- a/test/org/torproject/onionoo/DummyDocumentStore.java +++ b/test/org/torproject/onionoo/DummyDocumentStore.java @@ -30,6 +30,12 @@ public class DummyDocumentStore extends DocumentStore { fingerprint == null ? FINGERPRINT_NULL : fingerprint, document); } + public <T extends Document> T getDocument(Class<T> documentType, + String fingerprint) { + return documentType.cast(this.getStoredDocumentsByClass(documentType). + get(fingerprint == null ? FINGERPRINT_NULL : fingerprint)); + } + public void flushDocumentCache() { /* Nothing to do. */ } diff --git a/test/org/torproject/onionoo/DummyStatusEntry.java b/test/org/torproject/onionoo/DummyStatusEntry.java new file mode 100644 index 0000000..8fdc5cd --- /dev/null +++ b/test/org/torproject/onionoo/DummyStatusEntry.java @@ -0,0 +1,92 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.torproject.descriptor.NetworkStatusEntry; + +public class DummyStatusEntry implements NetworkStatusEntry { + + public DummyStatusEntry(String fingerprint) { + this.fingerprint = fingerprint; + } + + public byte[] getStatusEntryBytes() { + return null; + } + + @Override + public String getNickname() { + return null; + } + + private String fingerprint; + public String getFingerprint() { + return this.fingerprint; + } + + public String getDescriptor() { + return null; + } + + public long getPublishedMillis() { + return 0; + } + + public String getAddress() { + return null; + } + + public int getOrPort() { + return 0; + } + + public int getDirPort() { + return 0; + } + + public Set<String> getMicrodescriptorDigests() { + return null; + } + + public List<String> getOrAddresses() { + return null; + } + + private SortedSet<String> flags = new TreeSet<String>(); + public void addFlag(String flag) { + this.flags.add(flag); + } + public SortedSet<String> getFlags() { + return this.flags; + } + + public String getVersion() { + return null; + } + + public long getBandwidth() { + return 0; + } + + public long getMeasured() { + return 0; + } + + public boolean getUnmeasured() { + return false; + } + + public String getDefaultPolicy() { + return null; + } + + public String getPortList() { + return null; + } +} + diff --git a/test/org/torproject/onionoo/UptimeStatusUpdaterTest.java b/test/org/torproject/onionoo/UptimeStatusUpdaterTest.java new file mode 100644 index 0000000..4ca2245 --- /dev/null +++ b/test/org/torproject/onionoo/UptimeStatusUpdaterTest.java @@ -0,0 +1,176 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import static org.junit.Assert.assertEquals; + +import org.junit.Before; +import org.junit.Test; + +public class UptimeStatusUpdaterTest { + + private DummyDescriptorSource descriptorSource; + + @Before + public void createDummyDescriptorSource() { + this.descriptorSource = new DummyDescriptorSource(); + ApplicationFactory.setDescriptorSource(this.descriptorSource); + } + + private DummyDocumentStore documentStore; + + @Before + public void createDummyDocumentStore() { + this.documentStore = new DummyDocumentStore(); + ApplicationFactory.setDocumentStore(this.documentStore); + } + + @Test + public void testNoDescriptorsNoStatusFiles() { + UptimeStatusUpdater updater = new UptimeStatusUpdater(); + ApplicationFactory.getDescriptorSource().readDescriptors(); + updater.updateStatuses(); + assertEquals("Without providing any data, nothing should be written " + + "to disk.", 0, + this.documentStore.getPerformedStoreOperations()); + } + + private static final long VALID_AFTER_SAMPLE = + DateTimeHelper.parse("2014-03-21 20:00:00"); + + private static final String GABELMOO_FINGERPRINT = + "F2044413DAC2E02E3D6BCF4735A19BCA1DE97281"; + + private void addConsensusSample() { + DummyStatusEntry statusEntry = new DummyStatusEntry( + GABELMOO_FINGERPRINT); + statusEntry.addFlag("Running"); + DummyConsensus consensus = new DummyConsensus(); + consensus.setValidAfterMillis(VALID_AFTER_SAMPLE); + consensus.addStatusEntry(statusEntry); + this.descriptorSource.addDescriptor(DescriptorType.RELAY_CONSENSUSES, + consensus); + } + + @Test + public void testOneConsensusNoStatusFiles() { + this.addConsensusSample(); + UptimeStatusUpdater updater = new UptimeStatusUpdater(); + ApplicationFactory.getDescriptorSource().readDescriptors(); + updater.updateStatuses(); + assertEquals("Two status files should have been written to disk.", + 2, this.documentStore.getPerformedStoreOperations()); + for (String fingerprint : new String[] { GABELMOO_FINGERPRINT, + null }) { + UptimeStatus status = this.documentStore.getDocument( + UptimeStatus.class, fingerprint); + UptimeHistory history = status.getHistory().first(); + assertEquals("History must contain one entry.", 1, + status.getHistory().size()); + assertEquals("History not for relay.", true, history.isRelay()); + assertEquals("History start millis not as expected.", + VALID_AFTER_SAMPLE, history.getStartMillis()); + assertEquals("History uptime hours must be 1.", 1, + history.getUptimeHours()); + } + } + + private static final String ALL_RELAYS_AND_BRIDGES_FINGERPRINT = null; + + private static final String ALL_RELAYS_AND_BRIDGES_UPTIME_SAMPLE = + "r 2013-07-22-17 5811\n" /* ends 2014-03-21 20:00:00 */ + + "b 2013-07-22-17 5811\n"; /* ends 2014-03-21 20:00:00 */ + + private void addAllRelaysAndBridgesUptimeSample() { + UptimeStatus uptimeStatus = new UptimeStatus(); + uptimeStatus.fromDocumentString(ALL_RELAYS_AND_BRIDGES_UPTIME_SAMPLE); + this.documentStore.addDocument(uptimeStatus, + ALL_RELAYS_AND_BRIDGES_FINGERPRINT); + } + + @Test + public void testOneConsensusOneStatusFiles() { + this.addAllRelaysAndBridgesUptimeSample(); + this.addConsensusSample(); + UptimeStatusUpdater updater = new UptimeStatusUpdater(); + ApplicationFactory.getDescriptorSource().readDescriptors(); + updater.updateStatuses(); + assertEquals("Two status files should have been written to disk.", + 2, this.documentStore.getPerformedStoreOperations()); + UptimeStatus status = this.documentStore.getDocument( + UptimeStatus.class, ALL_RELAYS_AND_BRIDGES_FINGERPRINT); + assertEquals("History must contain two entries.", 2, + status.getHistory().size()); + UptimeHistory history = status.getHistory().first(); + assertEquals("History not for relay.", true, history.isRelay()); + assertEquals("History start millis not as expected.", + DateTimeHelper.parse("2013-07-22 17:00:00"), + history.getStartMillis()); + assertEquals("History uptime hours must be 5812.", 5812, + history.getUptimeHours()); + } + + private static final long PUBLISHED_SAMPLE = + DateTimeHelper.parse("2014-03-21 20:37:03"); + + private static final String NDNOP2_FINGERPRINT = + "DE6397A047ABE5F78B4C87AF725047831B221AAB"; + + private void addBridgeStatusSample() { + DummyStatusEntry statusEntry = new DummyStatusEntry( + NDNOP2_FINGERPRINT); + statusEntry.addFlag("Running"); + DummyBridgeStatus bridgeStatus = new DummyBridgeStatus(); + bridgeStatus.setPublishedMillis(PUBLISHED_SAMPLE); + bridgeStatus.addStatusEntry(statusEntry); + this.descriptorSource.addDescriptor(DescriptorType.BRIDGE_STATUSES, + bridgeStatus); + } + + @Test + public void testOneBridgeStatusNoStatusFiles() { + this.addBridgeStatusSample(); + UptimeStatusUpdater updater = new UptimeStatusUpdater(); + ApplicationFactory.getDescriptorSource().readDescriptors(); + updater.updateStatuses(); + assertEquals("Two status files should have been written to disk.", + 2, this.documentStore.getPerformedStoreOperations()); + for (String fingerprint : new String[] { NDNOP2_FINGERPRINT, + null }) { + UptimeStatus status = this.documentStore.getDocument( + UptimeStatus.class, fingerprint); + UptimeHistory history = status.getHistory().first(); + assertEquals("History must contain one entry.", 1, + status.getHistory().size()); + assertEquals("History not for bridge.", false, history.isRelay()); + assertEquals("History start millis not as expected.", + DateTimeHelper.parse("2014-03-21 20:00:00"), + history.getStartMillis()); + assertEquals("History uptime hours must be 1.", 1, + history.getUptimeHours()); + } + } + + @Test + public void testOneBridgeStatusOneStatusFiles() { + this.addAllRelaysAndBridgesUptimeSample(); + this.addBridgeStatusSample(); + UptimeStatusUpdater updater = new UptimeStatusUpdater(); + ApplicationFactory.getDescriptorSource().readDescriptors(); + updater.updateStatuses(); + assertEquals("Two status files should have been written to disk.", + 2, this.documentStore.getPerformedStoreOperations()); + UptimeStatus status = this.documentStore.getDocument( + UptimeStatus.class, ALL_RELAYS_AND_BRIDGES_FINGERPRINT); + assertEquals("History must contain two entries.", 2, + status.getHistory().size()); + UptimeHistory history = status.getHistory().last(); + assertEquals("History not for relay.", false, history.isRelay()); + assertEquals("History start millis not as expected.", + DateTimeHelper.parse("2013-07-22 17:00:00"), + history.getStartMillis()); + assertEquals("History uptime hours must be 5812.", 5812, + history.getUptimeHours()); + } +} +

1 0

[onionoo/master] Teach UptimeStatus how to load/save itself.
by karsten＠torproject.org 14 Apr '14

14 Apr '14

commit 1fe217042241376fee66936e0b4140baaf0e513d Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Mar 21 20:52:04 2014 +0100 Teach UptimeStatus how to load/save itself. --- src/org/torproject/onionoo/UptimeStatus.java | 33 +++++++++++++++++++- .../torproject/onionoo/UptimeStatusUpdater.java | 17 ++-------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/org/torproject/onionoo/UptimeStatus.java b/src/org/torproject/onionoo/UptimeStatus.java index 61b4fe0..1a222b0 100644 --- a/src/org/torproject/onionoo/UptimeStatus.java +++ b/src/org/torproject/onionoo/UptimeStatus.java @@ -95,6 +95,10 @@ class UptimeHistory class UptimeStatus extends Document { + private transient String fingerprint; + + private transient boolean isDirty = false; + private SortedSet<UptimeHistory> history = new TreeSet<UptimeHistory>(); public void setHistory(SortedSet<UptimeHistory> history) { this.history = history; @@ -103,6 +107,19 @@ class UptimeStatus extends Document { return this.history; } + public static UptimeStatus loadOrCreate(String fingerprint) { + UptimeStatus uptimeStatus = (fingerprint == null) ? + ApplicationFactory.getDocumentStore().retrieve( + UptimeStatus.class, true) : + ApplicationFactory.getDocumentStore().retrieve( + UptimeStatus.class, true, fingerprint); + if (uptimeStatus == null) { + uptimeStatus = new UptimeStatus(); + } + uptimeStatus.fingerprint = fingerprint; + return uptimeStatus; + } + public void fromDocumentString(String documentString) { Scanner s = new Scanner(documentString); while (s.hasNextLine()) { @@ -138,10 +155,24 @@ class UptimeStatus extends Document { } } this.history.add(interval); + this.isDirty = true; + } + } + + public void storeIfChanged() { + if (this.isDirty) { + this.compressHistory(); + if (fingerprint == null) { + ApplicationFactory.getDocumentStore().store(this); + } else { + ApplicationFactory.getDocumentStore().store(this, + this.fingerprint); + } + this.isDirty = false; } } - public void compressHistory() { + private void compressHistory() { SortedSet<UptimeHistory> compressedHistory = new TreeSet<UptimeHistory>(); UptimeHistory lastInterval = null; diff --git a/src/org/torproject/onionoo/UptimeStatusUpdater.java b/src/org/torproject/onionoo/UptimeStatusUpdater.java index 4430387..6dc0b94 100644 --- a/src/org/torproject/onionoo/UptimeStatusUpdater.java +++ b/src/org/torproject/onionoo/UptimeStatusUpdater.java @@ -18,11 +18,8 @@ public class UptimeStatusUpdater implements DescriptorListener, private DescriptorSource descriptorSource; - private DocumentStore documentStore; - public UptimeStatusUpdater() { this.descriptorSource = ApplicationFactory.getDescriptorSource(); - this.documentStore = ApplicationFactory.getDocumentStore(); this.registerDescriptorListeners(); } @@ -108,19 +105,9 @@ public class UptimeStatusUpdater implements DescriptorListener, private void updateStatus(boolean relay, String fingerprint, SortedSet<Long> newUptimeHours) { - UptimeStatus uptimeStatus = fingerprint == null ? - documentStore.retrieve(UptimeStatus.class, true) : - documentStore.retrieve(UptimeStatus.class, true, fingerprint); - if (uptimeStatus == null) { - uptimeStatus = new UptimeStatus(); - } + UptimeStatus uptimeStatus = UptimeStatus.loadOrCreate(fingerprint); uptimeStatus.addToHistory(relay, newUptimeHours); - uptimeStatus.compressHistory(); - if (fingerprint == null) { - this.documentStore.store(uptimeStatus); - } else { - this.documentStore.store(uptimeStatus, fingerprint); - } + uptimeStatus.storeIfChanged(); } public String getStatsString() {

1 0

[onionoo/master] Use String.indexOf(char) because it's faster.
by karsten＠torproject.org 14 Apr '14

14 Apr '14

commit dda3e54078182d9f2fa863e817bf55d8e4317947 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Mar 20 10:56:08 2014 +0100 Use String.indexOf(char) because it's faster. Found with SonarQube. --- src/org/torproject/onionoo/NodeStatus.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/torproject/onionoo/NodeStatus.java b/src/org/torproject/onionoo/NodeStatus.java index 30e1ed4..afe2ed3 100644 --- a/src/org/torproject/onionoo/NodeStatus.java +++ b/src/org/torproject/onionoo/NodeStatus.java @@ -338,7 +338,7 @@ public class NodeStatus extends Document { System.exit(1); } else if (orAddressAndPort.length() > 0) { String orAddress = orAddressAndPort.substring(0, - orAddressAndPort.lastIndexOf(":")); + orAddressAndPort.lastIndexOf(':')); if (this.exitAddresses.contains(orAddress)) { this.exitAddresses.remove(orAddress); }

1 0

[onionoo/master] Implement hashCode() whenever implementing equals().
by karsten＠torproject.org 14 Apr '14

14 Apr '14

commit ca390d2ab17897a45a24c6ab1f2ff2a710253c32 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Mar 20 10:47:37 2014 +0100 Implement hashCode() whenever implementing equals(). Found with SonarQube. --- src/org/torproject/onionoo/ClientsStatus.java | 4 ++++ src/org/torproject/onionoo/UptimeStatus.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/org/torproject/onionoo/ClientsStatus.java b/src/org/torproject/onionoo/ClientsStatus.java index e315fc3..c1ce80e 100644 --- a/src/org/torproject/onionoo/ClientsStatus.java +++ b/src/org/torproject/onionoo/ClientsStatus.java @@ -165,6 +165,10 @@ class ClientsHistory implements Comparable<ClientsHistory> { return other instanceof ClientsHistory && this.startMillis == ((ClientsHistory) other).startMillis; } + + public int hashCode() { + return (int) this.startMillis; + } } class ClientsStatus extends Document { diff --git a/src/org/torproject/onionoo/UptimeStatus.java b/src/org/torproject/onionoo/UptimeStatus.java index fedf1a1..4d8dca7 100644 --- a/src/org/torproject/onionoo/UptimeStatus.java +++ b/src/org/torproject/onionoo/UptimeStatus.java @@ -89,6 +89,10 @@ class UptimeHistory this.relay == ((UptimeHistory) other).relay && this.startMillis == ((UptimeHistory) other).startMillis; } + + public int hashCode() { + return (int) this.startMillis + (this.relay ? 1 : 0); + } } class UptimeStatus extends Document {

1 0

[onionoo/master] Don't store user-supplied array directly.
by karsten＠torproject.org 14 Apr '14

14 Apr '14

commit 9f62a8115e4b24b3551b33aa7271d7250100ca24 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Mar 20 10:42:13 2014 +0100 Don't store user-supplied array directly. Found with SonarQube. --- src/org/torproject/onionoo/ResponseBuilder.java | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java index b0775b1..1412d88 100644 --- a/src/org/torproject/onionoo/ResponseBuilder.java +++ b/src/org/torproject/onionoo/ResponseBuilder.java @@ -305,7 +305,8 @@ public class ResponseBuilder { this.running = running; } public void setSearch(String[] search) { - this.search = search; + this.search = new String[search.length]; + System.arraycopy(search, 0, this.search, 0, search.length); } public void setLookup(String lookup) { this.lookup = lookup; @@ -320,19 +321,26 @@ public class ResponseBuilder { this.flag = flag; } public void setFirstSeenDays(int[] firstSeenDays) { - this.firstSeenDays = firstSeenDays; + this.firstSeenDays = new int[firstSeenDays.length]; + System.arraycopy(firstSeenDays, 0, this.firstSeenDays, 0, + firstSeenDays.length); } public void setLastSeenDays(int[] lastSeenDays) { - this.lastSeenDays = lastSeenDays; + this.lastSeenDays = new int[lastSeenDays.length]; + System.arraycopy(lastSeenDays, 0, this.lastSeenDays, 0, + lastSeenDays.length); } public void setContact(String[] contact) { - this.contact = contact; + this.contact = new String[contact.length]; + System.arraycopy(contact, 0, this.contact, 0, contact.length); } public void setFields(String[] fields) { - this.fields = fields; + this.fields = new String[fields.length]; + System.arraycopy(fields, 0, this.fields, 0, fields.length); } public void setOrder(String[] order) { - this.order = order; + this.order = new String[order.length]; + System.arraycopy(order, 0, this.order, 0, order.length); } public void setOffset(String offset) { this.offset = offset;

1 0

[onionoo/master] Avoid inner assignment.
by karsten＠torproject.org 14 Apr '14

14 Apr '14

commit b66c87752098a3055a555174bb9cbe30f7ffc759 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Mar 20 10:57:23 2014 +0100 Avoid inner assignment. Found with SonarQube. --- src/org/torproject/onionoo/ResourceServlet.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java index de47d44..27ff414 100644 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ b/src/org/torproject/onionoo/ResourceServlet.java @@ -388,7 +388,8 @@ public class ResourceServlet extends HttpServlet { int x = 0, y = Integer.MAX_VALUE; try { if (!parameter.contains("-")) { - x = y = Integer.parseInt(parameter); + x = Integer.parseInt(parameter); + y = x; } else { String[] parts = parameter.split("-", 2); if (parts[0].length() > 0) {

1 0

[onionoo/master] Move date parsing and formatting to util class.
by karsten＠torproject.org 14 Apr '14

14 Apr '14

commit 14a6e590853aaf8e151125bf8ae60ed82b654cf3 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Apr 8 23:17:32 2014 +0200 Move date parsing and formatting to util class. Suggested by SonarQube to avoid code repetition. --- .../onionoo/BandwidthDocumentWriter.java | 11 +-- src/org/torproject/onionoo/BandwidthStatus.java | 85 ++++++++---------- .../torproject/onionoo/BandwidthStatusUpdater.java | 42 ++++----- .../torproject/onionoo/ClientsDocumentWriter.java | 9 +- src/org/torproject/onionoo/ClientsStatus.java | 25 ++---- .../torproject/onionoo/ClientsStatusUpdater.java | 7 +- src/org/torproject/onionoo/DateTimeHelper.java | 57 +++++++++++++ .../torproject/onionoo/DetailsDocumentWriter.java | 18 ++-- .../onionoo/NodeDetailsStatusUpdater.java | 18 +--- src/org/torproject/onionoo/NodeStatus.java | 55 ++++++------ src/org/torproject/onionoo/ResponseBuilder.java | 9 +- .../torproject/onionoo/UptimeDocumentWriter.java | 9 +- src/org/torproject/onionoo/UptimeStatus.java | 21 ++--- .../torproject/onionoo/WeightsDocumentWriter.java | 9 +- src/org/torproject/onionoo/WeightsStatus.java | 90 +++++++++----------- .../torproject/onionoo/WeightsStatusUpdater.java | 7 +- 16 files changed, 209 insertions(+), 263 deletions(-) diff --git a/src/org/torproject/onionoo/BandwidthDocumentWriter.java b/src/org/torproject/onionoo/BandwidthDocumentWriter.java index 754c8f3..68912c7 100644 --- a/src/org/torproject/onionoo/BandwidthDocumentWriter.java +++ b/src/org/torproject/onionoo/BandwidthDocumentWriter.java @@ -2,7 +2,6 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -10,7 +9,6 @@ import java.util.Locale; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; -import java.util.TimeZone; public class BandwidthDocumentWriter implements FingerprintListener, DocumentWriter{ @@ -21,16 +19,11 @@ public class BandwidthDocumentWriter implements FingerprintListener, private long now; - private SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - public BandwidthDocumentWriter(DescriptorSource descriptorSource, DocumentStore documentStore, Time time) { this.descriptorSource = descriptorSource; this.documentStore = documentStore; this.now = time.currentTimeMillis(); - this.dateTimeFormat.setLenient(false); - this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); this.registerFingerprintListeners(); } @@ -161,9 +154,9 @@ public class BandwidthDocumentWriter implements FingerprintListener, StringBuilder sb2 = new StringBuilder(); sb2.append("\"" + graphName + "\":{" + "\"first\":\"" - + this.dateTimeFormat.format(firstDataPointMillis) + "\"," + + DateTimeHelper.format(firstDataPointMillis) + "\"," + "\"last\":\"" - + this.dateTimeFormat.format(lastDataPointMillis) + "\"," + + DateTimeHelper.format(lastDataPointMillis) + "\"," +"\"interval\":" + String.valueOf(dataPointInterval / 1000L) + ",\"factor\":" + String.format(Locale.US, "%.3f", factor) + ",\"count\":" + String.valueOf(count) + ",\"values\":["); diff --git a/src/org/torproject/onionoo/BandwidthStatus.java b/src/org/torproject/onionoo/BandwidthStatus.java index fd3c36e..00e8ca8 100644 --- a/src/org/torproject/onionoo/BandwidthStatus.java +++ b/src/org/torproject/onionoo/BandwidthStatus.java @@ -2,11 +2,8 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.Scanner; import java.util.SortedMap; -import java.util.TimeZone; import java.util.TreeMap; class BandwidthStatus extends Document { @@ -16,62 +13,50 @@ class BandwidthStatus extends Document { SortedMap<Long, long[]> readHistory = new TreeMap<Long, long[]>(); public void fromDocumentString(String documentString) { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - try { - Scanner s = new Scanner(documentString); - while (s.hasNextLine()) { - String line = s.nextLine(); - String[] parts = line.split(" "); - if (parts.length != 6) { - System.err.println("Illegal line '" + line + "' in bandwidth " - + "history. Skipping this line."); - continue; - } - SortedMap<Long, long[]> history = parts[0].equals("r") - ? readHistory : writeHistory; - long startMillis = dateTimeFormat.parse(parts[1] + " " - + parts[2]).getTime(); - long endMillis = dateTimeFormat.parse(parts[3] + " " - + parts[4]).getTime(); - long bandwidth = Long.parseLong(parts[5]); - long previousEndMillis = history.headMap(startMillis).isEmpty() - ? startMillis - : history.get(history.headMap(startMillis).lastKey())[1]; - long nextStartMillis = history.tailMap(startMillis).isEmpty() - ? endMillis : history.tailMap(startMillis).firstKey(); - if (previousEndMillis <= startMillis && - nextStartMillis >= endMillis) { - history.put(startMillis, new long[] { startMillis, endMillis, - bandwidth }); - } + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + String[] parts = line.split(" "); + if (parts.length != 6) { + System.err.println("Illegal line '" + line + "' in bandwidth " + + "history. Skipping this line."); + continue; + } + SortedMap<Long, long[]> history = parts[0].equals("r") + ? readHistory : writeHistory; + long startMillis = DateTimeHelper.parse(parts[1] + " " + parts[2]); + long endMillis = DateTimeHelper.parse(parts[3] + " " + parts[4]); + if (startMillis < 0L || endMillis < 0L) { + System.err.println("Could not parse timestamp while reading " + + "bandwidth history. Skipping."); + break; + } + long bandwidth = Long.parseLong(parts[5]); + long previousEndMillis = history.headMap(startMillis).isEmpty() + ? startMillis + : history.get(history.headMap(startMillis).lastKey())[1]; + long nextStartMillis = history.tailMap(startMillis).isEmpty() + ? endMillis : history.tailMap(startMillis).firstKey(); + if (previousEndMillis <= startMillis && + nextStartMillis >= endMillis) { + history.put(startMillis, new long[] { startMillis, endMillis, + bandwidth }); } - s.close(); - } catch (ParseException e) { - System.err.println("Could not parse timestamp while reading " - + "bandwidth history. Skipping."); - e.printStackTrace(); } - + s.close(); } public String toDocumentString() { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); StringBuilder sb = new StringBuilder(); for (long[] v : writeHistory.values()) { - sb.append("w " + dateTimeFormat.format(v[0]) + " " - + dateTimeFormat.format(v[1]) + " " - + String.valueOf(v[2]) + "\n"); + sb.append("w " + DateTimeHelper.format(v[0]) + " " + + DateTimeHelper.format(v[1]) + " " + String.valueOf(v[2]) + + "\n"); } for (long[] v : readHistory.values()) { - sb.append("r " + dateTimeFormat.format(v[0]) + " " - + dateTimeFormat.format(v[1]) + " " - + String.valueOf(v[2]) + "\n"); + sb.append("r " + DateTimeHelper.format(v[0]) + " " + + DateTimeHelper.format(v[1]) + " " + String.valueOf(v[2]) + + "\n"); } return sb.toString(); } diff --git a/src/org/torproject/onionoo/BandwidthStatusUpdater.java b/src/org/torproject/onionoo/BandwidthStatusUpdater.java index 6254260..cce56bc 100644 --- a/src/org/torproject/onionoo/BandwidthStatusUpdater.java +++ b/src/org/torproject/onionoo/BandwidthStatusUpdater.java @@ -2,10 +2,7 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.SortedMap; -import java.util.TimeZone; import java.util.TreeMap; import org.torproject.descriptor.Descriptor; @@ -20,16 +17,11 @@ public class BandwidthStatusUpdater implements DescriptorListener, private long now; - private SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - public BandwidthStatusUpdater(DescriptorSource descriptorSource, DocumentStore documentStore, Time time) { this.descriptorSource = descriptorSource; this.documentStore = documentStore; this.now = time.currentTimeMillis(); - this.dateTimeFormat.setLenient(false); - this.dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); this.registerDescriptorListeners(); } @@ -76,23 +68,22 @@ public class BandwidthStatusUpdater implements DescriptorListener, if (parts.length < 6) { return; } - try { - long endMillis = this.dateTimeFormat.parse(parts[1] + " " - + parts[2]).getTime(); - long intervalMillis = Long.parseLong(parts[3].substring(1)) * 1000L; - String[] values = parts[5].split(","); - for (int i = values.length - 1; i >= 0; i--) { - long bandwidthValue = Long.parseLong(values[i]); - long startMillis = endMillis - intervalMillis; - /* TODO Should we first check whether an interval is already - * contained in history? */ - history.put(startMillis, new long[] { startMillis, endMillis, - bandwidthValue }); - endMillis -= intervalMillis; - } - } catch (ParseException e) { + long endMillis = DateTimeHelper.parse(parts[1] + " " + parts[2]); + if (endMillis < 0L) { System.err.println("Could not parse timestamp in line '" + line + "'. Skipping."); + return; + } + long intervalMillis = Long.parseLong(parts[3].substring(1)) * 1000L; + String[] values = parts[5].split(","); + for (int i = values.length - 1; i >= 0; i--) { + long bandwidthValue = Long.parseLong(values[i]); + long startMillis = endMillis - intervalMillis; + /* TODO Should we first check whether an interval is already + * contained in history? */ + history.put(startMillis, new long[] { startMillis, endMillis, + bandwidthValue }); + endMillis -= intervalMillis; } } @@ -101,8 +92,6 @@ public class BandwidthStatusUpdater implements DescriptorListener, new TreeMap<Long, long[]>(history); history.clear(); long lastStartMillis = 0L, lastEndMillis = 0L, lastBandwidth = 0L; - SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); String lastMonthString = "1970-01"; for (long[] v : uncompressedHistory.values()) { long startMillis = v[0], endMillis = v[1], bandwidth = v[2]; @@ -120,7 +109,8 @@ public class BandwidthStatusUpdater implements DescriptorListener, } else { intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L; } - String monthString = dateTimeFormat.format(startMillis); + String monthString = DateTimeHelper.format(startMillis, + DateTimeHelper.ISO_YEARMONTH_FORMAT); if (lastEndMillis == startMillis && ((lastEndMillis - 1L) / intervalLengthMillis) == ((endMillis - 1L) / intervalLengthMillis) && diff --git a/src/org/torproject/onionoo/ClientsDocumentWriter.java b/src/org/torproject/onionoo/ClientsDocumentWriter.java index 9d3b8dc..fe400d2 100644 --- a/src/org/torproject/onionoo/ClientsDocumentWriter.java +++ b/src/org/torproject/onionoo/ClientsDocumentWriter.java @@ -2,14 +2,12 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; -import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; @@ -221,12 +219,9 @@ public class ClientsDocumentWriter implements FingerprintListener, double factor = ((double) maxValue) / 999.0; int count = lastNonNullIndex - firstNonNullIndex + 1; StringBuilder sb = new StringBuilder(); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); sb.append("\"" + graphName + "\":{" - + "\"first\":\"" + dateTimeFormat.format(firstDataPointMillis) - + "\",\"last\":\"" + dateTimeFormat.format(lastDataPointMillis) + + "\"first\":\"" + DateTimeHelper.format(firstDataPointMillis) + + "\",\"last\":\"" + DateTimeHelper.format(lastDataPointMillis) + "\",\"interval\":" + String.valueOf(dataPointInterval / 1000L) + ",\"factor\":" + String.format(Locale.US, "%.9f", factor) + ",\"count\":" + String.valueOf(count) + ",\"values\":["); diff --git a/src/org/torproject/onionoo/ClientsStatus.java b/src/org/torproject/onionoo/ClientsStatus.java index c1ce80e..26f9eab 100644 --- a/src/org/torproject/onionoo/ClientsStatus.java +++ b/src/org/torproject/onionoo/ClientsStatus.java @@ -2,13 +2,10 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.Map; import java.util.Scanner; import java.util.SortedMap; import java.util.SortedSet; -import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; @@ -45,17 +42,9 @@ class ClientsHistory implements Comparable<ClientsHistory> { if (parts.length != 8) { return null; } - long startMillis = -1L, endMillis = -1L; - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - try { - startMillis = dateTimeFormat.parse(parts[0] + " " + parts[1]). - getTime(); - endMillis = dateTimeFormat.parse(parts[2] + " " + parts[3]). - getTime(); - } catch (ParseException e) { + long startMillis = DateTimeHelper.parse(parts[0] + " " + parts[1]); + long endMillis = DateTimeHelper.parse(parts[2] + " " + parts[3]); + if (startMillis < 0L || endMillis < 0L) { return null; } if (startMillis >= endMillis) { @@ -104,12 +93,8 @@ class ClientsHistory implements Comparable<ClientsHistory> { public String toString() { StringBuilder sb = new StringBuilder(); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - sb.append(dateTimeFormat.format(startMillis)); - sb.append(" " + dateTimeFormat.format(endMillis)); + sb.append(DateTimeHelper.format(startMillis)); + sb.append(" " + DateTimeHelper.format(endMillis)); sb.append(" " + String.format("%.3f", this.totalResponses)); this.appendResponses(sb, this.responsesByCountry); this.appendResponses(sb, this.responsesByTransport); diff --git a/src/org/torproject/onionoo/ClientsStatusUpdater.java b/src/org/torproject/onionoo/ClientsStatusUpdater.java index e15c11a..0ce550f 100644 --- a/src/org/torproject/onionoo/ClientsStatusUpdater.java +++ b/src/org/torproject/onionoo/ClientsStatusUpdater.java @@ -2,11 +2,9 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.SimpleDateFormat; import java.util.Map; import java.util.SortedMap; import java.util.SortedSet; -import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; @@ -177,8 +175,6 @@ public class ClientsStatusUpdater implements DescriptorListener, SortedSet<ClientsHistory> compressedHistory = new TreeSet<ClientsHistory>(); ClientsHistory lastResponses = null; - SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); String lastMonthString = "1970-01"; for (ClientsHistory responses : history) { long intervalLengthMillis; @@ -191,7 +187,8 @@ public class ClientsStatusUpdater implements DescriptorListener, } else { intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L; } - String monthString = dateTimeFormat.format(responses.startMillis); + String monthString = DateTimeHelper.format(responses.startMillis, + DateTimeHelper.ISO_YEARMONTH_FORMAT); if (lastResponses != null && lastResponses.endMillis == responses.startMillis && ((lastResponses.endMillis - 1L) / intervalLengthMillis) == diff --git a/src/org/torproject/onionoo/DateTimeHelper.java b/src/org/torproject/onionoo/DateTimeHelper.java new file mode 100644 index 0000000..f97c8c5 --- /dev/null +++ b/src/org/torproject/onionoo/DateTimeHelper.java @@ -0,0 +1,57 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.HashMap; +import java.util.Map; +import java.util.TimeZone; + +public class DateTimeHelper { + + public static final String ISO_DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; + + public static final String ISO_DATETIME_TAB_FORMAT = + "yyyy-MM-dd\tHH:mm:ss"; + + public static final String ISO_YEARMONTH_FORMAT = "yyyy-MM"; + + public static final String YEARHOUR_NOSPACE_FORMAT = "yyyy-MM-dd-HH"; + + private static Map<String, DateFormat> dateFormats = + new HashMap<String, DateFormat>(); + + private static DateFormat getDateFormat(String format) { + DateFormat dateFormat = dateFormats.get(format); + if (dateFormat == null) { + dateFormat = new SimpleDateFormat(format); + dateFormat.setLenient(false); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + dateFormats.put(format, dateFormat); + } + return dateFormat; + } + + public static String format(long millis, String format) { + return getDateFormat(format).format(millis); + } + + public static String format(long millis) { + return format(millis, ISO_DATETIME_FORMAT); + } + + public static long parse(String string, String format) { + try { + return getDateFormat(format).parse(string).getTime(); + } catch (ParseException e) { + return -1L; + } + } + + public static long parse(String string) { + return parse(string, ISO_DATETIME_FORMAT); + } +} + diff --git a/src/org/torproject/onionoo/DetailsDocumentWriter.java b/src/org/torproject/onionoo/DetailsDocumentWriter.java index 0fd47c5..57d0877 100644 --- a/src/org/torproject/onionoo/DetailsDocumentWriter.java +++ b/src/org/torproject/onionoo/DetailsDocumentWriter.java @@ -1,6 +1,5 @@ package org.torproject.onionoo; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -9,7 +8,6 @@ import java.util.Map; import java.util.Scanner; import java.util.Set; import java.util.SortedSet; -import java.util.TimeZone; import java.util.TreeSet; import org.apache.commons.lang.StringEscapeUtils; @@ -101,9 +99,6 @@ public class DetailsDocumentWriter implements DescriptorListener, } private void updateRelayDetailsFiles() { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); for (String fingerprint : this.newRelays) { /* Generate network-status-specific part. */ @@ -124,10 +119,10 @@ public class DetailsDocumentWriter implements DescriptorListener, (addressesWritten++ > 0 ? "," : "") + "\"" + orAddress.toLowerCase() + "\""); } - String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); - String firstSeen = dateTimeFormat.format( + String lastSeen = DateTimeHelper.format(entry.getLastSeenMillis()); + String firstSeen = DateTimeHelper.format( entry.getFirstSeenMillis()); - String lastChangedOrAddress = dateTimeFormat.format( + String lastChangedOrAddress = DateTimeHelper.format( entry.getLastChangedOrAddress()); String running = entry.getRunning() ? "true" : "false"; int dirPort = entry.getDirPort(); @@ -302,9 +297,6 @@ public class DetailsDocumentWriter implements DescriptorListener, } private void updateBridgeDetailsFiles() { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); for (String fingerprint : this.newBridges) { /* Generate network-status-specific part. */ @@ -314,8 +306,8 @@ public class DetailsDocumentWriter implements DescriptorListener, continue; } String nickname = entry.getNickname(); - String lastSeen = dateTimeFormat.format(entry.getLastSeenMillis()); - String firstSeen = dateTimeFormat.format( + String lastSeen = DateTimeHelper.format(entry.getLastSeenMillis()); + String firstSeen = DateTimeHelper.format( entry.getFirstSeenMillis()); String running = entry.getRunning() ? "true" : "false"; String address = entry.getAddress(); diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java index 3c67aea..aa245dc 100644 --- a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java +++ b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java @@ -2,7 +2,6 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -11,7 +10,6 @@ import java.util.Scanner; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; -import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; @@ -289,12 +287,8 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, String fingerprint = descriptor.getFingerprint(); DetailsStatus detailsStatus = this.documentStore.retrieve( DetailsStatus.class, false, fingerprint); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); String publishedDateTime = - dateTimeFormat.format(descriptor.getPublishedMillis()); + DateTimeHelper.format(descriptor.getPublishedMillis()); if (detailsStatus != null) { String detailsString = detailsStatus.documentString; String descPublishedLine = "\"desc_published\":\"" @@ -313,7 +307,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, s.close(); } StringBuilder sb = new StringBuilder(); - String lastRestartedString = dateTimeFormat.format( + String lastRestartedString = DateTimeHelper.format( descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L); int bandwidthRate = descriptor.getBandwidthRate(); int bandwidthBurst = descriptor.getBandwidthBurst(); @@ -377,12 +371,8 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, String fingerprint = descriptor.getFingerprint(); DetailsStatus detailsStatus = this.documentStore.retrieve( DetailsStatus.class, false, fingerprint); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); String publishedDateTime = - dateTimeFormat.format(descriptor.getPublishedMillis()); + DateTimeHelper.format(descriptor.getPublishedMillis()); String poolAssignmentLine = null; if (detailsStatus != null) { String detailsString = detailsStatus.documentString; @@ -401,7 +391,7 @@ public class NodeDetailsStatusUpdater implements DescriptorListener, s.close(); } StringBuilder sb = new StringBuilder(); - String lastRestartedString = dateTimeFormat.format( + String lastRestartedString = DateTimeHelper.format( descriptor.getPublishedMillis() - descriptor.getUptime() * 1000L); int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(), Math.min(descriptor.getBandwidthBurst(), diff --git a/src/org/torproject/onionoo/NodeStatus.java b/src/org/torproject/onionoo/NodeStatus.java index afe2ed3..1c3983e 100644 --- a/src/org/torproject/onionoo/NodeStatus.java +++ b/src/org/torproject/onionoo/NodeStatus.java @@ -2,18 +2,15 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.SortedSet; import java.util.SortedMap; -import java.util.TimeZone; -import java.util.TreeSet; +import java.util.SortedSet; import java.util.TreeMap; +import java.util.TreeSet; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; @@ -126,9 +123,6 @@ public class NodeStatus extends Document { int orPort = -1, dirPort = -1; Boolean recommendedVersion = null; try { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); String separator = documentString.contains("\t") ? "\t" : " "; String[] parts = documentString.trim().split(separator); isRelay = parts[0].equals("r"); @@ -161,8 +155,12 @@ public class NodeStatus extends Document { } else { address = addresses; } - lastSeenMillis = dateTimeFormat.parse(parts[4] + " " + parts[5]). - getTime(); + lastSeenMillis = DateTimeHelper.parse(parts[4] + " " + parts[5]); + if (lastSeenMillis < 0L) { + System.err.println("Parse exception while parsing node status " + + "line '" + documentString + "'. Skipping."); + return null; + } orPort = Integer.parseInt(parts[6]); dirPort = Integer.parseInt(parts[7]); relayFlags = new TreeSet<String>(); @@ -189,13 +187,23 @@ public class NodeStatus extends Document { } firstSeenMillis = lastSeenMillis; if (parts.length > 16) { - firstSeenMillis = dateTimeFormat.parse(parts[15] + " " - + parts[16]).getTime(); + firstSeenMillis = DateTimeHelper.parse(parts[15] + " " + + parts[16]); + if (firstSeenMillis < 0L) { + System.err.println("Parse exception while parsing node status " + + "line '" + documentString + "'. Skipping."); + return null; + } } lastChangedAddresses = lastSeenMillis; if (parts.length > 18 && !parts[17].equals("null")) { - lastChangedAddresses = dateTimeFormat.parse(parts[17] + " " - + parts[18]).getTime(); + lastChangedAddresses = DateTimeHelper.parse(parts[17] + " " + + parts[18]); + if (lastChangedAddresses < 0L) { + System.err.println("Parse exception while parsing node status " + + "line '" + documentString + "'. Skipping."); + return null; + } } if (parts.length > 19) { aSNumber = parts[19]; @@ -212,11 +220,6 @@ public class NodeStatus extends Document { + "status line '" + documentString + "': " + e.getMessage() + ". Skipping."); return null; - } catch (ParseException e) { - System.err.println("Parse exception while parsing node status " - + "line '" + documentString + "': " + e.getMessage() + ". " - + "Skipping."); - return null; } catch (Exception e) { /* This catch block is only here to handle yet unknown errors. It * should go away once we're sure what kind of errors can occur. */ @@ -260,9 +263,6 @@ public class NodeStatus extends Document { } public String toString() { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd\tHH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); StringBuilder sb = new StringBuilder(); sb.append(this.isRelay ? "r" : "b"); sb.append("\t" + this.nickname); @@ -280,7 +280,8 @@ public class NodeStatus extends Document { + exitAddress); } } - sb.append("\t" + dateTimeFormat.format(this.lastSeenMillis)); + sb.append("\t" + DateTimeHelper.format(this.lastSeenMillis, + DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); sb.append("\t" + this.orPort); sb.append("\t" + this.dirPort + "\t"); written = 0; @@ -299,10 +300,12 @@ public class NodeStatus extends Document { } else { sb.append("\t-1\t??\tnull\t-1\tnull\tnull"); } - sb.append("\t" + dateTimeFormat.format(this.firstSeenMillis)); + sb.append("\t" + DateTimeHelper.format(this.firstSeenMillis, + DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); if (this.isRelay) { - sb.append("\t" + dateTimeFormat.format( - this.getLastChangedOrAddress())); + sb.append("\t" + DateTimeHelper.format( + this.getLastChangedOrAddress(), + DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); sb.append("\t" + (this.aSNumber != null ? this.aSNumber : "null")); } else { sb.append("\tnull\tnull\tnull"); diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java index 1412d88..73ce683 100644 --- a/src/org/torproject/onionoo/ResponseBuilder.java +++ b/src/org/torproject/onionoo/ResponseBuilder.java @@ -3,7 +3,6 @@ package org.torproject.onionoo; import java.io.PrintWriter; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -14,7 +13,6 @@ import java.util.Scanner; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; -import java.util.TimeZone; import java.util.TreeMap; public class ResponseBuilder { @@ -114,12 +112,9 @@ public class ResponseBuilder { currentBridges.add(node); } } - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - newRelaysPublishedString = dateTimeFormat.format( + newRelaysPublishedString = DateTimeHelper.format( relaysLastValidAfterMillis); - newBridgesPublishedString = dateTimeFormat.format( + newBridgesPublishedString = DateTimeHelper.format( bridgesLastPublishedMillis); List<String> orderRelaysByConsensusWeight = new ArrayList<String>(); for (NodeStatus entry : currentRelays) { diff --git a/src/org/torproject/onionoo/UptimeDocumentWriter.java b/src/org/torproject/onionoo/UptimeDocumentWriter.java index 5b03153..c53c709 100644 --- a/src/org/torproject/onionoo/UptimeDocumentWriter.java +++ b/src/org/torproject/onionoo/UptimeDocumentWriter.java @@ -2,12 +2,10 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.SortedSet; -import java.util.TimeZone; import java.util.TreeSet; public class UptimeDocumentWriter implements FingerprintListener, @@ -257,12 +255,9 @@ public class UptimeDocumentWriter implements FingerprintListener, double factor = 1.0 / 999.0; int count = lastNonNullIndex - firstNonNullIndex + 1; StringBuilder sb = new StringBuilder(); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); sb.append("\"" + graphName + "\":{" - + "\"first\":\"" + dateTimeFormat.format(firstDataPointMillis) - + "\",\"last\":\"" + dateTimeFormat.format(lastDataPointMillis) + + "\"first\":\"" + DateTimeHelper.format(firstDataPointMillis) + + "\",\"last\":\"" + DateTimeHelper.format(lastDataPointMillis) + "\",\"interval\":" + String.valueOf(dataPointInterval / 1000L) + ",\"factor\":" + String.format(Locale.US, "%.9f", factor) + ",\"count\":" + String.valueOf(count) + ",\"values\":["); diff --git a/src/org/torproject/onionoo/UptimeStatus.java b/src/org/torproject/onionoo/UptimeStatus.java index 4d8dca7..0eb9dde 100644 --- a/src/org/torproject/onionoo/UptimeStatus.java +++ b/src/org/torproject/onionoo/UptimeStatus.java @@ -2,11 +2,8 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.Scanner; import java.util.SortedSet; -import java.util.TimeZone; import java.util.TreeSet; class UptimeHistory @@ -36,14 +33,9 @@ class UptimeHistory } else if (!parts[0].equals("b")) { return null; } - long startMillis = -1L; - SimpleDateFormat dateHourFormat = new SimpleDateFormat( - "yyyy-MM-dd-HH"); - dateHourFormat.setLenient(false); - dateHourFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - try { - startMillis = dateHourFormat.parse(parts[1]).getTime(); - } catch (ParseException e) { + long startMillis = DateTimeHelper.parse(parts[1], + DateTimeHelper.YEARHOUR_NOSPACE_FORMAT); + if (startMillis < 0L) { return null; } int uptimeHours = -1; @@ -57,12 +49,9 @@ class UptimeHistory public String toString() { StringBuilder sb = new StringBuilder(); - SimpleDateFormat dateHourFormat = new SimpleDateFormat( - "yyyy-MM-dd-HH"); - dateHourFormat.setLenient(false); - dateHourFormat.setTimeZone(TimeZone.getTimeZone("UTC")); sb.append(this.relay ? "r" : "b"); - sb.append(" " + dateHourFormat.format(this.startMillis)); + sb.append(" " + DateTimeHelper.format(this.startMillis, + DateTimeHelper.YEARHOUR_NOSPACE_FORMAT)); sb.append(" " + String.format("%d", this.uptimeHours)); return sb.toString(); } diff --git a/src/org/torproject/onionoo/WeightsDocumentWriter.java b/src/org/torproject/onionoo/WeightsDocumentWriter.java index 9e5355f..6e225f1 100644 --- a/src/org/torproject/onionoo/WeightsDocumentWriter.java +++ b/src/org/torproject/onionoo/WeightsDocumentWriter.java @@ -2,7 +2,6 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -11,7 +10,6 @@ import java.util.Map; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; -import java.util.TimeZone; public class WeightsDocumentWriter implements FingerprintListener, DocumentWriter { @@ -183,12 +181,9 @@ public class WeightsDocumentWriter implements FingerprintListener, double factor = ((double) maxValue) / 999.0; int count = lastNonNullIndex - firstNonNullIndex + 1; StringBuilder sb = new StringBuilder(); - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); sb.append("\"" + graphName + "\":{" - + "\"first\":\"" + dateTimeFormat.format(firstDataPointMillis) - + "\",\"last\":\"" + dateTimeFormat.format(lastDataPointMillis) + + "\"first\":\"" + DateTimeHelper.format(firstDataPointMillis) + + "\",\"last\":\"" + DateTimeHelper.format(lastDataPointMillis) + "\",\"interval\":" + String.valueOf(dataPointInterval / 1000L) + ",\"factor\":" + String.format(Locale.US, "%.9f", factor) + ",\"count\":" + String.valueOf(count) + ",\"values\":["); diff --git a/src/org/torproject/onionoo/WeightsStatus.java b/src/org/torproject/onionoo/WeightsStatus.java index 4d92f30..93f3087 100644 --- a/src/org/torproject/onionoo/WeightsStatus.java +++ b/src/org/torproject/onionoo/WeightsStatus.java @@ -1,13 +1,10 @@ package org.torproject.onionoo; -import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.Comparator; import java.util.HashMap; import java.util.Map; import java.util.Scanner; import java.util.SortedMap; -import java.util.TimeZone; import java.util.TreeMap; class WeightsStatus extends Document { @@ -23,52 +20,46 @@ class WeightsStatus extends Document { new HashMap<String, Integer>(); public void fromDocumentString(String documentString) { - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setLenient(false); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - try { - Scanner s = new Scanner(documentString); - while (s.hasNextLine()) { - String line = s.nextLine(); - String[] parts = line.split(" "); - if (parts.length == 2) { - String descriptorDigest = parts[0]; - int advertisedBandwidth = Integer.parseInt(parts[1]); - this.advertisedBandwidths.put(descriptorDigest, - advertisedBandwidth); - continue; - } - if (parts.length != 9) { - System.err.println("Illegal line '" + line + "' in weights " + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + String[] parts = line.split(" "); + if (parts.length == 2) { + String descriptorDigest = parts[0]; + int advertisedBandwidth = Integer.parseInt(parts[1]); + this.advertisedBandwidths.put(descriptorDigest, + advertisedBandwidth); + continue; + } + if (parts.length != 9) { + System.err.println("Illegal line '" + line + "' in weights " + "status file. Skipping this line."); - continue; - } - if (parts[4].equals("NaN")) { - /* Remove corrupt lines written on 2013-07-07 and the days - * after. */ - continue; - } - long validAfterMillis = dateTimeFormat.parse(parts[0] - + " " + parts[1]).getTime(); - long freshUntilMillis = dateTimeFormat.parse(parts[2] - + " " + parts[3]).getTime(); - long[] interval = new long[] { validAfterMillis, - freshUntilMillis }; - double[] weights = new double[] { - Double.parseDouble(parts[4]), - Double.parseDouble(parts[5]), - Double.parseDouble(parts[6]), - Double.parseDouble(parts[7]), - Double.parseDouble(parts[8]) }; - this.history.put(interval, weights); + continue; + } + if (parts[4].equals("NaN")) { + /* Remove corrupt lines written on 2013-07-07 and the days + * after. */ + continue; + } + long validAfterMillis = DateTimeHelper.parse(parts[0] + " " + + parts[1]); + long freshUntilMillis = DateTimeHelper.parse(parts[2] + " " + + parts[3]); + if (validAfterMillis < 0L || freshUntilMillis < 0L) { + System.err.println("Could not parse timestamp while reading " + + "weights status file. Skipping."); + break; } - s.close(); - } catch (ParseException e) { - System.err.println("Could not parse timestamp while reading " - + "weights status file. Skipping."); - e.printStackTrace(); + long[] interval = new long[] { validAfterMillis, freshUntilMillis }; + double[] weights = new double[] { + Double.parseDouble(parts[4]), + Double.parseDouble(parts[5]), + Double.parseDouble(parts[6]), + Double.parseDouble(parts[7]), + Double.parseDouble(parts[8]) }; + this.history.put(interval, weights); } + s.close(); } public String toDocumentString() { @@ -77,14 +68,11 @@ class WeightsStatus extends Document { this.advertisedBandwidths.entrySet()) { sb.append(e.getKey() + " " + String.valueOf(e.getValue()) + "\n"); } - SimpleDateFormat dateTimeFormat = new SimpleDateFormat( - "yyyy-MM-dd HH:mm:ss"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); for (Map.Entry<long[], double[]> e : history.entrySet()) { long[] fresh = e.getKey(); double[] weights = e.getValue(); - sb.append(dateTimeFormat.format(fresh[0]) + " " - + dateTimeFormat.format(fresh[1])); + sb.append(DateTimeHelper.format(fresh[0]) + " " + + DateTimeHelper.format(fresh[1])); for (double weight : weights) { sb.append(String.format(" %.12f", weight)); } diff --git a/src/org/torproject/onionoo/WeightsStatusUpdater.java b/src/org/torproject/onionoo/WeightsStatusUpdater.java index 79296d3..3438325 100644 --- a/src/org/torproject/onionoo/WeightsStatusUpdater.java +++ b/src/org/torproject/onionoo/WeightsStatusUpdater.java @@ -2,7 +2,6 @@ * See LICENSE for licensing information */ package org.torproject.onionoo; -import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -10,7 +9,6 @@ import java.util.Map; import java.util.Set; import java.util.SortedMap; import java.util.SortedSet; -import java.util.TimeZone; import java.util.TreeMap; import java.util.TreeSet; @@ -282,8 +280,6 @@ public class WeightsStatusUpdater implements DescriptorListener, new TreeMap<long[], double[]>(history.comparator()); long lastStartMillis = 0L, lastEndMillis = 0L; double[] lastWeights = null; - SimpleDateFormat dateTimeFormat = new SimpleDateFormat("yyyy-MM"); - dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC")); String lastMonthString = "1970-01"; for (Map.Entry<long[], double[]> e : history.entrySet()) { long startMillis = e.getKey()[0], endMillis = e.getKey()[1]; @@ -300,7 +296,8 @@ public class WeightsStatusUpdater implements DescriptorListener, } else { intervalLengthMillis = 10L * 24L * 60L * 60L * 1000L; } - String monthString = dateTimeFormat.format(startMillis); + String monthString = DateTimeHelper.format(startMillis, + DateTimeHelper.ISO_YEARMONTH_FORMAT); if (lastEndMillis == startMillis && ((lastEndMillis - 1L) / intervalLengthMillis) == ((endMillis - 1L) / intervalLengthMillis) &&

1 0

[onionoo/master] Make utility class constructors private.
by karsten＠torproject.org 14 Apr '14

14 Apr '14

commit c935ad720db9c9e2288aa738e607691a3a2a7fd4 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Apr 8 23:22:11 2014 +0200 Make utility class constructors private. Found with SonarQube. --- src/org/torproject/onionoo/DateTimeHelper.java | 3 +++ src/org/torproject/onionoo/Logger.java | 3 +++ src/org/torproject/onionoo/Main.java | 4 ++++ 3 files changed, 10 insertions(+) diff --git a/src/org/torproject/onionoo/DateTimeHelper.java b/src/org/torproject/onionoo/DateTimeHelper.java index f97c8c5..efe126b 100644 --- a/src/org/torproject/onionoo/DateTimeHelper.java +++ b/src/org/torproject/onionoo/DateTimeHelper.java @@ -11,6 +11,9 @@ import java.util.TimeZone; public class DateTimeHelper { + private DateTimeHelper() { + } + public static final String ISO_DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; public static final String ISO_DATETIME_TAB_FORMAT = diff --git a/src/org/torproject/onionoo/Logger.java b/src/org/torproject/onionoo/Logger.java index fa113f3..ccc7962 100644 --- a/src/org/torproject/onionoo/Logger.java +++ b/src/org/torproject/onionoo/Logger.java @@ -6,6 +6,9 @@ import java.util.Date; public class Logger { + private Logger() { + } + private static Time time; public static void setTime(Time timeParam) { diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index 355baac..a4e2d92 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -6,6 +6,10 @@ import java.io.File; /* Update search data and status data files. */ public class Main { + + private Main() { + } + public static void main(String[] args) { Time t = new Time();

1 0

[onionoo/master] Avoid parameter assignment.
by karsten＠torproject.org 14 Apr '14

14 Apr '14

commit f47edf0b1d5fcfb3176182a3d9c602398d98f7f7 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Mar 20 13:14:09 2014 +0100 Avoid parameter assignment. Found by SonarQube. --- src/org/torproject/onionoo/NodeStatus.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/org/torproject/onionoo/NodeStatus.java b/src/org/torproject/onionoo/NodeStatus.java index 1c3983e..78a08c2 100644 --- a/src/org/torproject/onionoo/NodeStatus.java +++ b/src/org/torproject/onionoo/NodeStatus.java @@ -508,9 +508,8 @@ public class NodeStatus extends Document { if (contact == null) { this.contact = null; } else { - contact = contact.toLowerCase(); StringBuilder sb = new StringBuilder(); - for (char c : contact.toCharArray()) { + for (char c : contact.toLowerCase().toCharArray()) { if (c >= 32 && c < 127) { sb.append(c); } else {

1 0