[tor-commits] [onionoo/master] Sort classes into six sub packages.

commit 43b563b5a0fe46cfaa899b67ca1fc6804166b468 Author: Karsten Loesing karsten.loesing@gmx.net Date: Wed Jul 23 17:34:36 2014 +0200

Sort classes into six sub packages. --- build.xml | 2 +- etc/web.xml.template | 4 +- src/org/torproject/onionoo/ApplicationFactory.java | 51 -- src/org/torproject/onionoo/BandwidthDocument.java | 27 - .../onionoo/BandwidthDocumentWriter.java | 190 ----- src/org/torproject/onionoo/BandwidthStatus.java | 78 -- .../torproject/onionoo/BandwidthStatusUpdater.java | 145 ---- src/org/torproject/onionoo/ClientsDocument.java | 22 - .../torproject/onionoo/ClientsDocumentWriter.java | 284 -------- .../torproject/onionoo/ClientsGraphHistory.java | 83 --- src/org/torproject/onionoo/ClientsStatus.java | 211 ------ .../torproject/onionoo/ClientsStatusUpdater.java | 224 ------ src/org/torproject/onionoo/DateTimeHelper.java | 92 --- src/org/torproject/onionoo/DescriptorSource.java | 665 ----------------- src/org/torproject/onionoo/DetailsDocument.java | 365 ---------- .../torproject/onionoo/DetailsDocumentWriter.java | 222 ------ src/org/torproject/onionoo/DetailsStatus.java | 141 ---- src/org/torproject/onionoo/Document.java | 24 - src/org/torproject/onionoo/DocumentStore.java | 744 ------------------- src/org/torproject/onionoo/DocumentWriter.java | 11 - src/org/torproject/onionoo/GraphHistory.java | 56 -- src/org/torproject/onionoo/LockFile.java | 43 -- src/org/torproject/onionoo/Logger.java | 81 --- src/org/torproject/onionoo/LookupService.java | 408 ----------- src/org/torproject/onionoo/Main.java | 119 ---- .../onionoo/NodeDetailsStatusUpdater.java | 620 ---------------- src/org/torproject/onionoo/NodeIndexer.java | 425 ----------- src/org/torproject/onionoo/NodeStatus.java | 581 --------------- src/org/torproject/onionoo/RequestHandler.java | 548 -------------- src/org/torproject/onionoo/ResourceServlet.java | 448 ------------ src/org/torproject/onionoo/ResponseBuilder.java | 311 -------- .../onionoo/ReverseDomainNameResolver.java | 174 ----- src/org/torproject/onionoo/StatusUpdater.java | 11 - src/org/torproject/onionoo/SummaryDocument.java | 201 ------ .../torproject/onionoo/SummaryDocumentWriter.java | 87 --- src/org/torproject/onionoo/Time.java | 14 - src/org/torproject/onionoo/UpdateStatus.java | 7 - src/org/torproject/onionoo/UptimeDocument.java | 23 - .../torproject/onionoo/UptimeDocumentWriter.java | 291 -------- src/org/torproject/onionoo/UptimeStatus.java | 226 ------ .../torproject/onionoo/UptimeStatusUpdater.java | 126 ---- src/org/torproject/onionoo/WeightsDocument.java | 64 -- .../torproject/onionoo/WeightsDocumentWriter.java | 222 ------ src/org/torproject/onionoo/WeightsStatus.java | 97 --- .../torproject/onionoo/WeightsStatusUpdater.java | 328 --------- src/org/torproject/onionoo/cron/Main.java | 140 ++++ .../torproject/onionoo/docs/BandwidthDocument.java | 27 + .../torproject/onionoo/docs/BandwidthStatus.java | 80 +++ .../torproject/onionoo/docs/ClientsDocument.java | 22 + .../onionoo/docs/ClientsGraphHistory.java | 83 +++ .../torproject/onionoo/docs/ClientsHistory.java | 174 +++++ src/org/torproject/onionoo/docs/ClientsStatus.java | 43 ++ .../torproject/onionoo/docs/DetailsDocument.java | 365 ++++++++++ src/org/torproject/onionoo/docs/DetailsStatus.java | 141 ++++ src/org/torproject/onionoo/docs/Document.java | 24 + src/org/torproject/onionoo/docs/DocumentStore.java | 748 ++++++++++++++++++++ src/org/torproject/onionoo/docs/GraphHistory.java | 56 ++ src/org/torproject/onionoo/docs/NodeStatus.java | 582 +++++++++++++++ .../torproject/onionoo/docs/SummaryDocument.java | 202 ++++++ src/org/torproject/onionoo/docs/UpdateStatus.java | 7 + .../torproject/onionoo/docs/UptimeDocument.java | 23 + src/org/torproject/onionoo/docs/UptimeHistory.java | 90 +++ src/org/torproject/onionoo/docs/UptimeStatus.java | 142 ++++ .../torproject/onionoo/docs/WeightsDocument.java | 64 ++ src/org/torproject/onionoo/docs/WeightsStatus.java | 99 +++ src/org/torproject/onionoo/server/NodeIndexer.java | 432 +++++++++++ .../torproject/onionoo/server/RequestHandler.java | 552 +++++++++++++++ .../torproject/onionoo/server/ResourceServlet.java | 451 ++++++++++++ .../torproject/onionoo/server/ResponseBuilder.java | 320 +++++++++ .../onionoo/updater/BandwidthStatusUpdater.java | 149 ++++ .../onionoo/updater/ClientsStatusUpdater.java | 230 ++++++ .../onionoo/updater/DescriptorListener.java | 7 + .../onionoo/updater/DescriptorSource.java | 646 +++++++++++++++++ .../torproject/onionoo/updater/DescriptorType.java | 15 + .../onionoo/updater/FingerprintListener.java | 10 + .../torproject/onionoo/updater/LookupResult.java | 70 ++ .../torproject/onionoo/updater/LookupService.java | 343 +++++++++ .../onionoo/updater/NodeDetailsStatusUpdater.java | 626 ++++++++++++++++ .../onionoo/updater/ReverseDomainNameResolver.java | 179 +++++ .../torproject/onionoo/updater/StatusUpdater.java | 11 + .../onionoo/updater/UptimeStatusUpdater.java | 130 ++++ .../onionoo/updater/WeightsStatusUpdater.java | 332 +++++++++ .../onionoo/util/ApplicationFactory.java | 55 ++ .../torproject/onionoo/util/DateTimeHelper.java | 92 +++ src/org/torproject/onionoo/util/LockFile.java | 43 ++ src/org/torproject/onionoo/util/Logger.java | 81 +++ src/org/torproject/onionoo/util/Time.java | 14 + .../onionoo/writer/BandwidthDocumentWriter.java | 201 ++++++ .../onionoo/writer/ClientsDocumentWriter.java | 296 ++++++++ .../onionoo/writer/DetailsDocumentWriter.java | 233 ++++++ .../torproject/onionoo/writer/DocumentWriter.java | 11 + .../onionoo/writer/SummaryDocumentWriter.java | 94 +++ .../onionoo/writer/UptimeDocumentWriter.java | 303 ++++++++ .../onionoo/writer/WeightsDocumentWriter.java | 233 ++++++ .../torproject/onionoo/DummyDescriptorSource.java | 4 + .../org/torproject/onionoo/DummyDocumentStore.java | 3 + test/org/torproject/onionoo/DummyTime.java | 2 + test/org/torproject/onionoo/LookupServiceTest.java | 2 + .../torproject/onionoo/ResourceServletTest.java | 44 +- .../onionoo/UptimeDocumentWriterTest.java | 7 + test/org/torproject/onionoo/UptimeStatusTest.java | 4 + .../onionoo/UptimeStatusUpdaterTest.java | 6 + 102 files changed, 9327 insertions(+), 9112 deletions(-)

diff --git a/build.xml b/build.xml index dccf374..a69bac2 100644 --- a/build.xml +++ b/build.xml @@ -93,7 +93,7 @@ <target name="run" depends="compile"> <java fork="true" maxmemory="4g" - classname="org.torproject.onionoo.Main" + classname="org.torproject.onionoo.cron.Main" error="errors"> <classpath refid="classpath"/> </java> diff --git a/etc/web.xml.template b/etc/web.xml.template index 988d47a..2c0b280 100644 --- a/etc/web.xml.template +++ b/etc/web.xml.template @@ -9,7 +9,7 @@ <servlet> <servlet-name>Resource</servlet-name> <servlet-class> - org.torproject.onionoo.ResourceServlet + org.torproject.onionoo.server.ResourceServlet </servlet-class> <init-param> <param-name>maintenance</param-name> @@ -48,7 +48,7 @@

<listener> <listener-class> - org.torproject.onionoo.NodeIndexer + org.torproject.onionoo.server.NodeIndexer </listener-class> </listener> </web-app> diff --git a/src/org/torproject/onionoo/ApplicationFactory.java b/src/org/torproject/onionoo/ApplicationFactory.java deleted file mode 100644 index 44f2c17..0000000 --- a/src/org/torproject/onionoo/ApplicationFactory.java +++ /dev/null @@ -1,51 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -public class ApplicationFactory { - - private static Time timeInstance; - public static void setTime(Time time) { - timeInstance = time; - } - public static Time getTime() { - if (timeInstance == null) { - timeInstance = new Time(); - } - return timeInstance; - } - - private static DescriptorSource descriptorSourceInstance; - public static void setDescriptorSource( - DescriptorSource descriptorSource) { - descriptorSourceInstance = descriptorSource; - } - public static DescriptorSource getDescriptorSource() { - if (descriptorSourceInstance == null) { - descriptorSourceInstance = new DescriptorSource(); - } - return descriptorSourceInstance; - } - - private static DocumentStore documentStoreInstance; - public static void setDocumentStore(DocumentStore documentStore) { - documentStoreInstance = documentStore; - } - public static DocumentStore getDocumentStore() { - if (documentStoreInstance == null) { - documentStoreInstance = new DocumentStore(); - } - return documentStoreInstance; - } - - private static NodeIndexer nodeIndexerInstance; - public static void setNodeIndexer(NodeIndexer nodeIndexer) { - nodeIndexerInstance = nodeIndexer; - } - public static NodeIndexer getNodeIndexer() { - if (nodeIndexerInstance == null) { - nodeIndexerInstance = new NodeIndexer(); - } - return nodeIndexerInstance; - } -} diff --git a/src/org/torproject/onionoo/BandwidthDocument.java b/src/org/torproject/onionoo/BandwidthDocument.java deleted file mode 100644 index 9c7cd4d..0000000 --- a/src/org/torproject/onionoo/BandwidthDocument.java +++ /dev/null @@ -1,27 +0,0 @@ -/* Copyright 2013 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Map; - -public class BandwidthDocument extends Document { - - @SuppressWarnings("unused") - private String fingerprint; - public void setFingerprint(String fingerprint) { - this.fingerprint = fingerprint; - } - - @SuppressWarnings("unused") - private Map<String, GraphHistory> write_history; - public void setWriteHistory(Map<String, GraphHistory> writeHistory) { - this.write_history = writeHistory; - } - - @SuppressWarnings("unused") - private Map<String, GraphHistory> read_history; - public void setReadHistory(Map<String, GraphHistory> readHistory) { - this.read_history = readHistory; - } -} - diff --git a/src/org/torproject/onionoo/BandwidthDocumentWriter.java b/src/org/torproject/onionoo/BandwidthDocumentWriter.java deleted file mode 100644 index 164ab30..0000000 --- a/src/org/torproject/onionoo/BandwidthDocumentWriter.java +++ /dev/null @@ -1,190 +0,0 @@ -/* Copyright 2011--2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; - -public class BandwidthDocumentWriter implements FingerprintListener, - DocumentWriter{ - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - private long now; - - public BandwidthDocumentWriter() { - this.descriptorSource = ApplicationFactory.getDescriptorSource(); - this.documentStore = ApplicationFactory.getDocumentStore(); - this.now = ApplicationFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_EXTRA_INFOS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS); - } - - private Set<String> updateBandwidthDocuments = new HashSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - this.updateBandwidthDocuments.addAll(fingerprints); - } - - public void writeDocuments() { - for (String fingerprint : this.updateBandwidthDocuments) { - BandwidthStatus bandwidthStatus = this.documentStore.retrieve( - BandwidthStatus.class, true, fingerprint); - if (bandwidthStatus == null) { - continue; - } - BandwidthDocument bandwidthDocument = this.compileBandwidthDocument( - fingerprint, bandwidthStatus); - this.documentStore.store(bandwidthDocument, fingerprint); - } - Logger.printStatusTime("Wrote bandwidth document files"); - } - - - private BandwidthDocument compileBandwidthDocument(String fingerprint, - BandwidthStatus bandwidthStatus) { - BandwidthDocument bandwidthDocument = new BandwidthDocument(); - bandwidthDocument.setFingerprint(fingerprint); - bandwidthDocument.setWriteHistory(this.compileGraphType( - bandwidthStatus.getWriteHistory())); - bandwidthDocument.setReadHistory(this.compileGraphType( - bandwidthStatus.getReadHistory())); - return bandwidthDocument; - } - - private String[] graphNames = new String[] { - "3_days", - "1_week", - "1_month", - "3_months", - "1_year", - "5_years" }; - - private long[] graphIntervals = new long[] { - DateTimeHelper.THREE_DAYS, - DateTimeHelper.ONE_WEEK, - DateTimeHelper.ROUGHLY_ONE_MONTH, - DateTimeHelper.ROUGHLY_THREE_MONTHS, - DateTimeHelper.ROUGHLY_ONE_YEAR, - DateTimeHelper.ROUGHLY_FIVE_YEARS }; - - private long[] dataPointIntervals = new long[] { - DateTimeHelper.FIFTEEN_MINUTES, - DateTimeHelper.ONE_HOUR, - DateTimeHelper.FOUR_HOURS, - DateTimeHelper.TWELVE_HOURS, - DateTimeHelper.TWO_DAYS, - DateTimeHelper.TEN_DAYS }; - - private Map<String, GraphHistory> compileGraphType( - SortedMap<Long, long[]> history) { - Map<String, GraphHistory> graphs = - new LinkedHashMap<String, GraphHistory>(); - for (int i = 0; i < this.graphIntervals.length; i++) { - String graphName = this.graphNames[i]; - long graphInterval = this.graphIntervals[i]; - long dataPointInterval = this.dataPointIntervals[i]; - List<Long> dataPoints = new ArrayList<Long>(); - long intervalStartMillis = ((this.now - graphInterval) - / dataPointInterval) * dataPointInterval; - long totalMillis = 0L, totalBandwidth = 0L; - for (long[] v : history.values()) { - long startMillis = v[0], endMillis = v[1], bandwidth = v[2]; - if (endMillis < intervalStartMillis) { - continue; - } - while ((intervalStartMillis / dataPointInterval) != - (endMillis / dataPointInterval)) { - dataPoints.add(totalMillis * 5L < dataPointInterval - ? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND) - / totalMillis); - totalBandwidth = 0L; - totalMillis = 0L; - intervalStartMillis += dataPointInterval; - } - totalBandwidth += bandwidth; - totalMillis += (endMillis - startMillis); - } - dataPoints.add(totalMillis * 5L < dataPointInterval - ? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND) - / totalMillis); - long maxValue = 1L; - int firstNonNullIndex = -1, lastNonNullIndex = -1; - for (int j = 0; j < dataPoints.size(); j++) { - long dataPoint = dataPoints.get(j); - if (dataPoint >= 0L) { - if (firstNonNullIndex < 0) { - firstNonNullIndex = j; - } - lastNonNullIndex = j; - if (dataPoint > maxValue) { - maxValue = dataPoint; - } - } - } - if (firstNonNullIndex < 0) { - continue; - } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) * dataPointInterval - + dataPointInterval / 2L; - if (i > 0 && - firstDataPointMillis >= this.now - graphIntervals[i - 1]) { - /* Skip bandwidth history object, because it doesn't contain - * anything new that wasn't already contained in the last - * bandwidth history object(s). */ - continue; - } - long lastDataPointMillis = firstDataPointMillis - + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; - double factor = ((double) maxValue) / 999.0; - int count = lastNonNullIndex - firstNonNullIndex + 1; - GraphHistory graphHistory = new GraphHistory(); - graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis)); - graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis)); - graphHistory.setInterval((int) (dataPointInterval - / DateTimeHelper.ONE_SECOND)); - graphHistory.setFactor(factor); - graphHistory.setCount(count); - int previousNonNullIndex = -2; - boolean foundTwoAdjacentDataPoints = false; - List<Integer> values = new ArrayList<Integer>(); - for (int j = firstNonNullIndex; j <= lastNonNullIndex; j++) { - long dataPoint = dataPoints.get(j); - if (dataPoint >= 0L) { - if (j - previousNonNullIndex == 1) { - foundTwoAdjacentDataPoints = true; - } - previousNonNullIndex = j; - } - values.add(dataPoint < 0L ? null : - (int) ((dataPoint * 999L) / maxValue)); - } - graphHistory.setValues(values); - if (foundTwoAdjacentDataPoints) { - graphs.put(graphName, graphHistory); - } - } - return graphs; - } - - public String getStatsString() { - /* TODO Add statistics string. */ - return null; - } -} diff --git a/src/org/torproject/onionoo/BandwidthStatus.java b/src/org/torproject/onionoo/BandwidthStatus.java deleted file mode 100644 index 3252d67..0000000 --- a/src/org/torproject/onionoo/BandwidthStatus.java +++ /dev/null @@ -1,78 +0,0 @@ -/* Copyright 2013--2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Scanner; -import java.util.SortedMap; -import java.util.TreeMap; - -public class BandwidthStatus extends Document { - - private SortedMap<Long, long[]> writeHistory = - new TreeMap<Long, long[]>(); - public void setWriteHistory(SortedMap<Long, long[]> writeHistory) { - this.writeHistory = writeHistory; - } - public SortedMap<Long, long[]> getWriteHistory() { - return this.writeHistory; - } - - private SortedMap<Long, long[]> readHistory = - new TreeMap<Long, long[]>(); - public void setReadHistory(SortedMap<Long, long[]> readHistory) { - this.readHistory = readHistory; - } - public SortedMap<Long, long[]> getReadHistory() { - return this.readHistory; - } - - public void fromDocumentString(String documentString) { - Scanner s = new Scanner(documentString); - while (s.hasNextLine()) { - String line = s.nextLine(); - String[] parts = line.split(" "); - if (parts.length != 6) { - System.err.println("Illegal line '" + line + "' in bandwidth " - + "history. Skipping this line."); - continue; - } - SortedMap<Long, long[]> history = parts[0].equals("r") - ? readHistory : writeHistory; - long startMillis = DateTimeHelper.parse(parts[1] + " " + parts[2]); - long endMillis = DateTimeHelper.parse(parts[3] + " " + parts[4]); - if (startMillis < 0L || endMillis < 0L) { - System.err.println("Could not parse timestamp while reading " - + "bandwidth history. Skipping."); - break; - } - long bandwidth = Long.parseLong(parts[5]); - long previousEndMillis = history.headMap(startMillis).isEmpty() - ? startMillis - : history.get(history.headMap(startMillis).lastKey())[1]; - long nextStartMillis = history.tailMap(startMillis).isEmpty() - ? endMillis : history.tailMap(startMillis).firstKey(); - if (previousEndMillis <= startMillis && - nextStartMillis >= endMillis) { - history.put(startMillis, new long[] { startMillis, endMillis, - bandwidth }); - } - } - s.close(); - } - - public String toDocumentString() { - StringBuilder sb = new StringBuilder(); - for (long[] v : writeHistory.values()) { - sb.append("w " + DateTimeHelper.format(v[0]) + " " - + DateTimeHelper.format(v[1]) + " " + String.valueOf(v[2]) - + "\n"); - } - for (long[] v : readHistory.values()) { - sb.append("r " + DateTimeHelper.format(v[0]) + " " - + DateTimeHelper.format(v[1]) + " " + String.valueOf(v[2]) - + "\n"); - } - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/BandwidthStatusUpdater.java b/src/org/torproject/onionoo/BandwidthStatusUpdater.java deleted file mode 100644 index 6320f6e..0000000 --- a/src/org/torproject/onionoo/BandwidthStatusUpdater.java +++ /dev/null @@ -1,145 +0,0 @@ -/* Copyright 2011--2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.SortedMap; -import java.util.TreeMap; - -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.ExtraInfoDescriptor; - -public class BandwidthStatusUpdater implements DescriptorListener, - StatusUpdater { - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - private long now; - - public BandwidthStatusUpdater() { - this.descriptorSource = ApplicationFactory.getDescriptorSource(); - this.documentStore = ApplicationFactory.getDocumentStore(); - this.now = ApplicationFactory.getTime().currentTimeMillis(); - this.registerDescriptorListeners(); - } - - private void registerDescriptorListeners() { - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_EXTRA_INFOS); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS); - } - - public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof ExtraInfoDescriptor) { - this.parseDescriptor((ExtraInfoDescriptor) descriptor); - } - } - - public void updateStatuses() { - /* Status files are already updated while processing descriptors. */ - } - - private void parseDescriptor(ExtraInfoDescriptor descriptor) { - String fingerprint = descriptor.getFingerprint(); - BandwidthStatus bandwidthStatus = this.documentStore.retrieve( - BandwidthStatus.class, true, fingerprint); - if (bandwidthStatus == null) { - bandwidthStatus = new BandwidthStatus(); - } - if (descriptor.getWriteHistory() != null) { - parseHistoryLine(descriptor.getWriteHistory().getLine(), - bandwidthStatus.getWriteHistory()); - } - if (descriptor.getReadHistory() != null) { - parseHistoryLine(descriptor.getReadHistory().getLine(), - bandwidthStatus.getReadHistory()); - } - this.compressHistory(bandwidthStatus.getWriteHistory()); - this.compressHistory(bandwidthStatus.getReadHistory()); - this.documentStore.store(bandwidthStatus, fingerprint); - } - - private void parseHistoryLine(String line, - SortedMap<Long, long[]> history) { - String[] parts = line.split(" "); - if (parts.length < 6) { - return; - } - long endMillis = DateTimeHelper.parse(parts[1] + " " + parts[2]); - if (endMillis < 0L) { - System.err.println("Could not parse timestamp in line '" + line - + "'. Skipping."); - return; - } - long intervalMillis = Long.parseLong(parts[3].substring(1)) - * DateTimeHelper.ONE_SECOND; - String[] values = parts[5].split(","); - for (int i = values.length - 1; i >= 0; i--) { - long bandwidthValue = Long.parseLong(values[i]); - long startMillis = endMillis - intervalMillis; - /* TODO Should we first check whether an interval is already - * contained in history? */ - history.put(startMillis, new long[] { startMillis, endMillis, - bandwidthValue }); - endMillis -= intervalMillis; - } - } - - private void compressHistory(SortedMap<Long, long[]> history) { - SortedMap<Long, long[]> uncompressedHistory = - new TreeMap<Long, long[]>(history); - history.clear(); - long lastStartMillis = 0L, lastEndMillis = 0L, lastBandwidth = 0L; - String lastMonthString = "1970-01"; - for (long[] v : uncompressedHistory.values()) { - long startMillis = v[0], endMillis = v[1], bandwidth = v[2]; - long intervalLengthMillis; - if (this.now - endMillis <= DateTimeHelper.THREE_DAYS) { - intervalLengthMillis = DateTimeHelper.FIFTEEN_MINUTES; - } else if (this.now - endMillis <= DateTimeHelper.ONE_WEEK) { - intervalLengthMillis = DateTimeHelper.ONE_HOUR; - } else if (this.now - endMillis <= - DateTimeHelper.ROUGHLY_ONE_MONTH) { - intervalLengthMillis = DateTimeHelper.FOUR_HOURS; - } else if (this.now - endMillis <= - DateTimeHelper.ROUGHLY_THREE_MONTHS) { - intervalLengthMillis = DateTimeHelper.TWELVE_HOURS; - } else if (this.now - endMillis <= - DateTimeHelper.ROUGHLY_ONE_YEAR) { - intervalLengthMillis = DateTimeHelper.TWO_DAYS; - } else { - intervalLengthMillis = DateTimeHelper.TEN_DAYS; - } - String monthString = DateTimeHelper.format(startMillis, - DateTimeHelper.ISO_YEARMONTH_FORMAT); - if (lastEndMillis == startMillis && - ((lastEndMillis - 1L) / intervalLengthMillis) == - ((endMillis - 1L) / intervalLengthMillis) && - lastMonthString.equals(monthString)) { - lastEndMillis = endMillis; - lastBandwidth += bandwidth; - } else { - if (lastStartMillis > 0L) { - history.put(lastStartMillis, new long[] { lastStartMillis, - lastEndMillis, lastBandwidth }); - } - lastStartMillis = startMillis; - lastEndMillis = endMillis; - lastBandwidth = bandwidth; - } - lastMonthString = monthString; - } - if (lastStartMillis > 0L) { - history.put(lastStartMillis, new long[] { lastStartMillis, - lastEndMillis, lastBandwidth }); - } - } - - public String getStatsString() { - /* TODO Add statistics string. */ - return null; - } -} - diff --git a/src/org/torproject/onionoo/ClientsDocument.java b/src/org/torproject/onionoo/ClientsDocument.java deleted file mode 100644 index e8e40ee..0000000 --- a/src/org/torproject/onionoo/ClientsDocument.java +++ /dev/null @@ -1,22 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Map; - -public class ClientsDocument extends Document { - - @SuppressWarnings("unused") - private String fingerprint; - public void setFingerprint(String fingerprint) { - this.fingerprint = fingerprint; - } - - @SuppressWarnings("unused") - private Map<String, ClientsGraphHistory> average_clients; - public void setAverageClients( - Map<String, ClientsGraphHistory> averageClients) { - this.average_clients = averageClients; - } -} - diff --git a/src/org/torproject/onionoo/ClientsDocumentWriter.java b/src/org/torproject/onionoo/ClientsDocumentWriter.java deleted file mode 100644 index 1fced6d..0000000 --- a/src/org/torproject/onionoo/ClientsDocumentWriter.java +++ /dev/null @@ -1,284 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; - -/* - * Clients status file produced as intermediate output: - * - * 2014-02-15 16:42:11 2014-02-16 00:00:00 - * 259.042 in=86.347,se=86.347 v4=259.042 - * 2014-02-16 00:00:00 2014-02-16 16:42:11 - * 592.958 in=197.653,se=197.653 v4=592.958 - * - * Clients document file produced as output: - * - * "1_month":{ - * "first":"2014-02-03 12:00:00", - * "last":"2014-02-28 12:00:00", - * "interval":86400, - * "factor":0.139049349, - * "count":26, - * "values":[371,354,349,374,432,null,485,458,493,536,null,null,524,576, - * 607,622,null,635,null,566,774,999,945,690,656,681], - * "countries":{"cn":0.0192,"in":0.1768,"ir":0.2487,"ru":0.0104, - * "se":0.1698,"sy":0.0325,"us":0.0406}, - * "transports":{"obfs2":0.4581}, - * "versions":{"v4":1.0000}} - */ -public class ClientsDocumentWriter implements FingerprintListener, - DocumentWriter { - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - private long now; - - public ClientsDocumentWriter() { - this.descriptorSource = ApplicationFactory.getDescriptorSource(); - this.documentStore = ApplicationFactory.getDocumentStore(); - this.now = ApplicationFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS); - } - - private SortedSet<String> updateDocuments = new TreeSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (!relay) { - this.updateDocuments.addAll(fingerprints); - } - } - - private int writtenDocuments = 0; - - public void writeDocuments() { - for (String hashedFingerprint : this.updateDocuments) { - ClientsStatus clientsStatus = this.documentStore.retrieve( - ClientsStatus.class, true, hashedFingerprint); - if (clientsStatus == null) { - continue; - } - SortedSet<ClientsHistory> history = clientsStatus.getHistory(); - ClientsDocument clientsDocument = this.compileClientsDocument( - hashedFingerprint, history); - this.documentStore.store(clientsDocument, hashedFingerprint); - this.writtenDocuments++; - } - Logger.printStatusTime("Wrote clients document files"); - } - - private String[] graphNames = new String[] { - "1_week", - "1_month", - "3_months", - "1_year", - "5_years" }; - - private long[] graphIntervals = new long[] { - DateTimeHelper.ONE_WEEK, - DateTimeHelper.ROUGHLY_ONE_MONTH, - DateTimeHelper.ROUGHLY_THREE_MONTHS, - DateTimeHelper.ROUGHLY_ONE_YEAR, - DateTimeHelper.ROUGHLY_FIVE_YEARS }; - - private long[] dataPointIntervals = new long[] { - DateTimeHelper.ONE_DAY, - DateTimeHelper.ONE_DAY, - DateTimeHelper.ONE_DAY, - DateTimeHelper.TWO_DAYS, - DateTimeHelper.TEN_DAYS }; - - private ClientsDocument compileClientsDocument(String hashedFingerprint, - SortedSet<ClientsHistory> history) { - ClientsDocument clientsDocument = new ClientsDocument(); - clientsDocument.setFingerprint(hashedFingerprint); - Map<String, ClientsGraphHistory> averageClients = - new LinkedHashMap<String, ClientsGraphHistory>(); - for (int graphIntervalIndex = 0; graphIntervalIndex < - this.graphIntervals.length; graphIntervalIndex++) { - String graphName = this.graphNames[graphIntervalIndex]; - ClientsGraphHistory graphHistory = this.compileClientsHistory( - graphIntervalIndex, history); - if (graphHistory != null) { - averageClients.put(graphName, graphHistory); - } - } - clientsDocument.setAverageClients(averageClients); - return clientsDocument; - } - - private ClientsGraphHistory compileClientsHistory( - int graphIntervalIndex, SortedSet<ClientsHistory> history) { - long graphInterval = this.graphIntervals[graphIntervalIndex]; - long dataPointInterval = - this.dataPointIntervals[graphIntervalIndex]; - List<Double> dataPoints = new ArrayList<Double>(); - long intervalStartMillis = ((this.now - graphInterval) - / dataPointInterval) * dataPointInterval; - long millis = 0L; - double responses = 0.0, totalResponses = 0.0; - SortedMap<String, Double> - totalResponsesByCountry = new TreeMap<String, Double>(), - totalResponsesByTransport = new TreeMap<String, Double>(), - totalResponsesByVersion = new TreeMap<String, Double>(); - for (ClientsHistory hist : history) { - if (hist.getEndMillis() < intervalStartMillis) { - continue; - } - while ((intervalStartMillis / dataPointInterval) != - (hist.getEndMillis() / dataPointInterval)) { - dataPoints.add(millis * 2L < dataPointInterval - ? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY) - / (((double) millis) * 10.0)); - responses = 0.0; - millis = 0L; - intervalStartMillis += dataPointInterval; - } - responses += hist.getTotalResponses(); - totalResponses += hist.getTotalResponses(); - for (Map.Entry<String, Double> e : - hist.getResponsesByCountry().entrySet()) { - if (!totalResponsesByCountry.containsKey(e.getKey())) { - totalResponsesByCountry.put(e.getKey(), 0.0); - } - totalResponsesByCountry.put(e.getKey(), e.getValue() - + totalResponsesByCountry.get(e.getKey())); - } - for (Map.Entry<String, Double> e : - hist.getResponsesByTransport().entrySet()) { - if (!totalResponsesByTransport.containsKey(e.getKey())) { - totalResponsesByTransport.put(e.getKey(), 0.0); - } - totalResponsesByTransport.put(e.getKey(), e.getValue() - + totalResponsesByTransport.get(e.getKey())); - } - for (Map.Entry<String, Double> e : - hist.getResponsesByVersion().entrySet()) { - if (!totalResponsesByVersion.containsKey(e.getKey())) { - totalResponsesByVersion.put(e.getKey(), 0.0); - } - totalResponsesByVersion.put(e.getKey(), e.getValue() - + totalResponsesByVersion.get(e.getKey())); - } - millis += (hist.getEndMillis() - hist.getStartMillis()); - } - dataPoints.add(millis * 2L < dataPointInterval - ? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY) - / (((double) millis) * 10.0)); - double maxValue = 0.0; - int firstNonNullIndex = -1, lastNonNullIndex = -1; - for (int dataPointIndex = 0; dataPointIndex < dataPoints.size(); - dataPointIndex++) { - double dataPoint = dataPoints.get(dataPointIndex); - if (dataPoint >= 0.0) { - if (firstNonNullIndex < 0) { - firstNonNullIndex = dataPointIndex; - } - lastNonNullIndex = dataPointIndex; - if (dataPoint > maxValue) { - maxValue = dataPoint; - } - } - } - if (firstNonNullIndex < 0) { - return null; - } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) * dataPointInterval - + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis >= - this.now - graphIntervals[graphIntervalIndex - 1]) { - /* Skip clients history object, because it doesn't contain - * anything new that wasn't already contained in the last - * clients history object(s). */ - return null; - } - long lastDataPointMillis = firstDataPointMillis - + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; - double factor = ((double) maxValue) / 999.0; - int count = lastNonNullIndex - firstNonNullIndex + 1; - ClientsGraphHistory graphHistory = new ClientsGraphHistory(); - graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis)); - graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis)); - graphHistory.setInterval((int) (dataPointInterval - / DateTimeHelper.ONE_SECOND)); - graphHistory.setFactor(factor); - graphHistory.setCount(count); - int previousNonNullIndex = -2; - boolean foundTwoAdjacentDataPoints = false; - List<Integer> values = new ArrayList<Integer>(); - for (int dataPointIndex = firstNonNullIndex; dataPointIndex <= - lastNonNullIndex; dataPointIndex++) { - double dataPoint = dataPoints.get(dataPointIndex); - if (dataPoint >= 0.0) { - if (dataPointIndex - previousNonNullIndex == 1) { - foundTwoAdjacentDataPoints = true; - } - previousNonNullIndex = dataPointIndex; - } - values.add(dataPoint < 0.0 ? null : - (int) ((dataPoint * 999.0) / maxValue)); - } - graphHistory.setValues(values); - if (!totalResponsesByCountry.isEmpty()) { - SortedMap<String, Float> countries = new TreeMap<String, Float>(); - for (Map.Entry<String, Double> e : - totalResponsesByCountry.entrySet()) { - if (e.getValue() > totalResponses / 100.0) { - countries.put(e.getKey(), - (float) (e.getValue() / totalResponses)); - } - } - graphHistory.setCountries(countries); - } - if (!totalResponsesByTransport.isEmpty()) { - SortedMap<String, Float> transports = new TreeMap<String, Float>(); - for (Map.Entry<String, Double> e : - totalResponsesByTransport.entrySet()) { - if (e.getValue() > totalResponses / 100.0) { - transports.put(e.getKey(), - (float) (e.getValue() / totalResponses)); - } - } - graphHistory.setTransports(transports); - } - if (!totalResponsesByVersion.isEmpty()) { - SortedMap<String, Float> versions = new TreeMap<String, Float>(); - for (Map.Entry<String, Double> e : - totalResponsesByVersion.entrySet()) { - if (e.getValue() > totalResponses / 100.0) { - versions.put(e.getKey(), - (float) (e.getValue() / totalResponses)); - } - } - graphHistory.setVersions(versions); - } - if (foundTwoAdjacentDataPoints) { - return graphHistory; - } else { - return null; - } - } - - public String getStatsString() { - StringBuilder sb = new StringBuilder(); - sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments) - + " clients document files updated\n"); - return sb.toString(); - } -} diff --git a/src/org/torproject/onionoo/ClientsGraphHistory.java b/src/org/torproject/onionoo/ClientsGraphHistory.java deleted file mode 100644 index b7b312b..0000000 --- a/src/org/torproject/onionoo/ClientsGraphHistory.java +++ /dev/null @@ -1,83 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.ArrayList; -import java.util.List; -import java.util.SortedMap; - -public class ClientsGraphHistory { - - private String first; - public void setFirst(String first) { - this.first = first; - } - public String getFirst() { - return this.first; - } - - private String last; - public void setLast(String last) { - this.last = last; - } - public String getLast() { - return this.last; - } - - private Integer interval; - public void setInterval(Integer interval) { - this.interval = interval; - } - public Integer getInterval() { - return this.interval; - } - - private Double factor; - public void setFactor(Double factor) { - this.factor = factor; - } - public Double getFactor() { - return this.factor; - } - - private Integer count; - public void setCount(Integer count) { - this.count = count; - } - public Integer getCount() { - return this.count; - } - - private List<Integer> values = new ArrayList<Integer>(); - public void setValues(List<Integer> values) { - this.values = values; - } - public List<Integer> getValues() { - return this.values; - } - - private SortedMap<String, Float> countries; - public void setCountries(SortedMap<String, Float> countries) { - this.countries = countries; - } - public SortedMap<String, Float> getCountries() { - return this.countries; - } - - private SortedMap<String, Float> transports; - public void setTransports(SortedMap<String, Float> transports) { - this.transports = transports; - } - public SortedMap<String, Float> getTransports() { - return this.transports; - } - - private SortedMap<String, Float> versions; - public void setVersions(SortedMap<String, Float> versions) { - this.versions = versions; - } - public SortedMap<String, Float> getVersions() { - return this.versions; - } -} - diff --git a/src/org/torproject/onionoo/ClientsStatus.java b/src/org/torproject/onionoo/ClientsStatus.java deleted file mode 100644 index 1ea16a7..0000000 --- a/src/org/torproject/onionoo/ClientsStatus.java +++ /dev/null @@ -1,211 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Map; -import java.util.Scanner; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; - -class ClientsHistory implements Comparable<ClientsHistory> { - - private long startMillis; - public long getStartMillis() { - return this.startMillis; - } - - private long endMillis; - public long getEndMillis() { - return this.endMillis; - } - - private double totalResponses; - public double getTotalResponses() { - return this.totalResponses; - } - - private SortedMap<String, Double> responsesByCountry; - public SortedMap<String, Double> getResponsesByCountry() { - return this.responsesByCountry; - } - - private SortedMap<String, Double> responsesByTransport; - public SortedMap<String, Double> getResponsesByTransport() { - return this.responsesByTransport; - } - - private SortedMap<String, Double> responsesByVersion; - public SortedMap<String, Double> getResponsesByVersion() { - return this.responsesByVersion; - } - - ClientsHistory(long startMillis, long endMillis, - double totalResponses, - SortedMap<String, Double> responsesByCountry, - SortedMap<String, Double> responsesByTransport, - SortedMap<String, Double> responsesByVersion) { - this.startMillis = startMillis; - this.endMillis = endMillis; - this.totalResponses = totalResponses; - this.responsesByCountry = responsesByCountry; - this.responsesByTransport = responsesByTransport; - this.responsesByVersion = responsesByVersion; - } - - public static ClientsHistory fromString( - String responseHistoryString) { - String[] parts = responseHistoryString.split(" ", 8); - if (parts.length != 8) { - return null; - } - long startMillis = DateTimeHelper.parse(parts[0] + " " + parts[1]); - long endMillis = DateTimeHelper.parse(parts[2] + " " + parts[3]); - if (startMillis < 0L || endMillis < 0L) { - return null; - } - if (startMillis >= endMillis) { - return null; - } - double totalResponses = 0.0; - try { - totalResponses = Double.parseDouble(parts[4]); - } catch (NumberFormatException e) { - return null; - } - SortedMap<String, Double> responsesByCountry = - parseResponses(parts[5]); - SortedMap<String, Double> responsesByTransport = - parseResponses(parts[6]); - SortedMap<String, Double> responsesByVersion = - parseResponses(parts[7]); - if (responsesByCountry == null || responsesByTransport == null || - responsesByVersion == null) { - return null; - } - return new ClientsHistory(startMillis, endMillis, totalResponses, - responsesByCountry, responsesByTransport, responsesByVersion); - } - - private static SortedMap<String, Double> parseResponses( - String responsesString) { - SortedMap<String, Double> responses = new TreeMap<String, Double>(); - if (responsesString.length() > 0) { - for (String pair : responsesString.split(",")) { - String[] keyValue = pair.split("="); - if (keyValue.length != 2) { - return null; - } - double value = 0.0; - try { - value = Double.parseDouble(keyValue[1]); - } catch (NumberFormatException e) { - return null; - } - responses.put(keyValue[0], value); - } - } - return responses; - } - - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(DateTimeHelper.format(startMillis)); - sb.append(" " + DateTimeHelper.format(endMillis)); - sb.append(" " + String.format("%.3f", this.totalResponses)); - this.appendResponses(sb, this.responsesByCountry); - this.appendResponses(sb, this.responsesByTransport); - this.appendResponses(sb, this.responsesByVersion); - return sb.toString(); - } - - private void appendResponses(StringBuilder sb, - SortedMap<String, Double> responses) { - sb.append(" "); - int written = 0; - for (Map.Entry<String, Double> e : responses.entrySet()) { - sb.append((written++ > 0 ? "," : "") + e.getKey() + "=" - + String.format("%.3f", e.getValue())); - } - } - - public void addResponses(ClientsHistory other) { - this.totalResponses += other.totalResponses; - this.addResponsesByCategory(this.responsesByCountry, - other.responsesByCountry); - this.addResponsesByCategory(this.responsesByTransport, - other.responsesByTransport); - this.addResponsesByCategory(this.responsesByVersion, - other.responsesByVersion); - if (this.startMillis > other.startMillis) { - this.startMillis = other.startMillis; - } - if (this.endMillis < other.endMillis) { - this.endMillis = other.endMillis; - } - } - - private void addResponsesByCategory( - SortedMap<String, Double> thisResponses, - SortedMap<String, Double> otherResponses) { - for (Map.Entry<String, Double> e : otherResponses.entrySet()) { - if (thisResponses.containsKey(e.getKey())) { - thisResponses.put(e.getKey(), thisResponses.get(e.getKey()) - + e.getValue()); - } else { - thisResponses.put(e.getKey(), e.getValue()); - } - } - } - - public int compareTo(ClientsHistory other) { - return this.startMillis < other.startMillis ? -1 : - this.startMillis > other.startMillis ? 1 : 0; - } - - public boolean equals(Object other) { - return other instanceof ClientsHistory && - this.startMillis == ((ClientsHistory) other).startMillis; - } - - public int hashCode() { - return (int) this.startMillis; - } -} - -public class ClientsStatus extends Document { - - private SortedSet<ClientsHistory> history = - new TreeSet<ClientsHistory>(); - public void setHistory(SortedSet<ClientsHistory> history) { - this.history = history; - } - public SortedSet<ClientsHistory> getHistory() { - return this.history; - } - - public void fromDocumentString(String documentString) { - Scanner s = new Scanner(documentString); - while (s.hasNextLine()) { - String line = s.nextLine(); - ClientsHistory parsedLine = ClientsHistory.fromString(line); - if (parsedLine != null) { - this.history.add(parsedLine); - } else { - System.err.println("Could not parse clients history line '" - + line + "'. Skipping."); - } - } - s.close(); - } - - public String toDocumentString() { - StringBuilder sb = new StringBuilder(); - for (ClientsHistory interval : this.history) { - sb.append(interval.toString() + "\n"); - } - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/ClientsStatusUpdater.java b/src/org/torproject/onionoo/ClientsStatusUpdater.java deleted file mode 100644 index 4fe2b2d..0000000 --- a/src/org/torproject/onionoo/ClientsStatusUpdater.java +++ /dev/null @@ -1,224 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Map; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.ExtraInfoDescriptor; - -/* - * Example extra-info descriptor used as input: - * - * extra-info ndnop2 DE6397A047ABE5F78B4C87AF725047831B221AAB - * dirreq-stats-end 2014-02-16 16:42:11 (86400 s) - * dirreq-v3-resp ok=856,not-enough-sigs=0,unavailable=0,not-found=0, - * not-modified=40,busy=0 - * bridge-stats-end 2014-02-16 16:42:17 (86400 s) - * bridge-ips ??=8,in=8,se=8 - * bridge-ip-versions v4=8,v6=0 - * - * Clients status file produced as intermediate output: - * - * 2014-02-15 16:42:11 2014-02-16 00:00:00 - * 259.042 in=86.347,se=86.347 v4=259.042 - * 2014-02-16 00:00:00 2014-02-16 16:42:11 - * 592.958 in=197.653,se=197.653 v4=592.958 - */ -public class ClientsStatusUpdater implements DescriptorListener, - StatusUpdater { - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - private long now; - - public ClientsStatusUpdater() { - this.descriptorSource = ApplicationFactory.getDescriptorSource(); - this.documentStore = ApplicationFactory.getDocumentStore(); - this.now = ApplicationFactory.getTime().currentTimeMillis(); - this.registerDescriptorListeners(); - } - - private void registerDescriptorListeners() { - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_EXTRA_INFOS); - } - - public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof ExtraInfoDescriptor && !relay) { - this.processBridgeExtraInfoDescriptor( - (ExtraInfoDescriptor) descriptor); - } - } - - private SortedMap<String, SortedSet<ClientsHistory>> newResponses = - new TreeMap<String, SortedSet<ClientsHistory>>(); - - private void processBridgeExtraInfoDescriptor( - ExtraInfoDescriptor descriptor) { - long dirreqStatsEndMillis = descriptor.getDirreqStatsEndMillis(); - long dirreqStatsIntervalLengthMillis = - descriptor.getDirreqStatsIntervalLength() - * DateTimeHelper.ONE_SECOND; - SortedMap<String, Integer> responses = descriptor.getDirreqV3Resp(); - if (dirreqStatsEndMillis < 0L || - dirreqStatsIntervalLengthMillis != DateTimeHelper.ONE_DAY || - responses == null || !responses.containsKey("ok")) { - return; - } - double okResponses = (double) (responses.get("ok") - 4); - if (okResponses < 0.0) { - return; - } - String hashedFingerprint = descriptor.getFingerprint().toUpperCase(); - long dirreqStatsStartMillis = dirreqStatsEndMillis - - dirreqStatsIntervalLengthMillis; - long utcBreakMillis = (dirreqStatsEndMillis / DateTimeHelper.ONE_DAY) - * DateTimeHelper.ONE_DAY; - for (int i = 0; i < 2; i++) { - long startMillis = i == 0 ? dirreqStatsStartMillis : utcBreakMillis; - long endMillis = i == 0 ? utcBreakMillis : dirreqStatsEndMillis; - if (startMillis >= endMillis) { - continue; - } - double totalResponses = okResponses - * ((double) (endMillis - startMillis)) - / ((double) DateTimeHelper.ONE_DAY); - SortedMap<String, Double> responsesByCountry = - this.weightResponsesWithUniqueIps(totalResponses, - descriptor.getBridgeIps(), "??"); - SortedMap<String, Double> responsesByTransport = - this.weightResponsesWithUniqueIps(totalResponses, - descriptor.getBridgeIpTransports(), "<??>"); - SortedMap<String, Double> responsesByVersion = - this.weightResponsesWithUniqueIps(totalResponses, - descriptor.getBridgeIpVersions(), ""); - ClientsHistory newResponseHistory = new ClientsHistory( - startMillis, endMillis, totalResponses, responsesByCountry, - responsesByTransport, responsesByVersion); - if (!this.newResponses.containsKey(hashedFingerprint)) { - this.newResponses.put(hashedFingerprint, - new TreeSet<ClientsHistory>()); - } - this.newResponses.get(hashedFingerprint).add( - newResponseHistory); - } - } - - private SortedMap<String, Double> weightResponsesWithUniqueIps( - double totalResponses, SortedMap<String, Integer> uniqueIps, - String omitString) { - SortedMap<String, Double> weightedResponses = - new TreeMap<String, Double>(); - int totalUniqueIps = 0; - if (uniqueIps != null) { - for (Map.Entry<String, Integer> e : uniqueIps.entrySet()) { - if (e.getValue() > 4) { - totalUniqueIps += e.getValue() - 4; - } - } - } - if (totalUniqueIps > 0) { - for (Map.Entry<String, Integer> e : uniqueIps.entrySet()) { - if (!e.getKey().equals(omitString) && e.getValue() > 4) { - weightedResponses.put(e.getKey(), - (((double) (e.getValue() - 4)) * totalResponses) - / ((double) totalUniqueIps)); - } - } - } - return weightedResponses; - } - - public void updateStatuses() { - for (Map.Entry<String, SortedSet<ClientsHistory>> e : - this.newResponses.entrySet()) { - String hashedFingerprint = e.getKey(); - ClientsStatus clientsStatus = this.documentStore.retrieve( - ClientsStatus.class, true, hashedFingerprint); - if (clientsStatus == null) { - clientsStatus = new ClientsStatus(); - } - this.addToHistory(clientsStatus, e.getValue()); - this.compressHistory(clientsStatus); - this.documentStore.store(clientsStatus, hashedFingerprint); - } - } - - private void addToHistory(ClientsStatus clientsStatus, - SortedSet<ClientsHistory> newIntervals) { - SortedSet<ClientsHistory> history = clientsStatus.getHistory(); - for (ClientsHistory interval : newIntervals) { - if ((history.headSet(interval).isEmpty() || - history.headSet(interval).last().getEndMillis() <= - interval.getStartMillis()) && - (history.tailSet(interval).isEmpty() || - history.tailSet(interval).first().getStartMillis() >= - interval.getEndMillis())) { - history.add(interval); - } - } - } - - private void compressHistory(ClientsStatus clientsStatus) { - SortedSet<ClientsHistory> history = clientsStatus.getHistory(); - SortedSet<ClientsHistory> compressedHistory = - new TreeSet<ClientsHistory>(); - ClientsHistory lastResponses = null; - String lastMonthString = "1970-01"; - for (ClientsHistory responses : history) { - long intervalLengthMillis; - if (this.now - responses.getEndMillis() <= - DateTimeHelper.ROUGHLY_THREE_MONTHS) { - intervalLengthMillis = DateTimeHelper.ONE_DAY; - } else if (this.now - responses.getEndMillis() <= - DateTimeHelper.ROUGHLY_ONE_YEAR) { - intervalLengthMillis = DateTimeHelper.TWO_DAYS; - } else { - intervalLengthMillis = DateTimeHelper.TEN_DAYS; - } - String monthString = DateTimeHelper.format( - responses.getStartMillis(), - DateTimeHelper.ISO_YEARMONTH_FORMAT); - if (lastResponses != null && - lastResponses.getEndMillis() == responses.getStartMillis() && - ((lastResponses.getEndMillis() - 1L) / intervalLengthMillis) == - ((responses.getEndMillis() - 1L) / intervalLengthMillis) && - lastMonthString.equals(monthString)) { - lastResponses.addResponses(responses); - } else { - if (lastResponses != null) { - compressedHistory.add(lastResponses); - } - lastResponses = responses; - } - lastMonthString = monthString; - } - if (lastResponses != null) { - compressedHistory.add(lastResponses); - } - clientsStatus.setHistory(compressedHistory); - } - - public String getStatsString() { - int newIntervals = 0; - for (SortedSet<ClientsHistory> hist : this.newResponses.values()) { - newIntervals += hist.size(); - } - StringBuilder sb = new StringBuilder(); - sb.append(" " - + Logger.formatDecimalNumber(newIntervals / 2) - + " client statistics processed from extra-info descriptors\n"); - sb.append(" " - + Logger.formatDecimalNumber(this.newResponses.size()) - + " client status files updated\n"); - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/DateTimeHelper.java b/src/org/torproject/onionoo/DateTimeHelper.java deleted file mode 100644 index 41ac70b..0000000 --- a/src/org/torproject/onionoo/DateTimeHelper.java +++ /dev/null @@ -1,92 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.HashMap; -import java.util.Map; -import java.util.TimeZone; - -public class DateTimeHelper { - - private DateTimeHelper() { - } - - public static final long ONE_SECOND = 1000L, - TEN_SECONDS = 10L * ONE_SECOND, - ONE_MINUTE = 60L * ONE_SECOND, - FIVE_MINUTES = 5L * ONE_MINUTE, - FIFTEEN_MINUTES = 15L * ONE_MINUTE, - ONE_HOUR = 60L * ONE_MINUTE, - FOUR_HOURS = 4L * ONE_HOUR, - SIX_HOURS = 6L * ONE_HOUR, - TWELVE_HOURS = 12L * ONE_HOUR, - ONE_DAY = 24L * ONE_HOUR, - TWO_DAYS = 2L * ONE_DAY, - THREE_DAYS = 3L * ONE_DAY, - ONE_WEEK = 7L * ONE_DAY, - TEN_DAYS = 10L * ONE_DAY, - ROUGHLY_ONE_MONTH = 31L * ONE_DAY, - ROUGHLY_THREE_MONTHS = 92L * ONE_DAY, - ROUGHLY_ONE_YEAR = 366L * ONE_DAY, - ROUGHLY_FIVE_YEARS = 5L * ROUGHLY_ONE_YEAR; - - public static final String ISO_DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; - - public static final String ISO_DATETIME_TAB_FORMAT = - "yyyy-MM-dd\tHH:mm:ss"; - - public static final String ISO_YEARMONTH_FORMAT = "yyyy-MM"; - - public static final String DATEHOUR_NOSPACE_FORMAT = "yyyy-MM-dd-HH"; - - private static ThreadLocal<Map<String, DateFormat>> dateFormats = - new ThreadLocal<Map<String, DateFormat>> () { - public Map<String, DateFormat> get() { - return super.get(); - } - protected Map<String, DateFormat> initialValue() { - return new HashMap<String, DateFormat>(); - } - public void remove() { - super.remove(); - } - public void set(Map<String, DateFormat> value) { - super.set(value); - } - }; - - private static DateFormat getDateFormat(String format) { - Map<String, DateFormat> threadDateFormats = dateFormats.get(); - if (!threadDateFormats.containsKey(format)) { - DateFormat dateFormat = new SimpleDateFormat(format); - dateFormat.setLenient(false); - dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); - threadDateFormats.put(format, dateFormat); - } - return threadDateFormats.get(format); - } - - public static String format(long millis, String format) { - return getDateFormat(format).format(millis); - } - - public static String format(long millis) { - return format(millis, ISO_DATETIME_FORMAT); - } - - public static long parse(String string, String format) { - try { - return getDateFormat(format).parse(string).getTime(); - } catch (ParseException e) { - return -1L; - } - } - - public static long parse(String string) { - return parse(string, ISO_DATETIME_FORMAT); - } -} - diff --git a/src/org/torproject/onionoo/DescriptorSource.java b/src/org/torproject/onionoo/DescriptorSource.java deleted file mode 100644 index 8246bba..0000000 --- a/src/org/torproject/onionoo/DescriptorSource.java +++ /dev/null @@ -1,665 +0,0 @@ -/* Copyright 2013, 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileOutputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.zip.GZIPInputStream; - -import org.torproject.descriptor.BridgeNetworkStatus; -import org.torproject.descriptor.BridgePoolAssignment; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; -import org.torproject.descriptor.DescriptorReader; -import org.torproject.descriptor.DescriptorSourceFactory; -import org.torproject.descriptor.ExitList; -import org.torproject.descriptor.ExitListEntry; -import org.torproject.descriptor.ExtraInfoDescriptor; -import org.torproject.descriptor.RelayNetworkStatusConsensus; -import org.torproject.descriptor.ServerDescriptor; - -enum DescriptorType { - RELAY_CONSENSUSES, - RELAY_SERVER_DESCRIPTORS, - RELAY_EXTRA_INFOS, - EXIT_LISTS, - BRIDGE_STATUSES, - BRIDGE_SERVER_DESCRIPTORS, - BRIDGE_EXTRA_INFOS, - BRIDGE_POOL_ASSIGNMENTS, -} - -interface DescriptorListener { - abstract void processDescriptor(Descriptor descriptor, boolean relay); -} - -interface FingerprintListener { - abstract void processFingerprints(SortedSet<String> fingerprints, - boolean relay); -} - -enum DescriptorHistory { - RELAY_CONSENSUS_HISTORY, - RELAY_SERVER_HISTORY, - RELAY_EXTRAINFO_HISTORY, - EXIT_LIST_HISTORY, - BRIDGE_STATUS_HISTORY, - BRIDGE_SERVER_HISTORY, - BRIDGE_EXTRAINFO_HISTORY, - BRIDGE_POOLASSIGN_HISTORY, -} - -class DescriptorDownloader { - - private final String protocolHostNameResourcePrefix = - "https://collector.torproject.org/recent/"; - - private String directory; - - private final File inDir = new File("in/recent"); - - public DescriptorDownloader(DescriptorType descriptorType) { - switch (descriptorType) { - case RELAY_CONSENSUSES: - this.directory = "relay-descriptors/consensuses/"; - break; - case RELAY_SERVER_DESCRIPTORS: - this.directory = "relay-descriptors/server-descriptors/"; - break; - case RELAY_EXTRA_INFOS: - this.directory = "relay-descriptors/extra-infos/"; - break; - case EXIT_LISTS: - this.directory = "exit-lists/"; - break; - case BRIDGE_STATUSES: - this.directory = "bridge-descriptors/statuses/"; - break; - case BRIDGE_SERVER_DESCRIPTORS: - this.directory = "bridge-descriptors/server-descriptors/"; - break; - case BRIDGE_EXTRA_INFOS: - this.directory = "bridge-descriptors/extra-infos/"; - break; - case BRIDGE_POOL_ASSIGNMENTS: - this.directory = "bridge-pool-assignments/"; - break; - default: - System.err.println("Unknown descriptor type."); - return; - } - } - - private SortedSet<String> localFiles = new TreeSet<String>(); - - public int statLocalFiles() { - File localDirectory = new File(this.inDir, this.directory); - if (localDirectory.exists()) { - for (File file : localDirectory.listFiles()) { - this.localFiles.add(file.getName()); - } - } - return this.localFiles.size(); - } - - private SortedSet<String> remoteFiles = new TreeSet<String>(); - - public int fetchRemoteDirectory() { - String directoryUrl = this.protocolHostNameResourcePrefix - + this.directory; - try { - URL u = new URL(directoryUrl); - HttpURLConnection huc = (HttpURLConnection) u.openConnection(); - huc.setRequestMethod("GET"); - huc.connect(); - if (huc.getResponseCode() != 200) { - System.err.println("Could not fetch " + directoryUrl - + ": " + huc.getResponseCode() + " " - + huc.getResponseMessage() + ". Skipping."); - return 0; - } - BufferedReader br = new BufferedReader(new InputStreamReader( - huc.getInputStream())); - String line; - while ((line = br.readLine()) != null) { - if (!line.trim().startsWith("<tr>") || - !line.contains("<a href=\"")) { - continue; - } - String linePart = line.substring( - line.indexOf("<a href=\"") + "<a href=\"".length()); - if (!linePart.contains("\"")) { - continue; - } - linePart = linePart.substring(0, linePart.indexOf("\"")); - if (linePart.endsWith("/")) { - continue; - } - this.remoteFiles.add(linePart); - } - br.close(); - } catch (IOException e) { - System.err.println("Could not fetch or parse " + directoryUrl - + ". Skipping."); - } - return this.remoteFiles.size(); - } - - public int fetchRemoteFiles() { - int fetchedFiles = 0; - for (String remoteFile : this.remoteFiles) { - if (this.localFiles.contains(remoteFile)) { - continue; - } - String fileUrl = this.protocolHostNameResourcePrefix - + this.directory + remoteFile; - File localTempFile = new File(this.inDir, this.directory - + remoteFile + ".tmp"); - File localFile = new File(this.inDir, this.directory + remoteFile); - try { - localFile.getParentFile().mkdirs(); - URL u = new URL(fileUrl); - HttpURLConnection huc = (HttpURLConnection) u.openConnection(); - huc.setRequestMethod("GET"); - huc.addRequestProperty("Accept-Encoding", "gzip"); - huc.connect(); - if (huc.getResponseCode() != 200) { - System.err.println("Could not fetch " + fileUrl - + ": " + huc.getResponseCode() + " " - + huc.getResponseMessage() + ". Skipping."); - continue; - } - long lastModified = huc.getHeaderFieldDate("Last-Modified", -1L); - InputStream is; - if (huc.getContentEncoding() != null && - huc.getContentEncoding().equalsIgnoreCase("gzip")) { - is = new GZIPInputStream(huc.getInputStream()); - } else { - is = huc.getInputStream(); - } - BufferedInputStream bis = new BufferedInputStream(is); - BufferedOutputStream bos = new BufferedOutputStream( - new FileOutputStream(localTempFile)); - int len; - byte[] data = new byte[1024]; - while ((len = bis.read(data, 0, 1024)) >= 0) { - bos.write(data, 0, len); - } - bis.close(); - bos.close(); - localTempFile.renameTo(localFile); - if (lastModified >= 0) { - localFile.setLastModified(lastModified); - } - fetchedFiles++; - } catch (IOException e) { - System.err.println("Could not fetch or store " + fileUrl - + ". Skipping."); - } - } - return fetchedFiles; - } - - public int deleteOldLocalFiles() { - int deletedFiles = 0; - for (String localFile : this.localFiles) { - if (!this.remoteFiles.contains(localFile)) { - new File(this.inDir, this.directory + localFile).delete(); - deletedFiles++; - } - } - return deletedFiles; - } -} - -class DescriptorQueue { - - private File inDir; - - private File statusDir; - - private DescriptorReader descriptorReader; - - private File historyFile; - - private Iterator<DescriptorFile> descriptorFiles; - - private List<Descriptor> descriptors; - - private int historySizeBefore; - public int getHistorySizeBefore() { - return this.historySizeBefore; - } - - private int historySizeAfter; - public int getHistorySizeAfter() { - return this.historySizeAfter; - } - - private long returnedDescriptors = 0L; - public long getReturnedDescriptors() { - return this.returnedDescriptors; - } - - private long returnedBytes = 0L; - public long getReturnedBytes() { - return this.returnedBytes; - } - - public DescriptorQueue(File inDir, File statusDir) { - this.inDir = inDir; - this.statusDir = statusDir; - this.descriptorReader = - DescriptorSourceFactory.createDescriptorReader(); - } - - public void addDirectory(DescriptorType descriptorType) { - String directoryName = null; - switch (descriptorType) { - case RELAY_CONSENSUSES: - directoryName = "relay-descriptors/consensuses"; - break; - case RELAY_SERVER_DESCRIPTORS: - directoryName = "relay-descriptors/server-descriptors"; - break; - case RELAY_EXTRA_INFOS: - directoryName = "relay-descriptors/extra-infos"; - break; - case BRIDGE_STATUSES: - directoryName = "bridge-descriptors/statuses"; - break; - case BRIDGE_SERVER_DESCRIPTORS: - directoryName = "bridge-descriptors/server-descriptors"; - break; - case BRIDGE_EXTRA_INFOS: - directoryName = "bridge-descriptors/extra-infos"; - break; - case BRIDGE_POOL_ASSIGNMENTS: - directoryName = "bridge-pool-assignments"; - break; - case EXIT_LISTS: - directoryName = "exit-lists"; - break; - default: - System.err.println("Unknown descriptor type. Not adding directory " - + "to descriptor reader."); - return; - } - File directory = new File(this.inDir, directoryName); - if (directory.exists() && directory.isDirectory()) { - this.descriptorReader.addDirectory(directory); - this.descriptorReader.setMaxDescriptorFilesInQueue(1); - } else { - System.err.println("Directory " + directory.getAbsolutePath() - + " either does not exist or is not a directory. Not adding " - + "to descriptor reader."); - } - } - - public void readHistoryFile(DescriptorHistory descriptorHistory) { - String historyFileName = null; - switch (descriptorHistory) { - case RELAY_EXTRAINFO_HISTORY: - historyFileName = "relay-extrainfo-history"; - break; - case BRIDGE_EXTRAINFO_HISTORY: - historyFileName = "bridge-extrainfo-history"; - break; - case EXIT_LIST_HISTORY: - historyFileName = "exit-list-history"; - break; - case BRIDGE_POOLASSIGN_HISTORY: - historyFileName = "bridge-poolassign-history"; - break; - case RELAY_CONSENSUS_HISTORY: - historyFileName = "relay-consensus-history"; - break; - case BRIDGE_STATUS_HISTORY: - historyFileName = "bridge-status-history"; - break; - case RELAY_SERVER_HISTORY: - historyFileName = "relay-server-history"; - break; - case BRIDGE_SERVER_HISTORY: - historyFileName = "bridge-server-history"; - break; - default: - System.err.println("Unknown descriptor history. Not excluding " - + "files."); - return; - } - this.historyFile = new File(this.statusDir, historyFileName); - if (this.historyFile.exists() && this.historyFile.isFile()) { - SortedMap<String, Long> excludedFiles = new TreeMap<String, Long>(); - try { - BufferedReader br = new BufferedReader(new FileReader( - this.historyFile)); - String line; - while ((line = br.readLine()) != null) { - try { - String[] parts = line.split(" ", 2); - excludedFiles.put(parts[1], Long.parseLong(parts[0])); - } catch (NumberFormatException e) { - System.err.println("Illegal line '" + line + "' in parse " - + "history. Skipping line."); - } - } - br.close(); - } catch (IOException e) { - System.err.println("Could not read history file '" - + this.historyFile.getAbsolutePath() + "'. Not excluding " - + "descriptors in this execution."); - e.printStackTrace(); - return; - } - this.historySizeBefore = excludedFiles.size(); - this.descriptorReader.setExcludedFiles(excludedFiles); - } - } - - public void writeHistoryFile() { - if (this.historyFile == null) { - return; - } - SortedMap<String, Long> excludedAndParsedFiles = - new TreeMap<String, Long>(); - excludedAndParsedFiles.putAll( - this.descriptorReader.getExcludedFiles()); - excludedAndParsedFiles.putAll(this.descriptorReader.getParsedFiles()); - this.historySizeAfter = excludedAndParsedFiles.size(); - try { - this.historyFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter( - this.historyFile)); - for (Map.Entry<String, Long> e : excludedAndParsedFiles.entrySet()) { - String absolutePath = e.getKey(); - long lastModifiedMillis = e.getValue(); - bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath - + "\n"); - } - bw.close(); - } catch (IOException e) { - System.err.println("Could not write history file '" - + this.historyFile.getAbsolutePath() + "'. Not excluding " - + "descriptors in next execution."); - return; - } - } - - public Descriptor nextDescriptor() { - Descriptor nextDescriptor = null; - if (this.descriptorFiles == null) { - this.descriptorFiles = this.descriptorReader.readDescriptors(); - } - while (this.descriptors == null && this.descriptorFiles.hasNext()) { - DescriptorFile descriptorFile = this.descriptorFiles.next(); - if (descriptorFile.getException() != null) { - System.err.println("Could not parse " - + descriptorFile.getFileName()); - descriptorFile.getException().printStackTrace(); - } - if (descriptorFile.getDescriptors() != null && - !descriptorFile.getDescriptors().isEmpty()) { - this.descriptors = descriptorFile.getDescriptors(); - } - } - if (this.descriptors != null) { - nextDescriptor = this.descriptors.remove(0); - this.returnedDescriptors++; - this.returnedBytes += nextDescriptor.getRawDescriptorBytes().length; - if (this.descriptors.isEmpty()) { - this.descriptors = null; - } - } - return nextDescriptor; - } -} - -public class DescriptorSource { - - private final File inDir = new File("in/recent"); - - private final File statusDir = new File("status"); - - private List<DescriptorQueue> descriptorQueues; - - public DescriptorSource() { - this.descriptorQueues = new ArrayList<DescriptorQueue>(); - this.descriptorListeners = - new HashMap<DescriptorType, Set<DescriptorListener>>(); - this.fingerprintListeners = - new HashMap<DescriptorType, Set<FingerprintListener>>(); - } - - private DescriptorQueue getDescriptorQueue( - DescriptorType descriptorType, - DescriptorHistory descriptorHistory) { - DescriptorQueue descriptorQueue = new DescriptorQueue(this.inDir, - this.statusDir); - descriptorQueue.addDirectory(descriptorType); - if (descriptorHistory != null) { - descriptorQueue.readHistoryFile(descriptorHistory); - } - this.descriptorQueues.add(descriptorQueue); - return descriptorQueue; - } - - private Map<DescriptorType, Set<DescriptorListener>> - descriptorListeners; - - private Map<DescriptorType, Set<FingerprintListener>> - fingerprintListeners; - - public void registerDescriptorListener(DescriptorListener listener, - DescriptorType descriptorType) { - if (!this.descriptorListeners.containsKey(descriptorType)) { - this.descriptorListeners.put(descriptorType, - new HashSet<DescriptorListener>()); - } - this.descriptorListeners.get(descriptorType).add(listener); - } - - public void registerFingerprintListener(FingerprintListener listener, - DescriptorType descriptorType) { - if (!this.fingerprintListeners.containsKey(descriptorType)) { - this.fingerprintListeners.put(descriptorType, - new HashSet<FingerprintListener>()); - } - this.fingerprintListeners.get(descriptorType).add(listener); - } - - public void downloadDescriptors() { - for (DescriptorType descriptorType : DescriptorType.values()) { - this.downloadDescriptors(descriptorType); - } - } - - private int localFilesBefore = 0, foundRemoteFiles = 0, - downloadedFiles = 0, deletedLocalFiles = 0; - - private void downloadDescriptors(DescriptorType descriptorType) { - if (!this.descriptorListeners.containsKey(descriptorType) && - !this.fingerprintListeners.containsKey(descriptorType)) { - return; - } - DescriptorDownloader descriptorDownloader = - new DescriptorDownloader(descriptorType); - this.localFilesBefore += descriptorDownloader.statLocalFiles(); - this.foundRemoteFiles += - descriptorDownloader.fetchRemoteDirectory(); - this.downloadedFiles += descriptorDownloader.fetchRemoteFiles(); - this.deletedLocalFiles += descriptorDownloader.deleteOldLocalFiles(); - } - - public void readDescriptors() { - /* Careful when changing the order of parsing descriptor types! The - * various status updaters may base assumptions on this order. */ - this.readDescriptors(DescriptorType.RELAY_SERVER_DESCRIPTORS, - DescriptorHistory.RELAY_SERVER_HISTORY, true); - this.readDescriptors(DescriptorType.RELAY_EXTRA_INFOS, - DescriptorHistory.RELAY_EXTRAINFO_HISTORY, true); - this.readDescriptors(DescriptorType.EXIT_LISTS, - DescriptorHistory.EXIT_LIST_HISTORY, true); - this.readDescriptors(DescriptorType.RELAY_CONSENSUSES, - DescriptorHistory.RELAY_CONSENSUS_HISTORY, true); - this.readDescriptors(DescriptorType.BRIDGE_SERVER_DESCRIPTORS, - DescriptorHistory.BRIDGE_SERVER_HISTORY, false); - this.readDescriptors(DescriptorType.BRIDGE_EXTRA_INFOS, - DescriptorHistory.BRIDGE_EXTRAINFO_HISTORY, false); - this.readDescriptors(DescriptorType.BRIDGE_POOL_ASSIGNMENTS, - DescriptorHistory.BRIDGE_POOLASSIGN_HISTORY, false); - this.readDescriptors(DescriptorType.BRIDGE_STATUSES, - DescriptorHistory.BRIDGE_STATUS_HISTORY, false); - } - - private void readDescriptors(DescriptorType descriptorType, - DescriptorHistory descriptorHistory, boolean relay) { - if (!this.descriptorListeners.containsKey(descriptorType) && - !this.fingerprintListeners.containsKey(descriptorType)) { - return; - } - Set<DescriptorListener> descriptorListeners = - this.descriptorListeners.get(descriptorType); - Set<FingerprintListener> fingerprintListeners = - this.fingerprintListeners.get(descriptorType); - DescriptorQueue descriptorQueue = this.getDescriptorQueue( - descriptorType, descriptorHistory); - Descriptor descriptor; - while ((descriptor = descriptorQueue.nextDescriptor()) != null) { - for (DescriptorListener descriptorListener : descriptorListeners) { - descriptorListener.processDescriptor(descriptor, relay); - } - if (fingerprintListeners == null) { - continue; - } - SortedSet<String> fingerprints = new TreeSet<String>(); - if (descriptorType == DescriptorType.RELAY_CONSENSUSES && - descriptor instanceof RelayNetworkStatusConsensus) { - fingerprints.addAll(((RelayNetworkStatusConsensus) descriptor). - getStatusEntries().keySet()); - } else if (descriptorType - == DescriptorType.RELAY_SERVER_DESCRIPTORS && - descriptor instanceof ServerDescriptor) { - fingerprints.add(((ServerDescriptor) descriptor). - getFingerprint()); - } else if (descriptorType == DescriptorType.RELAY_EXTRA_INFOS && - descriptor instanceof ExtraInfoDescriptor) { - fingerprints.add(((ExtraInfoDescriptor) descriptor). - getFingerprint()); - } else if (descriptorType == DescriptorType.EXIT_LISTS && - descriptor instanceof ExitList) { - for (ExitListEntry entry : - ((ExitList) descriptor).getExitListEntries()) { - fingerprints.add(entry.getFingerprint()); - } - } else if (descriptorType == DescriptorType.BRIDGE_STATUSES && - descriptor instanceof BridgeNetworkStatus) { - fingerprints.addAll(((BridgeNetworkStatus) descriptor). - getStatusEntries().keySet()); - } else if (descriptorType == - DescriptorType.BRIDGE_SERVER_DESCRIPTORS && - descriptor instanceof ServerDescriptor) { - fingerprints.add(((ServerDescriptor) descriptor). - getFingerprint()); - } else if (descriptorType == DescriptorType.BRIDGE_EXTRA_INFOS && - descriptor instanceof ExtraInfoDescriptor) { - fingerprints.add(((ExtraInfoDescriptor) descriptor). - getFingerprint()); - } else if (descriptorType == - DescriptorType.BRIDGE_POOL_ASSIGNMENTS && - descriptor instanceof BridgePoolAssignment) { - fingerprints.addAll(((BridgePoolAssignment) descriptor). - getEntries().keySet()); - } - for (FingerprintListener fingerprintListener : - fingerprintListeners) { - fingerprintListener.processFingerprints(fingerprints, relay); - } - } - switch (descriptorType) { - case RELAY_CONSENSUSES: - Logger.printStatusTime("Read relay network consensuses"); - break; - case RELAY_SERVER_DESCRIPTORS: - Logger.printStatusTime("Read relay server descriptors"); - break; - case RELAY_EXTRA_INFOS: - Logger.printStatusTime("Read relay extra-info descriptors"); - break; - case EXIT_LISTS: - Logger.printStatusTime("Read exit lists"); - break; - case BRIDGE_STATUSES: - Logger.printStatusTime("Read bridge network statuses"); - break; - case BRIDGE_SERVER_DESCRIPTORS: - Logger.printStatusTime("Read bridge server descriptors"); - break; - case BRIDGE_EXTRA_INFOS: - Logger.printStatusTime("Read bridge extra-info descriptors"); - break; - case BRIDGE_POOL_ASSIGNMENTS: - Logger.printStatusTime("Read bridge-pool assignments"); - break; - } - } - - public void writeHistoryFiles() { - for (DescriptorQueue descriptorQueue : this.descriptorQueues) { - descriptorQueue.writeHistoryFile(); - } - } - - public String getStatsString() { - StringBuilder sb = new StringBuilder(); - sb.append(" " + this.localFilesBefore + " descriptor files found " - + "locally\n"); - sb.append(" " + this.foundRemoteFiles + " descriptor files found " - + "remotely\n"); - sb.append(" " + this.downloadedFiles + " descriptor files " - + "downloaded from remote\n"); - sb.append(" " + this.deletedLocalFiles + " descriptor files " - + "deleted locally\n"); - sb.append(" " + this.descriptorQueues.size() + " descriptor " - + "queues created\n"); - int historySizeBefore = 0, historySizeAfter = 0; - long descriptors = 0L, bytes = 0L; - for (DescriptorQueue descriptorQueue : descriptorQueues) { - historySizeBefore += descriptorQueue.getHistorySizeBefore(); - historySizeAfter += descriptorQueue.getHistorySizeAfter(); - descriptors += descriptorQueue.getReturnedDescriptors(); - bytes += descriptorQueue.getReturnedBytes(); - } - sb.append(" " + Logger.formatDecimalNumber(historySizeBefore) - + " descriptors excluded from this execution\n"); - sb.append(" " + Logger.formatDecimalNumber(descriptors) - + " descriptors provided\n"); - sb.append(" " + Logger.formatBytes(bytes) + " provided\n"); - sb.append(" " + Logger.formatDecimalNumber(historySizeAfter) - + " descriptors excluded from next execution\n"); - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/DetailsDocument.java b/src/org/torproject/onionoo/DetailsDocument.java deleted file mode 100644 index 2df6e78..0000000 --- a/src/org/torproject/onionoo/DetailsDocument.java +++ /dev/null @@ -1,365 +0,0 @@ -/* Copyright 2013--2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.List; -import java.util.Map; - -import org.apache.commons.lang.StringEscapeUtils; - -public class DetailsDocument extends Document { - - /* We must ensure that details files only contain ASCII characters - * and no UTF-8 characters. While UTF-8 characters are perfectly - * valid in JSON, this would break compatibility with existing files - * pretty badly. We do this by escaping non-ASCII characters, e.g., - * \u00F2. Gson won't treat this as UTF-8, but will think that we want - * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing - * we'll have to do is to change back the '\\' that Gson writes for the - * '\'. */ - private static String escapeJSON(String s) { - return s == null ? null : - StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'"); - } - private static String unescapeJSON(String s) { - return s == null ? null : - StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'")); - } - - private String nickname; - public void setNickname(String nickname) { - this.nickname = nickname; - } - public String getNickname() { - return this.nickname; - } - - private String fingerprint; - public void setFingerprint(String fingerprint) { - this.fingerprint = fingerprint; - } - public String getFingerprint() { - return this.fingerprint; - } - - private String hashed_fingerprint; - public void setHashedFingerprint(String hashedFingerprint) { - this.hashed_fingerprint = hashedFingerprint; - } - public String getHashedFingerprint() { - return this.hashed_fingerprint; - } - - private List<String> or_addresses; - public void setOrAddresses(List<String> orAddresses) { - this.or_addresses = orAddresses; - } - public List<String> getOrAddresses() { - return this.or_addresses; - } - - private List<String> exit_addresses; - public void setExitAddresses(List<String> exitAddresses) { - this.exit_addresses = exitAddresses; - } - public List<String> getExitAddresses() { - return this.exit_addresses; - } - - private String dir_address; - public void setDirAddress(String dirAddress) { - this.dir_address = dirAddress; - } - public String getDirAddress() { - return this.dir_address; - } - - private String last_seen; - public void setLastSeen(String lastSeen) { - this.last_seen = lastSeen; - } - public String getLastSeen() { - return this.last_seen; - } - - private String last_changed_address_or_port; - public void setLastChangedAddressOrPort( - String lastChangedAddressOrPort) { - this.last_changed_address_or_port = lastChangedAddressOrPort; - } - public String getLastChangedAddressOrPort() { - return this.last_changed_address_or_port; - } - - private String first_seen; - public void setFirstSeen(String firstSeen) { - this.first_seen = firstSeen; - } - public String getFirstSeen() { - return this.first_seen; - } - - private Boolean running; - public void setRunning(Boolean running) { - this.running = running; - } - public Boolean getRunning() { - return this.running; - } - - private List<String> flags; - public void setFlags(List<String> flags) { - this.flags = flags; - } - public List<String> getFlags() { - return this.flags; - } - - private String country; - public void setCountry(String country) { - this.country = country; - } - public String getCountry() { - return this.country; - } - - private String country_name; - public void setCountryName(String countryName) { - this.country_name = escapeJSON(countryName); - } - public String getCountryName() { - return unescapeJSON(this.country_name); - } - - private String region_name; - public void setRegionName(String regionName) { - this.region_name = escapeJSON(regionName); - } - public String getRegionName() { - return unescapeJSON(this.region_name); - } - - private String city_name; - public void setCityName(String cityName) { - this.city_name = escapeJSON(cityName); - } - public String getCityName() { - return unescapeJSON(this.city_name); - } - - private Float latitude; - public void setLatitude(Float latitude) { - this.latitude = latitude; - } - public Float getLatitude() { - return this.latitude; - } - - private Float longitude; - public void setLongitude(Float longitude) { - this.longitude = longitude; - } - public Float getLongitude() { - return this.longitude; - } - - private String as_number; - public void setAsNumber(String asNumber) { - this.as_number = escapeJSON(asNumber); - } - public String getAsNumber() { - return unescapeJSON(this.as_number); - } - - private String as_name; - public void setAsName(String asName) { - this.as_name = escapeJSON(asName); - } - public String getAsName() { - return unescapeJSON(this.as_name); - } - - private Long consensus_weight; - public void setConsensusWeight(Long consensusWeight) { - this.consensus_weight = consensusWeight; - } - public Long getConsensusWeight() { - return this.consensus_weight; - } - - private String host_name; - public void setHostName(String hostName) { - this.host_name = escapeJSON(hostName); - } - public String getHostName() { - return unescapeJSON(this.host_name); - } - - private String last_restarted; - public void setLastRestarted(String lastRestarted) { - this.last_restarted = lastRestarted; - } - public String getLastRestarted() { - return this.last_restarted; - } - - private Integer bandwidth_rate; - public void setBandwidthRate(Integer bandwidthRate) { - this.bandwidth_rate = bandwidthRate; - } - public Integer getBandwidthRate() { - return this.bandwidth_rate; - } - - private Integer bandwidth_burst; - public void setBandwidthBurst(Integer bandwidthBurst) { - this.bandwidth_burst = bandwidthBurst; - } - public Integer getBandwidthBurst() { - return this.bandwidth_burst; - } - - private Integer observed_bandwidth; - public void setObservedBandwidth(Integer observedBandwidth) { - this.observed_bandwidth = observedBandwidth; - } - public Integer getObservedBandwidth() { - return this.observed_bandwidth; - } - - private Integer advertised_bandwidth; - public void setAdvertisedBandwidth(Integer advertisedBandwidth) { - this.advertised_bandwidth = advertisedBandwidth; - } - public Integer getAdvertisedBandwidth() { - return this.advertised_bandwidth; - } - - private List<String> exit_policy; - public void setExitPolicy(List<String> exitPolicy) { - this.exit_policy = exitPolicy; - } - public List<String> getExitPolicy() { - return this.exit_policy; - } - - private Map<String, List<String>> exit_policy_summary; - public void setExitPolicySummary( - Map<String, List<String>> exitPolicySummary) { - this.exit_policy_summary = exitPolicySummary; - } - public Map<String, List<String>> getExitPolicySummary() { - return this.exit_policy_summary; - } - - private Map<String, List<String>> exit_policy_v6_summary; - public void setExitPolicyV6Summary( - Map<String, List<String>> exitPolicyV6Summary) { - this.exit_policy_v6_summary = exitPolicyV6Summary; - } - public Map<String, List<String>> getExitPolicyV6Summary() { - return this.exit_policy_v6_summary; - } - - private String contact; - public void setContact(String contact) { - this.contact = escapeJSON(contact); - } - public String getContact() { - return unescapeJSON(this.contact); - } - - private String platform; - public void setPlatform(String platform) { - this.platform = escapeJSON(platform); - } - public String getPlatform() { - return unescapeJSON(this.platform); - } - - private List<String> family; - public void setFamily(List<String> family) { - this.family = family; - } - public List<String> getFamily() { - return this.family; - } - - private Float advertised_bandwidth_fraction; - public void setAdvertisedBandwidthFraction( - Float advertisedBandwidthFraction) { - if (advertisedBandwidthFraction == null || - advertisedBandwidthFraction >= 0.0) { - this.advertised_bandwidth_fraction = advertisedBandwidthFraction; - } - } - public Float getAdvertisedBandwidthFraction() { - return this.advertised_bandwidth_fraction; - } - - private Float consensus_weight_fraction; - public void setConsensusWeightFraction(Float consensusWeightFraction) { - if (consensusWeightFraction == null || - consensusWeightFraction >= 0.0) { - this.consensus_weight_fraction = consensusWeightFraction; - } - } - public Float getConsensusWeightFraction() { - return this.consensus_weight_fraction; - } - - private Float guard_probability; - public void setGuardProbability(Float guardProbability) { - if (guardProbability == null || guardProbability >= 0.0) { - this.guard_probability = guardProbability; - } - } - public Float getGuardProbability() { - return this.guard_probability; - } - - private Float middle_probability; - public void setMiddleProbability(Float middleProbability) { - if (middleProbability == null || middleProbability >= 0.0) { - this.middle_probability = middleProbability; - } - } - public Float getMiddleProbability() { - return this.middle_probability; - } - - private Float exit_probability; - public void setExitProbability(Float exitProbability) { - if (exitProbability == null || exitProbability >= 0.0) { - this.exit_probability = exitProbability; - } - } - public Float getExitProbability() { - return this.exit_probability; - } - - private Boolean recommended_version; - public void setRecommendedVersion(Boolean recommendedVersion) { - this.recommended_version = recommendedVersion; - } - public Boolean getRecommendedVersion() { - return this.recommended_version; - } - - private Boolean hibernating; - public void setHibernating(Boolean hibernating) { - this.hibernating = hibernating; - } - public Boolean getHibernating() { - return this.hibernating; - } - - private String pool_assignment; - public void setPoolAssignment(String poolAssignment) { - this.pool_assignment = poolAssignment; - } - public String getPoolAssignment() { - return this.pool_assignment; - } -} - diff --git a/src/org/torproject/onionoo/DetailsDocumentWriter.java b/src/org/torproject/onionoo/DetailsDocumentWriter.java deleted file mode 100644 index a70efd0..0000000 --- a/src/org/torproject/onionoo/DetailsDocumentWriter.java +++ /dev/null @@ -1,222 +0,0 @@ -package org.torproject.onionoo; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.SortedSet; -import java.util.TreeSet; - -public class DetailsDocumentWriter implements FingerprintListener, - DocumentWriter { - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - private long now; - - public DetailsDocumentWriter() { - this.descriptorSource = ApplicationFactory.getDescriptorSource(); - this.documentStore = ApplicationFactory.getDocumentStore(); - this.now = ApplicationFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_STATUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_SERVER_DESCRIPTORS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_POOL_ASSIGNMENTS); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.EXIT_LISTS); - } - - private SortedSet<String> newRelays = new TreeSet<String>(), - newBridges = new TreeSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (relay) { - this.newRelays.addAll(fingerprints); - } else { - this.newBridges.addAll(fingerprints); - } - } - - public void writeDocuments() { - this.updateRelayDetailsFiles(); - this.updateBridgeDetailsFiles(); - Logger.printStatusTime("Wrote details document files"); - } - - private void updateRelayDetailsFiles() { - for (String fingerprint : this.newRelays) { - - /* Generate network-status-specific part. */ - NodeStatus entry = this.documentStore.retrieve(NodeStatus.class, - true, fingerprint); - if (entry == null) { - continue; - } - DetailsDocument detailsDocument = new DetailsDocument(); - detailsDocument.setNickname(entry.getNickname()); - detailsDocument.setFingerprint(fingerprint); - List<String> orAddresses = new ArrayList<String>(); - orAddresses.add(entry.getAddress() + ":" + entry.getOrPort()); - for (String orAddress : entry.getOrAddressesAndPorts()) { - orAddresses.add(orAddress.toLowerCase()); - } - detailsDocument.setOrAddresses(orAddresses); - if (entry.getDirPort() != 0) { - detailsDocument.setDirAddress(entry.getAddress() + ":" - + entry.getDirPort()); - } - detailsDocument.setLastSeen(DateTimeHelper.format( - entry.getLastSeenMillis())); - detailsDocument.setFirstSeen(DateTimeHelper.format( - entry.getFirstSeenMillis())); - detailsDocument.setLastChangedAddressOrPort( - DateTimeHelper.format(entry.getLastChangedOrAddress())); - detailsDocument.setRunning(entry.getRunning()); - if (!entry.getRelayFlags().isEmpty()) { - detailsDocument.setFlags(new ArrayList<String>( - entry.getRelayFlags())); - } - detailsDocument.setCountry(entry.getCountryCode()); - detailsDocument.setLatitude(entry.getLatitude()); - detailsDocument.setLongitude(entry.getLongitude()); - detailsDocument.setCountryName(entry.getCountryName()); - detailsDocument.setRegionName(entry.getRegionName()); - detailsDocument.setCityName(entry.getCityName()); - detailsDocument.setAsNumber(entry.getASNumber()); - detailsDocument.setAsName(entry.getASName()); - detailsDocument.setConsensusWeight(entry.getConsensusWeight()); - detailsDocument.setHostName(entry.getHostName()); - detailsDocument.setAdvertisedBandwidthFraction( - (float) entry.getAdvertisedBandwidthFraction()); - detailsDocument.setConsensusWeightFraction( - (float) entry.getConsensusWeightFraction()); - detailsDocument.setGuardProbability( - (float) entry.getGuardProbability()); - detailsDocument.setMiddleProbability( - (float) entry.getMiddleProbability()); - detailsDocument.setExitProbability( - (float) entry.getExitProbability()); - String defaultPolicy = entry.getDefaultPolicy(); - String portList = entry.getPortList(); - if (defaultPolicy != null && (defaultPolicy.equals("accept") || - defaultPolicy.equals("reject")) && portList != null) { - Map<String, List<String>> exitPolicySummary = - new HashMap<String, List<String>>(); - List<String> portsOrPortRanges = Arrays.asList( - portList.split(",")); - exitPolicySummary.put(defaultPolicy, portsOrPortRanges); - detailsDocument.setExitPolicySummary(exitPolicySummary); - } - detailsDocument.setRecommendedVersion( - entry.getRecommendedVersion()); - - /* Append descriptor-specific part and exit addresses from details - * status file. */ - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus != null) { - detailsDocument.setLastRestarted( - detailsStatus.getLastRestarted()); - detailsDocument.setBandwidthRate( - detailsStatus.getBandwidthRate()); - detailsDocument.setBandwidthBurst( - detailsStatus.getBandwidthBurst()); - detailsDocument.setObservedBandwidth( - detailsStatus.getObservedBandwidth()); - detailsDocument.setAdvertisedBandwidth( - detailsStatus.getAdvertisedBandwidth()); - detailsDocument.setExitPolicy(detailsStatus.getExitPolicy()); - detailsDocument.setContact(detailsStatus.getContact()); - detailsDocument.setPlatform(detailsStatus.getPlatform()); - detailsDocument.setFamily(detailsStatus.getFamily()); - detailsDocument.setExitPolicyV6Summary( - detailsStatus.getExitPolicyV6Summary()); - detailsDocument.setHibernating(detailsStatus.getHibernating()); - if (detailsStatus.getExitAddresses() != null) { - SortedSet<String> exitAddresses = new TreeSet<String>(); - for (Map.Entry<String, Long> e : - detailsStatus.getExitAddresses().entrySet()) { - String exitAddress = e.getKey().toLowerCase(); - long scanMillis = e.getValue(); - if (!entry.getAddress().equals(exitAddress) && - !entry.getOrAddresses().contains(exitAddress) && - scanMillis >= this.now - DateTimeHelper.ONE_DAY) { - exitAddresses.add(exitAddress); - } - } - if (!exitAddresses.isEmpty()) { - detailsDocument.setExitAddresses(new ArrayList<String>( - exitAddresses)); - } - } - } - - /* Write details file to disk. */ - this.documentStore.store(detailsDocument, fingerprint); - } - } - - private void updateBridgeDetailsFiles() { - for (String fingerprint : this.newBridges) { - - /* Generate network-status-specific part. */ - NodeStatus entry = this.documentStore.retrieve(NodeStatus.class, - true, fingerprint); - if (entry == null) { - continue; - } - DetailsDocument detailsDocument = new DetailsDocument(); - detailsDocument.setNickname(entry.getNickname()); - detailsDocument.setHashedFingerprint(fingerprint); - String address = entry.getAddress(); - List<String> orAddresses = new ArrayList<String>(); - orAddresses.add(address + ":" + entry.getOrPort()); - for (String orAddress : entry.getOrAddressesAndPorts()) { - orAddresses.add(orAddress.toLowerCase()); - } - detailsDocument.setOrAddresses(orAddresses); - detailsDocument.setLastSeen(DateTimeHelper.format( - entry.getLastSeenMillis())); - detailsDocument.setFirstSeen(DateTimeHelper.format( - entry.getFirstSeenMillis())); - detailsDocument.setRunning(entry.getRunning()); - detailsDocument.setFlags(new ArrayList<String>( - entry.getRelayFlags())); - - /* Append descriptor-specific part from details status file. */ - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus != null) { - detailsDocument.setLastRestarted( - detailsStatus.getLastRestarted()); - detailsDocument.setAdvertisedBandwidth( - detailsStatus.getAdvertisedBandwidth()); - detailsDocument.setPlatform(detailsStatus.getPlatform()); - detailsDocument.setPoolAssignment( - detailsStatus.getPoolAssignment()); - } - - /* Write details file to disk. */ - this.documentStore.store(detailsDocument, fingerprint); - } - } - - public String getStatsString() { - /* TODO Add statistics string. */ - return null; - } -} diff --git a/src/org/torproject/onionoo/DetailsStatus.java b/src/org/torproject/onionoo/DetailsStatus.java deleted file mode 100644 index 1951c89..0000000 --- a/src/org/torproject/onionoo/DetailsStatus.java +++ /dev/null @@ -1,141 +0,0 @@ -/* Copyright 2013--2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.List; -import java.util.Map; - -import org.apache.commons.lang.StringEscapeUtils; - -public class DetailsStatus extends Document { - - /* We must ensure that details files only contain ASCII characters - * and no UTF-8 characters. While UTF-8 characters are perfectly - * valid in JSON, this would break compatibility with existing files - * pretty badly. We do this by escaping non-ASCII characters, e.g., - * \u00F2. Gson won't treat this as UTF-8, but will think that we want - * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing - * we'll have to do is to change back the '\\' that Gson writes for the - * '\'. */ - private static String escapeJSON(String s) { - return s == null ? null : - StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'"); - } - private static String unescapeJSON(String s) { - return s == null ? null : - StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'")); - } - - private String desc_published; - public void setDescPublished(String descPublished) { - this.desc_published = descPublished; - } - public String getDescPublished() { - return this.desc_published; - } - - private String last_restarted; - public void setLastRestarted(String lastRestarted) { - this.last_restarted = lastRestarted; - } - public String getLastRestarted() { - return this.last_restarted; - } - - private Integer bandwidth_rate; - public void setBandwidthRate(Integer bandwidthRate) { - this.bandwidth_rate = bandwidthRate; - } - public Integer getBandwidthRate() { - return this.bandwidth_rate; - } - - private Integer bandwidth_burst; - public void setBandwidthBurst(Integer bandwidthBurst) { - this.bandwidth_burst = bandwidthBurst; - } - public Integer getBandwidthBurst() { - return this.bandwidth_burst; - } - - private Integer observed_bandwidth; - public void setObservedBandwidth(Integer observedBandwidth) { - this.observed_bandwidth = observedBandwidth; - } - public Integer getObservedBandwidth() { - return this.observed_bandwidth; - } - - private Integer advertised_bandwidth; - public void setAdvertisedBandwidth(Integer advertisedBandwidth) { - this.advertised_bandwidth = advertisedBandwidth; - } - public Integer getAdvertisedBandwidth() { - return this.advertised_bandwidth; - } - - private List<String> exit_policy; - public void setExitPolicy(List<String> exitPolicy) { - this.exit_policy = exitPolicy; - } - public List<String> getExitPolicy() { - return this.exit_policy; - } - - private String contact; - public void setContact(String contact) { - this.contact = escapeJSON(contact); - } - public String getContact() { - return unescapeJSON(this.contact); - } - - private String platform; - public void setPlatform(String platform) { - this.platform = escapeJSON(platform); - } - public String getPlatform() { - return unescapeJSON(this.platform); - } - - private List<String> family; - public void setFamily(List<String> family) { - this.family = family; - } - public List<String> getFamily() { - return this.family; - } - - private Map<String, List<String>> exit_policy_v6_summary; - public void setExitPolicyV6Summary( - Map<String, List<String>> exitPolicyV6Summary) { - this.exit_policy_v6_summary = exitPolicyV6Summary; - } - public Map<String, List<String>> getExitPolicyV6Summary() { - return this.exit_policy_v6_summary; - } - - private Boolean hibernating; - public void setHibernating(Boolean hibernating) { - this.hibernating = hibernating; - } - public Boolean getHibernating() { - return this.hibernating; - } - - private String pool_assignment; - public void setPoolAssignment(String poolAssignment) { - this.pool_assignment = poolAssignment; - } - public String getPoolAssignment() { - return this.pool_assignment; - } - - private Map<String, Long> exit_addresses; - public void setExitAddresses(Map<String, Long> exitAddresses) { - this.exit_addresses = exitAddresses; - } - public Map<String, Long> getExitAddresses() { - return this.exit_addresses; - } -} diff --git a/src/org/torproject/onionoo/Document.java b/src/org/torproject/onionoo/Document.java deleted file mode 100644 index f49574c..0000000 --- a/src/org/torproject/onionoo/Document.java +++ /dev/null @@ -1,24 +0,0 @@ -/* Copyright 2013 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -public abstract class Document { - - private transient String documentString; - public void setDocumentString(String documentString) { - this.documentString = documentString; - } - public String getDocumentString() { - return this.documentString; - } - - public void fromDocumentString(String documentString) { - /* Subclasses may override this method to parse documentString. */ - } - - public String toDocumentString() { - /* Subclasses may override this method to write documentString. */ - return null; - } -} - diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java deleted file mode 100644 index 434ecb6..0000000 --- a/src/org/torproject/onionoo/DocumentStore.java +++ /dev/null @@ -1,744 +0,0 @@ -/* Copyright 2013 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.Stack; -import java.util.TreeMap; -import java.util.TreeSet; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonParseException; - -// TODO For later migration from disk to database, do the following: -// - read from database and then from disk if not found -// - write only to database, delete from disk once in database -// - move entirely to database once disk is "empty" -// TODO Also look into simple key-value stores instead of real databases. -public class DocumentStore { - - private final File statusDir = new File("status"); - - private File outDir = new File("out"); - public void setOutDir(File outDir) { - this.outDir = outDir; - } - - private Time time; - - public DocumentStore() { - this.time = ApplicationFactory.getTime(); - } - - private long listOperations = 0L, listedFiles = 0L, storedFiles = 0L, - storedBytes = 0L, retrievedFiles = 0L, retrievedBytes = 0L, - removedFiles = 0L; - - /* Node statuses and summary documents are cached in memory, as opposed - * to all other document types. These caches are initialized when first - * accessing or modifying a NodeStatus or SummaryDocument document, - * respectively. */ - private SortedMap<String, NodeStatus> cachedNodeStatuses; - private SortedMap<String, SummaryDocument> cachedSummaryDocuments; - - public <T extends Document> SortedSet<String> list( - Class<T> documentType) { - if (documentType.equals(NodeStatus.class)) { - return this.listNodeStatuses(); - } else if (documentType.equals(SummaryDocument.class)) { - return this.listSummaryDocuments(); - } else { - return this.listDocumentFiles(documentType); - } - } - - private SortedSet<String> listNodeStatuses() { - if (this.cachedNodeStatuses == null) { - this.cacheNodeStatuses(); - } - return new TreeSet<String>(this.cachedNodeStatuses.keySet()); - } - - private void cacheNodeStatuses() { - SortedMap<String, NodeStatus> parsedNodeStatuses = - new TreeMap<String, NodeStatus>(); - File directory = this.statusDir; - if (directory != null) { - File summaryFile = new File(directory, "summary"); - if (summaryFile.exists()) { - try { - BufferedReader br = new BufferedReader(new FileReader( - summaryFile)); - String line; - while ((line = br.readLine()) != null) { - if (line.length() == 0) { - continue; - } - NodeStatus node = NodeStatus.fromString(line); - if (node != null) { - parsedNodeStatuses.put(node.getFingerprint(), node); - } - } - br.close(); - this.listedFiles += parsedNodeStatuses.size(); - this.listOperations++; - } catch (IOException e) { - System.err.println("Could not read file '" - + summaryFile.getAbsolutePath() + "'."); - e.printStackTrace(); - } - } - } - this.cachedNodeStatuses = parsedNodeStatuses; - } - - private SortedSet<String> listSummaryDocuments() { - if (this.cachedSummaryDocuments == null) { - this.cacheSummaryDocuments(); - } - return new TreeSet<String>(this.cachedSummaryDocuments.keySet()); - } - - private void cacheSummaryDocuments() { - SortedMap<String, SummaryDocument> parsedSummaryDocuments = - new TreeMap<String, SummaryDocument>(); - File directory = this.outDir; - if (directory != null) { - File summaryFile = new File(directory, "summary"); - if (summaryFile.exists()) { - String line = null; - try { - Gson gson = new Gson(); - BufferedReader br = new BufferedReader(new FileReader( - summaryFile)); - while ((line = br.readLine()) != null) { - if (line.length() == 0) { - continue; - } - SummaryDocument summaryDocument = gson.fromJson(line, - SummaryDocument.class); - if (summaryDocument != null) { - parsedSummaryDocuments.put(summaryDocument.getFingerprint(), - summaryDocument); - } - } - br.close(); - this.listedFiles += parsedSummaryDocuments.size(); - this.listOperations++; - } catch (IOException e) { - System.err.println("Could not read file '" - + summaryFile.getAbsolutePath() + "'."); - e.printStackTrace(); - } catch (JsonParseException e) { - System.err.println("Could not parse summary document '" + line - + "' in file '" + summaryFile.getAbsolutePath() + "'."); - e.printStackTrace(); - } - } - } - this.cachedSummaryDocuments = parsedSummaryDocuments; - } - - private <T extends Document> SortedSet<String> listDocumentFiles( - Class<T> documentType) { - SortedSet<String> fingerprints = new TreeSet<String>(); - File directory = null; - String subdirectory = null; - if (documentType.equals(DetailsStatus.class)) { - directory = this.statusDir; - subdirectory = "details"; - } else if (documentType.equals(BandwidthStatus.class)) { - directory = this.statusDir; - subdirectory = "bandwidth"; - } else if (documentType.equals(WeightsStatus.class)) { - directory = this.statusDir; - subdirectory = "weights"; - } else if (documentType.equals(ClientsStatus.class)) { - directory = this.statusDir; - subdirectory = "clients"; - } else if (documentType.equals(UptimeStatus.class)) { - directory = this.statusDir; - subdirectory = "uptimes"; - } else if (documentType.equals(DetailsDocument.class)) { - directory = this.outDir; - subdirectory = "details"; - } else if (documentType.equals(BandwidthDocument.class)) { - directory = this.outDir; - subdirectory = "bandwidth"; - } else if (documentType.equals(WeightsDocument.class)) { - directory = this.outDir; - subdirectory = "weights"; - } else if (documentType.equals(ClientsDocument.class)) { - directory = this.outDir; - subdirectory = "clients"; - } else if (documentType.equals(UptimeDocument.class)) { - directory = this.outDir; - subdirectory = "uptimes"; - } - if (directory != null && subdirectory != null) { - Stack<File> files = new Stack<File>(); - files.add(new File(directory, subdirectory)); - while (!files.isEmpty()) { - File file = files.pop(); - if (file.isDirectory()) { - files.addAll(Arrays.asList(file.listFiles())); - } else if (file.getName().length() == 40) { - fingerprints.add(file.getName()); - } - } - } - this.listOperations++; - this.listedFiles += fingerprints.size(); - return fingerprints; - } - - public <T extends Document> boolean store(T document) { - return this.store(document, null); - } - - public <T extends Document> boolean store(T document, - String fingerprint) { - if (document instanceof NodeStatus) { - return this.storeNodeStatus((NodeStatus) document, fingerprint); - } else if (document instanceof SummaryDocument) { - return this.storeSummaryDocument((SummaryDocument) document, - fingerprint); - } else { - return this.storeDocumentFile(document, fingerprint); - } - } - - private <T extends Document> boolean storeNodeStatus( - NodeStatus nodeStatus, String fingerprint) { - if (this.cachedNodeStatuses == null) { - this.cacheNodeStatuses(); - } - this.cachedNodeStatuses.put(fingerprint, nodeStatus); - return true; - } - - private <T extends Document> boolean storeSummaryDocument( - SummaryDocument summaryDocument, String fingerprint) { - if (this.cachedSummaryDocuments == null) { - this.cacheSummaryDocuments(); - } - this.cachedSummaryDocuments.put(fingerprint, summaryDocument); - return true; - } - - private <T extends Document> boolean storeDocumentFile(T document, - String fingerprint) { - File documentFile = this.getDocumentFile(document.getClass(), - fingerprint); - if (documentFile == null) { - return false; - } - String documentString; - if (document.getDocumentString() != null) { - documentString = document.getDocumentString(); - } else if (document instanceof BandwidthDocument || - document instanceof WeightsDocument || - document instanceof ClientsDocument || - document instanceof UptimeDocument) { - Gson gson = new Gson(); - documentString = gson.toJson(document); - } else if (document instanceof DetailsStatus || - document instanceof DetailsDocument) { - /* Don't escape HTML characters, like < and >, contained in - * strings. */ - Gson gson = new GsonBuilder().disableHtmlEscaping().create(); - /* We must ensure that details files only contain ASCII characters - * and no UTF-8 characters. While UTF-8 characters are perfectly - * valid in JSON, this would break compatibility with existing files - * pretty badly. We already make sure that all strings in details - * objects are escaped JSON, e.g., \u00F2. When Gson serlializes - * this string, it escapes the \ to \\, hence writes \\u00F2. We - * need to undo this and change \\u00F2 back to \u00F2. */ - documentString = gson.toJson(document).replaceAll("\\\\\\\\u", - "\\\\u"); - /* Existing details statuses don't contain opening and closing curly - * brackets, so we should remove them from new details statuses, - * too. */ - if (document instanceof DetailsStatus) { - documentString = documentString.substring( - documentString.indexOf("{") + 1, - documentString.lastIndexOf("}")); - } - } else if (document instanceof BandwidthStatus || - document instanceof WeightsStatus || - document instanceof ClientsStatus || - document instanceof UptimeStatus) { - documentString = document.toDocumentString(); - } else { - System.err.println("Serializing is not supported for type " - + document.getClass().getName() + "."); - return false; - } - try { - documentFile.getParentFile().mkdirs(); - File documentTempFile = new File( - documentFile.getAbsolutePath() + ".tmp"); - BufferedWriter bw = new BufferedWriter(new FileWriter( - documentTempFile)); - bw.write(documentString); - bw.close(); - documentFile.delete(); - documentTempFile.renameTo(documentFile); - this.storedFiles++; - this.storedBytes += documentString.length(); - } catch (IOException e) { - System.err.println("Could not write file '" - + documentFile.getAbsolutePath() + "'."); - e.printStackTrace(); - return false; - } - return true; - } - - public <T extends Document> T retrieve(Class<T> documentType, - boolean parse) { - return this.retrieve(documentType, parse, null); - } - - public <T extends Document> T retrieve(Class<T> documentType, - boolean parse, String fingerprint) { - if (documentType.equals(NodeStatus.class)) { - return documentType.cast(this.retrieveNodeStatus(fingerprint)); - } else if (documentType.equals(SummaryDocument.class)) { - return documentType.cast(this.retrieveSummaryDocument(fingerprint)); - } else { - return this.retrieveDocumentFile(documentType, parse, fingerprint); - } - } - - private NodeStatus retrieveNodeStatus(String fingerprint) { - if (this.cachedNodeStatuses == null) { - this.cacheNodeStatuses(); - } - return this.cachedNodeStatuses.get(fingerprint); - } - - private SummaryDocument retrieveSummaryDocument(String fingerprint) { - if (this.cachedSummaryDocuments == null) { - this.cacheSummaryDocuments(); - } - if (this.cachedSummaryDocuments.containsKey(fingerprint)) { - return this.cachedSummaryDocuments.get(fingerprint); - } - /* TODO This is an evil hack to support looking up relays or bridges - * that haven't been running for a week without having to load - * 500,000 NodeStatus instances into memory. Maybe there's a better - * way? Or do we need to switch to a real database for this? */ - DetailsDocument detailsDocument = this.retrieveDocumentFile( - DetailsDocument.class, true, fingerprint); - if (detailsDocument == null) { - return null; - } - boolean isRelay = detailsDocument.getHashedFingerprint() == null; - boolean running = false; - String nickname = detailsDocument.getNickname(); - List<String> addresses = new ArrayList<String>(); - String countryCode = null, aSNumber = null, contact = null; - for (String orAddressAndPort : detailsDocument.getOrAddresses()) { - if (!orAddressAndPort.contains(":")) { - return null; - } - String orAddress = orAddressAndPort.substring(0, - orAddressAndPort.lastIndexOf(":")); - if (!addresses.contains(orAddress)) { - addresses.add(orAddress); - } - } - if (detailsDocument.getExitAddresses() != null) { - for (String exitAddress : detailsDocument.getExitAddresses()) { - if (!addresses.contains(exitAddress)) { - addresses.add(exitAddress); - } - } - } - SortedSet<String> relayFlags = new TreeSet<String>(), family = null; - long lastSeenMillis = -1L, consensusWeight = -1L, - firstSeenMillis = -1L; - SummaryDocument summaryDocument = new SummaryDocument(isRelay, - nickname, fingerprint, addresses, lastSeenMillis, running, - relayFlags, consensusWeight, countryCode, firstSeenMillis, - aSNumber, contact, family); - return summaryDocument; - } - - private <T extends Document> T retrieveDocumentFile( - Class<T> documentType, boolean parse, String fingerprint) { - File documentFile = this.getDocumentFile(documentType, fingerprint); - if (documentFile == null || !documentFile.exists()) { - return null; - } else if (documentFile.isDirectory()) { - System.err.println("Could not read file '" - + documentFile.getAbsolutePath() + "', because it is a " - + "directory."); - return null; - } - String documentString = null; - try { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - BufferedInputStream bis = new BufferedInputStream( - new FileInputStream(documentFile)); - int len; - byte[] data = new byte[1024]; - while ((len = bis.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - bis.close(); - byte[] allData = baos.toByteArray(); - if (allData.length == 0) { - return null; - } - documentString = new String(allData, "US-ASCII"); - this.retrievedFiles++; - this.retrievedBytes += documentString.length(); - } catch (IOException e) { - System.err.println("Could not read file '" - + documentFile.getAbsolutePath() + "'."); - e.printStackTrace(); - return null; - } - T result = null; - if (!parse) { - return this.retrieveUnparsedDocumentFile(documentType, - documentString); - } else if (documentType.equals(DetailsDocument.class) || - documentType.equals(BandwidthDocument.class) || - documentType.equals(WeightsDocument.class) || - documentType.equals(ClientsDocument.class) || - documentType.equals(UptimeDocument.class)) { - return this.retrieveParsedDocumentFile(documentType, - documentString); - } else if (documentType.equals(BandwidthStatus.class) || - documentType.equals(WeightsStatus.class) || - documentType.equals(ClientsStatus.class) || - documentType.equals(UptimeStatus.class)) { - return this.retrieveParsedStatusFile(documentType, documentString); - } else if (documentType.equals(DetailsStatus.class)) { - return this.retrieveParsedDocumentFile(documentType, "{" - + documentString + "}"); - } else { - System.err.println("Parsing is not supported for type " - + documentType.getName() + "."); - } - return result; - } - - private <T extends Document> T retrieveParsedStatusFile( - Class<T> documentType, String documentString) { - T result = null; - try { - result = documentType.newInstance(); - result.fromDocumentString(documentString); - } catch (InstantiationException e) { - /* Handle below. */ - e.printStackTrace(); - } catch (IllegalAccessException e) { - /* Handle below. */ - e.printStackTrace(); - } - if (result == null) { - System.err.println("Could not initialize parsed status file of " - + "type " + documentType.getName() + "."); - } - return result; - } - - private <T extends Document> T retrieveParsedDocumentFile( - Class<T> documentType, String documentString) { - T result = null; - Gson gson = new Gson(); - try { - result = gson.fromJson(documentString, documentType); - } catch (JsonParseException e) { - /* Handle below. */ - e.printStackTrace(); - } - if (result == null) { - System.err.println("Could not initialize parsed document of type " - + documentType.getName() + "."); - } - return result; - } - - private <T extends Document> T retrieveUnparsedDocumentFile( - Class<T> documentType, String documentString) { - T result = null; - try { - result = documentType.newInstance(); - result.setDocumentString(documentString); - } catch (InstantiationException e) { - /* Handle below. */ - e.printStackTrace(); - } catch (IllegalAccessException e) { - /* Handle below. */ - e.printStackTrace(); - } - if (result == null) { - System.err.println("Could not initialize unparsed document of type " - + documentType.getName() + "."); - } - return result; - } - - public <T extends Document> boolean remove(Class<T> documentType) { - return this.remove(documentType, null); - } - - public <T extends Document> boolean remove(Class<T> documentType, - String fingerprint) { - if (documentType.equals(NodeStatus.class)) { - return this.removeNodeStatus(fingerprint); - } else if (documentType.equals(SummaryDocument.class)) { - return this.removeSummaryDocument(fingerprint); - } else { - return this.removeDocumentFile(documentType, fingerprint); - } - } - - private boolean removeNodeStatus(String fingerprint) { - if (this.cachedNodeStatuses == null) { - this.cacheNodeStatuses(); - } - return this.cachedNodeStatuses.remove(fingerprint) != null; - } - - private boolean removeSummaryDocument(String fingerprint) { - if (this.cachedSummaryDocuments == null) { - this.cacheSummaryDocuments(); - } - return this.cachedSummaryDocuments.remove(fingerprint) != null; - } - - private <T extends Document> boolean removeDocumentFile( - Class<T> documentType, String fingerprint) { - File documentFile = this.getDocumentFile(documentType, fingerprint); - if (documentFile == null || !documentFile.delete()) { - System.err.println("Could not delete file '" - + documentFile.getAbsolutePath() + "'."); - return false; - } - this.removedFiles++; - return true; - } - - private <T extends Document> File getDocumentFile(Class<T> documentType, - String fingerprint) { - File documentFile = null; - if (fingerprint == null && !documentType.equals(UpdateStatus.class) && - !documentType.equals(UptimeStatus.class)) { - // TODO Instead of using the update file workaround, add new method - // lastModified(Class<T> documentType) that serves a similar - // purpose. - return null; - } - File directory = null; - String fileName = null; - if (documentType.equals(DetailsStatus.class)) { - directory = this.statusDir; - fileName = String.format("details/%s/%s/%s", - fingerprint.substring(0, 1), fingerprint.substring(1, 2), - fingerprint); - } else if (documentType.equals(BandwidthStatus.class)) { - directory = this.statusDir; - fileName = String.format("bandwidth/%s/%s/%s", - fingerprint.substring(0, 1), fingerprint.substring(1, 2), - fingerprint); - } else if (documentType.equals(WeightsStatus.class)) { - directory = this.statusDir; - fileName = String.format("weights/%s/%s/%s", - fingerprint.substring(0, 1), fingerprint.substring(1, 2), - fingerprint); - } else if (documentType.equals(ClientsStatus.class)) { - directory = this.statusDir; - fileName = String.format("clients/%s/%s/%s", - fingerprint.substring(0, 1), fingerprint.substring(1, 2), - fingerprint); - } else if (documentType.equals(UptimeStatus.class)) { - directory = this.statusDir; - if (fingerprint == null) { - fileName = "uptime"; - } else { - fileName = String.format("uptimes/%s/%s/%s", - fingerprint.substring(0, 1), fingerprint.substring(1, 2), - fingerprint); - } - } else if (documentType.equals(UpdateStatus.class)) { - directory = this.outDir; - fileName = "update"; - } else if (documentType.equals(DetailsDocument.class)) { - directory = this.outDir; - fileName = String.format("details/%s", fingerprint); - } else if (documentType.equals(BandwidthDocument.class)) { - directory = this.outDir; - fileName = String.format("bandwidth/%s", fingerprint); - } else if (documentType.equals(WeightsDocument.class)) { - directory = this.outDir; - fileName = String.format("weights/%s", fingerprint); - } else if (documentType.equals(ClientsDocument.class)) { - directory = this.outDir; - fileName = String.format("clients/%s", fingerprint); - } else if (documentType.equals(UptimeDocument.class)) { - directory = this.outDir; - fileName = String.format("uptimes/%s", fingerprint); - } - if (directory != null && fileName != null) { - documentFile = new File(directory, fileName); - } - return documentFile; - } - - public void flushDocumentCache() { - /* Write cached node statuses to disk, and write update file - * containing current time. It's important to write the update file - * now, not earlier, because the front-end should not read new node - * statuses until all details, bandwidths, and weights are ready. */ - if (this.cachedNodeStatuses != null || - this.cachedSummaryDocuments != null) { - if (this.cachedNodeStatuses != null) { - this.writeNodeStatuses(); - } - if (this.cachedSummaryDocuments != null) { - this.writeSummaryDocuments(); - } - this.writeUpdateStatus(); - } - } - - private void writeNodeStatuses() { - File directory = this.statusDir; - if (directory == null) { - return; - } - File summaryFile = new File(directory, "summary"); - SortedMap<String, NodeStatus> - cachedRelays = new TreeMap<String, NodeStatus>(), - cachedBridges = new TreeMap<String, NodeStatus>(); - for (Map.Entry<String, NodeStatus> e : - this.cachedNodeStatuses.entrySet()) { - if (e.getValue().isRelay()) { - cachedRelays.put(e.getKey(), e.getValue()); - } else { - cachedBridges.put(e.getKey(), e.getValue()); - } - } - StringBuilder sb = new StringBuilder(); - for (NodeStatus relay : cachedRelays.values()) { - String line = relay.toString(); - if (line != null) { - sb.append(line + "\n"); - } else { - System.err.println("Could not serialize relay node status '" - + relay.getFingerprint() + "'"); - } - } - for (NodeStatus bridge : cachedBridges.values()) { - String line = bridge.toString(); - if (line != null) { - sb.append(line + "\n"); - } else { - System.err.println("Could not serialize bridge node status '" - + bridge.getFingerprint() + "'"); - } - } - String documentString = sb.toString(); - try { - summaryFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); - bw.write(documentString); - bw.close(); - this.storedFiles++; - this.storedBytes += documentString.length(); - } catch (IOException e) { - System.err.println("Could not write file '" - + summaryFile.getAbsolutePath() + "'."); - e.printStackTrace(); - } - } - - private void writeSummaryDocuments() { - StringBuilder sb = new StringBuilder(); - Gson gson = new Gson(); - for (SummaryDocument summaryDocument : - this.cachedSummaryDocuments.values()) { - String line = gson.toJson(summaryDocument); - if (line != null) { - sb.append(line + "\n"); - } else { - System.err.println("Could not serialize relay summary document '" - + summaryDocument.getFingerprint() + "'"); - } - } - String documentString = sb.toString(); - File summaryFile = new File(this.outDir, "summary"); - try { - summaryFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); - bw.write(documentString); - bw.close(); - this.storedFiles++; - this.storedBytes += documentString.length(); - } catch (IOException e) { - System.err.println("Could not write file '" - + summaryFile.getAbsolutePath() + "'."); - e.printStackTrace(); - } - } - - private void writeUpdateStatus() { - if (this.outDir == null) { - return; - } - File updateFile = new File(this.outDir, "update"); - String documentString = String.valueOf(this.time.currentTimeMillis()); - try { - updateFile.getParentFile().mkdirs(); - BufferedWriter bw = new BufferedWriter(new FileWriter(updateFile)); - bw.write(documentString); - bw.close(); - this.storedFiles++; - this.storedBytes += documentString.length(); - } catch (IOException e) { - System.err.println("Could not write file '" - + updateFile.getAbsolutePath() + "'."); - e.printStackTrace(); - } - } - - public String getStatsString() { - StringBuilder sb = new StringBuilder(); - sb.append(" " + Logger.formatDecimalNumber(listOperations) - + " list operations performed\n"); - sb.append(" " + Logger.formatDecimalNumber(listedFiles) - + " files listed\n"); - sb.append(" " + Logger.formatDecimalNumber(storedFiles) - + " files stored\n"); - sb.append(" " + Logger.formatBytes(storedBytes) + " stored\n"); - sb.append(" " + Logger.formatDecimalNumber(retrievedFiles) - + " files retrieved\n"); - sb.append(" " + Logger.formatBytes(retrievedBytes) - + " retrieved\n"); - sb.append(" " + Logger.formatDecimalNumber(removedFiles) - + " files removed\n"); - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/DocumentWriter.java b/src/org/torproject/onionoo/DocumentWriter.java deleted file mode 100644 index 4cdeef9..0000000 --- a/src/org/torproject/onionoo/DocumentWriter.java +++ /dev/null @@ -1,11 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -public interface DocumentWriter { - - public abstract void writeDocuments(); - - public abstract String getStatsString(); -} - diff --git a/src/org/torproject/onionoo/GraphHistory.java b/src/org/torproject/onionoo/GraphHistory.java deleted file mode 100644 index f03be58..0000000 --- a/src/org/torproject/onionoo/GraphHistory.java +++ /dev/null @@ -1,56 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.List; - -public class GraphHistory { - - private String first; - public void setFirst(String first) { - this.first = first; - } - public String getFirst() { - return this.first; - } - - private String last; - public void setLast(String last) { - this.last = last; - } - public String getLast() { - return this.last; - } - - private Integer interval; - public void setInterval(Integer interval) { - this.interval = interval; - } - public Integer getInterval() { - return this.interval; - } - - private Double factor; - public void setFactor(Double factor) { - this.factor = factor; - } - public Double getFactor() { - return this.factor; - } - - private Integer count; - public void setCount(Integer count) { - this.count = count; - } - public Integer getCount() { - return this.count; - } - - private List<Integer> values; - public void setValues(List<Integer> values) { - this.values = values; - } - public List<Integer> getValues() { - return this.values; - } -} diff --git a/src/org/torproject/onionoo/LockFile.java b/src/org/torproject/onionoo/LockFile.java deleted file mode 100644 index 768db53..0000000 --- a/src/org/torproject/onionoo/LockFile.java +++ /dev/null @@ -1,43 +0,0 @@ -/* Copyright 2013 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; - -public class LockFile { - - private final File lockFile = new File("lock"); - - public boolean acquireLock() { - Time time = ApplicationFactory.getTime(); - try { - if (this.lockFile.exists()) { - return false; - } - if (this.lockFile.getParentFile() != null) { - this.lockFile.getParentFile().mkdirs(); - } - BufferedWriter bw = new BufferedWriter(new FileWriter( - this.lockFile)); - bw.append("" + time.currentTimeMillis() + "\n"); - bw.close(); - return true; - } catch (IOException e) { - System.err.println("Caught exception while trying to acquire " - + "lock!"); - e.printStackTrace(); - return false; - } - } - - public boolean releaseLock() { - if (this.lockFile.exists()) { - this.lockFile.delete(); - } - return !this.lockFile.exists(); - } -} - diff --git a/src/org/torproject/onionoo/Logger.java b/src/org/torproject/onionoo/Logger.java deleted file mode 100644 index 6906a6a..0000000 --- a/src/org/torproject/onionoo/Logger.java +++ /dev/null @@ -1,81 +0,0 @@ -/* Copyright 2013 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Date; - -public class Logger { - - private Logger() { - } - - private static Time time; - - public static void setTime() { - time = ApplicationFactory.getTime(); - } - - private static long currentTimeMillis() { - if (time == null) { - return System.currentTimeMillis(); - } else { - return time.currentTimeMillis(); - } - } - - public static String formatDecimalNumber(long decimalNumber) { - return String.format("%,d", decimalNumber); - } - - public static String formatMillis(long millis) { - return String.format("%02d:%02d.%03d minutes", - millis / DateTimeHelper.ONE_MINUTE, - (millis % DateTimeHelper.ONE_MINUTE) / DateTimeHelper.ONE_SECOND, - millis % DateTimeHelper.ONE_SECOND); - } - - public static String formatBytes(long bytes) { - if (bytes < 1024) { - return bytes + " B"; - } else { - int exp = (int) (Math.log(bytes) / Math.log(1024)); - return String.format("%.1f %siB", bytes / Math.pow(1024, exp), - "KMGTPE".charAt(exp-1)); - } - } - - private static long printedLastStatusMessage = -1L; - - public static void printStatus(String message) { - System.out.println(new Date() + ": " + message); - printedLastStatusMessage = currentTimeMillis(); - } - - public static void printStatistics(String component, String message) { - System.out.print(" " + component + " statistics:\n" + message); - } - - public static void printStatusTime(String message) { - printStatusOrErrorTime(message, false); - } - - public static void printErrorTime(String message) { - printStatusOrErrorTime(message, true); - } - - private static void printStatusOrErrorTime(String message, - boolean printToSystemErr) { - long now = currentTimeMillis(); - long millis = printedLastStatusMessage < 0 ? 0 : - now - printedLastStatusMessage; - String line = " " + message + " (" + Logger.formatMillis(millis) - + ")."; - if (printToSystemErr) { - System.err.println(line); - } else { - System.out.println(line); - } - printedLastStatusMessage = now; - } -} - diff --git a/src/org/torproject/onionoo/LookupService.java b/src/org/torproject/onionoo/LookupService.java deleted file mode 100644 index a968bc0..0000000 --- a/src/org/torproject/onionoo/LookupService.java +++ /dev/null @@ -1,408 +0,0 @@ -/* Copyright 2013 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.regex.Pattern; - -class LookupResult { - - private String countryCode; - public void setCountryCode(String countryCode) { - this.countryCode = countryCode; - } - public String getCountryCode() { - return this.countryCode; - } - - private String countryName; - public void setCountryName(String countryName) { - this.countryName = countryName; - } - public String getCountryName() { - return this.countryName; - } - - private String regionName; - public void setRegionName(String regionName) { - this.regionName = regionName; - } - public String getRegionName() { - return this.regionName; - } - - private String cityName; - public void setCityName(String cityName) { - this.cityName = cityName; - } - public String getCityName() { - return this.cityName; - } - - private Float latitude; - public void setLatitude(Float latitude) { - this.latitude = latitude; - } - public Float getLatitude() { - return this.latitude; - } - - private Float longitude; - public void setLongitude(Float longitude) { - this.longitude = longitude; - } - public Float getLongitude() { - return this.longitude; - } - - private String asNumber; - public void setAsNumber(String asNumber) { - this.asNumber = asNumber; - } - public String getAsNumber() { - return this.asNumber; - } - - private String asName; - public void setAsName(String asName) { - this.asName = asName; - } - public String getAsName() { - return this.asName; - } -} - -public class LookupService { - - private File geoipDir; - private File geoLite2CityBlocksCsvFile; - private File geoLite2CityLocationsCsvFile; - private File geoIPASNum2CsvFile; - private boolean hasAllFiles = false; - public LookupService(File geoipDir) { - this.geoipDir = geoipDir; - this.findRequiredCsvFiles(); - } - - /* Make sure we have all required .csv files. */ - private void findRequiredCsvFiles() { - this.geoLite2CityBlocksCsvFile = new File(this.geoipDir, - "GeoLite2-City-Blocks.csv"); - if (!this.geoLite2CityBlocksCsvFile.exists()) { - System.err.println("No GeoLite2-City-Blocks.csv file in geoip/."); - return; - } - this.geoLite2CityLocationsCsvFile = new File(this.geoipDir, - "GeoLite2-City-Locations.csv"); - if (!this.geoLite2CityLocationsCsvFile.exists()) { - System.err.println("No GeoLite2-City-Locations.csv file in " - + "geoip/."); - return; - } - this.geoIPASNum2CsvFile = new File(this.geoipDir, "GeoIPASNum2.csv"); - if (!this.geoIPASNum2CsvFile.exists()) { - System.err.println("No GeoIPASNum2.csv file in geoip/."); - return; - } - this.hasAllFiles = true; - } - - private Pattern ipv4Pattern = Pattern.compile("^[0-9\\.]{7,15}$"); - private long parseAddressString(String addressString) { - long addressNumber = -1L; - if (ipv4Pattern.matcher(addressString).matches()) { - String[] parts = addressString.split("\\.", 4); - if (parts.length == 4) { - addressNumber = 0L; - for (int i = 0; i < 4; i++) { - addressNumber *= 256L; - int octetValue = -1; - try { - octetValue = Integer.parseInt(parts[i]); - } catch (NumberFormatException e) { - } - if (octetValue < 0 || octetValue > 255) { - addressNumber = -1L; - break; - } - addressNumber += octetValue; - } - } - } - return addressNumber; - } - - public SortedMap<String, LookupResult> lookup( - SortedSet<String> addressStrings) { - - SortedMap<String, LookupResult> lookupResults = - new TreeMap<String, LookupResult>(); - - if (!this.hasAllFiles) { - return lookupResults; - } - - /* Obtain a map from relay IP address strings to numbers. */ - Map<String, Long> addressStringNumbers = new HashMap<String, Long>(); - for (String addressString : addressStrings) { - long addressNumber = this.parseAddressString(addressString); - if (addressNumber >= 0L) { - addressStringNumbers.put(addressString, addressNumber); - } - } - if (addressStringNumbers.isEmpty()) { - return lookupResults; - } - - /* Obtain a map from IP address numbers to blocks and to latitudes and - longitudes. */ - Map<Long, Long> addressNumberBlocks = new HashMap<Long, Long>(); - Map<Long, Float[]> addressNumberLatLong = - new HashMap<Long, Float[]>(); - try { - SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>( - addressStringNumbers.values()); - BufferedReader br = new BufferedReader(new InputStreamReader( - new FileInputStream(geoLite2CityBlocksCsvFile), "ISO-8859-1")); - String line = br.readLine(); - while ((line = br.readLine()) != null) { - if (!line.startsWith("::ffff:")) { - /* TODO Make this less hacky and IPv6-ready at some point. */ - continue; - } - String[] parts = line.replaceAll("\"", "").split(",", 10); - if (parts.length != 10) { - System.err.println("Illegal line '" + line + "' in " - + geoLite2CityBlocksCsvFile.getAbsolutePath() + "."); - br.close(); - return lookupResults; - } - try { - String startAddressString = parts[0].substring(7); /* ::ffff: */ - long startIpNum = this.parseAddressString(startAddressString); - if (startIpNum < 0L) { - System.err.println("Illegal IP address in '" + line - + "' in " + geoLite2CityBlocksCsvFile.getAbsolutePath() - + "."); - br.close(); - return lookupResults; - } - int networkMaskLength = Integer.parseInt(parts[1]); - if (networkMaskLength < 96 || networkMaskLength > 128) { - System.err.println("Illegal network mask in '" + line - + "' in " + geoLite2CityBlocksCsvFile.getAbsolutePath() - + "."); - br.close(); - return lookupResults; - } - if (parts[2].length() == 0 && parts[3].length() == 0) { - continue; - } - long endIpNum = startIpNum + (1 << (128 - networkMaskLength)) - - 1; - for (long addressNumber : sortedAddressNumbers. - tailSet(startIpNum).headSet(endIpNum + 1L)) { - String blockString = parts[2].length() > 0 ? parts[2] : - parts[3]; - long blockNumber = Long.parseLong(blockString); - addressNumberBlocks.put(addressNumber, blockNumber); - if (parts[6].length() > 0 && parts[7].length() > 0) { - addressNumberLatLong.put(addressNumber, - new Float[] { Float.parseFloat(parts[6]), - Float.parseFloat(parts[7]) }); - } - } - } catch (NumberFormatException e) { - System.err.println("Number format exception while parsing line " - + "'" + line + "' in " - + geoLite2CityBlocksCsvFile.getAbsolutePath() + "."); - br.close(); - return lookupResults; - } - } - br.close(); - } catch (IOException e) { - System.err.println("I/O exception while reading " - + geoLite2CityBlocksCsvFile.getAbsolutePath() + "."); - return lookupResults; - } - - /* Obtain a map from relevant blocks to location lines. */ - Map<Long, String> blockLocations = new HashMap<Long, String>(); - try { - Set<Long> blockNumbers = new HashSet<Long>( - addressNumberBlocks.values()); - BufferedReader br = new BufferedReader(new InputStreamReader( - new FileInputStream(geoLite2CityLocationsCsvFile), - "ISO-8859-1")); - String line = br.readLine(); - while ((line = br.readLine()) != null) { - String[] parts = line.replaceAll("\"", "").split(",", 10); - if (parts.length != 10) { - System.err.println("Illegal line '" + line + "' in " - + geoLite2CityLocationsCsvFile.getAbsolutePath() + "."); - br.close(); - return lookupResults; - } - try { - long locId = Long.parseLong(parts[0]); - if (blockNumbers.contains(locId)) { - blockLocations.put(locId, line); - } - } catch (NumberFormatException e) { - System.err.println("Number format exception while parsing line " - + "'" + line + "' in " - + geoLite2CityLocationsCsvFile.getAbsolutePath() + "."); - br.close(); - return lookupResults; - } - } - br.close(); - } catch (IOException e) { - System.err.println("I/O exception while reading " - + geoLite2CityLocationsCsvFile.getAbsolutePath() + "."); - return lookupResults; - } - - /* Obtain a map from IP address numbers to ASN. */ - Map<Long, String> addressNumberASN = new HashMap<Long, String>(); - try { - SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>( - addressStringNumbers.values()); - long firstAddressNumber = sortedAddressNumbers.first(); - BufferedReader br = new BufferedReader(new InputStreamReader( - new FileInputStream(geoIPASNum2CsvFile), "ISO-8859-1")); - String line; - long previousStartIpNum = -1L; - while ((line = br.readLine()) != null) { - String[] parts = line.replaceAll("\"", "").split(",", 3); - if (parts.length != 3) { - System.err.println("Illegal line '" + line + "' in " - + geoIPASNum2CsvFile.getAbsolutePath() + "."); - br.close(); - return lookupResults; - } - try { - long startIpNum = Long.parseLong(parts[0]); - if (startIpNum <= previousStartIpNum) { - System.err.println("Line '" + line + "' not sorted in " - + geoIPASNum2CsvFile.getAbsolutePath() + "."); - br.close(); - return lookupResults; - } - previousStartIpNum = startIpNum; - while (firstAddressNumber < startIpNum && - firstAddressNumber != -1L) { - sortedAddressNumbers.remove(firstAddressNumber); - if (sortedAddressNumbers.isEmpty()) { - firstAddressNumber = -1L; - } else { - firstAddressNumber = sortedAddressNumbers.first(); - } - } - long endIpNum = Long.parseLong(parts[1]); - while (firstAddressNumber <= endIpNum && - firstAddressNumber != -1L) { - if (parts[2].startsWith("AS") && - parts[2].split(" ", 2).length == 2) { - addressNumberASN.put(firstAddressNumber, parts[2]); - } - sortedAddressNumbers.remove(firstAddressNumber); - if (sortedAddressNumbers.isEmpty()) { - firstAddressNumber = -1L; - } else { - firstAddressNumber = sortedAddressNumbers.first(); - } - } - if (firstAddressNumber == -1L) { - break; - } - } - catch (NumberFormatException e) { - System.err.println("Number format exception while parsing line " - + "'" + line + "' in " - + geoIPASNum2CsvFile.getAbsolutePath() + "."); - br.close(); - return lookupResults; - } - } - br.close(); - } catch (IOException e) { - System.err.println("I/O exception while reading " - + geoIPASNum2CsvFile.getAbsolutePath() + "."); - return lookupResults; - } - - /* Finally, put together lookup results. */ - for (String addressString : addressStrings) { - if (!addressStringNumbers.containsKey(addressString)) { - continue; - } - long addressNumber = addressStringNumbers.get(addressString); - if (!addressNumberBlocks.containsKey(addressNumber) && - !addressNumberLatLong.containsKey(addressNumber) && - !addressNumberASN.containsKey(addressNumber)) { - continue; - } - LookupResult lookupResult = new LookupResult(); - if (addressNumberBlocks.containsKey(addressNumber)) { - long blockNumber = addressNumberBlocks.get(addressNumber); - if (blockLocations.containsKey(blockNumber)) { - String[] parts = blockLocations.get(blockNumber). - replaceAll("\"", "").split(",", -1); - lookupResult.setCountryCode(parts[3].toLowerCase()); - if (parts[4].length() > 0) { - lookupResult.setCountryName(parts[4]); - } - if (parts[6].length() > 0) { - lookupResult.setRegionName(parts[6]); - } - if (parts[7].length() > 0) { - lookupResult.setCityName(parts[7]); - } - } - } - if (addressNumberLatLong.containsKey(addressNumber)) { - Float[] latLong = addressNumberLatLong.get(addressNumber); - lookupResult.setLatitude(latLong[0]); - lookupResult.setLongitude(latLong[1]); - } - if (addressNumberASN.containsKey(addressNumber)) { - String[] parts = addressNumberASN.get(addressNumber).split(" ", - 2); - lookupResult.setAsNumber(parts[0]); - lookupResult.setAsName(parts[1]); - } - lookupResults.put(addressString, lookupResult); - } - - /* Keep statistics. */ - this.addressesLookedUp += addressStrings.size(); - this.addressesResolved += lookupResults.size(); - - return lookupResults; - } - - private int addressesLookedUp = 0, addressesResolved = 0; - - public String getStatsString() { - StringBuilder sb = new StringBuilder(); - sb.append(" " + Logger.formatDecimalNumber(addressesLookedUp) - + " addresses looked up\n"); - sb.append(" " + Logger.formatDecimalNumber(addressesResolved) - + " addresses resolved\n"); - return sb.toString(); - } -} diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java deleted file mode 100644 index e9afe2f..0000000 --- a/src/org/torproject/onionoo/Main.java +++ /dev/null @@ -1,119 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.io.File; - -/* Update search data and status data files. */ -public class Main { - - private Main() { - } - - public static void main(String[] args) { - - LockFile lf = new LockFile(); - Logger.setTime(); - Logger.printStatus("Initializing."); - if (lf.acquireLock()) { - Logger.printStatusTime("Acquired lock"); - } else { - Logger.printErrorTime("Could not acquire lock. Is Onionoo " - + "already running? Terminating"); - return; - } - - DescriptorSource dso = ApplicationFactory.getDescriptorSource(); - Logger.printStatusTime("Initialized descriptor source"); - DocumentStore ds = ApplicationFactory.getDocumentStore(); - Logger.printStatusTime("Initialized document store"); - LookupService ls = new LookupService(new File("geoip")); - Logger.printStatusTime("Initialized Geoip lookup service"); - ReverseDomainNameResolver rdnr = new ReverseDomainNameResolver(); - Logger.printStatusTime("Initialized reverse domain name resolver"); - NodeDetailsStatusUpdater ndsu = new NodeDetailsStatusUpdater(rdnr, - ls); - Logger.printStatusTime("Initialized node data writer"); - BandwidthStatusUpdater bsu = new BandwidthStatusUpdater(); - Logger.printStatusTime("Initialized bandwidth status updater"); - WeightsStatusUpdater wsu = new WeightsStatusUpdater(); - Logger.printStatusTime("Initialized weights status updater"); - ClientsStatusUpdater csu = new ClientsStatusUpdater(); - Logger.printStatusTime("Initialized clients status updater"); - UptimeStatusUpdater usu = new UptimeStatusUpdater(); - Logger.printStatusTime("Initialized uptime status updater"); - StatusUpdater[] sus = new StatusUpdater[] { ndsu, bsu, wsu, csu, - usu }; - - SummaryDocumentWriter sdw = new SummaryDocumentWriter(); - Logger.printStatusTime("Initialized summary document writer"); - DetailsDocumentWriter ddw = new DetailsDocumentWriter(); - Logger.printStatusTime("Initialized details document writer"); - BandwidthDocumentWriter bdw = new BandwidthDocumentWriter(); - Logger.printStatusTime("Initialized bandwidth document writer"); - WeightsDocumentWriter wdw = new WeightsDocumentWriter(); - Logger.printStatusTime("Initialized weights document writer"); - ClientsDocumentWriter cdw = new ClientsDocumentWriter(); - Logger.printStatusTime("Initialized clients document writer"); - UptimeDocumentWriter udw = new UptimeDocumentWriter(); - Logger.printStatusTime("Initialized uptime document writer"); - DocumentWriter[] dws = new DocumentWriter[] { sdw, ddw, bdw, wdw, cdw, - udw }; - - Logger.printStatus("Downloading descriptors."); - dso.downloadDescriptors(); - - Logger.printStatus("Reading descriptors."); - dso.readDescriptors(); - - Logger.printStatus("Updating internal status files."); - for (StatusUpdater su : sus) { - su.updateStatuses(); - Logger.printStatusTime(su.getClass().getSimpleName() - + " updated status files"); - } - - Logger.printStatus("Updating document files."); - for (DocumentWriter dw : dws) { - dw.writeDocuments(); - } - - Logger.printStatus("Shutting down."); - dso.writeHistoryFiles(); - Logger.printStatusTime("Wrote parse histories"); - ds.flushDocumentCache(); - Logger.printStatusTime("Flushed document cache"); - - Logger.printStatus("Gathering statistics."); - for (StatusUpdater su : sus) { - String statsString = su.getStatsString(); - if (statsString != null) { - Logger.printStatistics(su.getClass().getSimpleName(), - statsString); - } - } - for (DocumentWriter dw : dws) { - String statsString = dw.getStatsString(); - if (statsString != null) { - Logger.printStatistics(dw.getClass().getSimpleName(), - statsString); - } - } - Logger.printStatistics("Descriptor source", dso.getStatsString()); - Logger.printStatistics("Document store", ds.getStatsString()); - Logger.printStatistics("GeoIP lookup service", ls.getStatsString()); - Logger.printStatistics("Reverse domain name resolver", - rdnr.getStatsString()); - - Logger.printStatus("Releasing lock."); - if (lf.releaseLock()) { - Logger.printStatusTime("Released lock"); - } else { - Logger.printErrorTime("Could not release lock. The next " - + "execution may not start as expected"); - } - - Logger.printStatus("Terminating."); - } -} - diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java deleted file mode 100644 index a1149a5..0000000 --- a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java +++ /dev/null @@ -1,620 +0,0 @@ -/* Copyright 2011--2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.torproject.descriptor.BridgeNetworkStatus; -import org.torproject.descriptor.BridgePoolAssignment; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.ExitList; -import org.torproject.descriptor.ExitListEntry; -import org.torproject.descriptor.NetworkStatusEntry; -import org.torproject.descriptor.RelayNetworkStatusConsensus; -import org.torproject.descriptor.ServerDescriptor; - -public class NodeDetailsStatusUpdater implements DescriptorListener, - StatusUpdater { - - private DescriptorSource descriptorSource; - - private ReverseDomainNameResolver reverseDomainNameResolver; - - private LookupService lookupService; - - private DocumentStore documentStore; - - private long now; - - private SortedMap<String, NodeStatus> knownNodes = - new TreeMap<String, NodeStatus>(); - - private SortedMap<String, NodeStatus> relays; - - private SortedMap<String, NodeStatus> bridges; - - private long relaysLastValidAfterMillis = -1L; - - private long bridgesLastPublishedMillis = -1L; - - private SortedMap<String, Integer> lastBandwidthWeights = null; - - private int relayConsensusesProcessed = 0, bridgeStatusesProcessed = 0; - - public NodeDetailsStatusUpdater( - ReverseDomainNameResolver reverseDomainNameResolver, - LookupService lookupService) { - this.descriptorSource = ApplicationFactory.getDescriptorSource(); - this.reverseDomainNameResolver = reverseDomainNameResolver; - this.lookupService = lookupService; - this.documentStore = ApplicationFactory.getDocumentStore(); - this.now = ApplicationFactory.getTime().currentTimeMillis(); - this.registerDescriptorListeners(); - } - - private void registerDescriptorListeners() { - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_STATUSES); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_SERVER_DESCRIPTORS); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_POOL_ASSIGNMENTS); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.EXIT_LISTS); - } - - public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof ServerDescriptor && relay) { - this.processRelayServerDescriptor((ServerDescriptor) descriptor); - } else if (descriptor instanceof ExitList) { - this.processExitList((ExitList) descriptor); - } else if (descriptor instanceof RelayNetworkStatusConsensus) { - this.processRelayNetworkStatusConsensus( - (RelayNetworkStatusConsensus) descriptor); - } else if (descriptor instanceof ServerDescriptor && !relay) { - this.processBridgeServerDescriptor((ServerDescriptor) descriptor); - } else if (descriptor instanceof BridgePoolAssignment) { - this.processBridgePoolAssignment((BridgePoolAssignment) descriptor); - } else if (descriptor instanceof BridgeNetworkStatus) { - this.processBridgeNetworkStatus((BridgeNetworkStatus) descriptor); - } - } - - private void processRelayServerDescriptor( - ServerDescriptor descriptor) { - String fingerprint = descriptor.getFingerprint(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - String publishedDateTime = - DateTimeHelper.format(descriptor.getPublishedMillis()); - if (detailsStatus == null) { - detailsStatus = new DetailsStatus(); - } else if (detailsStatus.getDescPublished() != null && - publishedDateTime.compareTo( - detailsStatus.getDescPublished()) < 0) { - return; - } - String lastRestartedString = DateTimeHelper.format( - descriptor.getPublishedMillis() - descriptor.getUptime() - * DateTimeHelper.ONE_SECOND); - int bandwidthRate = descriptor.getBandwidthRate(); - int bandwidthBurst = descriptor.getBandwidthBurst(); - int observedBandwidth = descriptor.getBandwidthObserved(); - int advertisedBandwidth = Math.min(bandwidthRate, - Math.min(bandwidthBurst, observedBandwidth)); - detailsStatus.setDescPublished(publishedDateTime); - detailsStatus.setLastRestarted(lastRestartedString); - detailsStatus.setBandwidthRate(bandwidthRate); - detailsStatus.setBandwidthBurst(bandwidthBurst); - detailsStatus.setObservedBandwidth(observedBandwidth); - detailsStatus.setAdvertisedBandwidth(advertisedBandwidth); - detailsStatus.setExitPolicy(descriptor.getExitPolicyLines()); - detailsStatus.setContact(descriptor.getContact()); - detailsStatus.setPlatform(descriptor.getPlatform()); - detailsStatus.setFamily(descriptor.getFamilyEntries()); - if (descriptor.getIpv6DefaultPolicy() != null && - (descriptor.getIpv6DefaultPolicy().equals("accept") || - descriptor.getIpv6DefaultPolicy().equals("reject")) && - descriptor.getIpv6PortList() != null) { - Map<String, List<String>> exitPolicyV6Summary = - new HashMap<String, List<String>>(); - List<String> portsOrPortRanges = Arrays.asList( - descriptor.getIpv6PortList().split(",")); - exitPolicyV6Summary.put(descriptor.getIpv6DefaultPolicy(), - portsOrPortRanges); - detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary); - } - if (descriptor.isHibernating()) { - detailsStatus.setHibernating(true); - } - this.documentStore.store(detailsStatus, fingerprint); - } - - private Map<String, Map<String, Long>> exitListEntries = - new HashMap<String, Map<String, Long>>(); - - private void processExitList(ExitList exitList) { - for (ExitListEntry exitListEntry : exitList.getExitListEntries()) { - String fingerprint = exitListEntry.getFingerprint(); - if (exitListEntry.getScanMillis() < - this.now - DateTimeHelper.ONE_DAY) { - continue; - } - if (!this.exitListEntries.containsKey(fingerprint)) { - this.exitListEntries.put(fingerprint, - new HashMap<String, Long>()); - } - String exitAddress = exitListEntry.getExitAddress(); - long scanMillis = exitListEntry.getScanMillis(); - if (!this.exitListEntries.get(fingerprint).containsKey(exitAddress) - || this.exitListEntries.get(fingerprint).get(exitAddress) - < scanMillis) { - this.exitListEntries.get(fingerprint).put(exitAddress, - scanMillis); - } - } - } - - private void processRelayNetworkStatusConsensus( - RelayNetworkStatusConsensus consensus) { - long validAfterMillis = consensus.getValidAfterMillis(); - if (validAfterMillis > this.relaysLastValidAfterMillis) { - this.relaysLastValidAfterMillis = validAfterMillis; - } - Set<String> recommendedVersions = null; - if (consensus.getRecommendedServerVersions() != null) { - recommendedVersions = new HashSet<String>(); - for (String recommendedVersion : - consensus.getRecommendedServerVersions()) { - recommendedVersions.add("Tor " + recommendedVersion); - } - } - for (NetworkStatusEntry entry : - consensus.getStatusEntries().values()) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); - String address = entry.getAddress(); - SortedSet<String> orAddressesAndPorts = new TreeSet<String>( - entry.getOrAddresses()); - int orPort = entry.getOrPort(); - int dirPort = entry.getDirPort(); - SortedSet<String> relayFlags = entry.getFlags(); - long consensusWeight = entry.getBandwidth(); - String defaultPolicy = entry.getDefaultPolicy(); - String portList = entry.getPortList(); - Boolean recommendedVersion = (recommendedVersions == null || - entry.getVersion() == null) ? null : - recommendedVersions.contains(entry.getVersion()); - NodeStatus newNodeStatus = new NodeStatus(true, nickname, - fingerprint, address, orAddressesAndPorts, null, - validAfterMillis, orPort, dirPort, relayFlags, consensusWeight, - null, null, -1L, defaultPolicy, portList, validAfterMillis, - validAfterMillis, null, null, recommendedVersion, null); - if (this.knownNodes.containsKey(fingerprint)) { - this.knownNodes.get(fingerprint).update(newNodeStatus); - } else { - this.knownNodes.put(fingerprint, newNodeStatus); - } - } - this.relayConsensusesProcessed++; - if (this.relaysLastValidAfterMillis == validAfterMillis) { - this.lastBandwidthWeights = consensus.getBandwidthWeights(); - } - } - - private void processBridgeServerDescriptor( - ServerDescriptor descriptor) { - String fingerprint = descriptor.getFingerprint(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - String publishedDateTime = - DateTimeHelper.format(descriptor.getPublishedMillis()); - if (detailsStatus == null) { - detailsStatus = new DetailsStatus(); - } else if (detailsStatus.getDescPublished() != null && - publishedDateTime.compareTo( - detailsStatus.getDescPublished()) < 0) { - return; - } - String lastRestartedString = DateTimeHelper.format( - descriptor.getPublishedMillis() - descriptor.getUptime() - * DateTimeHelper.ONE_SECOND); - int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(), - Math.min(descriptor.getBandwidthBurst(), - descriptor.getBandwidthObserved())); - detailsStatus.setDescPublished(publishedDateTime); - detailsStatus.setLastRestarted(lastRestartedString); - detailsStatus.setAdvertisedBandwidth(advertisedBandwidth); - detailsStatus.setPlatform(descriptor.getPlatform()); - this.documentStore.store(detailsStatus, fingerprint); - } - - private void processBridgePoolAssignment( - BridgePoolAssignment bridgePoolAssignment) { - for (Map.Entry<String, String> e : - bridgePoolAssignment.getEntries().entrySet()) { - String fingerprint = e.getKey(); - String details = e.getValue(); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus == null) { - detailsStatus = new DetailsStatus(); - } else if (details.equals(detailsStatus.getPoolAssignment())) { - continue; - } - detailsStatus.setPoolAssignment(details); - this.documentStore.store(detailsStatus, fingerprint); - } - } - - private void processBridgeNetworkStatus(BridgeNetworkStatus status) { - long publishedMillis = status.getPublishedMillis(); - if (publishedMillis > this.bridgesLastPublishedMillis) { - this.bridgesLastPublishedMillis = publishedMillis; - } - for (NetworkStatusEntry entry : status.getStatusEntries().values()) { - String nickname = entry.getNickname(); - String fingerprint = entry.getFingerprint(); - String address = entry.getAddress(); - SortedSet<String> orAddressesAndPorts = new TreeSet<String>( - entry.getOrAddresses()); - int orPort = entry.getOrPort(); - int dirPort = entry.getDirPort(); - SortedSet<String> relayFlags = entry.getFlags(); - NodeStatus newNodeStatus = new NodeStatus(false, nickname, - fingerprint, address, orAddressesAndPorts, null, - publishedMillis, orPort, dirPort, relayFlags, -1L, "??", null, - -1L, null, null, publishedMillis, -1L, null, null, null, null); - if (this.knownNodes.containsKey(fingerprint)) { - this.knownNodes.get(fingerprint).update(newNodeStatus); - } else { - this.knownNodes.put(fingerprint, newNodeStatus); - } - } - this.bridgeStatusesProcessed++; - } - - public void updateStatuses() { - this.readStatusSummary(); - Logger.printStatusTime("Read status summary"); - this.setCurrentNodes(); - Logger.printStatusTime("Set current node fingerprints"); - this.startReverseDomainNameLookups(); - Logger.printStatusTime("Started reverse domain name lookups"); - this.lookUpCitiesAndASes(); - Logger.printStatusTime("Looked up cities and ASes"); - this.setDescriptorPartsOfNodeStatus(); - Logger.printStatusTime("Set descriptor parts of node statuses."); - this.calculatePathSelectionProbabilities(); - Logger.printStatusTime("Calculated path selection probabilities"); - this.finishReverseDomainNameLookups(); - Logger.printStatusTime("Finished reverse domain name lookups"); - this.writeStatusSummary(); - Logger.printStatusTime("Wrote status summary"); - this.updateDetailsStatuses(); - Logger.printStatusTime("Updated exit addresses in details statuses"); - } - - private void readStatusSummary() { - SortedSet<String> fingerprints = this.documentStore.list( - NodeStatus.class); - for (String fingerprint : fingerprints) { - NodeStatus node = this.documentStore.retrieve(NodeStatus.class, - true, fingerprint); - if (node.isRelay()) { - this.relaysLastValidAfterMillis = Math.max( - this.relaysLastValidAfterMillis, node.getLastSeenMillis()); - } else { - this.bridgesLastPublishedMillis = Math.max( - this.bridgesLastPublishedMillis, node.getLastSeenMillis()); - } - if (this.knownNodes.containsKey(fingerprint)) { - this.knownNodes.get(fingerprint).update(node); - } else { - this.knownNodes.put(fingerprint, node); - } - } - } - - private void setCurrentNodes() { - long cutoff = Math.max(this.relaysLastValidAfterMillis, - this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L; - SortedMap<String, NodeStatus> currentNodes = - new TreeMap<String, NodeStatus>(); - for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { - if (e.getValue().getLastSeenMillis() >= cutoff) { - currentNodes.put(e.getKey(), e.getValue()); - } - } - this.relays = new TreeMap<String, NodeStatus>(); - this.bridges = new TreeMap<String, NodeStatus>(); - for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) { - if (e.getValue().isRelay()) { - this.relays.put(e.getKey(), e.getValue()); - } else { - this.bridges.put(e.getKey(), e.getValue()); - } - } - } - - private void startReverseDomainNameLookups() { - Map<String, Long> addressLastLookupTimes = - new HashMap<String, Long>(); - for (NodeStatus relay : relays.values()) { - addressLastLookupTimes.put(relay.getAddress(), - relay.getLastRdnsLookup()); - } - this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes); - this.reverseDomainNameResolver.startReverseDomainNameLookups(); - } - - private void lookUpCitiesAndASes() { - SortedSet<String> addressStrings = new TreeSet<String>(); - for (NodeStatus node : this.knownNodes.values()) { - if (node.isRelay()) { - addressStrings.add(node.getAddress()); - } - } - if (addressStrings.isEmpty()) { - System.err.println("No relay IP addresses to resolve to cities or " - + "ASN."); - return; - } - SortedMap<String, LookupResult> lookupResults = - this.lookupService.lookup(addressStrings); - for (NodeStatus node : knownNodes.values()) { - if (!node.isRelay()) { - continue; - } - String addressString = node.getAddress(); - if (lookupResults.containsKey(addressString)) { - LookupResult lookupResult = lookupResults.get(addressString); - node.setCountryCode(lookupResult.getCountryCode()); - node.setCountryName(lookupResult.getCountryName()); - node.setRegionName(lookupResult.getRegionName()); - node.setCityName(lookupResult.getCityName()); - node.setLatitude(lookupResult.getLatitude()); - node.setLongitude(lookupResult.getLongitude()); - node.setASNumber(lookupResult.getAsNumber()); - node.setASName(lookupResult.getAsName()); - } - } - } - - private void setDescriptorPartsOfNodeStatus() { - for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus node = e.getValue(); - if (node.isRelay()) { - if (node.getRelayFlags().contains("Running") && - node.getLastSeenMillis() == this.relaysLastValidAfterMillis) { - node.setRunning(true); - } - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus != null) { - node.setContact(detailsStatus.getContact()); - if (detailsStatus.getExitAddresses() != null) { - for (Map.Entry<String, Long> ea : - detailsStatus.getExitAddresses().entrySet()) { - if (ea.getValue() >= this.now - DateTimeHelper.ONE_DAY) { - node.addExitAddress(ea.getKey()); - } - } - } - if (detailsStatus.getFamily() != null && - !detailsStatus.getFamily().isEmpty()) { - SortedSet<String> familyFingerprints = new TreeSet<String>(); - for (String familyMember : detailsStatus.getFamily()) { - if (familyMember.startsWith("$") && - familyMember.length() == 41) { - familyFingerprints.add(familyMember.substring(1)); - } - } - if (!familyFingerprints.isEmpty()) { - node.setFamilyFingerprints(familyFingerprints); - } - } - } - } - if (!node.isRelay() && node.getRelayFlags().contains("Running") && - node.getLastSeenMillis() == this.bridgesLastPublishedMillis) { - node.setRunning(true); - } - } - } - - private void calculatePathSelectionProbabilities() { - boolean consensusContainsBandwidthWeights = false; - double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0, - wmd = 0.0, wee = 0.0, wed = 0.0; - if (this.lastBandwidthWeights != null) { - SortedSet<String> weightKeys = new TreeSet<String>(Arrays.asList( - "Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(","))); - weightKeys.removeAll(this.lastBandwidthWeights.keySet()); - if (weightKeys.isEmpty()) { - consensusContainsBandwidthWeights = true; - wgg = ((double) this.lastBandwidthWeights.get("Wgg")) / 10000.0; - wgd = ((double) this.lastBandwidthWeights.get("Wgd")) / 10000.0; - wmg = ((double) this.lastBandwidthWeights.get("Wmg")) / 10000.0; - wmm = ((double) this.lastBandwidthWeights.get("Wmm")) / 10000.0; - wme = ((double) this.lastBandwidthWeights.get("Wme")) / 10000.0; - wmd = ((double) this.lastBandwidthWeights.get("Wmd")) / 10000.0; - wee = ((double) this.lastBandwidthWeights.get("Wee")) / 10000.0; - wed = ((double) this.lastBandwidthWeights.get("Wed")) / 10000.0; - } - } else { - System.err.println("Could not determine most recent Wxx parameter " - + "values, probably because we didn't parse a consensus in " - + "this execution. All relays' guard/middle/exit weights are " - + "going to be 0.0."); - } - SortedMap<String, Double> - advertisedBandwidths = new TreeMap<String, Double>(), - consensusWeights = new TreeMap<String, Double>(), - guardWeights = new TreeMap<String, Double>(), - middleWeights = new TreeMap<String, Double>(), - exitWeights = new TreeMap<String, Double>(); - double totalAdvertisedBandwidth = 0.0; - double totalConsensusWeight = 0.0; - double totalGuardWeight = 0.0; - double totalMiddleWeight = 0.0; - double totalExitWeight = 0.0; - for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus relay = e.getValue(); - if (!relay.getRunning()) { - continue; - } - boolean isExit = relay.getRelayFlags().contains("Exit") && - !relay.getRelayFlags().contains("BadExit"); - boolean isGuard = relay.getRelayFlags().contains("Guard"); - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus != null) { - double advertisedBandwidth = - detailsStatus.getAdvertisedBandwidth(); - if (advertisedBandwidth >= 0.0) { - advertisedBandwidths.put(fingerprint, advertisedBandwidth); - totalAdvertisedBandwidth += advertisedBandwidth; - } - } - double consensusWeight = (double) relay.getConsensusWeight(); - consensusWeights.put(fingerprint, consensusWeight); - totalConsensusWeight += consensusWeight; - if (consensusContainsBandwidthWeights) { - double guardWeight = consensusWeight, - middleWeight = consensusWeight, - exitWeight = consensusWeight; - if (isGuard && isExit) { - guardWeight *= wgd; - middleWeight *= wmd; - exitWeight *= wed; - } else if (isGuard) { - guardWeight *= wgg; - middleWeight *= wmg; - exitWeight = 0.0; - } else if (isExit) { - guardWeight = 0.0; - middleWeight *= wme; - exitWeight *= wee; - } else { - guardWeight = 0.0; - middleWeight *= wmm; - exitWeight = 0.0; - } - guardWeights.put(fingerprint, guardWeight); - middleWeights.put(fingerprint, middleWeight); - exitWeights.put(fingerprint, exitWeight); - totalGuardWeight += guardWeight; - totalMiddleWeight += middleWeight; - totalExitWeight += exitWeight; - } - } - for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { - String fingerprint = e.getKey(); - NodeStatus relay = e.getValue(); - if (advertisedBandwidths.containsKey(fingerprint)) { - relay.setAdvertisedBandwidthFraction(advertisedBandwidths.get( - fingerprint) / totalAdvertisedBandwidth); - } - if (consensusWeights.containsKey(fingerprint)) { - relay.setConsensusWeightFraction(consensusWeights.get(fingerprint) - / totalConsensusWeight); - } - if (guardWeights.containsKey(fingerprint)) { - relay.setGuardProbability(guardWeights.get(fingerprint) - / totalGuardWeight); - } - if (middleWeights.containsKey(fingerprint)) { - relay.setMiddleProbability(middleWeights.get(fingerprint) - / totalMiddleWeight); - } - if (exitWeights.containsKey(fingerprint)) { - relay.setExitProbability(exitWeights.get(fingerprint) - / totalExitWeight); - } - } - } - - private void finishReverseDomainNameLookups() { - this.reverseDomainNameResolver.finishReverseDomainNameLookups(); - Map<String, String> lookupResults = - this.reverseDomainNameResolver.getLookupResults(); - long startedRdnsLookups = - this.reverseDomainNameResolver.getLookupStartMillis(); - for (NodeStatus relay : relays.values()) { - if (lookupResults.containsKey(relay.getAddress())) { - relay.setHostName(lookupResults.get(relay.getAddress())); - relay.setLastRdnsLookup(startedRdnsLookups); - } - } - } - - private void writeStatusSummary() { - for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { - this.documentStore.store(e.getValue(), e.getKey()); - } - } - - private void updateDetailsStatuses() { - SortedSet<String> fingerprints = new TreeSet<String>(); - fingerprints.addAll(this.exitListEntries.keySet()); - for (String fingerprint : fingerprints) { - DetailsStatus detailsStatus = this.documentStore.retrieve( - DetailsStatus.class, true, fingerprint); - if (detailsStatus == null) { - detailsStatus = new DetailsStatus(); - } - Map<String, Long> exitAddresses = new HashMap<String, Long>(); - if (detailsStatus.getExitAddresses() != null) { - for (Map.Entry<String, Long> e : - detailsStatus.getExitAddresses().entrySet()) { - if (e.getValue() >= this.now - DateTimeHelper.ONE_DAY) { - exitAddresses.put(e.getKey(), e.getValue()); - } - } - } - if (this.exitListEntries.containsKey(fingerprint)) { - for (Map.Entry<String, Long> e : - this.exitListEntries.get(fingerprint).entrySet()) { - if (!exitAddresses.containsKey(e.getKey()) || - exitAddresses.get(e.getKey()) < e.getValue()) { - exitAddresses.put(e.getKey(), e.getValue()); - } - } - } - if (this.knownNodes.containsKey(fingerprint)) { - for (String orAddress : - this.knownNodes.get(fingerprint).getOrAddresses()) { - this.exitListEntries.remove(orAddress); - } - } - detailsStatus.setExitAddresses(exitAddresses); - this.documentStore.store(detailsStatus, fingerprint); - } - } - - public String getStatsString() { - StringBuilder sb = new StringBuilder(); - sb.append(" " + Logger.formatDecimalNumber( - relayConsensusesProcessed) + " relay consensuses processed\n"); - sb.append(" " + Logger.formatDecimalNumber(bridgeStatusesProcessed) - + " bridge statuses processed\n"); - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/NodeIndexer.java b/src/org/torproject/onionoo/NodeIndexer.java deleted file mode 100644 index d24284e..0000000 --- a/src/org/torproject/onionoo/NodeIndexer.java +++ /dev/null @@ -1,425 +0,0 @@ -package org.torproject.onionoo; - -import java.io.File; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; - -import javax.servlet.ServletContext; -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; - -class NodeIndex { - - private String relaysPublishedString; - public void setRelaysPublishedString(String relaysPublishedString) { - this.relaysPublishedString = relaysPublishedString; - } - public String getRelaysPublishedString() { - return relaysPublishedString; - } - - private String bridgesPublishedString; - public void setBridgesPublishedString(String bridgesPublishedString) { - this.bridgesPublishedString = bridgesPublishedString; - } - public String getBridgesPublishedString() { - return bridgesPublishedString; - } - - private List<String> relaysByConsensusWeight; - public void setRelaysByConsensusWeight( - List<String> relaysByConsensusWeight) { - this.relaysByConsensusWeight = relaysByConsensusWeight; - } - public List<String> getRelaysByConsensusWeight() { - return relaysByConsensusWeight; - } - - - private Map<String, SummaryDocument> relayFingerprintSummaryLines; - public void setRelayFingerprintSummaryLines( - Map<String, SummaryDocument> relayFingerprintSummaryLines) { - this.relayFingerprintSummaryLines = relayFingerprintSummaryLines; - } - public Map<String, SummaryDocument> getRelayFingerprintSummaryLines() { - return this.relayFingerprintSummaryLines; - } - - private Map<String, SummaryDocument> bridgeFingerprintSummaryLines; - public void setBridgeFingerprintSummaryLines( - Map<String, SummaryDocument> bridgeFingerprintSummaryLines) { - this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines; - } - public Map<String, SummaryDocument> getBridgeFingerprintSummaryLines() { - return this.bridgeFingerprintSummaryLines; - } - - private Map<String, Set<String>> relaysByCountryCode = null; - public void setRelaysByCountryCode( - Map<String, Set<String>> relaysByCountryCode) { - this.relaysByCountryCode = relaysByCountryCode; - } - public Map<String, Set<String>> getRelaysByCountryCode() { - return relaysByCountryCode; - } - - private Map<String, Set<String>> relaysByASNumber = null; - public void setRelaysByASNumber( - Map<String, Set<String>> relaysByASNumber) { - this.relaysByASNumber = relaysByASNumber; - } - public Map<String, Set<String>> getRelaysByASNumber() { - return relaysByASNumber; - } - - private Map<String, Set<String>> relaysByFlag = null; - public void setRelaysByFlag(Map<String, Set<String>> relaysByFlag) { - this.relaysByFlag = relaysByFlag; - } - public Map<String, Set<String>> getRelaysByFlag() { - return relaysByFlag; - } - - private Map<String, Set<String>> bridgesByFlag = null; - public void setBridgesByFlag(Map<String, Set<String>> bridgesByFlag) { - this.bridgesByFlag = bridgesByFlag; - } - public Map<String, Set<String>> getBridgesByFlag() { - return bridgesByFlag; - } - - private Map<String, Set<String>> relaysByContact = null; - public void setRelaysByContact( - Map<String, Set<String>> relaysByContact) { - this.relaysByContact = relaysByContact; - } - public Map<String, Set<String>> getRelaysByContact() { - return relaysByContact; - } - - private Map<String, Set<String>> relaysByFamily = null; - public void setRelaysByFamily(Map<String, Set<String>> relaysByFamily) { - this.relaysByFamily = relaysByFamily; - } - public Map<String, Set<String>> getRelaysByFamily() { - return this.relaysByFamily; - } - - private SortedMap<Integer, Set<String>> relaysByFirstSeenDays; - public void setRelaysByFirstSeenDays( - SortedMap<Integer, Set<String>> relaysByFirstSeenDays) { - this.relaysByFirstSeenDays = relaysByFirstSeenDays; - } - public SortedMap<Integer, Set<String>> getRelaysByFirstSeenDays() { - return relaysByFirstSeenDays; - } - - private SortedMap<Integer, Set<String>> bridgesByFirstSeenDays; - public void setBridgesByFirstSeenDays( - SortedMap<Integer, Set<String>> bridgesByFirstSeenDays) { - this.bridgesByFirstSeenDays = bridgesByFirstSeenDays; - } - public SortedMap<Integer, Set<String>> getBridgesByFirstSeenDays() { - return bridgesByFirstSeenDays; - } - - private SortedMap<Integer, Set<String>> relaysByLastSeenDays; - public void setRelaysByLastSeenDays( - SortedMap<Integer, Set<String>> relaysByLastSeenDays) { - this.relaysByLastSeenDays = relaysByLastSeenDays; - } - public SortedMap<Integer, Set<String>> getRelaysByLastSeenDays() { - return relaysByLastSeenDays; - } - - private SortedMap<Integer, Set<String>> bridgesByLastSeenDays; - public void setBridgesByLastSeenDays( - SortedMap<Integer, Set<String>> bridgesByLastSeenDays) { - this.bridgesByLastSeenDays = bridgesByLastSeenDays; - } - public SortedMap<Integer, Set<String>> getBridgesByLastSeenDays() { - return bridgesByLastSeenDays; - } -} - -public class NodeIndexer implements ServletContextListener, Runnable { - - public void contextInitialized(ServletContextEvent contextEvent) { - ServletContext servletContext = contextEvent.getServletContext(); - File outDir = new File(servletContext.getInitParameter("outDir")); - DocumentStore documentStore = ApplicationFactory.getDocumentStore(); - documentStore.setOutDir(outDir); - /* The servlet container created us, and we need to avoid that - * ApplicationFactory creates another instance of us. */ - ApplicationFactory.setNodeIndexer(this); - this.startIndexing(); - } - - public void contextDestroyed(ServletContextEvent contextEvent) { - this.stopIndexing(); - } - - private long lastIndexed = -1L; - - private NodeIndex latestNodeIndex = null; - - private Thread nodeIndexerThread = null; - - public synchronized long getLastIndexed(long timeoutMillis) { - if (this.lastIndexed == -1L && this.nodeIndexerThread != null && - timeoutMillis > 0L) { - try { - this.wait(timeoutMillis); - } catch (InterruptedException e) { - } - } - return this.lastIndexed; - } - - public synchronized NodeIndex getLatestNodeIndex(long timeoutMillis) { - if (this.latestNodeIndex == null && this.nodeIndexerThread != null && - timeoutMillis > 0L) { - try { - this.wait(timeoutMillis); - } catch (InterruptedException e) { - } - } - return this.latestNodeIndex; - } - - public synchronized void startIndexing() { - if (this.nodeIndexerThread == null) { - this.nodeIndexerThread = new Thread(this); - this.nodeIndexerThread.setDaemon(true); - this.nodeIndexerThread.start(); - } - } - - public void run() { - while (this.nodeIndexerThread != null) { - this.indexNodeStatuses(); - try { - Thread.sleep(DateTimeHelper.ONE_MINUTE); - } catch (InterruptedException e) { - } - } - } - - public synchronized void stopIndexing() { - Thread indexerThread = this.nodeIndexerThread; - this.nodeIndexerThread = null; - indexerThread.interrupt(); - } - - private void indexNodeStatuses() { - long updateStatusMillis = -1L; - DocumentStore documentStore = ApplicationFactory.getDocumentStore(); - UpdateStatus updateStatus = documentStore.retrieve(UpdateStatus.class, - false); - if (updateStatus != null && - updateStatus.getDocumentString() != null) { - String updateString = updateStatus.getDocumentString(); - try { - updateStatusMillis = Long.parseLong(updateString.trim()); - } catch (NumberFormatException e) { - /* Handle below. */ - } - } - synchronized (this) { - if (updateStatusMillis <= this.lastIndexed) { - return; - } - } - List<String> newRelaysByConsensusWeight = new ArrayList<String>(); - Map<String, SummaryDocument> - newRelayFingerprintSummaryLines = - new HashMap<String, SummaryDocument>(), - newBridgeFingerprintSummaryLines = - new HashMap<String, SummaryDocument>(); - Map<String, Set<String>> - newRelaysByCountryCode = new HashMap<String, Set<String>>(), - newRelaysByASNumber = new HashMap<String, Set<String>>(), - newRelaysByFlag = new HashMap<String, Set<String>>(), - newBridgesByFlag = new HashMap<String, Set<String>>(), - newRelaysByContact = new HashMap<String, Set<String>>(), - newRelaysByFamily = new HashMap<String, Set<String>>(); - SortedMap<Integer, Set<String>> - newRelaysByFirstSeenDays = new TreeMap<Integer, Set<String>>(), - newBridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(), - newRelaysByLastSeenDays = new TreeMap<Integer, Set<String>>(), - newBridgesByLastSeenDays = new TreeMap<Integer, Set<String>>(); - Set<SummaryDocument> currentRelays = new HashSet<SummaryDocument>(), - currentBridges = new HashSet<SummaryDocument>(); - SortedSet<String> fingerprints = documentStore.list( - SummaryDocument.class); - long relaysLastValidAfterMillis = 0L, bridgesLastPublishedMillis = 0L; - for (String fingerprint : fingerprints) { - SummaryDocument node = documentStore.retrieve(SummaryDocument.class, - true, fingerprint); - if (node.isRelay()) { - relaysLastValidAfterMillis = Math.max( - relaysLastValidAfterMillis, node.getLastSeenMillis()); - currentRelays.add(node); - } else { - bridgesLastPublishedMillis = Math.max( - bridgesLastPublishedMillis, node.getLastSeenMillis()); - currentBridges.add(node); - } - } - Time time = ApplicationFactory.getTime(); - List<String> orderRelaysByConsensusWeight = new ArrayList<String>(); - for (SummaryDocument entry : currentRelays) { - String fingerprint = entry.getFingerprint().toUpperCase(); - String hashedFingerprint = entry.getHashedFingerprint(). - toUpperCase(); - newRelayFingerprintSummaryLines.put(fingerprint, entry); - newRelayFingerprintSummaryLines.put(hashedFingerprint, entry); - long consensusWeight = entry.getConsensusWeight(); - orderRelaysByConsensusWeight.add(String.format("%020d %s", - consensusWeight, fingerprint)); - orderRelaysByConsensusWeight.add(String.format("%020d %s", - consensusWeight, hashedFingerprint)); - if (entry.getCountryCode() != null) { - String countryCode = entry.getCountryCode(); - if (!newRelaysByCountryCode.containsKey(countryCode)) { - newRelaysByCountryCode.put(countryCode, - new HashSet<String>()); - } - newRelaysByCountryCode.get(countryCode).add(fingerprint); - newRelaysByCountryCode.get(countryCode).add(hashedFingerprint); - } - if (entry.getASNumber() != null) { - String aSNumber = entry.getASNumber(); - if (!newRelaysByASNumber.containsKey(aSNumber)) { - newRelaysByASNumber.put(aSNumber, new HashSet<String>()); - } - newRelaysByASNumber.get(aSNumber).add(fingerprint); - newRelaysByASNumber.get(aSNumber).add(hashedFingerprint); - } - for (String flag : entry.getRelayFlags()) { - String flagLowerCase = flag.toLowerCase(); - if (!newRelaysByFlag.containsKey(flagLowerCase)) { - newRelaysByFlag.put(flagLowerCase, new HashSet<String>()); - } - newRelaysByFlag.get(flagLowerCase).add(fingerprint); - newRelaysByFlag.get(flagLowerCase).add(hashedFingerprint); - } - if (entry.getFamilyFingerprints() != null) { - newRelaysByFamily.put(fingerprint, entry.getFamilyFingerprints()); - } - int daysSinceFirstSeen = (int) ((time.currentTimeMillis() - - entry.getFirstSeenMillis()) / DateTimeHelper.ONE_DAY); - if (!newRelaysByFirstSeenDays.containsKey(daysSinceFirstSeen)) { - newRelaysByFirstSeenDays.put(daysSinceFirstSeen, - new HashSet<String>()); - } - newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add(fingerprint); - newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add( - hashedFingerprint); - int daysSinceLastSeen = (int) ((time.currentTimeMillis() - - entry.getLastSeenMillis()) / DateTimeHelper.ONE_DAY); - if (!newRelaysByLastSeenDays.containsKey(daysSinceLastSeen)) { - newRelaysByLastSeenDays.put(daysSinceLastSeen, - new HashSet<String>()); - } - newRelaysByLastSeenDays.get(daysSinceLastSeen).add(fingerprint); - newRelaysByLastSeenDays.get(daysSinceLastSeen).add( - hashedFingerprint); - String contact = entry.getContact(); - if (!newRelaysByContact.containsKey(contact)) { - newRelaysByContact.put(contact, new HashSet<String>()); - } - newRelaysByContact.get(contact).add(fingerprint); - newRelaysByContact.get(contact).add(hashedFingerprint); - } - Collections.sort(orderRelaysByConsensusWeight); - newRelaysByConsensusWeight = new ArrayList<String>(); - for (String relay : orderRelaysByConsensusWeight) { - newRelaysByConsensusWeight.add(relay.split(" ")[1]); - } - for (Map.Entry<String, Set<String>> e : - newRelaysByFamily.entrySet()) { - String fingerprint = e.getKey(); - Set<String> inMutualFamilyRelation = new HashSet<String>(); - for (String otherFingerprint : e.getValue()) { - if (newRelaysByFamily.containsKey(otherFingerprint) && - newRelaysByFamily.get(otherFingerprint).contains( - fingerprint)) { - inMutualFamilyRelation.add(otherFingerprint); - } - } - e.getValue().retainAll(inMutualFamilyRelation); - } - for (SummaryDocument entry : currentBridges) { - String hashedFingerprint = entry.getFingerprint().toUpperCase(); - String hashedHashedFingerprint = entry.getHashedFingerprint(). - toUpperCase(); - newBridgeFingerprintSummaryLines.put(hashedFingerprint, entry); - newBridgeFingerprintSummaryLines.put(hashedHashedFingerprint, - entry); - for (String flag : entry.getRelayFlags()) { - String flagLowerCase = flag.toLowerCase(); - if (!newBridgesByFlag.containsKey(flagLowerCase)) { - newBridgesByFlag.put(flagLowerCase, new HashSet<String>()); - } - newBridgesByFlag.get(flagLowerCase).add(hashedFingerprint); - newBridgesByFlag.get(flagLowerCase).add( - hashedHashedFingerprint); - } - int daysSinceFirstSeen = (int) ((time.currentTimeMillis() - - entry.getFirstSeenMillis()) / DateTimeHelper.ONE_DAY); - if (!newBridgesByFirstSeenDays.containsKey(daysSinceFirstSeen)) { - newBridgesByFirstSeenDays.put(daysSinceFirstSeen, - new HashSet<String>()); - } - newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add( - hashedFingerprint); - newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add( - hashedHashedFingerprint); - int daysSinceLastSeen = (int) ((time.currentTimeMillis() - - entry.getLastSeenMillis()) / DateTimeHelper.ONE_DAY); - if (!newBridgesByLastSeenDays.containsKey(daysSinceLastSeen)) { - newBridgesByLastSeenDays.put(daysSinceLastSeen, - new HashSet<String>()); - } - newBridgesByLastSeenDays.get(daysSinceLastSeen).add( - hashedFingerprint); - newBridgesByLastSeenDays.get(daysSinceLastSeen).add( - hashedHashedFingerprint); - } - NodeIndex newNodeIndex = new NodeIndex(); - newNodeIndex.setRelaysByConsensusWeight(newRelaysByConsensusWeight); - newNodeIndex.setRelayFingerprintSummaryLines( - newRelayFingerprintSummaryLines); - newNodeIndex.setBridgeFingerprintSummaryLines( - newBridgeFingerprintSummaryLines); - newNodeIndex.setRelaysByCountryCode(newRelaysByCountryCode); - newNodeIndex.setRelaysByASNumber(newRelaysByASNumber); - newNodeIndex.setRelaysByFlag(newRelaysByFlag); - newNodeIndex.setBridgesByFlag(newBridgesByFlag); - newNodeIndex.setRelaysByContact(newRelaysByContact); - newNodeIndex.setRelaysByFamily(newRelaysByFamily); - newNodeIndex.setRelaysByFirstSeenDays(newRelaysByFirstSeenDays); - newNodeIndex.setRelaysByLastSeenDays(newRelaysByLastSeenDays); - newNodeIndex.setBridgesByFirstSeenDays(newBridgesByFirstSeenDays); - newNodeIndex.setBridgesByLastSeenDays(newBridgesByLastSeenDays); - newNodeIndex.setRelaysPublishedString(DateTimeHelper.format( - relaysLastValidAfterMillis)); - newNodeIndex.setBridgesPublishedString(DateTimeHelper.format( - bridgesLastPublishedMillis)); - synchronized (this) { - this.lastIndexed = updateStatusMillis; - this.latestNodeIndex = newNodeIndex; - this.notifyAll(); - } - } -} - diff --git a/src/org/torproject/onionoo/NodeStatus.java b/src/org/torproject/onionoo/NodeStatus.java deleted file mode 100644 index 1bc74f9..0000000 --- a/src/org/torproject/onionoo/NodeStatus.java +++ /dev/null @@ -1,581 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; - -/* Store search data of a single relay that was running in the past seven - * days. */ -public class NodeStatus extends Document { - - private boolean isRelay; - public boolean isRelay() { - return this.isRelay; - } - - private String fingerprint; - public String getFingerprint() { - return this.fingerprint; - } - - private String hashedFingerprint; - public String getHashedFingerprint() { - return this.hashedFingerprint; - } - - private String nickname; - public String getNickname() { - return this.nickname; - } - - private String address; - public String getAddress() { - return this.address; - } - - private SortedSet<String> orAddresses; - private SortedSet<String> orAddressesAndPorts; - public SortedSet<String> getOrAddresses() { - return new TreeSet<String>(this.orAddresses); - } - public void addOrAddressAndPort(String orAddressAndPort) { - if (orAddressAndPort.contains(":") && orAddressAndPort.length() > 0) { - String orAddress = orAddressAndPort.substring(0, - orAddressAndPort.lastIndexOf(':')); - if (this.exitAddresses.contains(orAddress)) { - this.exitAddresses.remove(orAddress); - } - this.orAddresses.add(orAddress); - this.orAddressesAndPorts.add(orAddressAndPort); - } - } - public SortedSet<String> getOrAddressesAndPorts() { - return new TreeSet<String>(this.orAddressesAndPorts); - } - - private SortedSet<String> exitAddresses; - public void addExitAddress(String exitAddress) { - if (exitAddress.length() > 0 && !this.address.equals(exitAddress) && - !this.orAddresses.contains(exitAddress)) { - this.exitAddresses.add(exitAddress); - } - } - public SortedSet<String> getExitAddresses() { - return new TreeSet<String>(this.exitAddresses); - } - - private Float latitude; - public void setLatitude(Float latitude) { - this.latitude = latitude; - } - public Float getLatitude() { - return this.latitude; - } - - private Float longitude; - public void setLongitude(Float longitude) { - this.longitude = longitude; - } - public Float getLongitude() { - return this.longitude; - } - - private String countryCode; - public void setCountryCode(String countryCode) { - this.countryCode = countryCode; - } - public String getCountryCode() { - return this.countryCode; - } - - private String countryName; - public void setCountryName(String countryName) { - this.countryName = countryName; - } - public String getCountryName() { - return this.countryName; - } - - private String regionName; - public void setRegionName(String regionName) { - this.regionName = regionName; - } - public String getRegionName() { - return this.regionName; - } - - private String cityName; - public void setCityName(String cityName) { - this.cityName = cityName; - } - public String getCityName() { - return this.cityName; - } - - private String aSName; - public void setASName(String aSName) { - this.aSName = aSName; - } - public String getASName() { - return this.aSName; - } - - private String aSNumber; - public void setASNumber(String aSNumber) { - this.aSNumber = aSNumber; - } - public String getASNumber() { - return this.aSNumber; - } - - private long firstSeenMillis; - public long getFirstSeenMillis() { - return this.firstSeenMillis; - } - - private long lastSeenMillis; - public long getLastSeenMillis() { - return this.lastSeenMillis; - } - - private int orPort; - public int getOrPort() { - return this.orPort; - } - - private int dirPort; - public int getDirPort() { - return this.dirPort; - } - - private SortedSet<String> relayFlags; - public SortedSet<String> getRelayFlags() { - return this.relayFlags; - } - - private long consensusWeight; - public long getConsensusWeight() { - return this.consensusWeight; - } - - private boolean running; - public void setRunning(boolean running) { - this.running = running; - } - public boolean getRunning() { - return this.running; - } - - private String hostName; - public void setHostName(String hostName) { - this.hostName = hostName; - } - public String getHostName() { - return this.hostName; - } - - private long lastRdnsLookup = -1L; - public void setLastRdnsLookup(long lastRdnsLookup) { - this.lastRdnsLookup = lastRdnsLookup; - } - public long getLastRdnsLookup() { - return this.lastRdnsLookup; - } - - private double advertisedBandwidthFraction = -1.0; - public void setAdvertisedBandwidthFraction( - double advertisedBandwidthFraction) { - this.advertisedBandwidthFraction = advertisedBandwidthFraction; - } - public double getAdvertisedBandwidthFraction() { - return this.advertisedBandwidthFraction; - } - - private double consensusWeightFraction = -1.0; - public void setConsensusWeightFraction(double consensusWeightFraction) { - this.consensusWeightFraction = consensusWeightFraction; - } - public double getConsensusWeightFraction() { - return this.consensusWeightFraction; - } - - private double guardProbability = -1.0; - public void setGuardProbability(double guardProbability) { - this.guardProbability = guardProbability; - } - public double getGuardProbability() { - return this.guardProbability; - } - - private double middleProbability = -1.0; - public void setMiddleProbability(double middleProbability) { - this.middleProbability = middleProbability; - } - public double getMiddleProbability() { - return this.middleProbability; - } - - private double exitProbability = -1.0; - public void setExitProbability(double exitProbability) { - this.exitProbability = exitProbability; - } - public double getExitProbability() { - return this.exitProbability; - } - - private String defaultPolicy; - public String getDefaultPolicy() { - return this.defaultPolicy; - } - - private String portList; - public String getPortList() { - return this.portList; - } - - private SortedMap<Long, Set<String>> lastAddresses; - public SortedMap<Long, Set<String>> getLastAddresses() { - return this.lastAddresses == null ? null : - new TreeMap<Long, Set<String>>(this.lastAddresses); - } - public long getLastChangedOrAddress() { - long lastChangedAddressesMillis = -1L; - if (this.lastAddresses != null) { - Set<String> lastAddresses = null; - for (Map.Entry<Long, Set<String>> e : this.lastAddresses.entrySet()) { - if (lastAddresses != null) { - for (String address : e.getValue()) { - if (!lastAddresses.contains(address)) { - return lastChangedAddressesMillis; - } - } - } - lastChangedAddressesMillis = e.getKey(); - lastAddresses = e.getValue(); - } - } - return lastChangedAddressesMillis; - } - - private String contact; - public void setContact(String contact) { - if (contact == null) { - this.contact = null; - } else { - StringBuilder sb = new StringBuilder(); - for (char c : contact.toLowerCase().toCharArray()) { - if (c >= 32 && c < 127) { - sb.append(c); - } else { - sb.append(" "); - } - } - this.contact = sb.toString(); - } - } - public String getContact() { - return this.contact; - } - - private Boolean recommendedVersion; - public Boolean getRecommendedVersion() { - return this.recommendedVersion; - } - - private SortedSet<String> familyFingerprints; - public void setFamilyFingerprints( - SortedSet<String> familyFingerprints) { - this.familyFingerprints = familyFingerprints; - } - public SortedSet<String> getFamilyFingerprints() { - return this.familyFingerprints; - } - - public NodeStatus(boolean isRelay, String nickname, String fingerprint, - String address, SortedSet<String> orAddressesAndPorts, - SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, - int dirPort, SortedSet<String> relayFlags, long consensusWeight, - String countryCode, String hostName, long lastRdnsLookup, - String defaultPolicy, String portList, long firstSeenMillis, - long lastChangedAddresses, String aSNumber, String contact, - Boolean recommendedVersion, SortedSet<String> familyFingerprints) { - this.isRelay = isRelay; - this.nickname = nickname; - this.fingerprint = fingerprint; - try { - this.hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex( - this.fingerprint.toCharArray())).toUpperCase(); - } catch (DecoderException e) { - throw new IllegalArgumentException("Fingerprint '" + fingerprint - + "' is not a valid fingerprint.", e); - } - this.address = address; - this.exitAddresses = new TreeSet<String>(); - if (exitAddresses != null) { - this.exitAddresses.addAll(exitAddresses); - } - this.exitAddresses.remove(this.address); - this.orAddresses = new TreeSet<String>(); - this.orAddressesAndPorts = new TreeSet<String>(); - if (orAddressesAndPorts != null) { - for (String orAddressAndPort : orAddressesAndPorts) { - this.addOrAddressAndPort(orAddressAndPort); - } - } - this.lastSeenMillis = lastSeenMillis; - this.orPort = orPort; - this.dirPort = dirPort; - this.relayFlags = relayFlags; - this.consensusWeight = consensusWeight; - this.countryCode = countryCode; - this.hostName = hostName; - this.lastRdnsLookup = lastRdnsLookup; - this.defaultPolicy = defaultPolicy; - this.portList = portList; - this.firstSeenMillis = firstSeenMillis; - this.lastAddresses = - new TreeMap<Long, Set<String>>(Collections.reverseOrder()); - Set<String> addresses = new HashSet<String>(); - addresses.add(address + ":" + orPort); - if (dirPort > 0) { - addresses.add(address + ":" + dirPort); - } - addresses.addAll(orAddressesAndPorts); - this.lastAddresses.put(lastChangedAddresses, addresses); - this.aSNumber = aSNumber; - this.contact = contact; - this.recommendedVersion = recommendedVersion; - this.familyFingerprints = familyFingerprints; - } - - public static NodeStatus fromString(String documentString) { - boolean isRelay = false; - String nickname = null, fingerprint = null, address = null, - countryCode = null, hostName = null, defaultPolicy = null, - portList = null, aSNumber = null, contact = null; - SortedSet<String> orAddressesAndPorts = null, exitAddresses = null, - relayFlags = null, familyFingerprints = null; - long lastSeenMillis = -1L, consensusWeight = -1L, - lastRdnsLookup = -1L, firstSeenMillis = -1L, - lastChangedAddresses = -1L; - int orPort = -1, dirPort = -1; - Boolean recommendedVersion = null; - try { - String separator = documentString.contains("\t") ? "\t" : " "; - String[] parts = documentString.trim().split(separator); - isRelay = parts[0].equals("r"); - if (parts.length < 9) { - System.err.println("Too few space-separated values in line '" - + documentString.trim() + "'. Skipping."); - return null; - } - nickname = parts[1]; - fingerprint = parts[2]; - orAddressesAndPorts = new TreeSet<String>(); - exitAddresses = new TreeSet<String>(); - String addresses = parts[3]; - if (addresses.contains(";")) { - String[] addressParts = addresses.split(";", -1); - if (addressParts.length != 3) { - System.err.println("Invalid addresses entry in line '" - + documentString.trim() + "'. Skipping."); - return null; - } - address = addressParts[0]; - if (addressParts[1].length() > 0) { - orAddressesAndPorts.addAll(Arrays.asList( - addressParts[1].split("\\+"))); - } - if (addressParts[2].length() > 0) { - exitAddresses.addAll(Arrays.asList( - addressParts[2].split("\\+"))); - } - } else { - address = addresses; - } - lastSeenMillis = DateTimeHelper.parse(parts[4] + " " + parts[5]); - if (lastSeenMillis < 0L) { - System.err.println("Parse exception while parsing node status " - + "line '" + documentString + "'. Skipping."); - return null; - } - orPort = Integer.parseInt(parts[6]); - dirPort = Integer.parseInt(parts[7]); - relayFlags = new TreeSet<String>(); - if (parts[8].length() > 0) { - relayFlags.addAll(Arrays.asList(parts[8].split(","))); - } - if (parts.length > 9) { - consensusWeight = Long.parseLong(parts[9]); - } - if (parts.length > 10) { - countryCode = parts[10]; - } - if (parts.length > 12) { - hostName = parts[11].equals("null") ? null : parts[11]; - lastRdnsLookup = Long.parseLong(parts[12]); - } - if (parts.length > 14) { - if (!parts[13].equals("null")) { - defaultPolicy = parts[13]; - } - if (!parts[14].equals("null")) { - portList = parts[14]; - } - } - firstSeenMillis = lastSeenMillis; - if (parts.length > 16) { - firstSeenMillis = DateTimeHelper.parse(parts[15] + " " - + parts[16]); - if (firstSeenMillis < 0L) { - System.err.println("Parse exception while parsing node status " - + "line '" + documentString + "'. Skipping."); - return null; - } - } - lastChangedAddresses = lastSeenMillis; - if (parts.length > 18 && !parts[17].equals("null")) { - lastChangedAddresses = DateTimeHelper.parse(parts[17] + " " - + parts[18]); - if (lastChangedAddresses < 0L) { - System.err.println("Parse exception while parsing node status " - + "line '" + documentString + "'. Skipping."); - return null; - } - } - if (parts.length > 19) { - aSNumber = parts[19]; - } - if (parts.length > 20) { - contact = parts[20]; - } - if (parts.length > 21) { - recommendedVersion = parts[21].equals("null") ? null : - parts[21].equals("true"); - } - if (parts.length > 22 && !parts[22].equals("null")) { - familyFingerprints = new TreeSet<String>(Arrays.asList( - parts[22].split(";"))); - } - } catch (NumberFormatException e) { - System.err.println("Number format exception while parsing node " - + "status line '" + documentString + "': " + e.getMessage() - + ". Skipping."); - return null; - } catch (Exception e) { - /* This catch block is only here to handle yet unknown errors. It - * should go away once we're sure what kind of errors can occur. */ - System.err.println("Unknown exception while parsing node status " - + "line '" + documentString + "': " + e.getMessage() + ". " - + "Skipping."); - return null; - } - NodeStatus newNodeStatus = new NodeStatus(isRelay, nickname, - fingerprint, address, orAddressesAndPorts, exitAddresses, - lastSeenMillis, orPort, dirPort, relayFlags, consensusWeight, - countryCode, hostName, lastRdnsLookup, defaultPolicy, portList, - firstSeenMillis, lastChangedAddresses, aSNumber, contact, - recommendedVersion, familyFingerprints); - return newNodeStatus; - } - - public void update(NodeStatus newNodeStatus) { - if (newNodeStatus.lastSeenMillis > this.lastSeenMillis) { - this.nickname = newNodeStatus.nickname; - this.address = newNodeStatus.address; - this.orAddressesAndPorts = newNodeStatus.orAddressesAndPorts; - this.lastSeenMillis = newNodeStatus.lastSeenMillis; - this.orPort = newNodeStatus.orPort; - this.dirPort = newNodeStatus.dirPort; - this.relayFlags = newNodeStatus.relayFlags; - this.consensusWeight = newNodeStatus.consensusWeight; - this.countryCode = newNodeStatus.countryCode; - this.defaultPolicy = newNodeStatus.defaultPolicy; - this.portList = newNodeStatus.portList; - this.aSNumber = newNodeStatus.aSNumber; - this.recommendedVersion = newNodeStatus.recommendedVersion; - } - if (this.isRelay && newNodeStatus.isRelay) { - this.lastAddresses.putAll(newNodeStatus.lastAddresses); - } - this.firstSeenMillis = Math.min(newNodeStatus.firstSeenMillis, - this.firstSeenMillis); - } - - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(this.isRelay ? "r" : "b"); - sb.append("\t" + this.nickname); - sb.append("\t" + this.fingerprint); - sb.append("\t" + this.address + ";"); - int written = 0; - for (String orAddressAndPort : this.orAddressesAndPorts) { - sb.append((written++ > 0 ? "+" : "") + orAddressAndPort); - } - sb.append(";"); - if (this.isRelay) { - written = 0; - for (String exitAddress : this.exitAddresses) { - sb.append((written++ > 0 ? "+" : "") - + exitAddress); - } - } - sb.append("\t" + DateTimeHelper.format(this.lastSeenMillis, - DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); - sb.append("\t" + this.orPort); - sb.append("\t" + this.dirPort + "\t"); - written = 0; - for (String relayFlag : this.relayFlags) { - sb.append((written++ > 0 ? "," : "") + relayFlag); - } - if (this.isRelay) { - sb.append("\t" + String.valueOf(this.consensusWeight)); - sb.append("\t" - + (this.countryCode != null ? this.countryCode : "??")); - sb.append("\t" + (this.hostName != null ? this.hostName : "null")); - sb.append("\t" + String.valueOf(this.lastRdnsLookup)); - sb.append("\t" + (this.defaultPolicy != null ? this.defaultPolicy - : "null")); - sb.append("\t" + (this.portList != null ? this.portList : "null")); - } else { - sb.append("\t-1\t??\tnull\t-1\tnull\tnull"); - } - sb.append("\t" + DateTimeHelper.format(this.firstSeenMillis, - DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); - if (this.isRelay) { - sb.append("\t" + DateTimeHelper.format( - this.getLastChangedOrAddress(), - DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); - sb.append("\t" + (this.aSNumber != null ? this.aSNumber : "null")); - } else { - sb.append("\tnull\tnull\tnull"); - } - sb.append("\t" + (this.contact != null ? this.contact : "")); - sb.append("\t" + (this.recommendedVersion == null ? "null" : - this.recommendedVersion ? "true" : "false")); - if (this.familyFingerprints == null || - this.familyFingerprints.isEmpty()) { - sb.append("\tnull"); - } else { - sb.append("\t"); - written = 0; - for (String familyFingerprint : this.familyFingerprints) { - sb.append((written++ > 0 ? ";" : "") + familyFingerprint); - } - } - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/RequestHandler.java b/src/org/torproject/onionoo/RequestHandler.java deleted file mode 100644 index 58ec17a..0000000 --- a/src/org/torproject/onionoo/RequestHandler.java +++ /dev/null @@ -1,548 +0,0 @@ -/* Copyright 2011--2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; - -public class RequestHandler { - - private NodeIndex nodeIndex; - - private DocumentStore documentStore; - - public RequestHandler(NodeIndex nodeIndex) { - this.nodeIndex = nodeIndex; - this.documentStore = ApplicationFactory.getDocumentStore(); - } - - private String resourceType; - public void setResourceType(String resourceType) { - this.resourceType = resourceType; - } - - private String type; - public void setType(String type) { - this.type = type; - } - - private String running; - public void setRunning(String running) { - this.running = running; - } - - private String[] search; - public void setSearch(String[] search) { - this.search = new String[search.length]; - System.arraycopy(search, 0, this.search, 0, search.length); - } - - private String lookup; - public void setLookup(String lookup) { - this.lookup = lookup; - } - - private String fingerprint; - public void setFingerprint(String fingerprint) { - this.fingerprint = fingerprint; - } - - private String country; - public void setCountry(String country) { - this.country = country; - } - - private String as; - public void setAs(String as) { - this.as = as; - } - - private String flag; - public void setFlag(String flag) { - this.flag = flag; - } - - private String[] contact; - public void setContact(String[] contact) { - this.contact = new String[contact.length]; - System.arraycopy(contact, 0, this.contact, 0, contact.length); - } - - private String[] order; - public void setOrder(String[] order) { - this.order = new String[order.length]; - System.arraycopy(order, 0, this.order, 0, order.length); - } - - private String offset; - public void setOffset(String offset) { - this.offset = offset; - } - - private String limit; - public void setLimit(String limit) { - this.limit = limit; - } - - private int[] firstSeenDays; - public void setFirstSeenDays(int[] firstSeenDays) { - this.firstSeenDays = new int[firstSeenDays.length]; - System.arraycopy(firstSeenDays, 0, this.firstSeenDays, 0, - firstSeenDays.length); - } - - private int[] lastSeenDays; - public void setLastSeenDays(int[] lastSeenDays) { - this.lastSeenDays = new int[lastSeenDays.length]; - System.arraycopy(lastSeenDays, 0, this.lastSeenDays, 0, - lastSeenDays.length); - } - - private String family; - public void setFamily(String family) { - this.family = family; - } - - private Map<String, SummaryDocument> filteredRelays = - new HashMap<String, SummaryDocument>(); - - private Map<String, SummaryDocument> filteredBridges = - new HashMap<String, SummaryDocument>(); - - public void handleRequest() { - this.filteredRelays.putAll( - this.nodeIndex.getRelayFingerprintSummaryLines()); - this.filteredBridges.putAll( - this.nodeIndex.getBridgeFingerprintSummaryLines()); - this.filterByResourceType(); - this.filterByType(); - this.filterByRunning(); - this.filterBySearchTerms(); - this.filterByLookup(); - this.filterByFingerprint(); - this.filterByCountryCode(); - this.filterByASNumber(); - this.filterByFlag(); - this.filterNodesByFirstSeenDays(); - this.filterNodesByLastSeenDays(); - this.filterByContact(); - this.filterByFamily(); - this.order(); - this.offset(); - this.limit(); - } - - - private void filterByResourceType() { - if (this.resourceType.equals("clients")) { - this.filteredRelays.clear(); - } - if (this.resourceType.equals("weights")) { - this.filteredBridges.clear(); - } - } - - private void filterByType() { - if (this.type == null) { - return; - } else if (this.type.equals("relay")) { - this.filteredBridges.clear(); - } else { - this.filteredRelays.clear(); - } - } - - private void filterByRunning() { - if (this.running == null) { - return; - } - boolean runningRequested = this.running.equals("true"); - Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, SummaryDocument> e : - filteredRelays.entrySet()) { - if (e.getValue().isRunning() != runningRequested) { - removeRelays.add(e.getKey()); - } - } - for (String fingerprint : removeRelays) { - this.filteredRelays.remove(fingerprint); - } - Set<String> removeBridges = new HashSet<String>(); - for (Map.Entry<String, SummaryDocument> e : - filteredBridges.entrySet()) { - if (e.getValue().isRunning() != runningRequested) { - removeBridges.add(e.getKey()); - } - } - for (String fingerprint : removeBridges) { - this.filteredBridges.remove(fingerprint); - } - } - - private void filterBySearchTerms() { - if (this.search == null) { - return; - } - for (String searchTerm : this.search) { - filterBySearchTerm(searchTerm); - } - } - - private void filterBySearchTerm(String searchTerm) { - Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, SummaryDocument> e : - filteredRelays.entrySet()) { - String fingerprint = e.getKey(); - SummaryDocument entry = e.getValue(); - boolean lineMatches = false; - String nickname = entry.getNickname() != null ? - entry.getNickname().toLowerCase() : "unnamed"; - if (searchTerm.startsWith("$")) { - /* Search is for $-prefixed fingerprint. */ - if (fingerprint.startsWith( - searchTerm.substring(1).toUpperCase())) { - /* $-prefixed fingerprint matches. */ - lineMatches = true; - } - } else if (nickname.contains(searchTerm.toLowerCase())) { - /* Nickname matches. */ - lineMatches = true; - } else if (fingerprint.startsWith(searchTerm.toUpperCase())) { - /* Non-$-prefixed fingerprint matches. */ - lineMatches = true; - } else { - List<String> addresses = entry.getAddresses(); - for (String address : addresses) { - if (address.startsWith(searchTerm.toLowerCase())) { - /* Address matches. */ - lineMatches = true; - break; - } - } - } - if (!lineMatches) { - removeRelays.add(e.getKey()); - } - } - for (String fingerprint : removeRelays) { - this.filteredRelays.remove(fingerprint); - } - Set<String> removeBridges = new HashSet<String>(); - for (Map.Entry<String, SummaryDocument> e : - filteredBridges.entrySet()) { - String hashedFingerprint = e.getKey(); - SummaryDocument entry = e.getValue(); - boolean lineMatches = false; - String nickname = entry.getNickname() != null ? - entry.getNickname().toLowerCase() : "unnamed"; - if (searchTerm.startsWith("$")) { - /* Search is for $-prefixed hashed fingerprint. */ - if (hashedFingerprint.startsWith( - searchTerm.substring(1).toUpperCase())) { - /* $-prefixed hashed fingerprint matches. */ - lineMatches = true; - } - } else if (nickname.contains(searchTerm.toLowerCase())) { - /* Nickname matches. */ - lineMatches = true; - } else if (hashedFingerprint.startsWith(searchTerm.toUpperCase())) { - /* Non-$-prefixed hashed fingerprint matches. */ - lineMatches = true; - } - if (!lineMatches) { - removeBridges.add(e.getKey()); - } - } - for (String fingerprint : removeBridges) { - this.filteredBridges.remove(fingerprint); - } - } - - private void filterByLookup() { - if (this.lookup == null) { - return; - } - String fingerprint = this.lookup; - SummaryDocument relayLine = this.filteredRelays.get(fingerprint); - this.filteredRelays.clear(); - if (relayLine != null) { - this.filteredRelays.put(fingerprint, relayLine); - } - SummaryDocument bridgeLine = this.filteredBridges.get(fingerprint); - this.filteredBridges.clear(); - if (bridgeLine != null) { - this.filteredBridges.put(fingerprint, bridgeLine); - } - } - - private void filterByFingerprint() { - if (this.fingerprint == null) { - return; - } - this.filteredRelays.clear(); - this.filteredBridges.clear(); - String fingerprint = this.fingerprint; - SummaryDocument entry = this.documentStore.retrieve( - SummaryDocument.class, true, fingerprint); - if (entry != null) { - if (entry.isRelay()) { - this.filteredRelays.put(fingerprint, entry); - } else { - this.filteredBridges.put(fingerprint, entry); - } - } - } - - private void filterByCountryCode() { - if (this.country == null) { - return; - } - String countryCode = this.country.toLowerCase(); - if (!this.nodeIndex.getRelaysByCountryCode().containsKey( - countryCode)) { - this.filteredRelays.clear(); - } else { - Set<String> relaysWithCountryCode = - this.nodeIndex.getRelaysByCountryCode().get(countryCode); - Set<String> removeRelays = new HashSet<String>(); - for (String fingerprint : this.filteredRelays.keySet()) { - if (!relaysWithCountryCode.contains(fingerprint)) { - removeRelays.add(fingerprint); - } - } - for (String fingerprint : removeRelays) { - this.filteredRelays.remove(fingerprint); - } - } - this.filteredBridges.clear(); - } - - private void filterByASNumber() { - if (this.as == null) { - return; - } - String aSNumber = this.as.toUpperCase(); - if (!aSNumber.startsWith("AS")) { - aSNumber = "AS" + aSNumber; - } - if (!this.nodeIndex.getRelaysByASNumber().containsKey(aSNumber)) { - this.filteredRelays.clear(); - } else { - Set<String> relaysWithASNumber = - this.nodeIndex.getRelaysByASNumber().get(aSNumber); - Set<String> removeRelays = new HashSet<String>(); - for (String fingerprint : this.filteredRelays.keySet()) { - if (!relaysWithASNumber.contains(fingerprint)) { - removeRelays.add(fingerprint); - } - } - for (String fingerprint : removeRelays) { - this.filteredRelays.remove(fingerprint); - } - } - this.filteredBridges.clear(); - } - - private void filterByFlag() { - if (this.flag == null) { - return; - } - String flag = this.flag.toLowerCase(); - if (!this.nodeIndex.getRelaysByFlag().containsKey(flag)) { - this.filteredRelays.clear(); - } else { - Set<String> relaysWithFlag = this.nodeIndex.getRelaysByFlag().get( - flag); - Set<String> removeRelays = new HashSet<String>(); - for (String fingerprint : this.filteredRelays.keySet()) { - if (!relaysWithFlag.contains(fingerprint)) { - removeRelays.add(fingerprint); - } - } - for (String fingerprint : removeRelays) { - this.filteredRelays.remove(fingerprint); - } - } - if (!this.nodeIndex.getBridgesByFlag().containsKey(flag)) { - this.filteredBridges.clear(); - } else { - Set<String> bridgesWithFlag = this.nodeIndex.getBridgesByFlag().get( - flag); - Set<String> removeBridges = new HashSet<String>(); - for (String fingerprint : this.filteredBridges.keySet()) { - if (!bridgesWithFlag.contains(fingerprint)) { - removeBridges.add(fingerprint); - } - } - for (String fingerprint : removeBridges) { - this.filteredBridges.remove(fingerprint); - } - } - } - - private void filterNodesByFirstSeenDays() { - if (this.firstSeenDays == null) { - return; - } - filterNodesByDays(this.filteredRelays, - this.nodeIndex.getRelaysByFirstSeenDays(), this.firstSeenDays); - filterNodesByDays(this.filteredBridges, - this.nodeIndex.getBridgesByFirstSeenDays(), this.firstSeenDays); - } - - private void filterNodesByLastSeenDays() { - if (this.lastSeenDays == null) { - return; - } - filterNodesByDays(this.filteredRelays, - this.nodeIndex.getRelaysByLastSeenDays(), this.lastSeenDays); - filterNodesByDays(this.filteredBridges, - this.nodeIndex.getBridgesByLastSeenDays(), this.lastSeenDays); - } - - private void filterNodesByDays( - Map<String, SummaryDocument> filteredNodes, - SortedMap<Integer, Set<String>> nodesByDays, int[] days) { - Set<String> removeNodes = new HashSet<String>(); - for (Set<String> nodes : nodesByDays.headMap(days[0]).values()) { - removeNodes.addAll(nodes); - } - if (days[1] < Integer.MAX_VALUE) { - for (Set<String> nodes : - nodesByDays.tailMap(days[1] + 1).values()) { - removeNodes.addAll(nodes); - } - } - for (String fingerprint : removeNodes) { - filteredNodes.remove(fingerprint); - } - } - - private void filterByContact() { - if (this.contact == null) { - return; - } - Set<String> removeRelays = new HashSet<String>(); - for (Map.Entry<String, Set<String>> e : - this.nodeIndex.getRelaysByContact().entrySet()) { - String contact = e.getKey(); - for (String contactPart : this.contact) { - if (contact == null || - !contact.contains(contactPart.toLowerCase())) { - removeRelays.addAll(e.getValue()); - break; - } - } - } - for (String fingerprint : removeRelays) { - this.filteredRelays.remove(fingerprint); - } - this.filteredBridges.clear(); - } - - private void filterByFamily() { - if (this.family == null) { - return; - } - Set<String> removeRelays = new HashSet<String>( - this.filteredRelays.keySet()); - removeRelays.remove(this.family); - if (this.nodeIndex.getRelaysByFamily().containsKey(this.family)) { - removeRelays.removeAll(this.nodeIndex.getRelaysByFamily(). - get(this.family)); - } - for (String fingerprint : removeRelays) { - this.filteredRelays.remove(fingerprint); - } - this.filteredBridges.clear(); - } - - private void order() { - if (this.order != null && this.order.length == 1) { - List<String> orderBy = new ArrayList<String>( - this.nodeIndex.getRelaysByConsensusWeight()); - if (this.order[0].startsWith("-")) { - Collections.reverse(orderBy); - } - for (String relay : orderBy) { - if (this.filteredRelays.containsKey(relay) && - !this.orderedRelays.contains(filteredRelays.get(relay))) { - this.orderedRelays.add(this.filteredRelays.remove(relay)); - } - } - for (String relay : this.filteredRelays.keySet()) { - if (!this.orderedRelays.contains(this.filteredRelays.get(relay))) { - this.orderedRelays.add(this.filteredRelays.remove(relay)); - } - } - Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>( - this.filteredBridges.values()); - this.orderedBridges.addAll(uniqueBridges); - } else { - Set<SummaryDocument> uniqueRelays = new HashSet<SummaryDocument>( - this.filteredRelays.values()); - this.orderedRelays.addAll(uniqueRelays); - Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>( - this.filteredBridges.values()); - this.orderedBridges.addAll(uniqueBridges); - } - } - - private void offset() { - if (this.offset == null) { - return; - } - int offsetValue = Integer.parseInt(this.offset); - while (offsetValue-- > 0 && - (!this.orderedRelays.isEmpty() || - !this.orderedBridges.isEmpty())) { - if (!this.orderedRelays.isEmpty()) { - this.orderedRelays.remove(0); - } else { - this.orderedBridges.remove(0); - } - } - } - - private void limit() { - if (this.limit == null) { - return; - } - int limitValue = Integer.parseInt(this.limit); - while (!this.orderedRelays.isEmpty() && - limitValue < this.orderedRelays.size()) { - this.orderedRelays.remove(this.orderedRelays.size() - 1); - } - limitValue -= this.orderedRelays.size(); - while (!this.orderedBridges.isEmpty() && - limitValue < this.orderedBridges.size()) { - this.orderedBridges.remove(this.orderedBridges.size() - 1); - } - } - - private List<SummaryDocument> orderedRelays = - new ArrayList<SummaryDocument>(); - public List<SummaryDocument> getOrderedRelays() { - return this.orderedRelays; - } - - private List<SummaryDocument> orderedBridges = - new ArrayList<SummaryDocument>(); - public List<SummaryDocument> getOrderedBridges() { - return this.orderedBridges; - } - - public String getRelaysPublishedString() { - return this.nodeIndex.getRelaysPublishedString(); - } - - public String getBridgesPublishedString() { - return this.nodeIndex.getBridgesPublishedString(); - } -} diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java deleted file mode 100644 index efce86e..0000000 --- a/src/org/torproject/onionoo/ResourceServlet.java +++ /dev/null @@ -1,448 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.io.IOException; -import java.io.PrintWriter; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; - -import javax.servlet.ServletConfig; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -public class ResourceServlet extends HttpServlet { - - private static final long serialVersionUID = 7236658979947465319L; - - private boolean maintenanceMode = false; - - /* Called by servlet container, not by test class. */ - public void init(ServletConfig config) throws ServletException { - super.init(config); - this.maintenanceMode = - config.getInitParameter("maintenance") != null && - config.getInitParameter("maintenance").equals("1"); - } - - public long getLastModified(HttpServletRequest request) { - if (this.maintenanceMode) { - return super.getLastModified(request); - } else { - return ApplicationFactory.getNodeIndexer().getLastIndexed( - DateTimeHelper.TEN_SECONDS); - } - } - - protected static class HttpServletRequestWrapper { - private HttpServletRequest request; - protected HttpServletRequestWrapper(HttpServletRequest request) { - this.request = request; - } - protected String getRequestURI() { - return this.request.getRequestURI(); - } - @SuppressWarnings("rawtypes") - protected Map getParameterMap() { - return this.request.getParameterMap(); - } - protected String[] getParameterValues(String parameterKey) { - return this.request.getParameterValues(parameterKey); - } - } - - protected static class HttpServletResponseWrapper { - private HttpServletResponse response = null; - protected HttpServletResponseWrapper(HttpServletResponse response) { - this.response = response; - } - protected void sendError(int errorStatusCode) throws IOException { - this.response.sendError(errorStatusCode); - } - protected void setHeader(String headerName, String headerValue) { - this.response.setHeader(headerName, headerValue); - } - protected void setContentType(String contentType) { - this.response.setContentType(contentType); - } - protected void setCharacterEncoding(String characterEncoding) { - this.response.setCharacterEncoding(characterEncoding); - } - protected PrintWriter getWriter() throws IOException { - return this.response.getWriter(); - } - } - - public void doGet(HttpServletRequest request, - HttpServletResponse response) throws IOException, ServletException { - HttpServletRequestWrapper requestWrapper = - new HttpServletRequestWrapper(request); - HttpServletResponseWrapper responseWrapper = - new HttpServletResponseWrapper(response); - this.doGet(requestWrapper, responseWrapper); - } - - public void doGet(HttpServletRequestWrapper request, - HttpServletResponseWrapper response) throws IOException { - - if (this.maintenanceMode) { - response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); - return; - } - - if (ApplicationFactory.getNodeIndexer().getLastIndexed( - DateTimeHelper.TEN_SECONDS) + DateTimeHelper.SIX_HOURS - < ApplicationFactory.getTime().currentTimeMillis()) { - response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - return; - } - - NodeIndex nodeIndex = ApplicationFactory.getNodeIndexer(). - getLatestNodeIndex(DateTimeHelper.TEN_SECONDS); - if (nodeIndex == null) { - response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - return; - } - - String uri = request.getRequestURI(); - if (uri.startsWith("/onionoo/")) { - uri = uri.substring("/onionoo".length()); - } - String resourceType = null; - if (uri.startsWith("/summary")) { - resourceType = "summary"; - } else if (uri.startsWith("/details")) { - resourceType = "details"; - } else if (uri.startsWith("/bandwidth")) { - resourceType = "bandwidth"; - } else if (uri.startsWith("/weights")) { - resourceType = "weights"; - } else if (uri.startsWith("/clients")) { - resourceType = "clients"; - } else if (uri.startsWith("/uptime")) { - resourceType = "uptime"; - } else { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - - RequestHandler rh = new RequestHandler(nodeIndex); - rh.setResourceType(resourceType); - - /* Extract parameters either from the old-style URI or from request - * parameters. */ - Map<String, String> parameterMap = new HashMap<String, String>(); - for (Object parameterKey : request.getParameterMap().keySet()) { - String[] parameterValues = - request.getParameterValues((String) parameterKey); - parameterMap.put((String) parameterKey, parameterValues[0]); - } - - /* Make sure that the request doesn't contain any unknown - * parameters. */ - Set<String> knownParameters = new HashSet<String>(Arrays.asList(( - "type,running,search,lookup,fingerprint,country,as,flag," - + "first_seen_days,last_seen_days,contact,order,limit,offset," - + "fields,family").split(","))); - for (String parameterKey : parameterMap.keySet()) { - if (!knownParameters.contains(parameterKey)) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - } - - /* Filter relays and bridges matching the request. */ - if (parameterMap.containsKey("type")) { - String typeParameterValue = parameterMap.get("type").toLowerCase(); - boolean relaysRequested = true; - if (typeParameterValue.equals("bridge")) { - relaysRequested = false; - } else if (!typeParameterValue.equals("relay")) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - rh.setType(relaysRequested ? "relay" : "bridge"); - } - if (parameterMap.containsKey("running")) { - String runningParameterValue = - parameterMap.get("running").toLowerCase(); - boolean runningRequested = true; - if (runningParameterValue.equals("false")) { - runningRequested = false; - } else if (!runningParameterValue.equals("true")) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - rh.setRunning(runningRequested ? "true" : "false"); - } - if (parameterMap.containsKey("search")) { - String[] searchTerms = this.parseSearchParameters( - parameterMap.get("search")); - if (searchTerms == null) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - rh.setSearch(searchTerms); - } - if (parameterMap.containsKey("lookup")) { - String lookupParameter = this.parseFingerprintParameter( - parameterMap.get("lookup")); - if (lookupParameter == null) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - String fingerprint = lookupParameter.toUpperCase(); - rh.setLookup(fingerprint); - } - if (parameterMap.containsKey("fingerprint")) { - String fingerprintParameter = this.parseFingerprintParameter( - parameterMap.get("fingerprint")); - if (fingerprintParameter == null) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - String fingerprint = fingerprintParameter.toUpperCase(); - rh.setFingerprint(fingerprint); - } - if (parameterMap.containsKey("country")) { - String countryCodeParameter = this.parseCountryCodeParameter( - parameterMap.get("country")); - if (countryCodeParameter == null) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - rh.setCountry(countryCodeParameter); - } - if (parameterMap.containsKey("as")) { - String aSNumberParameter = this.parseASNumberParameter( - parameterMap.get("as")); - if (aSNumberParameter == null) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - rh.setAs(aSNumberParameter); - } - if (parameterMap.containsKey("flag")) { - String flagParameter = this.parseFlagParameter( - parameterMap.get("flag")); - if (flagParameter == null) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - rh.setFlag(flagParameter); - } - if (parameterMap.containsKey("first_seen_days")) { - int[] days = this.parseDaysParameter( - parameterMap.get("first_seen_days")); - if (days == null) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - rh.setFirstSeenDays(days); - } - if (parameterMap.containsKey("last_seen_days")) { - int[] days = this.parseDaysParameter( - parameterMap.get("last_seen_days")); - if (days == null) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - rh.setLastSeenDays(days); - } - if (parameterMap.containsKey("contact")) { - String[] contactParts = this.parseContactParameter( - parameterMap.get("contact")); - if (contactParts == null) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - rh.setContact(contactParts); - } - if (parameterMap.containsKey("order")) { - String orderParameter = parameterMap.get("order").toLowerCase(); - String orderByField = orderParameter; - if (orderByField.startsWith("-")) { - orderByField = orderByField.substring(1); - } - if (!orderByField.equals("consensus_weight")) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - rh.setOrder(new String[] { orderParameter }); - } - if (parameterMap.containsKey("offset")) { - String offsetParameter = parameterMap.get("offset"); - if (offsetParameter.length() > 6) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - try { - Integer.parseInt(offsetParameter); - } catch (NumberFormatException e) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - rh.setOffset(offsetParameter); - } - if (parameterMap.containsKey("limit")) { - String limitParameter = parameterMap.get("limit"); - if (limitParameter.length() > 6) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - try { - Integer.parseInt(limitParameter); - } catch (NumberFormatException e) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - rh.setLimit(limitParameter); - } - if (parameterMap.containsKey("family")) { - String familyParameter = this.parseFingerprintParameter( - parameterMap.get("family")); - if (familyParameter == null) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - String family = familyParameter.toUpperCase(); - rh.setFamily(family); - } - rh.handleRequest(); - - ResponseBuilder rb = new ResponseBuilder(); - rb.setResourceType(resourceType); - rb.setRelaysPublishedString(rh.getRelaysPublishedString()); - rb.setBridgesPublishedString(rh.getBridgesPublishedString()); - rb.setOrderedRelays(rh.getOrderedRelays()); - rb.setOrderedBridges(rh.getOrderedBridges()); - String[] fields = null; - if (parameterMap.containsKey("fields")) { - fields = this.parseFieldsParameter(parameterMap.get("fields")); - if (fields == null) { - response.sendError(HttpServletResponse.SC_BAD_REQUEST); - return; - } - rb.setFields(fields); - } - - response.setHeader("Access-Control-Allow-Origin", "*"); - response.setContentType("application/json"); - response.setCharacterEncoding("utf-8"); - PrintWriter pw = response.getWriter(); - rb.buildResponse(pw); - pw.flush(); - pw.close(); - } - - private static Pattern searchParameterPattern = - Pattern.compile("^\\$?[0-9a-fA-F]{1,40}$|" /* Fingerprint. */ - + "^[0-9a-zA-Z\\.]{1,19}$|" /* Nickname or IPv4 address. */ - + "^\\[[0-9a-fA-F:\\.]{1,39}\\]?$"); /* IPv6 address. */ - private String[] parseSearchParameters(String parameter) { - String[] searchParameters; - if (parameter.contains(" ")) { - searchParameters = parameter.split(" "); - } else { - searchParameters = new String[] { parameter }; - } - for (String searchParameter : searchParameters) { - if (!searchParameterPattern.matcher(searchParameter).matches()) { - return null; - } - } - return searchParameters; - } - - private static Pattern fingerprintParameterPattern = - Pattern.compile("^[0-9a-zA-Z]{1,40}$"); - private String parseFingerprintParameter(String parameter) { - if (!fingerprintParameterPattern.matcher(parameter).matches()) { - return null; - } - if (parameter.length() != 40) { - return null; - } - return parameter; - } - - private static Pattern countryCodeParameterPattern = - Pattern.compile("^[0-9a-zA-Z]{2}$"); - private String parseCountryCodeParameter(String parameter) { - if (!countryCodeParameterPattern.matcher(parameter).matches()) { - return null; - } - return parameter; - } - - private static Pattern aSNumberParameterPattern = - Pattern.compile("^[asAS]{0,2}[0-9]{1,10}$"); - private String parseASNumberParameter(String parameter) { - if (!aSNumberParameterPattern.matcher(parameter).matches()) { - return null; - } - return parameter; - } - - private static Pattern flagPattern = - Pattern.compile("^[a-zA-Z0-9]{1,20}$"); - private String parseFlagParameter(String parameter) { - if (!flagPattern.matcher(parameter).matches()) { - return null; - } - return parameter; - } - - private static Pattern daysPattern = Pattern.compile("^[0-9-]{1,10}$"); - private int[] parseDaysParameter(String parameter) { - if (!daysPattern.matcher(parameter).matches()) { - return null; - } - int x = 0, y = Integer.MAX_VALUE; - try { - if (!parameter.contains("-")) { - x = Integer.parseInt(parameter); - y = x; - } else { - String[] parts = parameter.split("-", 2); - if (parts[0].length() > 0) { - x = Integer.parseInt(parts[0]); - } - if (parts.length > 1 && parts[1].length() > 0) { - y = Integer.parseInt(parts[1]); - } - } - } catch (NumberFormatException e) { - return null; - } - if (x > y) { - return null; - } - return new int[] { x, y }; - } - - private String[] parseContactParameter(String parameter) { - for (char c : parameter.toCharArray()) { - if (c < 32 || c >= 127) { - return null; - } - } - return parameter.split(" "); - } - - private static Pattern fieldsParameterPattern = - Pattern.compile("^[0-9a-zA-Z_,]*$"); - private String[] parseFieldsParameter(String parameter) { - if (!fieldsParameterPattern.matcher(parameter).matches()) { - return null; - } - return parameter.split(","); - } -} - diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java deleted file mode 100644 index 2086ea8..0000000 --- a/src/org/torproject/onionoo/ResponseBuilder.java +++ /dev/null @@ -1,311 +0,0 @@ -/* Copyright 2011--2013 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.List; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -public class ResponseBuilder { - - private DocumentStore documentStore; - - public ResponseBuilder() { - this.documentStore = ApplicationFactory.getDocumentStore(); - } - - private String resourceType; - public void setResourceType(String resourceType) { - this.resourceType = resourceType; - } - - private String relaysPublishedString; - public void setRelaysPublishedString(String relaysPublishedString) { - this.relaysPublishedString = relaysPublishedString; - } - - private String bridgesPublishedString; - public void setBridgesPublishedString(String bridgesPublishedString) { - this.bridgesPublishedString = bridgesPublishedString; - } - - private List<SummaryDocument> orderedRelays = - new ArrayList<SummaryDocument>(); - public void setOrderedRelays(List<SummaryDocument> orderedRelays) { - this.orderedRelays = orderedRelays; - } - - private List<SummaryDocument> orderedBridges = - new ArrayList<SummaryDocument>(); - public void setOrderedBridges(List<SummaryDocument> orderedBridges) { - this.orderedBridges = orderedBridges; - } - - private String[] fields; - public void setFields(String[] fields) { - this.fields = new String[fields.length]; - System.arraycopy(fields, 0, this.fields, 0, fields.length); - } - - public void buildResponse(PrintWriter pw) { - writeRelays(orderedRelays, pw); - writeBridges(orderedBridges, pw); - } - - private void writeRelays(List<SummaryDocument> relays, PrintWriter pw) { - pw.write("{\"relays_published\":\"" + relaysPublishedString - + "\",\n\"relays\":["); - int written = 0; - for (SummaryDocument entry : relays) { - String lines = this.formatNodeStatus(entry); - if (lines.length() > 0) { - pw.print((written++ > 0 ? ",\n" : "\n") + lines); - } - } - pw.print("\n],\n"); - } - - private void writeBridges(List<SummaryDocument> bridges, - PrintWriter pw) { - pw.write("\"bridges_published\":\"" + bridgesPublishedString - + "\",\n\"bridges\":["); - int written = 0; - for (SummaryDocument entry : bridges) { - String lines = this.formatNodeStatus(entry); - if (lines.length() > 0) { - pw.print((written++ > 0 ? ",\n" : "\n") + lines); - } - } - pw.print("\n]}\n"); - } - - private String formatNodeStatus(SummaryDocument entry) { - if (this.resourceType == null) { - return ""; - } else if (this.resourceType.equals("summary")) { - return this.writeSummaryLine(entry); - } else if (this.resourceType.equals("details")) { - return this.writeDetailsLines(entry); - } else if (this.resourceType.equals("bandwidth")) { - return this.writeBandwidthLines(entry); - } else if (this.resourceType.equals("weights")) { - return this.writeWeightsLines(entry); - } else if (this.resourceType.equals("clients")) { - return this.writeClientsLines(entry); - } else if (this.resourceType.equals("uptime")) { - return this.writeUptimeLines(entry); - } else { - return ""; - } - } - - private String writeSummaryLine(SummaryDocument entry) { - return entry.isRelay() ? writeRelaySummaryLine(entry) - : writeBridgeSummaryLine(entry); - } - - private String writeRelaySummaryLine(SummaryDocument entry) { - String nickname = !entry.getNickname().equals("Unnamed") ? - entry.getNickname() : null; - String fingerprint = entry.getFingerprint(); - String running = entry.isRunning() ? "true" : "false"; - List<String> addresses = entry.getAddresses(); - StringBuilder addressesBuilder = new StringBuilder(); - int written = 0; - for (String address : addresses) { - addressesBuilder.append((written++ > 0 ? "," : "") + "\"" - + address.toLowerCase() + "\""); - } - return String.format("{%s\"f\":\"%s\",\"a\":[%s],\"r\":%s}", - (nickname == null ? "" : "\"n\":\"" + nickname + "\","), - fingerprint, addressesBuilder.toString(), running); - } - - private String writeBridgeSummaryLine(SummaryDocument entry) { - String nickname = !entry.getNickname().equals("Unnamed") ? - entry.getNickname() : null; - String hashedFingerprint = entry.getFingerprint(); - String running = entry.isRunning() ? "true" : "false"; - return String.format("{%s\"h\":\"%s\",\"r\":%s}", - (nickname == null ? "" : "\"n\":\"" + nickname + "\","), - hashedFingerprint, running); - } - - private String writeDetailsLines(SummaryDocument entry) { - String fingerprint = entry.getFingerprint(); - if (this.fields != null) { - /* TODO Maybe there's a more elegant way (more maintainable, more - * efficient, etc.) to implement this? */ - DetailsDocument detailsDocument = documentStore.retrieve( - DetailsDocument.class, true, fingerprint); - if (detailsDocument != null) { - DetailsDocument dd = new DetailsDocument(); - for (String field : this.fields) { - if (field.equals("nickname")) { - dd.setNickname(detailsDocument.getNickname()); - } else if (field.equals("fingerprint")) { - dd.setFingerprint(detailsDocument.getFingerprint()); - } else if (field.equals("hashed_fingerprint")) { - dd.setHashedFingerprint( - detailsDocument.getHashedFingerprint()); - } else if (field.equals("or_addresses")) { - dd.setOrAddresses(detailsDocument.getOrAddresses()); - } else if (field.equals("exit_addresses")) { - dd.setExitAddresses(detailsDocument.getExitAddresses()); - } else if (field.equals("dir_address")) { - dd.setDirAddress(detailsDocument.getDirAddress()); - } else if (field.equals("last_seen")) { - dd.setLastSeen(detailsDocument.getLastSeen()); - } else if (field.equals("last_changed_address_or_port")) { - dd.setLastChangedAddressOrPort( - detailsDocument.getLastChangedAddressOrPort()); - } else if (field.equals("first_seen")) { - dd.setFirstSeen(detailsDocument.getFirstSeen()); - } else if (field.equals("running")) { - dd.setRunning(detailsDocument.getRunning()); - } else if (field.equals("flags")) { - dd.setFlags(detailsDocument.getFlags()); - } else if (field.equals("country")) { - dd.setCountry(detailsDocument.getCountry()); - } else if (field.equals("country_name")) { - dd.setCountryName(detailsDocument.getCountryName()); - } else if (field.equals("region_name")) { - dd.setRegionName(detailsDocument.getRegionName()); - } else if (field.equals("city_name")) { - dd.setCityName(detailsDocument.getCityName()); - } else if (field.equals("latitude")) { - dd.setLatitude(detailsDocument.getLatitude()); - } else if (field.equals("longitude")) { - dd.setLongitude(detailsDocument.getLongitude()); - } else if (field.equals("as_number")) { - dd.setAsNumber(detailsDocument.getAsNumber()); - } else if (field.equals("as_name")) { - dd.setAsName(detailsDocument.getAsName()); - } else if (field.equals("consensus_weight")) { - dd.setConsensusWeight(detailsDocument.getConsensusWeight()); - } else if (field.equals("host_name")) { - dd.setHostName(detailsDocument.getHostName()); - } else if (field.equals("last_restarted")) { - dd.setLastRestarted(detailsDocument.getLastRestarted()); - } else if (field.equals("bandwidth_rate")) { - dd.setBandwidthRate(detailsDocument.getBandwidthRate()); - } else if (field.equals("bandwidth_burst")) { - dd.setBandwidthBurst(detailsDocument.getBandwidthBurst()); - } else if (field.equals("observed_bandwidth")) { - dd.setObservedBandwidth( - detailsDocument.getObservedBandwidth()); - } else if (field.equals("advertised_bandwidth")) { - dd.setAdvertisedBandwidth( - detailsDocument.getAdvertisedBandwidth()); - } else if (field.equals("exit_policy")) { - dd.setExitPolicy(detailsDocument.getExitPolicy()); - } else if (field.equals("exit_policy_summary")) { - dd.setExitPolicySummary( - detailsDocument.getExitPolicySummary()); - } else if (field.equals("exit_policy_v6_summary")) { - dd.setExitPolicyV6Summary( - detailsDocument.getExitPolicyV6Summary()); - } else if (field.equals("contact")) { - dd.setContact(detailsDocument.getContact()); - } else if (field.equals("platform")) { - dd.setPlatform(detailsDocument.getPlatform()); - } else if (field.equals("family")) { - dd.setFamily(detailsDocument.getFamily()); - } else if (field.equals("advertised_bandwidth_fraction")) { - dd.setAdvertisedBandwidthFraction( - detailsDocument.getAdvertisedBandwidthFraction()); - } else if (field.equals("consensus_weight_fraction")) { - dd.setConsensusWeightFraction( - detailsDocument.getConsensusWeightFraction()); - } else if (field.equals("guard_probability")) { - dd.setGuardProbability(detailsDocument.getGuardProbability()); - } else if (field.equals("middle_probability")) { - dd.setMiddleProbability( - detailsDocument.getMiddleProbability()); - } else if (field.equals("exit_probability")) { - dd.setExitProbability(detailsDocument.getExitProbability()); - } else if (field.equals("recommended_version")) { - dd.setRecommendedVersion( - detailsDocument.getRecommendedVersion()); - } else if (field.equals("hibernating")) { - dd.setHibernating(detailsDocument.getHibernating()); - } else if (field.equals("pool_assignment")) { - dd.setPoolAssignment(detailsDocument.getPoolAssignment()); - } - } - /* Don't escape HTML characters, like < and >, contained in - * strings. */ - Gson gson = new GsonBuilder().disableHtmlEscaping().create(); - /* Whenever we provide Gson with a string containing an escaped - * non-ASCII character like \u00F2, it escapes the \ to \\, which - * we need to undo before including the string in a response. */ - return gson.toJson(dd).replaceAll("\\\\\\\\u", "\\\\u"); - } else { - // TODO We should probably log that we didn't find a details - // document that we expected to exist. - return ""; - } - } else { - DetailsDocument detailsDocument = documentStore.retrieve( - DetailsDocument.class, false, fingerprint); - if (detailsDocument != null) { - return detailsDocument.getDocumentString(); - } else { - // TODO We should probably log that we didn't find a details - // document that we expected to exist. - return ""; - } - } - } - - private String writeBandwidthLines(SummaryDocument entry) { - String fingerprint = entry.getFingerprint(); - BandwidthDocument bandwidthDocument = this.documentStore.retrieve( - BandwidthDocument.class, false, fingerprint); - if (bandwidthDocument != null && - bandwidthDocument.getDocumentString() != null) { - return bandwidthDocument.getDocumentString(); - } else { - return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}"; - } - } - - private String writeWeightsLines(SummaryDocument entry) { - String fingerprint = entry.getFingerprint(); - WeightsDocument weightsDocument = this.documentStore.retrieve( - WeightsDocument.class, false, fingerprint); - if (weightsDocument != null && - weightsDocument.getDocumentString() != null) { - return weightsDocument.getDocumentString(); - } else { - return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}"; - } - } - - private String writeClientsLines(SummaryDocument entry) { - String fingerprint = entry.getFingerprint(); - ClientsDocument clientsDocument = this.documentStore.retrieve( - ClientsDocument.class, false, fingerprint); - if (clientsDocument != null && - clientsDocument.getDocumentString() != null) { - return clientsDocument.getDocumentString(); - } else { - return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}"; - } - } - - private String writeUptimeLines(SummaryDocument entry) { - String fingerprint = entry.getFingerprint(); - UptimeDocument uptimeDocument = this.documentStore.retrieve( - UptimeDocument.class, false, fingerprint); - if (uptimeDocument != null && - uptimeDocument.getDocumentString() != null) { - return uptimeDocument.getDocumentString(); - } else { - return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}"; - } - } -} diff --git a/src/org/torproject/onionoo/ReverseDomainNameResolver.java b/src/org/torproject/onionoo/ReverseDomainNameResolver.java deleted file mode 100644 index 3dbf9d1..0000000 --- a/src/org/torproject/onionoo/ReverseDomainNameResolver.java +++ /dev/null @@ -1,174 +0,0 @@ -/* Copyright 2013 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public class ReverseDomainNameResolver { - - private class RdnsLookupWorker extends Thread { - public void run() { - while (time.currentTimeMillis() - RDNS_LOOKUP_MAX_DURATION_MILLIS - <= startedRdnsLookups) { - String rdnsLookupJob = null; - synchronized (rdnsLookupJobs) { - for (String job : rdnsLookupJobs) { - rdnsLookupJob = job; - rdnsLookupJobs.remove(job); - break; - } - } - if (rdnsLookupJob == null) { - break; - } - RdnsLookupRequest request = new RdnsLookupRequest(this, - rdnsLookupJob); - request.setDaemon(true); - request.start(); - try { - Thread.sleep(RDNS_LOOKUP_MAX_REQUEST_MILLIS); - } catch (InterruptedException e) { - /* Getting interrupted should be the default case. */ - } - String hostName = request.getHostName(); - if (hostName != null) { - synchronized (rdnsLookupResults) { - rdnsLookupResults.put(rdnsLookupJob, hostName); - } - } - long lookupMillis = request.getLookupMillis(); - if (lookupMillis >= 0L) { - synchronized (rdnsLookupMillis) { - rdnsLookupMillis.add(lookupMillis); - } - } - } - } - } - - private class RdnsLookupRequest extends Thread { - private RdnsLookupWorker parent; - private String address, hostName; - private long lookupStartedMillis = -1L, lookupCompletedMillis = -1L; - public RdnsLookupRequest(RdnsLookupWorker parent, String address) { - this.parent = parent; - this.address = address; - } - public void run() { - this.lookupStartedMillis = time.currentTimeMillis(); - try { - String result = InetAddress.getByName(this.address).getHostName(); - synchronized (this) { - this.hostName = result; - } - } catch (UnknownHostException e) { - /* We'll try again the next time. */ - } - this.lookupCompletedMillis = time.currentTimeMillis(); - this.parent.interrupt(); - } - public synchronized String getHostName() { - return hostName; - } - public synchronized long getLookupMillis() { - return this.lookupCompletedMillis - this.lookupStartedMillis; - } - } - - private Time time; - - public ReverseDomainNameResolver() { - this.time = ApplicationFactory.getTime(); - } - - private static final long RDNS_LOOKUP_MAX_REQUEST_MILLIS = - DateTimeHelper.TEN_SECONDS; - private static final long RDNS_LOOKUP_MAX_DURATION_MILLIS = - DateTimeHelper.FIVE_MINUTES; - private static final long RDNS_LOOKUP_MAX_AGE_MILLIS = - DateTimeHelper.TWELVE_HOURS; - private static final int RDNS_LOOKUP_WORKERS_NUM = 5; - - private Map<String, Long> addressLastLookupTimes; - - private Set<String> rdnsLookupJobs; - - private Map<String, String> rdnsLookupResults; - - private List<Long> rdnsLookupMillis; - - private long startedRdnsLookups; - - private List<RdnsLookupWorker> rdnsLookupWorkers; - - public void setAddresses(Map<String, Long> addressLastLookupTimes) { - this.addressLastLookupTimes = addressLastLookupTimes; - } - - public void startReverseDomainNameLookups() { - this.startedRdnsLookups = this.time.currentTimeMillis(); - this.rdnsLookupJobs = new HashSet<String>(); - for (Map.Entry<String, Long> e : - this.addressLastLookupTimes.entrySet()) { - if (e.getValue() < this.startedRdnsLookups - - RDNS_LOOKUP_MAX_AGE_MILLIS) { - this.rdnsLookupJobs.add(e.getKey()); - } - } - this.rdnsLookupResults = new HashMap<String, String>(); - this.rdnsLookupMillis = new ArrayList<Long>(); - this.rdnsLookupWorkers = new ArrayList<RdnsLookupWorker>(); - for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) { - RdnsLookupWorker rdnsLookupWorker = new RdnsLookupWorker(); - this.rdnsLookupWorkers.add(rdnsLookupWorker); - rdnsLookupWorker.setDaemon(true); - rdnsLookupWorker.start(); - } - } - - public void finishReverseDomainNameLookups() { - for (RdnsLookupWorker rdnsLookupWorker : this.rdnsLookupWorkers) { - try { - rdnsLookupWorker.join(); - } catch (InterruptedException e) { - /* This is not something that we can take care of. Just leave the - * worker thread alone. */ - } - } - } - - public Map<String, String> getLookupResults() { - synchronized (this.rdnsLookupResults) { - return new HashMap<String, String>(this.rdnsLookupResults); - } - } - - public long getLookupStartMillis() { - return this.startedRdnsLookups; - } - - public String getStatsString() { - StringBuilder sb = new StringBuilder(); - sb.append(" " + Logger.formatDecimalNumber(rdnsLookupMillis.size()) - + " lookups performed\n"); - if (rdnsLookupMillis.size() > 0) { - Collections.sort(rdnsLookupMillis); - sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get(0)) - + " minimum lookup time\n"); - sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get( - rdnsLookupMillis.size() / 2)) + " median lookup time\n"); - sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get( - rdnsLookupMillis.size() - 1)) + " maximum lookup time\n"); - } - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/StatusUpdater.java b/src/org/torproject/onionoo/StatusUpdater.java deleted file mode 100644 index fb82182..0000000 --- a/src/org/torproject/onionoo/StatusUpdater.java +++ /dev/null @@ -1,11 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -public interface StatusUpdater { - - public abstract void updateStatuses(); - - public abstract String getStatsString(); -} - diff --git a/src/org/torproject/onionoo/SummaryDocument.java b/src/org/torproject/onionoo/SummaryDocument.java deleted file mode 100644 index 61a3ec6..0000000 --- a/src/org/torproject/onionoo/SummaryDocument.java +++ /dev/null @@ -1,201 +0,0 @@ -/* Copyright 2013--2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.regex.Pattern; - -import org.apache.commons.codec.DecoderException; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.codec.digest.DigestUtils; - -public class SummaryDocument extends Document { - - private boolean t; - public void setRelay(boolean isRelay) { - this.t = isRelay; - } - public boolean isRelay() { - return this.t; - } - - private String f; - public void setFingerprint(String fingerprint) { - if (fingerprint != null) { - Pattern fingerprintPattern = Pattern.compile("^[0-9a-fA-F]{40}$"); - if (!fingerprintPattern.matcher(fingerprint).matches()) { - throw new IllegalArgumentException("Fingerprint '" + fingerprint - + "' is not a valid fingerprint."); - } - } - this.f = fingerprint; - } - public String getFingerprint() { - return this.f; - } - - public String getHashedFingerprint() { - String hashedFingerprint = null; - if (this.f != null) { - try { - hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex( - this.f.toCharArray())).toUpperCase(); - } catch (DecoderException e) { - /* Format tested in setFingerprint(). */ - } - } - return hashedFingerprint; - } - - private String n; - public void setNickname(String nickname) { - if (nickname == null || nickname.equals("Unnamed")) { - this.n = null; - } else { - this.n = nickname; - } - } - public String getNickname() { - return this.n == null ? "Unnamed" : this.n; - } - - private String[] ad; - public void setAddresses(List<String> addresses) { - this.ad = this.collectionToStringArray(addresses); - } - public List<String> getAddresses() { - return this.stringArrayToList(this.ad); - } - - private String[] collectionToStringArray( - Collection<String> collection) { - String[] stringArray = null; - if (collection != null && !collection.isEmpty()) { - stringArray = new String[collection.size()]; - int i = 0; - for (String string : collection) { - stringArray[i++] = string; - } - } - return stringArray; - } - private List<String> stringArrayToList(String[] stringArray) { - List<String> list; - if (stringArray == null) { - list = new ArrayList<String>(); - } else { - list = Arrays.asList(stringArray); - } - return list; - } - private SortedSet<String> stringArrayToSortedSet(String[] stringArray) { - SortedSet<String> sortedSet = new TreeSet<String>(); - if (stringArray != null) { - sortedSet.addAll(Arrays.asList(stringArray)); - } - return sortedSet; - } - - private String cc; - public void setCountryCode(String countryCode) { - this.cc = countryCode; - } - public String getCountryCode() { - return this.cc; - } - - private String as; - public void setASNumber(String aSNumber) { - this.as = aSNumber; - } - public String getASNumber() { - return this.as; - } - - private String fs; - public void setFirstSeenMillis(long firstSeenMillis) { - this.fs = DateTimeHelper.format(firstSeenMillis); - } - public long getFirstSeenMillis() { - return DateTimeHelper.parse(this.fs); - } - - private String ls; - public void setLastSeenMillis(long lastSeenMillis) { - this.ls = DateTimeHelper.format(lastSeenMillis); - } - public long getLastSeenMillis() { - return DateTimeHelper.parse(this.ls); - } - - private String[] rf; - public void setRelayFlags(SortedSet<String> relayFlags) { - this.rf = this.collectionToStringArray(relayFlags); - } - public SortedSet<String> getRelayFlags() { - return this.stringArrayToSortedSet(this.rf); - } - - private long cw; - public void setConsensusWeight(long consensusWeight) { - this.cw = consensusWeight; - } - public long getConsensusWeight() { - return this.cw; - } - - private boolean r; - public void setRunning(boolean isRunning) { - this.r = isRunning; - } - public boolean isRunning() { - return this.r; - } - - private String c; - public void setContact(String contact) { - if (contact != null && contact.length() == 0) { - this.c = null; - } else { - this.c = contact; - } - } - public String getContact() { - return this.c; - } - - private String[] ff; - public void setFamilyFingerprints( - SortedSet<String> familyFingerprints) { - this.ff = this.collectionToStringArray(familyFingerprints); - } - public SortedSet<String> getFamilyFingerprints() { - return this.stringArrayToSortedSet(this.ff); - } - - public SummaryDocument(boolean isRelay, String nickname, - String fingerprint, List<String> addresses, long lastSeenMillis, - boolean running, SortedSet<String> relayFlags, long consensusWeight, - String countryCode, long firstSeenMillis, String aSNumber, - String contact, SortedSet<String> familyFingerprints) { - this.setRelay(isRelay); - this.setNickname(nickname); - this.setFingerprint(fingerprint); - this.setAddresses(addresses); - this.setLastSeenMillis(lastSeenMillis); - this.setRunning(running); - this.setRelayFlags(relayFlags); - this.setConsensusWeight(consensusWeight); - this.setCountryCode(countryCode); - this.setFirstSeenMillis(firstSeenMillis); - this.setASNumber(aSNumber); - this.setContact(contact); - this.setFamilyFingerprints(familyFingerprints); - } -} - diff --git a/src/org/torproject/onionoo/SummaryDocumentWriter.java b/src/org/torproject/onionoo/SummaryDocumentWriter.java deleted file mode 100644 index 7b257f5..0000000 --- a/src/org/torproject/onionoo/SummaryDocumentWriter.java +++ /dev/null @@ -1,87 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.ArrayList; -import java.util.List; -import java.util.SortedSet; - -public class SummaryDocumentWriter implements DocumentWriter { - - private DocumentStore documentStore; - - public SummaryDocumentWriter() { - this.documentStore = ApplicationFactory.getDocumentStore(); - } - - private int writtenDocuments = 0, deletedDocuments = 0; - - public void writeDocuments() { - long maxLastSeenMillis = 0L; - for (String fingerprint : this.documentStore.list(NodeStatus.class)) { - NodeStatus nodeStatus = this.documentStore.retrieve( - NodeStatus.class, true, fingerprint); - if (nodeStatus != null && - nodeStatus.getLastSeenMillis() > maxLastSeenMillis) { - maxLastSeenMillis = nodeStatus.getLastSeenMillis(); - } - } - long cutoff = maxLastSeenMillis - DateTimeHelper.ONE_WEEK; - for (String fingerprint : this.documentStore.list(NodeStatus.class)) { - NodeStatus nodeStatus = this.documentStore.retrieve( - NodeStatus.class, - true, fingerprint); - if (nodeStatus == null) { - continue; - } - if (nodeStatus.getLastSeenMillis() < cutoff) { - if (this.documentStore.remove(SummaryDocument.class, - fingerprint)) { - this.deletedDocuments++; - } - continue; - } - boolean isRelay = nodeStatus.isRelay(); - String nickname = nodeStatus.getNickname(); - List<String> addresses = new ArrayList<String>(); - addresses.add(nodeStatus.getAddress()); - for (String orAddress : nodeStatus.getOrAddresses()) { - if (!addresses.contains(orAddress)) { - addresses.add(orAddress); - } - } - for (String exitAddress : nodeStatus.getExitAddresses()) { - if (!addresses.contains(exitAddress)) { - addresses.add(exitAddress); - } - } - long lastSeenMillis = nodeStatus.getLastSeenMillis(); - boolean running = nodeStatus.getRunning(); - SortedSet<String> relayFlags = nodeStatus.getRelayFlags(); - long consensusWeight = nodeStatus.getConsensusWeight(); - String countryCode = nodeStatus.getCountryCode(); - long firstSeenMillis = nodeStatus.getFirstSeenMillis(); - String aSNumber = nodeStatus.getASNumber(); - String contact = nodeStatus.getContact(); - SortedSet<String> familyFingerprints = - nodeStatus.getFamilyFingerprints(); - SummaryDocument summaryDocument = new SummaryDocument(isRelay, - nickname, fingerprint, addresses, lastSeenMillis, running, - relayFlags, consensusWeight, countryCode, firstSeenMillis, - aSNumber, contact, familyFingerprints); - if (this.documentStore.store(summaryDocument, fingerprint)) { - this.writtenDocuments++; - }; - } - Logger.printStatusTime("Wrote summary document files"); - } - - public String getStatsString() { - StringBuilder sb = new StringBuilder(); - sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments) - + " summary document files written\n"); - sb.append(" " + Logger.formatDecimalNumber(this.deletedDocuments) - + " summary document files deleted\n"); - return sb.toString(); - } -} diff --git a/src/org/torproject/onionoo/Time.java b/src/org/torproject/onionoo/Time.java deleted file mode 100644 index c969556..0000000 --- a/src/org/torproject/onionoo/Time.java +++ /dev/null @@ -1,14 +0,0 @@ -/* Copyright 2013 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -/* - * Wrapper for System.currentTimeMillis() that can be replaced with a - * custom time source for testing. - */ -public class Time { - public long currentTimeMillis() { - return System.currentTimeMillis(); - } -} - diff --git a/src/org/torproject/onionoo/UpdateStatus.java b/src/org/torproject/onionoo/UpdateStatus.java deleted file mode 100644 index a697cbf..0000000 --- a/src/org/torproject/onionoo/UpdateStatus.java +++ /dev/null @@ -1,7 +0,0 @@ -/* Copyright 2013 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -public class UpdateStatus extends Document { -} - diff --git a/src/org/torproject/onionoo/UptimeDocument.java b/src/org/torproject/onionoo/UptimeDocument.java deleted file mode 100644 index 1946e2c..0000000 --- a/src/org/torproject/onionoo/UptimeDocument.java +++ /dev/null @@ -1,23 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Map; - -public class UptimeDocument extends Document { - - @SuppressWarnings("unused") - private String fingerprint; - public void setFingerprint(String fingerprint) { - this.fingerprint = fingerprint; - } - - private Map<String, GraphHistory> uptime; - public void setUptime(Map<String, GraphHistory> uptime) { - this.uptime = uptime; - } - public Map<String, GraphHistory> getUptime() { - return this.uptime; - } -} - diff --git a/src/org/torproject/onionoo/UptimeDocumentWriter.java b/src/org/torproject/onionoo/UptimeDocumentWriter.java deleted file mode 100644 index 8293f77..0000000 --- a/src/org/torproject/onionoo/UptimeDocumentWriter.java +++ /dev/null @@ -1,291 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.SortedSet; -import java.util.TreeSet; - -public class UptimeDocumentWriter implements FingerprintListener, - DocumentWriter { - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - private long now; - - public UptimeDocumentWriter() { - this.descriptorSource = ApplicationFactory.getDescriptorSource(); - this.documentStore = ApplicationFactory.getDocumentStore(); - this.now = ApplicationFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.BRIDGE_STATUSES); - } - - private SortedSet<String> newRelayFingerprints = new TreeSet<String>(), - newBridgeFingerprints = new TreeSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (relay) { - this.newRelayFingerprints.addAll(fingerprints); - } else { - this.newBridgeFingerprints.addAll(fingerprints); - } - } - - public void writeDocuments() { - UptimeStatus uptimeStatus = this.documentStore.retrieve( - UptimeStatus.class, true); - if (uptimeStatus == null) { - return; - } - for (String fingerprint : this.newRelayFingerprints) { - this.updateDocument(true, fingerprint, - uptimeStatus.getRelayHistory()); - } - for (String fingerprint : this.newBridgeFingerprints) { - this.updateDocument(false, fingerprint, - uptimeStatus.getBridgeHistory()); - } - Logger.printStatusTime("Wrote uptime document files"); - } - - private int writtenDocuments = 0; - - private void updateDocument(boolean relay, String fingerprint, - SortedSet<UptimeHistory> knownStatuses) { - UptimeStatus uptimeStatus = this.documentStore.retrieve( - UptimeStatus.class, true, fingerprint); - if (uptimeStatus != null) { - SortedSet<UptimeHistory> history = relay - ? uptimeStatus.getRelayHistory() - : uptimeStatus.getBridgeHistory(); - UptimeDocument uptimeDocument = this.compileUptimeDocument(relay, - fingerprint, history, knownStatuses); - this.documentStore.store(uptimeDocument, fingerprint); - this.writtenDocuments++; - } - } - - private String[] graphNames = new String[] { - "1_week", - "1_month", - "3_months", - "1_year", - "5_years" }; - - private long[] graphIntervals = new long[] { - DateTimeHelper.ONE_WEEK, - DateTimeHelper.ROUGHLY_ONE_MONTH, - DateTimeHelper.ROUGHLY_THREE_MONTHS, - DateTimeHelper.ROUGHLY_ONE_YEAR, - DateTimeHelper.ROUGHLY_FIVE_YEARS }; - - private long[] dataPointIntervals = new long[] { - DateTimeHelper.ONE_HOUR, - DateTimeHelper.FOUR_HOURS, - DateTimeHelper.TWELVE_HOURS, - DateTimeHelper.TWO_DAYS, - DateTimeHelper.TEN_DAYS }; - - private UptimeDocument compileUptimeDocument(boolean relay, - String fingerprint, SortedSet<UptimeHistory> history, - SortedSet<UptimeHistory> knownStatuses) { - UptimeDocument uptimeDocument = new UptimeDocument(); - uptimeDocument.setFingerprint(fingerprint); - Map<String, GraphHistory> uptime = - new LinkedHashMap<String, GraphHistory>(); - for (int graphIntervalIndex = 0; graphIntervalIndex < - this.graphIntervals.length; graphIntervalIndex++) { - String graphName = this.graphNames[graphIntervalIndex]; - GraphHistory graphHistory = this.compileUptimeHistory( - graphIntervalIndex, relay, history, knownStatuses); - if (graphHistory != null) { - uptime.put(graphName, graphHistory); - } - } - uptimeDocument.setUptime(uptime); - return uptimeDocument; - } - - private GraphHistory compileUptimeHistory(int graphIntervalIndex, - boolean relay, SortedSet<UptimeHistory> history, - SortedSet<UptimeHistory> knownStatuses) { - long graphInterval = this.graphIntervals[graphIntervalIndex]; - long dataPointInterval = - this.dataPointIntervals[graphIntervalIndex]; - int dataPointIntervalHours = (int) (dataPointInterval - / DateTimeHelper.ONE_HOUR); - List<Integer> uptimeDataPoints = new ArrayList<Integer>(); - long intervalStartMillis = ((this.now - graphInterval) - / dataPointInterval) * dataPointInterval; - int uptimeHours = 0; - long firstStatusStartMillis = -1L; - for (UptimeHistory hist : history) { - if (hist.isRelay() != relay) { - continue; - } - if (firstStatusStartMillis < 0L) { - firstStatusStartMillis = hist.getStartMillis(); - } - long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR - * hist.getUptimeHours(); - if (histEndMillis < intervalStartMillis) { - continue; - } - while (hist.getStartMillis() >= intervalStartMillis - + dataPointInterval) { - if (firstStatusStartMillis < intervalStartMillis - + dataPointInterval) { - uptimeDataPoints.add(uptimeHours); - } else { - uptimeDataPoints.add(-1); - } - uptimeHours = 0; - intervalStartMillis += dataPointInterval; - } - while (histEndMillis >= intervalStartMillis + dataPointInterval) { - uptimeHours += (int) ((intervalStartMillis + dataPointInterval - - Math.max(hist.getStartMillis(), intervalStartMillis)) - / DateTimeHelper.ONE_HOUR); - uptimeDataPoints.add(uptimeHours); - uptimeHours = 0; - intervalStartMillis += dataPointInterval; - } - uptimeHours += (int) ((histEndMillis - Math.max( - hist.getStartMillis(), intervalStartMillis)) - / DateTimeHelper.ONE_HOUR); - } - uptimeDataPoints.add(uptimeHours); - List<Integer> statusDataPoints = new ArrayList<Integer>(); - intervalStartMillis = ((this.now - graphInterval) - / dataPointInterval) * dataPointInterval; - int statusHours = -1; - for (UptimeHistory hist : knownStatuses) { - if (hist.isRelay() != relay) { - continue; - } - long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR - * hist.getUptimeHours(); - if (histEndMillis < intervalStartMillis) { - continue; - } - while (hist.getStartMillis() >= intervalStartMillis - + dataPointInterval) { - statusDataPoints.add(statusHours * 5 > dataPointIntervalHours - ? statusHours : -1); - statusHours = -1; - intervalStartMillis += dataPointInterval; - } - while (histEndMillis >= intervalStartMillis + dataPointInterval) { - if (statusHours < 0) { - statusHours = 0; - } - statusHours += (int) ((intervalStartMillis + dataPointInterval - - Math.max(Math.max(hist.getStartMillis(), - firstStatusStartMillis), intervalStartMillis)) - / DateTimeHelper.ONE_HOUR); - statusDataPoints.add(statusHours * 5 > dataPointIntervalHours - ? statusHours : -1); - statusHours = -1; - intervalStartMillis += dataPointInterval; - } - if (statusHours < 0) { - statusHours = 0; - } - statusHours += (int) ((histEndMillis - Math.max(Math.max( - hist.getStartMillis(), firstStatusStartMillis), - intervalStartMillis)) / DateTimeHelper.ONE_HOUR); - } - if (statusHours > 0) { - statusDataPoints.add(statusHours * 5 > dataPointIntervalHours - ? statusHours : -1); - } - List<Double> dataPoints = new ArrayList<Double>(); - for (int dataPointIndex = 0; dataPointIndex < statusDataPoints.size(); - dataPointIndex++) { - if (dataPointIndex >= uptimeDataPoints.size()) { - dataPoints.add(0.0); - } else if (uptimeDataPoints.get(dataPointIndex) >= 0 && - statusDataPoints.get(dataPointIndex) > 0) { - dataPoints.add(((double) uptimeDataPoints.get(dataPointIndex)) - / ((double) statusDataPoints.get(dataPointIndex))); - } else { - dataPoints.add(-1.0); - } - } - int firstNonNullIndex = -1, lastNonNullIndex = -1; - for (int dataPointIndex = 0; dataPointIndex < dataPoints.size(); - dataPointIndex++) { - double dataPoint = dataPoints.get(dataPointIndex); - if (dataPoint >= 0.0) { - if (firstNonNullIndex < 0) { - firstNonNullIndex = dataPointIndex; - } - lastNonNullIndex = dataPointIndex; - } - } - if (firstNonNullIndex < 0) { - return null; - } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) - * dataPointInterval + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis >= - this.now - graphIntervals[graphIntervalIndex - 1]) { - /* Skip uptime history object, because it doesn't contain - * anything new that wasn't already contained in the last - * uptime history object(s). */ - return null; - } - long lastDataPointMillis = firstDataPointMillis - + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; - int count = lastNonNullIndex - firstNonNullIndex + 1; - GraphHistory graphHistory = new GraphHistory(); - graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis)); - graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis)); - graphHistory.setInterval((int) (dataPointInterval - / DateTimeHelper.ONE_SECOND)); - graphHistory.setFactor(1.0 / 999.0); - graphHistory.setCount(count); - int previousNonNullIndex = -2; - boolean foundTwoAdjacentDataPoints = false; - List<Integer> values = new ArrayList<Integer>(); - for (int dataPointIndex = firstNonNullIndex; dataPointIndex <= - lastNonNullIndex; dataPointIndex++) { - double dataPoint = dataPoints.get(dataPointIndex); - if (dataPoint >= 0.0) { - if (dataPointIndex - previousNonNullIndex == 1) { - foundTwoAdjacentDataPoints = true; - } - previousNonNullIndex = dataPointIndex; - } - values.add(dataPoint < -0.5 ? null : ((int) (dataPoint * 999.0))); - } - graphHistory.setValues(values); - if (foundTwoAdjacentDataPoints) { - return graphHistory; - } else { - return null; - } - } - - public String getStatsString() { - StringBuilder sb = new StringBuilder(); - sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments) - + " uptime document files written\n"); - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/UptimeStatus.java b/src/org/torproject/onionoo/UptimeStatus.java deleted file mode 100644 index 92ca629..0000000 --- a/src/org/torproject/onionoo/UptimeStatus.java +++ /dev/null @@ -1,226 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Scanner; -import java.util.SortedSet; -import java.util.TreeSet; - -class UptimeHistory - implements Comparable<UptimeHistory> { - - private boolean relay; - public boolean isRelay() { - return this.relay; - } - - private long startMillis; - public long getStartMillis() { - return this.startMillis; - } - - private int uptimeHours; - public int getUptimeHours() { - return this.uptimeHours; - } - - UptimeHistory(boolean relay, long startMillis, - int uptimeHours) { - this.relay = relay; - this.startMillis = startMillis; - this.uptimeHours = uptimeHours; - } - - public static UptimeHistory fromString(String uptimeHistoryString) { - String[] parts = uptimeHistoryString.split(" ", 3); - if (parts.length != 3) { - return null; - } - boolean relay = false; - if (parts[0].equals("r")) { - relay = true; - } else if (!parts[0].equals("b")) { - return null; - } - long startMillis = DateTimeHelper.parse(parts[1], - DateTimeHelper.DATEHOUR_NOSPACE_FORMAT); - if (startMillis < 0L) { - return null; - } - int uptimeHours = -1; - try { - uptimeHours = Integer.parseInt(parts[2]); - } catch (NumberFormatException e) { - return null; - } - return new UptimeHistory(relay, startMillis, uptimeHours); - } - - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(this.relay ? "r" : "b"); - sb.append(" " + DateTimeHelper.format(this.startMillis, - DateTimeHelper.DATEHOUR_NOSPACE_FORMAT)); - sb.append(" " + String.format("%d", this.uptimeHours)); - return sb.toString(); - } - - public void addUptime(UptimeHistory other) { - this.uptimeHours += other.uptimeHours; - if (this.startMillis > other.startMillis) { - this.startMillis = other.startMillis; - } - } - - public int compareTo(UptimeHistory other) { - if (this.relay && !other.relay) { - return -1; - } else if (!this.relay && other.relay) { - return 1; - } - return this.startMillis < other.startMillis ? -1 : - this.startMillis > other.startMillis ? 1 : 0; - } - - public boolean equals(Object other) { - return other instanceof UptimeHistory && - this.relay == ((UptimeHistory) other).relay && - this.startMillis == ((UptimeHistory) other).startMillis; - } - - public int hashCode() { - return (int) this.startMillis + (this.relay ? 1 : 0); - } -} - -public class UptimeStatus extends Document { - - private transient String fingerprint; - - private transient boolean isDirty = false; - - private SortedSet<UptimeHistory> relayHistory = - new TreeSet<UptimeHistory>(); - public void setRelayHistory(SortedSet<UptimeHistory> relayHistory) { - this.relayHistory = relayHistory; - } - public SortedSet<UptimeHistory> getRelayHistory() { - return this.relayHistory; - } - - private SortedSet<UptimeHistory> bridgeHistory = - new TreeSet<UptimeHistory>(); - public void setBridgeHistory(SortedSet<UptimeHistory> bridgeHistory) { - this.bridgeHistory = bridgeHistory; - } - public SortedSet<UptimeHistory> getBridgeHistory() { - return this.bridgeHistory; - } - - public static UptimeStatus loadOrCreate(String fingerprint) { - UptimeStatus uptimeStatus = (fingerprint == null) ? - ApplicationFactory.getDocumentStore().retrieve( - UptimeStatus.class, true) : - ApplicationFactory.getDocumentStore().retrieve( - UptimeStatus.class, true, fingerprint); - if (uptimeStatus == null) { - uptimeStatus = new UptimeStatus(); - } - uptimeStatus.fingerprint = fingerprint; - return uptimeStatus; - } - - public void fromDocumentString(String documentString) { - Scanner s = new Scanner(documentString); - while (s.hasNextLine()) { - String line = s.nextLine(); - UptimeHistory parsedLine = UptimeHistory.fromString(line); - if (parsedLine != null) { - if (parsedLine.isRelay()) { - this.relayHistory.add(parsedLine); - } else { - this.bridgeHistory.add(parsedLine); - } - } else { - System.err.println("Could not parse uptime history line '" - + line + "'. Skipping."); - } - } - s.close(); - } - - public void addToHistory(boolean relay, SortedSet<Long> newIntervals) { - for (long startMillis : newIntervals) { - SortedSet<UptimeHistory> history = relay ? this.relayHistory - : this.bridgeHistory; - UptimeHistory interval = new UptimeHistory(relay, startMillis, 1); - if (!history.headSet(interval).isEmpty()) { - UptimeHistory prev = history.headSet(interval).last(); - if (prev.isRelay() == interval.isRelay() && - prev.getStartMillis() + DateTimeHelper.ONE_HOUR - * prev.getUptimeHours() > interval.getStartMillis()) { - continue; - } - } - if (!history.tailSet(interval).isEmpty()) { - UptimeHistory next = history.tailSet(interval).first(); - if (next.isRelay() == interval.isRelay() && - next.getStartMillis() < interval.getStartMillis() - + DateTimeHelper.ONE_HOUR) { - continue; - } - } - history.add(interval); - this.isDirty = true; - } - } - - public void storeIfChanged() { - if (this.isDirty) { - this.compressHistory(this.relayHistory); - this.compressHistory(this.bridgeHistory); - if (fingerprint == null) { - ApplicationFactory.getDocumentStore().store(this); - } else { - ApplicationFactory.getDocumentStore().store(this, - this.fingerprint); - } - this.isDirty = false; - } - } - - private void compressHistory(SortedSet<UptimeHistory> history) { - SortedSet<UptimeHistory> uncompressedHistory = - new TreeSet<UptimeHistory>(history); - history.clear(); - UptimeHistory lastInterval = null; - for (UptimeHistory interval : uncompressedHistory) { - if (lastInterval != null && - lastInterval.getStartMillis() + DateTimeHelper.ONE_HOUR - * lastInterval.getUptimeHours() == interval.getStartMillis() && - lastInterval.isRelay() == interval.isRelay()) { - lastInterval.addUptime(interval); - } else { - if (lastInterval != null) { - history.add(lastInterval); - } - lastInterval = interval; - } - } - if (lastInterval != null) { - history.add(lastInterval); - } - } - - public String toDocumentString() { - StringBuilder sb = new StringBuilder(); - for (UptimeHistory interval : this.relayHistory) { - sb.append(interval.toString() + "\n"); - } - for (UptimeHistory interval : this.bridgeHistory) { - sb.append(interval.toString() + "\n"); - } - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/UptimeStatusUpdater.java b/src/org/torproject/onionoo/UptimeStatusUpdater.java deleted file mode 100644 index eccc2f2..0000000 --- a/src/org/torproject/onionoo/UptimeStatusUpdater.java +++ /dev/null @@ -1,126 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Map; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.torproject.descriptor.BridgeNetworkStatus; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.NetworkStatusEntry; -import org.torproject.descriptor.RelayNetworkStatusConsensus; - -public class UptimeStatusUpdater implements DescriptorListener, - StatusUpdater { - - private DescriptorSource descriptorSource; - - public UptimeStatusUpdater() { - this.descriptorSource = ApplicationFactory.getDescriptorSource(); - this.registerDescriptorListeners(); - } - - private void registerDescriptorListeners() { - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.BRIDGE_STATUSES); - } - - public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof RelayNetworkStatusConsensus) { - this.processRelayNetworkStatusConsensus( - (RelayNetworkStatusConsensus) descriptor); - } else if (descriptor instanceof BridgeNetworkStatus) { - this.processBridgeNetworkStatus( - (BridgeNetworkStatus) descriptor); - } - } - - private SortedSet<Long> newRelayStatuses = new TreeSet<Long>(), - newBridgeStatuses = new TreeSet<Long>(); - private SortedMap<String, SortedSet<Long>> - newRunningRelays = new TreeMap<String, SortedSet<Long>>(), - newRunningBridges = new TreeMap<String, SortedSet<Long>>(); - - private void processRelayNetworkStatusConsensus( - RelayNetworkStatusConsensus consensus) { - SortedSet<String> fingerprints = new TreeSet<String>(); - for (NetworkStatusEntry entry : - consensus.getStatusEntries().values()) { - if (entry.getFlags().contains("Running")) { - fingerprints.add(entry.getFingerprint()); - } - } - if (!fingerprints.isEmpty()) { - long dateHourMillis = (consensus.getValidAfterMillis() - / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR; - for (String fingerprint : fingerprints) { - if (!this.newRunningRelays.containsKey(fingerprint)) { - this.newRunningRelays.put(fingerprint, new TreeSet<Long>()); - } - this.newRunningRelays.get(fingerprint).add(dateHourMillis); - } - this.newRelayStatuses.add(dateHourMillis); - } - } - - private void processBridgeNetworkStatus(BridgeNetworkStatus status) { - SortedSet<String> fingerprints = new TreeSet<String>(); - for (NetworkStatusEntry entry : - status.getStatusEntries().values()) { - if (entry.getFlags().contains("Running")) { - fingerprints.add(entry.getFingerprint()); - } - } - if (!fingerprints.isEmpty()) { - long dateHourMillis = (status.getPublishedMillis() - / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR; - for (String fingerprint : fingerprints) { - if (!this.newRunningBridges.containsKey(fingerprint)) { - this.newRunningBridges.put(fingerprint, new TreeSet<Long>()); - } - this.newRunningBridges.get(fingerprint).add(dateHourMillis); - } - this.newBridgeStatuses.add(dateHourMillis); - } - } - - public void updateStatuses() { - for (Map.Entry<String, SortedSet<Long>> e : - this.newRunningRelays.entrySet()) { - this.updateStatus(true, e.getKey(), e.getValue()); - } - this.updateStatus(true, null, this.newRelayStatuses); - for (Map.Entry<String, SortedSet<Long>> e : - this.newRunningBridges.entrySet()) { - this.updateStatus(false, e.getKey(), e.getValue()); - } - this.updateStatus(false, null, this.newBridgeStatuses); - } - - private void updateStatus(boolean relay, String fingerprint, - SortedSet<Long> newUptimeHours) { - UptimeStatus uptimeStatus = UptimeStatus.loadOrCreate(fingerprint); - uptimeStatus.addToHistory(relay, newUptimeHours); - uptimeStatus.storeIfChanged(); - } - - public String getStatsString() { - StringBuilder sb = new StringBuilder(); - sb.append(" " + Logger.formatDecimalNumber( - this.newRelayStatuses.size()) + " hours of relay uptimes " - + "processed\n"); - sb.append(" " + Logger.formatDecimalNumber( - this.newBridgeStatuses.size()) + " hours of bridge uptimes " - + "processed\n"); - sb.append(" " + Logger.formatDecimalNumber( - this.newRunningRelays.size() + this.newRunningBridges.size()) - + " uptime status files updated\n"); - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/WeightsDocument.java b/src/org/torproject/onionoo/WeightsDocument.java deleted file mode 100644 index 4cd0021..0000000 --- a/src/org/torproject/onionoo/WeightsDocument.java +++ /dev/null @@ -1,64 +0,0 @@ -/* Copyright 2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Map; - -public class WeightsDocument extends Document { - - @SuppressWarnings("unused") - private String fingerprint; - public void setFingerprint(String fingerprint) { - this.fingerprint = fingerprint; - } - - @SuppressWarnings("unused") - private Map<String, GraphHistory> advertised_bandwidth_fraction; - public void setAdvertisedBandwidthFraction( - Map<String, GraphHistory> advertisedBandwidthFraction) { - this.advertised_bandwidth_fraction = advertisedBandwidthFraction; - } - - @SuppressWarnings("unused") - private Map<String, GraphHistory> consensus_weight_fraction; - public void setConsensusWeightFraction( - Map<String, GraphHistory> consensusWeightFraction) { - this.consensus_weight_fraction = consensusWeightFraction; - } - - @SuppressWarnings("unused") - private Map<String, GraphHistory> guard_probability; - public void setGuardProbability( - Map<String, GraphHistory> guardProbability) { - this.guard_probability = guardProbability; - } - - @SuppressWarnings("unused") - private Map<String, GraphHistory> middle_probability; - public void setMiddleProbability( - Map<String, GraphHistory> middleProbability) { - this.middle_probability = middleProbability; - } - - @SuppressWarnings("unused") - private Map<String, GraphHistory> exit_probability; - public void setExitProbability( - Map<String, GraphHistory> exitProbability) { - this.exit_probability = exitProbability; - } - - @SuppressWarnings("unused") - private Map<String, GraphHistory> advertised_bandwidth; - public void setAdvertisedBandwidth( - Map<String, GraphHistory> advertisedBandwidth) { - this.advertised_bandwidth = advertisedBandwidth; - } - - @SuppressWarnings("unused") - private Map<String, GraphHistory> consensus_weight; - public void setConsensusWeight( - Map<String, GraphHistory> consensusWeight) { - this.consensus_weight = consensusWeight; - } -} - diff --git a/src/org/torproject/onionoo/WeightsDocumentWriter.java b/src/org/torproject/onionoo/WeightsDocumentWriter.java deleted file mode 100644 index 2e0d465..0000000 --- a/src/org/torproject/onionoo/WeightsDocumentWriter.java +++ /dev/null @@ -1,222 +0,0 @@ -/* Copyright 2012--2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.SortedSet; - -public class WeightsDocumentWriter implements FingerprintListener, - DocumentWriter { - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - private long now; - - public WeightsDocumentWriter() { - this.descriptorSource = ApplicationFactory.getDescriptorSource(); - this.documentStore = ApplicationFactory.getDocumentStore(); - this.now = ApplicationFactory.getTime().currentTimeMillis(); - this.registerFingerprintListeners(); - } - - private void registerFingerprintListeners() { - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerFingerprintListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); - } - - private Set<String> updateWeightsDocuments = new HashSet<String>(); - - public void processFingerprints(SortedSet<String> fingerprints, - boolean relay) { - if (relay) { - this.updateWeightsDocuments.addAll(fingerprints); - } - } - - public void writeDocuments() { - this.writeWeightsDataFiles(); - Logger.printStatusTime("Wrote weights document files"); - } - - private void writeWeightsDataFiles() { - for (String fingerprint : this.updateWeightsDocuments) { - WeightsStatus weightsStatus = this.documentStore.retrieve( - WeightsStatus.class, true, fingerprint); - if (weightsStatus == null) { - continue; - } - SortedMap<long[], double[]> history = weightsStatus.getHistory(); - WeightsDocument weightsDocument = this.compileWeightsDocument( - fingerprint, history); - this.documentStore.store(weightsDocument, fingerprint); - } - Logger.printStatusTime("Wrote weights document files"); - } - - private String[] graphNames = new String[] { - "1_week", - "1_month", - "3_months", - "1_year", - "5_years" }; - - private long[] graphIntervals = new long[] { - DateTimeHelper.ONE_WEEK, - DateTimeHelper.ROUGHLY_ONE_MONTH, - DateTimeHelper.ROUGHLY_THREE_MONTHS, - DateTimeHelper.ROUGHLY_ONE_YEAR, - DateTimeHelper.ROUGHLY_FIVE_YEARS }; - - private long[] dataPointIntervals = new long[] { - DateTimeHelper.ONE_HOUR, - DateTimeHelper.FOUR_HOURS, - DateTimeHelper.TWELVE_HOURS, - DateTimeHelper.TWO_DAYS, - DateTimeHelper.TEN_DAYS }; - - private WeightsDocument compileWeightsDocument(String fingerprint, - SortedMap<long[], double[]> history) { - WeightsDocument weightsDocument = new WeightsDocument(); - weightsDocument.setFingerprint(fingerprint); - weightsDocument.setAdvertisedBandwidthFraction( - this.compileGraphType(history, 0)); - weightsDocument.setConsensusWeightFraction( - this.compileGraphType(history, 1)); - weightsDocument.setGuardProbability( - this.compileGraphType(history, 2)); - weightsDocument.setMiddleProbability( - this.compileGraphType(history, 3)); - weightsDocument.setExitProbability( - this.compileGraphType(history, 4)); - weightsDocument.setAdvertisedBandwidth( - this.compileGraphType(history, 5)); - weightsDocument.setConsensusWeight( - this.compileGraphType(history, 6)); - return weightsDocument; - } - - private Map<String, GraphHistory> compileGraphType( - SortedMap<long[], double[]> history, int graphTypeIndex) { - Map<String, GraphHistory> graphs = - new LinkedHashMap<String, GraphHistory>(); - for (int graphIntervalIndex = 0; graphIntervalIndex < - this.graphIntervals.length; graphIntervalIndex++) { - String graphName = this.graphNames[graphIntervalIndex]; - GraphHistory graphHistory = this.compileWeightsHistory( - graphTypeIndex, graphIntervalIndex, history); - if (graphHistory != null) { - graphs.put(graphName, graphHistory); - } - } - return graphs; - } - - private GraphHistory compileWeightsHistory(int graphTypeIndex, - int graphIntervalIndex, SortedMap<long[], double[]> history) { - long graphInterval = this.graphIntervals[graphIntervalIndex]; - long dataPointInterval = - this.dataPointIntervals[graphIntervalIndex]; - List<Double> dataPoints = new ArrayList<Double>(); - long intervalStartMillis = ((this.now - graphInterval) - / dataPointInterval) * dataPointInterval; - long totalMillis = 0L; - double totalWeightTimesMillis = 0.0; - for (Map.Entry<long[], double[]> e : history.entrySet()) { - long startMillis = e.getKey()[0], endMillis = e.getKey()[1]; - double weight = e.getValue()[graphTypeIndex]; - if (endMillis < intervalStartMillis) { - continue; - } - while ((intervalStartMillis / dataPointInterval) != - (endMillis / dataPointInterval)) { - dataPoints.add(totalMillis * 5L < dataPointInterval - ? -1.0 : totalWeightTimesMillis / (double) totalMillis); - totalWeightTimesMillis = 0.0; - totalMillis = 0L; - intervalStartMillis += dataPointInterval; - } - if (weight >= 0.0) { - totalWeightTimesMillis += weight - * ((double) (endMillis - startMillis)); - totalMillis += (endMillis - startMillis); - } - } - dataPoints.add(totalMillis * 5L < dataPointInterval - ? -1.0 : totalWeightTimesMillis / (double) totalMillis); - double maxValue = 0.0; - int firstNonNullIndex = -1, lastNonNullIndex = -1; - for (int dataPointIndex = 0; dataPointIndex < dataPoints.size(); - dataPointIndex++) { - double dataPoint = dataPoints.get(dataPointIndex); - if (dataPoint >= 0.0) { - if (firstNonNullIndex < 0) { - firstNonNullIndex = dataPointIndex; - } - lastNonNullIndex = dataPointIndex; - if (dataPoint > maxValue) { - maxValue = dataPoint; - } - } - } - if (firstNonNullIndex < 0) { - return null; - } - long firstDataPointMillis = (((this.now - graphInterval) - / dataPointInterval) + firstNonNullIndex) * dataPointInterval - + dataPointInterval / 2L; - if (graphIntervalIndex > 0 && firstDataPointMillis >= - this.now - graphIntervals[graphIntervalIndex - 1]) { - /* Skip weights history object, because it doesn't contain - * anything new that wasn't already contained in the last - * weights history object(s). */ - return null; - } - long lastDataPointMillis = firstDataPointMillis - + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; - double factor = ((double) maxValue) / 999.0; - int count = lastNonNullIndex - firstNonNullIndex + 1; - GraphHistory graphHistory = new GraphHistory(); - graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis)); - graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis)); - graphHistory.setInterval((int) (dataPointInterval - / DateTimeHelper.ONE_SECOND)); - graphHistory.setFactor(factor); - graphHistory.setCount(count); - int previousNonNullIndex = -2; - boolean foundTwoAdjacentDataPoints = false; - List<Integer> values = new ArrayList<Integer>(); - for (int dataPointIndex = firstNonNullIndex; dataPointIndex <= - lastNonNullIndex; dataPointIndex++) { - double dataPoint = dataPoints.get(dataPointIndex); - if (dataPoint >= 0.0) { - if (dataPointIndex - previousNonNullIndex == 1) { - foundTwoAdjacentDataPoints = true; - } - previousNonNullIndex = dataPointIndex; - } - values.add(dataPoint < 0.0 ? null : - (int) ((dataPoint * 999.0) / maxValue)); - } - graphHistory.setValues(values); - if (foundTwoAdjacentDataPoints) { - return graphHistory; - } else { - return null; - } - } - - public String getStatsString() { - /* TODO Add statistics string. */ - return null; - } -} diff --git a/src/org/torproject/onionoo/WeightsStatus.java b/src/org/torproject/onionoo/WeightsStatus.java deleted file mode 100644 index 6d06ca4..0000000 --- a/src/org/torproject/onionoo/WeightsStatus.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.torproject.onionoo; - -import java.util.Comparator; -import java.util.HashMap; -import java.util.Map; -import java.util.Scanner; -import java.util.SortedMap; -import java.util.TreeMap; - -public class WeightsStatus extends Document { - - private SortedMap<long[], double[]> history = - new TreeMap<long[], double[]>(new Comparator<long[]>() { - public int compare(long[] a, long[] b) { - return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0; - } - }); - public void setHistory(SortedMap<long[], double[]> history) { - this.history = history; - } - public SortedMap<long[], double[]> getHistory() { - return this.history; - } - - private Map<String, Integer> advertisedBandwidths = - new HashMap<String, Integer>(); - public Map<String, Integer> getAdvertisedBandwidths() { - return this.advertisedBandwidths; - } - - public void fromDocumentString(String documentString) { - Scanner s = new Scanner(documentString); - while (s.hasNextLine()) { - String line = s.nextLine(); - String[] parts = line.split(" "); - if (parts.length == 2) { - String descriptorDigest = parts[0]; - int advertisedBandwidth = Integer.parseInt(parts[1]); - this.advertisedBandwidths.put(descriptorDigest, - advertisedBandwidth); - continue; - } - if (parts.length != 9 && parts.length != 11) { - System.err.println("Illegal line '" + line + "' in weights " - + "status file. Skipping this line."); - continue; - } - if (parts[4].equals("NaN")) { - /* Remove corrupt lines written on 2013-07-07 and the days - * after. */ - continue; - } - long validAfterMillis = DateTimeHelper.parse(parts[0] + " " - + parts[1]); - long freshUntilMillis = DateTimeHelper.parse(parts[2] + " " - + parts[3]); - if (validAfterMillis < 0L || freshUntilMillis < 0L) { - System.err.println("Could not parse timestamp while reading " - + "weights status file. Skipping."); - break; - } - long[] interval = new long[] { validAfterMillis, freshUntilMillis }; - double[] weights = new double[] { - Double.parseDouble(parts[4]), - Double.parseDouble(parts[5]), - Double.parseDouble(parts[6]), - Double.parseDouble(parts[7]), - Double.parseDouble(parts[8]), -1.0, -1.0 }; - if (parts.length == 11) { - weights[5] = Double.parseDouble(parts[9]); - weights[6] = Double.parseDouble(parts[10]); - } - this.history.put(interval, weights); - } - s.close(); - } - - public String toDocumentString() { - StringBuilder sb = new StringBuilder(); - for (Map.Entry<String, Integer> e : - this.advertisedBandwidths.entrySet()) { - sb.append(e.getKey() + " " + String.valueOf(e.getValue()) + "\n"); - } - for (Map.Entry<long[], double[]> e : history.entrySet()) { - long[] fresh = e.getKey(); - double[] weights = e.getValue(); - sb.append(DateTimeHelper.format(fresh[0]) + " " - + DateTimeHelper.format(fresh[1])); - for (double weight : weights) { - sb.append(String.format(" %.12f", weight)); - } - sb.append("\n"); - } - return sb.toString(); - } -} - diff --git a/src/org/torproject/onionoo/WeightsStatusUpdater.java b/src/org/torproject/onionoo/WeightsStatusUpdater.java deleted file mode 100644 index 5932da6..0000000 --- a/src/org/torproject/onionoo/WeightsStatusUpdater.java +++ /dev/null @@ -1,328 +0,0 @@ -/* Copyright 2012--2014 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.onionoo; - -import java.util.Arrays; -import java.util.Map; -import java.util.SortedMap; -import java.util.SortedSet; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.NetworkStatusEntry; -import org.torproject.descriptor.RelayNetworkStatusConsensus; -import org.torproject.descriptor.ServerDescriptor; - -public class WeightsStatusUpdater implements DescriptorListener, - StatusUpdater { - - private DescriptorSource descriptorSource; - - private DocumentStore documentStore; - - private long now; - - public WeightsStatusUpdater() { - this.descriptorSource = ApplicationFactory.getDescriptorSource(); - this.documentStore = ApplicationFactory.getDocumentStore(); - this.now = ApplicationFactory.getTime().currentTimeMillis(); - this.registerDescriptorListeners(); - } - - private void registerDescriptorListeners() { - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_CONSENSUSES); - this.descriptorSource.registerDescriptorListener(this, - DescriptorType.RELAY_SERVER_DESCRIPTORS); - } - - public void processDescriptor(Descriptor descriptor, boolean relay) { - if (descriptor instanceof ServerDescriptor) { - this.processRelayServerDescriptor((ServerDescriptor) descriptor); - } else if (descriptor instanceof RelayNetworkStatusConsensus) { - this.processRelayNetworkConsensus( - (RelayNetworkStatusConsensus) descriptor); - } - } - - public void updateStatuses() { - /* Nothing to do. */ - } - - private void processRelayNetworkConsensus( - RelayNetworkStatusConsensus consensus) { - long validAfterMillis = consensus.getValidAfterMillis(), - freshUntilMillis = consensus.getFreshUntilMillis(); - SortedMap<String, double[]> pathSelectionWeights = - this.calculatePathSelectionProbabilities(consensus); - this.updateWeightsHistory(validAfterMillis, freshUntilMillis, - pathSelectionWeights); - } - - private void processRelayServerDescriptor( - ServerDescriptor serverDescriptor) { - String digest = serverDescriptor.getServerDescriptorDigest(). - toUpperCase(); - int advertisedBandwidth = Math.min(Math.min( - serverDescriptor.getBandwidthBurst(), - serverDescriptor.getBandwidthObserved()), - serverDescriptor.getBandwidthRate()); - String fingerprint = serverDescriptor.getFingerprint(); - WeightsStatus weightsStatus = this.documentStore.retrieve( - WeightsStatus.class, true, fingerprint); - if (weightsStatus == null) { - weightsStatus = new WeightsStatus(); - } - weightsStatus.getAdvertisedBandwidths().put(digest, - advertisedBandwidth); - this.documentStore.store(weightsStatus, fingerprint); -} - - private void updateWeightsHistory(long validAfterMillis, - long freshUntilMillis, - SortedMap<String, double[]> pathSelectionWeights) { - String fingerprint = null; - double[] weights = null; - do { - fingerprint = null; - synchronized (pathSelectionWeights) { - if (!pathSelectionWeights.isEmpty()) { - fingerprint = pathSelectionWeights.firstKey(); - weights = pathSelectionWeights.remove(fingerprint); - } - } - if (fingerprint != null) { - this.addToHistory(fingerprint, validAfterMillis, - freshUntilMillis, weights); - } - } while (fingerprint != null); - } - - private SortedMap<String, double[]> calculatePathSelectionProbabilities( - RelayNetworkStatusConsensus consensus) { - boolean containsBandwidthWeights = false; - double wgg = 1.0, wgd = 1.0, wmg = 1.0, wmm = 1.0, wme = 1.0, - wmd = 1.0, wee = 1.0, wed = 1.0; - SortedMap<String, Integer> bandwidthWeights = - consensus.getBandwidthWeights(); - if (bandwidthWeights != null) { - SortedSet<String> missingWeightKeys = new TreeSet<String>( - Arrays.asList("Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(","))); - missingWeightKeys.removeAll(bandwidthWeights.keySet()); - if (missingWeightKeys.isEmpty()) { - wgg = ((double) bandwidthWeights.get("Wgg")) / 10000.0; - wgd = ((double) bandwidthWeights.get("Wgd")) / 10000.0; - wmg = ((double) bandwidthWeights.get("Wmg")) / 10000.0; - wmm = ((double) bandwidthWeights.get("Wmm")) / 10000.0; - wme = ((double) bandwidthWeights.get("Wme")) / 10000.0; - wmd = ((double) bandwidthWeights.get("Wmd")) / 10000.0; - wee = ((double) bandwidthWeights.get("Wee")) / 10000.0; - wed = ((double) bandwidthWeights.get("Wed")) / 10000.0; - containsBandwidthWeights = true; - } - } - SortedMap<String, Double> - advertisedBandwidths = new TreeMap<String, Double>(), - consensusWeights = new TreeMap<String, Double>(), - guardWeights = new TreeMap<String, Double>(), - middleWeights = new TreeMap<String, Double>(), - exitWeights = new TreeMap<String, Double>(); - double totalAdvertisedBandwidth = 0.0; - double totalConsensusWeight = 0.0; - double totalGuardWeight = 0.0; - double totalMiddleWeight = 0.0; - double totalExitWeight = 0.0; - for (NetworkStatusEntry relay : - consensus.getStatusEntries().values()) { - String fingerprint = relay.getFingerprint(); - if (!relay.getFlags().contains("Running")) { - continue; - } - String digest = relay.getDescriptor().toUpperCase(); - WeightsStatus weightsStatus = this.documentStore.retrieve( - WeightsStatus.class, true, fingerprint); - if (weightsStatus != null && - weightsStatus.getAdvertisedBandwidths() != null && - weightsStatus.getAdvertisedBandwidths().containsKey(digest)) { - /* Read advertised bandwidth from weights status file. Server - * descriptors are parsed before consensuses, so we're sure that - * if there's a server descriptor for this relay, it'll be - * contained in the weights status file by now. */ - double advertisedBandwidth = - (double) weightsStatus.getAdvertisedBandwidths().get(digest); - advertisedBandwidths.put(fingerprint, advertisedBandwidth); - totalAdvertisedBandwidth += advertisedBandwidth; - } - if (relay.getBandwidth() >= 0L) { - double consensusWeight = (double) relay.getBandwidth(); - consensusWeights.put(fingerprint, consensusWeight); - totalConsensusWeight += consensusWeight; - if (containsBandwidthWeights) { - double guardWeight = (double) relay.getBandwidth(); - double middleWeight = (double) relay.getBandwidth(); - double exitWeight = (double) relay.getBandwidth(); - boolean isExit = relay.getFlags().contains("Exit") && - !relay.getFlags().contains("BadExit"); - boolean isGuard = relay.getFlags().contains("Guard"); - if (isGuard && isExit) { - guardWeight *= wgd; - middleWeight *= wmd; - exitWeight *= wed; - } else if (isGuard) { - guardWeight *= wgg; - middleWeight *= wmg; - exitWeight = 0.0; - } else if (isExit) { - guardWeight = 0.0; - middleWeight *= wme; - exitWeight *= wee; - } else { - guardWeight = 0.0; - middleWeight *= wmm; - exitWeight = 0.0; - } - guardWeights.put(fingerprint, guardWeight); - middleWeights.put(fingerprint, middleWeight); - exitWeights.put(fingerprint, exitWeight); - totalGuardWeight += guardWeight; - totalMiddleWeight += middleWeight; - totalExitWeight += exitWeight; - } - } - } - SortedMap<String, double[]> pathSelectionProbabilities = - new TreeMap<String, double[]>(); - SortedSet<String> fingerprints = new TreeSet<String>(); - fingerprints.addAll(consensusWeights.keySet()); - fingerprints.addAll(advertisedBandwidths.keySet()); - for (String fingerprint : fingerprints) { - double[] probabilities = new double[] { -1.0, -1.0, -1.0, -1.0, - -1.0, -1.0, -1.0 }; - if (consensusWeights.containsKey(fingerprint) && - totalConsensusWeight > 0.0) { - probabilities[1] = consensusWeights.get(fingerprint) / - totalConsensusWeight; - probabilities[6] = consensusWeights.get(fingerprint); - } - if (guardWeights.containsKey(fingerprint) && - totalGuardWeight > 0.0) { - probabilities[2] = guardWeights.get(fingerprint) / - totalGuardWeight; - } - if (middleWeights.containsKey(fingerprint) && - totalMiddleWeight > 0.0) { - probabilities[3] = middleWeights.get(fingerprint) / - totalMiddleWeight; - } - if (exitWeights.containsKey(fingerprint) && - totalExitWeight > 0.0) { - probabilities[4] = exitWeights.get(fingerprint) / - totalExitWeight; - } - if (advertisedBandwidths.containsKey(fingerprint) && - totalAdvertisedBandwidth > 0.0) { - probabilities[0] = advertisedBandwidths.get(fingerprint) - / totalAdvertisedBandwidth; - probabilities[5] = advertisedBandwidths.get(fingerprint); - } - pathSelectionProbabilities.put(fingerprint, probabilities); - } - return pathSelectionProbabilities; - } - - private void addToHistory(String fingerprint, long validAfterMillis, - long freshUntilMillis, double[] weights) { - WeightsStatus weightsStatus = this.documentStore.retrieve( - WeightsStatus.class, true, fingerprint); - if (weightsStatus == null) { - weightsStatus = new WeightsStatus(); - } - SortedMap<long[], double[]> history = weightsStatus.getHistory(); - long[] interval = new long[] { validAfterMillis, freshUntilMillis }; - if ((history.headMap(interval).isEmpty() || - history.headMap(interval).lastKey()[1] <= validAfterMillis) && - (history.tailMap(interval).isEmpty() || - history.tailMap(interval).firstKey()[0] >= freshUntilMillis)) { - history.put(interval, weights); - this.compressHistory(weightsStatus); - this.documentStore.store(weightsStatus, fingerprint); - } - } - - private void compressHistory(WeightsStatus weightsStatus) { - SortedMap<long[], double[]> history = weightsStatus.getHistory(); - SortedMap<long[], double[]> compressedHistory = - new TreeMap<long[], double[]>(history.comparator()); - long lastStartMillis = 0L, lastEndMillis = 0L; - double[] lastWeights = null; - String lastMonthString = "1970-01"; - int lastMissingValues = -1; - for (Map.Entry<long[], double[]> e : history.entrySet()) { - long startMillis = e.getKey()[0], endMillis = e.getKey()[1]; - double[] weights = e.getValue(); - long intervalLengthMillis; - if (this.now - endMillis <= DateTimeHelper.ONE_WEEK) { - intervalLengthMillis = DateTimeHelper.ONE_HOUR; - } else if (this.now - endMillis <= - DateTimeHelper.ROUGHLY_ONE_MONTH) { - intervalLengthMillis = DateTimeHelper.FOUR_HOURS; - } else if (this.now - endMillis <= - DateTimeHelper.ROUGHLY_THREE_MONTHS) { - intervalLengthMillis = DateTimeHelper.TWELVE_HOURS; - } else if (this.now - endMillis <= - DateTimeHelper.ROUGHLY_ONE_YEAR) { - intervalLengthMillis = DateTimeHelper.TWO_DAYS; - } else { - intervalLengthMillis = DateTimeHelper.TEN_DAYS; - } - String monthString = DateTimeHelper.format(startMillis, - DateTimeHelper.ISO_YEARMONTH_FORMAT); - int missingValues = 0; - for (int i = 0; i < weights.length; i++) { - if (weights[i] < -0.5) { - missingValues += 1 << i; - } - } - if (lastEndMillis == startMillis && - ((lastEndMillis - 1L) / intervalLengthMillis) == - ((endMillis - 1L) / intervalLengthMillis) && - lastMonthString.equals(monthString) && - lastMissingValues == missingValues) { - double lastIntervalInHours = (double) ((lastEndMillis - - lastStartMillis) / DateTimeHelper.ONE_HOUR); - double currentIntervalInHours = (double) ((endMillis - - startMillis) / DateTimeHelper.ONE_HOUR); - double newIntervalInHours = (double) ((endMillis - - lastStartMillis) / DateTimeHelper.ONE_HOUR); - for (int i = 0; i < lastWeights.length; i++) { - lastWeights[i] *= lastIntervalInHours; - lastWeights[i] += weights[i] * currentIntervalInHours; - lastWeights[i] /= newIntervalInHours; - } - lastEndMillis = endMillis; - } else { - if (lastStartMillis > 0L) { - compressedHistory.put(new long[] { lastStartMillis, - lastEndMillis }, lastWeights); - } - lastStartMillis = startMillis; - lastEndMillis = endMillis; - lastWeights = weights; - } - lastMonthString = monthString; - lastMissingValues = missingValues; - } - if (lastStartMillis > 0L) { - compressedHistory.put(new long[] { lastStartMillis, lastEndMillis }, - lastWeights); - } - weightsStatus.setHistory(compressedHistory); - } - - public String getStatsString() { - /* TODO Add statistics string. */ - return null; - } -} - diff --git a/src/org/torproject/onionoo/cron/Main.java b/src/org/torproject/onionoo/cron/Main.java new file mode 100644 index 0000000..94dfa00 --- /dev/null +++ b/src/org/torproject/onionoo/cron/Main.java @@ -0,0 +1,140 @@ +/* Copyright 2011, 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.cron; + +import java.io.File; + +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.updater.BandwidthStatusUpdater; +import org.torproject.onionoo.updater.ClientsStatusUpdater; +import org.torproject.onionoo.updater.DescriptorSource; +import org.torproject.onionoo.updater.LookupService; +import org.torproject.onionoo.updater.NodeDetailsStatusUpdater; +import org.torproject.onionoo.updater.ReverseDomainNameResolver; +import org.torproject.onionoo.updater.StatusUpdater; +import org.torproject.onionoo.updater.UptimeStatusUpdater; +import org.torproject.onionoo.updater.WeightsStatusUpdater; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.LockFile; +import org.torproject.onionoo.util.Logger; +import org.torproject.onionoo.writer.BandwidthDocumentWriter; +import org.torproject.onionoo.writer.ClientsDocumentWriter; +import org.torproject.onionoo.writer.DetailsDocumentWriter; +import org.torproject.onionoo.writer.DocumentWriter; +import org.torproject.onionoo.writer.SummaryDocumentWriter; +import org.torproject.onionoo.writer.UptimeDocumentWriter; +import org.torproject.onionoo.writer.WeightsDocumentWriter; + +/* Update search data and status data files. */ +public class Main { + + private Main() { + } + + public static void main(String[] args) { + + LockFile lf = new LockFile(); + Logger.setTime(); + Logger.printStatus("Initializing."); + if (lf.acquireLock()) { + Logger.printStatusTime("Acquired lock"); + } else { + Logger.printErrorTime("Could not acquire lock. Is Onionoo " + + "already running? Terminating"); + return; + } + + DescriptorSource dso = ApplicationFactory.getDescriptorSource(); + Logger.printStatusTime("Initialized descriptor source"); + DocumentStore ds = ApplicationFactory.getDocumentStore(); + Logger.printStatusTime("Initialized document store"); + LookupService ls = new LookupService(new File("geoip")); + Logger.printStatusTime("Initialized Geoip lookup service"); + ReverseDomainNameResolver rdnr = new ReverseDomainNameResolver(); + Logger.printStatusTime("Initialized reverse domain name resolver"); + NodeDetailsStatusUpdater ndsu = new NodeDetailsStatusUpdater(rdnr, + ls); + Logger.printStatusTime("Initialized node data writer"); + BandwidthStatusUpdater bsu = new BandwidthStatusUpdater(); + Logger.printStatusTime("Initialized bandwidth status updater"); + WeightsStatusUpdater wsu = new WeightsStatusUpdater(); + Logger.printStatusTime("Initialized weights status updater"); + ClientsStatusUpdater csu = new ClientsStatusUpdater(); + Logger.printStatusTime("Initialized clients status updater"); + UptimeStatusUpdater usu = new UptimeStatusUpdater(); + Logger.printStatusTime("Initialized uptime status updater"); + StatusUpdater[] sus = new StatusUpdater[] { ndsu, bsu, wsu, csu, + usu }; + + SummaryDocumentWriter sdw = new SummaryDocumentWriter(); + Logger.printStatusTime("Initialized summary document writer"); + DetailsDocumentWriter ddw = new DetailsDocumentWriter(); + Logger.printStatusTime("Initialized details document writer"); + BandwidthDocumentWriter bdw = new BandwidthDocumentWriter(); + Logger.printStatusTime("Initialized bandwidth document writer"); + WeightsDocumentWriter wdw = new WeightsDocumentWriter(); + Logger.printStatusTime("Initialized weights document writer"); + ClientsDocumentWriter cdw = new ClientsDocumentWriter(); + Logger.printStatusTime("Initialized clients document writer"); + UptimeDocumentWriter udw = new UptimeDocumentWriter(); + Logger.printStatusTime("Initialized uptime document writer"); + DocumentWriter[] dws = new DocumentWriter[] { sdw, ddw, bdw, wdw, cdw, + udw }; + + Logger.printStatus("Downloading descriptors."); + dso.downloadDescriptors(); + + Logger.printStatus("Reading descriptors."); + dso.readDescriptors(); + + Logger.printStatus("Updating internal status files."); + for (StatusUpdater su : sus) { + su.updateStatuses(); + Logger.printStatusTime(su.getClass().getSimpleName() + + " updated status files"); + } + + Logger.printStatus("Updating document files."); + for (DocumentWriter dw : dws) { + dw.writeDocuments(); + } + + Logger.printStatus("Shutting down."); + dso.writeHistoryFiles(); + Logger.printStatusTime("Wrote parse histories"); + ds.flushDocumentCache(); + Logger.printStatusTime("Flushed document cache"); + + Logger.printStatus("Gathering statistics."); + for (StatusUpdater su : sus) { + String statsString = su.getStatsString(); + if (statsString != null) { + Logger.printStatistics(su.getClass().getSimpleName(), + statsString); + } + } + for (DocumentWriter dw : dws) { + String statsString = dw.getStatsString(); + if (statsString != null) { + Logger.printStatistics(dw.getClass().getSimpleName(), + statsString); + } + } + Logger.printStatistics("Descriptor source", dso.getStatsString()); + Logger.printStatistics("Document store", ds.getStatsString()); + Logger.printStatistics("GeoIP lookup service", ls.getStatsString()); + Logger.printStatistics("Reverse domain name resolver", + rdnr.getStatsString()); + + Logger.printStatus("Releasing lock."); + if (lf.releaseLock()) { + Logger.printStatusTime("Released lock"); + } else { + Logger.printErrorTime("Could not release lock. The next " + + "execution may not start as expected"); + } + + Logger.printStatus("Terminating."); + } +} + diff --git a/src/org/torproject/onionoo/docs/BandwidthDocument.java b/src/org/torproject/onionoo/docs/BandwidthDocument.java new file mode 100644 index 0000000..ea20a5e --- /dev/null +++ b/src/org/torproject/onionoo/docs/BandwidthDocument.java @@ -0,0 +1,27 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.Map; + +public class BandwidthDocument extends Document { + + @SuppressWarnings("unused") + private String fingerprint; + public void setFingerprint(String fingerprint) { + this.fingerprint = fingerprint; + } + + @SuppressWarnings("unused") + private Map<String, GraphHistory> write_history; + public void setWriteHistory(Map<String, GraphHistory> writeHistory) { + this.write_history = writeHistory; + } + + @SuppressWarnings("unused") + private Map<String, GraphHistory> read_history; + public void setReadHistory(Map<String, GraphHistory> readHistory) { + this.read_history = readHistory; + } +} + diff --git a/src/org/torproject/onionoo/docs/BandwidthStatus.java b/src/org/torproject/onionoo/docs/BandwidthStatus.java new file mode 100644 index 0000000..a2980e5 --- /dev/null +++ b/src/org/torproject/onionoo/docs/BandwidthStatus.java @@ -0,0 +1,80 @@ +/* Copyright 2013--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.Scanner; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.torproject.onionoo.util.DateTimeHelper; + +public class BandwidthStatus extends Document { + + private SortedMap<Long, long[]> writeHistory = + new TreeMap<Long, long[]>(); + public void setWriteHistory(SortedMap<Long, long[]> writeHistory) { + this.writeHistory = writeHistory; + } + public SortedMap<Long, long[]> getWriteHistory() { + return this.writeHistory; + } + + private SortedMap<Long, long[]> readHistory = + new TreeMap<Long, long[]>(); + public void setReadHistory(SortedMap<Long, long[]> readHistory) { + this.readHistory = readHistory; + } + public SortedMap<Long, long[]> getReadHistory() { + return this.readHistory; + } + + public void fromDocumentString(String documentString) { + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + String[] parts = line.split(" "); + if (parts.length != 6) { + System.err.println("Illegal line '" + line + "' in bandwidth " + + "history. Skipping this line."); + continue; + } + SortedMap<Long, long[]> history = parts[0].equals("r") + ? readHistory : writeHistory; + long startMillis = DateTimeHelper.parse(parts[1] + " " + parts[2]); + long endMillis = DateTimeHelper.parse(parts[3] + " " + parts[4]); + if (startMillis < 0L || endMillis < 0L) { + System.err.println("Could not parse timestamp while reading " + + "bandwidth history. Skipping."); + break; + } + long bandwidth = Long.parseLong(parts[5]); + long previousEndMillis = history.headMap(startMillis).isEmpty() + ? startMillis + : history.get(history.headMap(startMillis).lastKey())[1]; + long nextStartMillis = history.tailMap(startMillis).isEmpty() + ? endMillis : history.tailMap(startMillis).firstKey(); + if (previousEndMillis <= startMillis && + nextStartMillis >= endMillis) { + history.put(startMillis, new long[] { startMillis, endMillis, + bandwidth }); + } + } + s.close(); + } + + public String toDocumentString() { + StringBuilder sb = new StringBuilder(); + for (long[] v : writeHistory.values()) { + sb.append("w " + DateTimeHelper.format(v[0]) + " " + + DateTimeHelper.format(v[1]) + " " + String.valueOf(v[2]) + + "\n"); + } + for (long[] v : readHistory.values()) { + sb.append("r " + DateTimeHelper.format(v[0]) + " " + + DateTimeHelper.format(v[1]) + " " + String.valueOf(v[2]) + + "\n"); + } + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/docs/ClientsDocument.java b/src/org/torproject/onionoo/docs/ClientsDocument.java new file mode 100644 index 0000000..27b1588 --- /dev/null +++ b/src/org/torproject/onionoo/docs/ClientsDocument.java @@ -0,0 +1,22 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.Map; + +public class ClientsDocument extends Document { + + @SuppressWarnings("unused") + private String fingerprint; + public void setFingerprint(String fingerprint) { + this.fingerprint = fingerprint; + } + + @SuppressWarnings("unused") + private Map<String, ClientsGraphHistory> average_clients; + public void setAverageClients( + Map<String, ClientsGraphHistory> averageClients) { + this.average_clients = averageClients; + } +} + diff --git a/src/org/torproject/onionoo/docs/ClientsGraphHistory.java b/src/org/torproject/onionoo/docs/ClientsGraphHistory.java new file mode 100644 index 0000000..e1db663 --- /dev/null +++ b/src/org/torproject/onionoo/docs/ClientsGraphHistory.java @@ -0,0 +1,83 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.ArrayList; +import java.util.List; +import java.util.SortedMap; + +public class ClientsGraphHistory { + + private String first; + public void setFirst(String first) { + this.first = first; + } + public String getFirst() { + return this.first; + } + + private String last; + public void setLast(String last) { + this.last = last; + } + public String getLast() { + return this.last; + } + + private Integer interval; + public void setInterval(Integer interval) { + this.interval = interval; + } + public Integer getInterval() { + return this.interval; + } + + private Double factor; + public void setFactor(Double factor) { + this.factor = factor; + } + public Double getFactor() { + return this.factor; + } + + private Integer count; + public void setCount(Integer count) { + this.count = count; + } + public Integer getCount() { + return this.count; + } + + private List<Integer> values = new ArrayList<Integer>(); + public void setValues(List<Integer> values) { + this.values = values; + } + public List<Integer> getValues() { + return this.values; + } + + private SortedMap<String, Float> countries; + public void setCountries(SortedMap<String, Float> countries) { + this.countries = countries; + } + public SortedMap<String, Float> getCountries() { + return this.countries; + } + + private SortedMap<String, Float> transports; + public void setTransports(SortedMap<String, Float> transports) { + this.transports = transports; + } + public SortedMap<String, Float> getTransports() { + return this.transports; + } + + private SortedMap<String, Float> versions; + public void setVersions(SortedMap<String, Float> versions) { + this.versions = versions; + } + public SortedMap<String, Float> getVersions() { + return this.versions; + } +} + diff --git a/src/org/torproject/onionoo/docs/ClientsHistory.java b/src/org/torproject/onionoo/docs/ClientsHistory.java new file mode 100644 index 0000000..446dd10 --- /dev/null +++ b/src/org/torproject/onionoo/docs/ClientsHistory.java @@ -0,0 +1,174 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.Map; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.torproject.onionoo.util.DateTimeHelper; + +public class ClientsHistory implements Comparable<ClientsHistory> { + + private long startMillis; + public long getStartMillis() { + return this.startMillis; + } + + private long endMillis; + public long getEndMillis() { + return this.endMillis; + } + + private double totalResponses; + public double getTotalResponses() { + return this.totalResponses; + } + + private SortedMap<String, Double> responsesByCountry; + public SortedMap<String, Double> getResponsesByCountry() { + return this.responsesByCountry; + } + + private SortedMap<String, Double> responsesByTransport; + public SortedMap<String, Double> getResponsesByTransport() { + return this.responsesByTransport; + } + + private SortedMap<String, Double> responsesByVersion; + public SortedMap<String, Double> getResponsesByVersion() { + return this.responsesByVersion; + } + + public ClientsHistory(long startMillis, long endMillis, + double totalResponses, + SortedMap<String, Double> responsesByCountry, + SortedMap<String, Double> responsesByTransport, + SortedMap<String, Double> responsesByVersion) { + this.startMillis = startMillis; + this.endMillis = endMillis; + this.totalResponses = totalResponses; + this.responsesByCountry = responsesByCountry; + this.responsesByTransport = responsesByTransport; + this.responsesByVersion = responsesByVersion; + } + + public static ClientsHistory fromString( + String responseHistoryString) { + String[] parts = responseHistoryString.split(" ", 8); + if (parts.length != 8) { + return null; + } + long startMillis = DateTimeHelper.parse(parts[0] + " " + parts[1]); + long endMillis = DateTimeHelper.parse(parts[2] + " " + parts[3]); + if (startMillis < 0L || endMillis < 0L) { + return null; + } + if (startMillis >= endMillis) { + return null; + } + double totalResponses = 0.0; + try { + totalResponses = Double.parseDouble(parts[4]); + } catch (NumberFormatException e) { + return null; + } + SortedMap<String, Double> responsesByCountry = + parseResponses(parts[5]); + SortedMap<String, Double> responsesByTransport = + parseResponses(parts[6]); + SortedMap<String, Double> responsesByVersion = + parseResponses(parts[7]); + if (responsesByCountry == null || responsesByTransport == null || + responsesByVersion == null) { + return null; + } + return new ClientsHistory(startMillis, endMillis, totalResponses, + responsesByCountry, responsesByTransport, responsesByVersion); + } + + private static SortedMap<String, Double> parseResponses( + String responsesString) { + SortedMap<String, Double> responses = new TreeMap<String, Double>(); + if (responsesString.length() > 0) { + for (String pair : responsesString.split(",")) { + String[] keyValue = pair.split("="); + if (keyValue.length != 2) { + return null; + } + double value = 0.0; + try { + value = Double.parseDouble(keyValue[1]); + } catch (NumberFormatException e) { + return null; + } + responses.put(keyValue[0], value); + } + } + return responses; + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(DateTimeHelper.format(startMillis)); + sb.append(" " + DateTimeHelper.format(endMillis)); + sb.append(" " + String.format("%.3f", this.totalResponses)); + this.appendResponses(sb, this.responsesByCountry); + this.appendResponses(sb, this.responsesByTransport); + this.appendResponses(sb, this.responsesByVersion); + return sb.toString(); + } + + private void appendResponses(StringBuilder sb, + SortedMap<String, Double> responses) { + sb.append(" "); + int written = 0; + for (Map.Entry<String, Double> e : responses.entrySet()) { + sb.append((written++ > 0 ? "," : "") + e.getKey() + "=" + + String.format("%.3f", e.getValue())); + } + } + + public void addResponses(ClientsHistory other) { + this.totalResponses += other.totalResponses; + this.addResponsesByCategory(this.responsesByCountry, + other.responsesByCountry); + this.addResponsesByCategory(this.responsesByTransport, + other.responsesByTransport); + this.addResponsesByCategory(this.responsesByVersion, + other.responsesByVersion); + if (this.startMillis > other.startMillis) { + this.startMillis = other.startMillis; + } + if (this.endMillis < other.endMillis) { + this.endMillis = other.endMillis; + } + } + + private void addResponsesByCategory( + SortedMap<String, Double> thisResponses, + SortedMap<String, Double> otherResponses) { + for (Map.Entry<String, Double> e : otherResponses.entrySet()) { + if (thisResponses.containsKey(e.getKey())) { + thisResponses.put(e.getKey(), thisResponses.get(e.getKey()) + + e.getValue()); + } else { + thisResponses.put(e.getKey(), e.getValue()); + } + } + } + + public int compareTo(ClientsHistory other) { + return this.startMillis < other.startMillis ? -1 : + this.startMillis > other.startMillis ? 1 : 0; + } + + public boolean equals(Object other) { + return other instanceof ClientsHistory && + this.startMillis == ((ClientsHistory) other).startMillis; + } + + public int hashCode() { + return (int) this.startMillis; + } +} \ No newline at end of file diff --git a/src/org/torproject/onionoo/docs/ClientsStatus.java b/src/org/torproject/onionoo/docs/ClientsStatus.java new file mode 100644 index 0000000..2bd2168 --- /dev/null +++ b/src/org/torproject/onionoo/docs/ClientsStatus.java @@ -0,0 +1,43 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.Scanner; +import java.util.SortedSet; +import java.util.TreeSet; + +public class ClientsStatus extends Document { + + private SortedSet<ClientsHistory> history = + new TreeSet<ClientsHistory>(); + public void setHistory(SortedSet<ClientsHistory> history) { + this.history = history; + } + public SortedSet<ClientsHistory> getHistory() { + return this.history; + } + + public void fromDocumentString(String documentString) { + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + ClientsHistory parsedLine = ClientsHistory.fromString(line); + if (parsedLine != null) { + this.history.add(parsedLine); + } else { + System.err.println("Could not parse clients history line '" + + line + "'. Skipping."); + } + } + s.close(); + } + + public String toDocumentString() { + StringBuilder sb = new StringBuilder(); + for (ClientsHistory interval : this.history) { + sb.append(interval.toString() + "\n"); + } + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/docs/DetailsDocument.java b/src/org/torproject/onionoo/docs/DetailsDocument.java new file mode 100644 index 0000000..142b591 --- /dev/null +++ b/src/org/torproject/onionoo/docs/DetailsDocument.java @@ -0,0 +1,365 @@ +/* Copyright 2013--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.StringEscapeUtils; + +public class DetailsDocument extends Document { + + /* We must ensure that details files only contain ASCII characters + * and no UTF-8 characters. While UTF-8 characters are perfectly + * valid in JSON, this would break compatibility with existing files + * pretty badly. We do this by escaping non-ASCII characters, e.g., + * \u00F2. Gson won't treat this as UTF-8, but will think that we want + * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing + * we'll have to do is to change back the '\\' that Gson writes for the + * '\'. */ + private static String escapeJSON(String s) { + return s == null ? null : + StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'"); + } + private static String unescapeJSON(String s) { + return s == null ? null : + StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'")); + } + + private String nickname; + public void setNickname(String nickname) { + this.nickname = nickname; + } + public String getNickname() { + return this.nickname; + } + + private String fingerprint; + public void setFingerprint(String fingerprint) { + this.fingerprint = fingerprint; + } + public String getFingerprint() { + return this.fingerprint; + } + + private String hashed_fingerprint; + public void setHashedFingerprint(String hashedFingerprint) { + this.hashed_fingerprint = hashedFingerprint; + } + public String getHashedFingerprint() { + return this.hashed_fingerprint; + } + + private List<String> or_addresses; + public void setOrAddresses(List<String> orAddresses) { + this.or_addresses = orAddresses; + } + public List<String> getOrAddresses() { + return this.or_addresses; + } + + private List<String> exit_addresses; + public void setExitAddresses(List<String> exitAddresses) { + this.exit_addresses = exitAddresses; + } + public List<String> getExitAddresses() { + return this.exit_addresses; + } + + private String dir_address; + public void setDirAddress(String dirAddress) { + this.dir_address = dirAddress; + } + public String getDirAddress() { + return this.dir_address; + } + + private String last_seen; + public void setLastSeen(String lastSeen) { + this.last_seen = lastSeen; + } + public String getLastSeen() { + return this.last_seen; + } + + private String last_changed_address_or_port; + public void setLastChangedAddressOrPort( + String lastChangedAddressOrPort) { + this.last_changed_address_or_port = lastChangedAddressOrPort; + } + public String getLastChangedAddressOrPort() { + return this.last_changed_address_or_port; + } + + private String first_seen; + public void setFirstSeen(String firstSeen) { + this.first_seen = firstSeen; + } + public String getFirstSeen() { + return this.first_seen; + } + + private Boolean running; + public void setRunning(Boolean running) { + this.running = running; + } + public Boolean getRunning() { + return this.running; + } + + private List<String> flags; + public void setFlags(List<String> flags) { + this.flags = flags; + } + public List<String> getFlags() { + return this.flags; + } + + private String country; + public void setCountry(String country) { + this.country = country; + } + public String getCountry() { + return this.country; + } + + private String country_name; + public void setCountryName(String countryName) { + this.country_name = escapeJSON(countryName); + } + public String getCountryName() { + return unescapeJSON(this.country_name); + } + + private String region_name; + public void setRegionName(String regionName) { + this.region_name = escapeJSON(regionName); + } + public String getRegionName() { + return unescapeJSON(this.region_name); + } + + private String city_name; + public void setCityName(String cityName) { + this.city_name = escapeJSON(cityName); + } + public String getCityName() { + return unescapeJSON(this.city_name); + } + + private Float latitude; + public void setLatitude(Float latitude) { + this.latitude = latitude; + } + public Float getLatitude() { + return this.latitude; + } + + private Float longitude; + public void setLongitude(Float longitude) { + this.longitude = longitude; + } + public Float getLongitude() { + return this.longitude; + } + + private String as_number; + public void setAsNumber(String asNumber) { + this.as_number = escapeJSON(asNumber); + } + public String getAsNumber() { + return unescapeJSON(this.as_number); + } + + private String as_name; + public void setAsName(String asName) { + this.as_name = escapeJSON(asName); + } + public String getAsName() { + return unescapeJSON(this.as_name); + } + + private Long consensus_weight; + public void setConsensusWeight(Long consensusWeight) { + this.consensus_weight = consensusWeight; + } + public Long getConsensusWeight() { + return this.consensus_weight; + } + + private String host_name; + public void setHostName(String hostName) { + this.host_name = escapeJSON(hostName); + } + public String getHostName() { + return unescapeJSON(this.host_name); + } + + private String last_restarted; + public void setLastRestarted(String lastRestarted) { + this.last_restarted = lastRestarted; + } + public String getLastRestarted() { + return this.last_restarted; + } + + private Integer bandwidth_rate; + public void setBandwidthRate(Integer bandwidthRate) { + this.bandwidth_rate = bandwidthRate; + } + public Integer getBandwidthRate() { + return this.bandwidth_rate; + } + + private Integer bandwidth_burst; + public void setBandwidthBurst(Integer bandwidthBurst) { + this.bandwidth_burst = bandwidthBurst; + } + public Integer getBandwidthBurst() { + return this.bandwidth_burst; + } + + private Integer observed_bandwidth; + public void setObservedBandwidth(Integer observedBandwidth) { + this.observed_bandwidth = observedBandwidth; + } + public Integer getObservedBandwidth() { + return this.observed_bandwidth; + } + + private Integer advertised_bandwidth; + public void setAdvertisedBandwidth(Integer advertisedBandwidth) { + this.advertised_bandwidth = advertisedBandwidth; + } + public Integer getAdvertisedBandwidth() { + return this.advertised_bandwidth; + } + + private List<String> exit_policy; + public void setExitPolicy(List<String> exitPolicy) { + this.exit_policy = exitPolicy; + } + public List<String> getExitPolicy() { + return this.exit_policy; + } + + private Map<String, List<String>> exit_policy_summary; + public void setExitPolicySummary( + Map<String, List<String>> exitPolicySummary) { + this.exit_policy_summary = exitPolicySummary; + } + public Map<String, List<String>> getExitPolicySummary() { + return this.exit_policy_summary; + } + + private Map<String, List<String>> exit_policy_v6_summary; + public void setExitPolicyV6Summary( + Map<String, List<String>> exitPolicyV6Summary) { + this.exit_policy_v6_summary = exitPolicyV6Summary; + } + public Map<String, List<String>> getExitPolicyV6Summary() { + return this.exit_policy_v6_summary; + } + + private String contact; + public void setContact(String contact) { + this.contact = escapeJSON(contact); + } + public String getContact() { + return unescapeJSON(this.contact); + } + + private String platform; + public void setPlatform(String platform) { + this.platform = escapeJSON(platform); + } + public String getPlatform() { + return unescapeJSON(this.platform); + } + + private List<String> family; + public void setFamily(List<String> family) { + this.family = family; + } + public List<String> getFamily() { + return this.family; + } + + private Float advertised_bandwidth_fraction; + public void setAdvertisedBandwidthFraction( + Float advertisedBandwidthFraction) { + if (advertisedBandwidthFraction == null || + advertisedBandwidthFraction >= 0.0) { + this.advertised_bandwidth_fraction = advertisedBandwidthFraction; + } + } + public Float getAdvertisedBandwidthFraction() { + return this.advertised_bandwidth_fraction; + } + + private Float consensus_weight_fraction; + public void setConsensusWeightFraction(Float consensusWeightFraction) { + if (consensusWeightFraction == null || + consensusWeightFraction >= 0.0) { + this.consensus_weight_fraction = consensusWeightFraction; + } + } + public Float getConsensusWeightFraction() { + return this.consensus_weight_fraction; + } + + private Float guard_probability; + public void setGuardProbability(Float guardProbability) { + if (guardProbability == null || guardProbability >= 0.0) { + this.guard_probability = guardProbability; + } + } + public Float getGuardProbability() { + return this.guard_probability; + } + + private Float middle_probability; + public void setMiddleProbability(Float middleProbability) { + if (middleProbability == null || middleProbability >= 0.0) { + this.middle_probability = middleProbability; + } + } + public Float getMiddleProbability() { + return this.middle_probability; + } + + private Float exit_probability; + public void setExitProbability(Float exitProbability) { + if (exitProbability == null || exitProbability >= 0.0) { + this.exit_probability = exitProbability; + } + } + public Float getExitProbability() { + return this.exit_probability; + } + + private Boolean recommended_version; + public void setRecommendedVersion(Boolean recommendedVersion) { + this.recommended_version = recommendedVersion; + } + public Boolean getRecommendedVersion() { + return this.recommended_version; + } + + private Boolean hibernating; + public void setHibernating(Boolean hibernating) { + this.hibernating = hibernating; + } + public Boolean getHibernating() { + return this.hibernating; + } + + private String pool_assignment; + public void setPoolAssignment(String poolAssignment) { + this.pool_assignment = poolAssignment; + } + public String getPoolAssignment() { + return this.pool_assignment; + } +} + diff --git a/src/org/torproject/onionoo/docs/DetailsStatus.java b/src/org/torproject/onionoo/docs/DetailsStatus.java new file mode 100644 index 0000000..a19b4b9 --- /dev/null +++ b/src/org/torproject/onionoo/docs/DetailsStatus.java @@ -0,0 +1,141 @@ +/* Copyright 2013--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.StringEscapeUtils; + +public class DetailsStatus extends Document { + + /* We must ensure that details files only contain ASCII characters + * and no UTF-8 characters. While UTF-8 characters are perfectly + * valid in JSON, this would break compatibility with existing files + * pretty badly. We do this by escaping non-ASCII characters, e.g., + * \u00F2. Gson won't treat this as UTF-8, but will think that we want + * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing + * we'll have to do is to change back the '\\' that Gson writes for the + * '\'. */ + private static String escapeJSON(String s) { + return s == null ? null : + StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'"); + } + private static String unescapeJSON(String s) { + return s == null ? null : + StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'")); + } + + private String desc_published; + public void setDescPublished(String descPublished) { + this.desc_published = descPublished; + } + public String getDescPublished() { + return this.desc_published; + } + + private String last_restarted; + public void setLastRestarted(String lastRestarted) { + this.last_restarted = lastRestarted; + } + public String getLastRestarted() { + return this.last_restarted; + } + + private Integer bandwidth_rate; + public void setBandwidthRate(Integer bandwidthRate) { + this.bandwidth_rate = bandwidthRate; + } + public Integer getBandwidthRate() { + return this.bandwidth_rate; + } + + private Integer bandwidth_burst; + public void setBandwidthBurst(Integer bandwidthBurst) { + this.bandwidth_burst = bandwidthBurst; + } + public Integer getBandwidthBurst() { + return this.bandwidth_burst; + } + + private Integer observed_bandwidth; + public void setObservedBandwidth(Integer observedBandwidth) { + this.observed_bandwidth = observedBandwidth; + } + public Integer getObservedBandwidth() { + return this.observed_bandwidth; + } + + private Integer advertised_bandwidth; + public void setAdvertisedBandwidth(Integer advertisedBandwidth) { + this.advertised_bandwidth = advertisedBandwidth; + } + public Integer getAdvertisedBandwidth() { + return this.advertised_bandwidth; + } + + private List<String> exit_policy; + public void setExitPolicy(List<String> exitPolicy) { + this.exit_policy = exitPolicy; + } + public List<String> getExitPolicy() { + return this.exit_policy; + } + + private String contact; + public void setContact(String contact) { + this.contact = escapeJSON(contact); + } + public String getContact() { + return unescapeJSON(this.contact); + } + + private String platform; + public void setPlatform(String platform) { + this.platform = escapeJSON(platform); + } + public String getPlatform() { + return unescapeJSON(this.platform); + } + + private List<String> family; + public void setFamily(List<String> family) { + this.family = family; + } + public List<String> getFamily() { + return this.family; + } + + private Map<String, List<String>> exit_policy_v6_summary; + public void setExitPolicyV6Summary( + Map<String, List<String>> exitPolicyV6Summary) { + this.exit_policy_v6_summary = exitPolicyV6Summary; + } + public Map<String, List<String>> getExitPolicyV6Summary() { + return this.exit_policy_v6_summary; + } + + private Boolean hibernating; + public void setHibernating(Boolean hibernating) { + this.hibernating = hibernating; + } + public Boolean getHibernating() { + return this.hibernating; + } + + private String pool_assignment; + public void setPoolAssignment(String poolAssignment) { + this.pool_assignment = poolAssignment; + } + public String getPoolAssignment() { + return this.pool_assignment; + } + + private Map<String, Long> exit_addresses; + public void setExitAddresses(Map<String, Long> exitAddresses) { + this.exit_addresses = exitAddresses; + } + public Map<String, Long> getExitAddresses() { + return this.exit_addresses; + } +} diff --git a/src/org/torproject/onionoo/docs/Document.java b/src/org/torproject/onionoo/docs/Document.java new file mode 100644 index 0000000..a581795 --- /dev/null +++ b/src/org/torproject/onionoo/docs/Document.java @@ -0,0 +1,24 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +public abstract class Document { + + private transient String documentString; + public void setDocumentString(String documentString) { + this.documentString = documentString; + } + public String getDocumentString() { + return this.documentString; + } + + public void fromDocumentString(String documentString) { + /* Subclasses may override this method to parse documentString. */ + } + + public String toDocumentString() { + /* Subclasses may override this method to write documentString. */ + return null; + } +} + diff --git a/src/org/torproject/onionoo/docs/DocumentStore.java b/src/org/torproject/onionoo/docs/DocumentStore.java new file mode 100644 index 0000000..c4fe965 --- /dev/null +++ b/src/org/torproject/onionoo/docs/DocumentStore.java @@ -0,0 +1,748 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.Stack; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.Logger; +import org.torproject.onionoo.util.Time; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonParseException; + +// TODO For later migration from disk to database, do the following: +// - read from database and then from disk if not found +// - write only to database, delete from disk once in database +// - move entirely to database once disk is "empty" +// TODO Also look into simple key-value stores instead of real databases. +public class DocumentStore { + + private final File statusDir = new File("status"); + + private File outDir = new File("out"); + public void setOutDir(File outDir) { + this.outDir = outDir; + } + + private Time time; + + public DocumentStore() { + this.time = ApplicationFactory.getTime(); + } + + private long listOperations = 0L, listedFiles = 0L, storedFiles = 0L, + storedBytes = 0L, retrievedFiles = 0L, retrievedBytes = 0L, + removedFiles = 0L; + + /* Node statuses and summary documents are cached in memory, as opposed + * to all other document types. These caches are initialized when first + * accessing or modifying a NodeStatus or SummaryDocument document, + * respectively. */ + private SortedMap<String, NodeStatus> cachedNodeStatuses; + private SortedMap<String, SummaryDocument> cachedSummaryDocuments; + + public <T extends Document> SortedSet<String> list( + Class<T> documentType) { + if (documentType.equals(NodeStatus.class)) { + return this.listNodeStatuses(); + } else if (documentType.equals(SummaryDocument.class)) { + return this.listSummaryDocuments(); + } else { + return this.listDocumentFiles(documentType); + } + } + + private SortedSet<String> listNodeStatuses() { + if (this.cachedNodeStatuses == null) { + this.cacheNodeStatuses(); + } + return new TreeSet<String>(this.cachedNodeStatuses.keySet()); + } + + private void cacheNodeStatuses() { + SortedMap<String, NodeStatus> parsedNodeStatuses = + new TreeMap<String, NodeStatus>(); + File directory = this.statusDir; + if (directory != null) { + File summaryFile = new File(directory, "summary"); + if (summaryFile.exists()) { + try { + BufferedReader br = new BufferedReader(new FileReader( + summaryFile)); + String line; + while ((line = br.readLine()) != null) { + if (line.length() == 0) { + continue; + } + NodeStatus node = NodeStatus.fromString(line); + if (node != null) { + parsedNodeStatuses.put(node.getFingerprint(), node); + } + } + br.close(); + this.listedFiles += parsedNodeStatuses.size(); + this.listOperations++; + } catch (IOException e) { + System.err.println("Could not read file '" + + summaryFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } + } + } + this.cachedNodeStatuses = parsedNodeStatuses; + } + + private SortedSet<String> listSummaryDocuments() { + if (this.cachedSummaryDocuments == null) { + this.cacheSummaryDocuments(); + } + return new TreeSet<String>(this.cachedSummaryDocuments.keySet()); + } + + private void cacheSummaryDocuments() { + SortedMap<String, SummaryDocument> parsedSummaryDocuments = + new TreeMap<String, SummaryDocument>(); + File directory = this.outDir; + if (directory != null) { + File summaryFile = new File(directory, "summary"); + if (summaryFile.exists()) { + String line = null; + try { + Gson gson = new Gson(); + BufferedReader br = new BufferedReader(new FileReader( + summaryFile)); + while ((line = br.readLine()) != null) { + if (line.length() == 0) { + continue; + } + SummaryDocument summaryDocument = gson.fromJson(line, + SummaryDocument.class); + if (summaryDocument != null) { + parsedSummaryDocuments.put(summaryDocument.getFingerprint(), + summaryDocument); + } + } + br.close(); + this.listedFiles += parsedSummaryDocuments.size(); + this.listOperations++; + } catch (IOException e) { + System.err.println("Could not read file '" + + summaryFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } catch (JsonParseException e) { + System.err.println("Could not parse summary document '" + line + + "' in file '" + summaryFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } + } + } + this.cachedSummaryDocuments = parsedSummaryDocuments; + } + + private <T extends Document> SortedSet<String> listDocumentFiles( + Class<T> documentType) { + SortedSet<String> fingerprints = new TreeSet<String>(); + File directory = null; + String subdirectory = null; + if (documentType.equals(DetailsStatus.class)) { + directory = this.statusDir; + subdirectory = "details"; + } else if (documentType.equals(BandwidthStatus.class)) { + directory = this.statusDir; + subdirectory = "bandwidth"; + } else if (documentType.equals(WeightsStatus.class)) { + directory = this.statusDir; + subdirectory = "weights"; + } else if (documentType.equals(ClientsStatus.class)) { + directory = this.statusDir; + subdirectory = "clients"; + } else if (documentType.equals(UptimeStatus.class)) { + directory = this.statusDir; + subdirectory = "uptimes"; + } else if (documentType.equals(DetailsDocument.class)) { + directory = this.outDir; + subdirectory = "details"; + } else if (documentType.equals(BandwidthDocument.class)) { + directory = this.outDir; + subdirectory = "bandwidth"; + } else if (documentType.equals(WeightsDocument.class)) { + directory = this.outDir; + subdirectory = "weights"; + } else if (documentType.equals(ClientsDocument.class)) { + directory = this.outDir; + subdirectory = "clients"; + } else if (documentType.equals(UptimeDocument.class)) { + directory = this.outDir; + subdirectory = "uptimes"; + } + if (directory != null && subdirectory != null) { + Stack<File> files = new Stack<File>(); + files.add(new File(directory, subdirectory)); + while (!files.isEmpty()) { + File file = files.pop(); + if (file.isDirectory()) { + files.addAll(Arrays.asList(file.listFiles())); + } else if (file.getName().length() == 40) { + fingerprints.add(file.getName()); + } + } + } + this.listOperations++; + this.listedFiles += fingerprints.size(); + return fingerprints; + } + + public <T extends Document> boolean store(T document) { + return this.store(document, null); + } + + public <T extends Document> boolean store(T document, + String fingerprint) { + if (document instanceof NodeStatus) { + return this.storeNodeStatus((NodeStatus) document, fingerprint); + } else if (document instanceof SummaryDocument) { + return this.storeSummaryDocument((SummaryDocument) document, + fingerprint); + } else { + return this.storeDocumentFile(document, fingerprint); + } + } + + private <T extends Document> boolean storeNodeStatus( + NodeStatus nodeStatus, String fingerprint) { + if (this.cachedNodeStatuses == null) { + this.cacheNodeStatuses(); + } + this.cachedNodeStatuses.put(fingerprint, nodeStatus); + return true; + } + + private <T extends Document> boolean storeSummaryDocument( + SummaryDocument summaryDocument, String fingerprint) { + if (this.cachedSummaryDocuments == null) { + this.cacheSummaryDocuments(); + } + this.cachedSummaryDocuments.put(fingerprint, summaryDocument); + return true; + } + + private <T extends Document> boolean storeDocumentFile(T document, + String fingerprint) { + File documentFile = this.getDocumentFile(document.getClass(), + fingerprint); + if (documentFile == null) { + return false; + } + String documentString; + if (document.getDocumentString() != null) { + documentString = document.getDocumentString(); + } else if (document instanceof BandwidthDocument || + document instanceof WeightsDocument || + document instanceof ClientsDocument || + document instanceof UptimeDocument) { + Gson gson = new Gson(); + documentString = gson.toJson(document); + } else if (document instanceof DetailsStatus || + document instanceof DetailsDocument) { + /* Don't escape HTML characters, like < and >, contained in + * strings. */ + Gson gson = new GsonBuilder().disableHtmlEscaping().create(); + /* We must ensure that details files only contain ASCII characters + * and no UTF-8 characters. While UTF-8 characters are perfectly + * valid in JSON, this would break compatibility with existing files + * pretty badly. We already make sure that all strings in details + * objects are escaped JSON, e.g., \u00F2. When Gson serlializes + * this string, it escapes the \ to \\, hence writes \\u00F2. We + * need to undo this and change \\u00F2 back to \u00F2. */ + documentString = gson.toJson(document).replaceAll("\\\\\\\\u", + "\\\\u"); + /* Existing details statuses don't contain opening and closing curly + * brackets, so we should remove them from new details statuses, + * too. */ + if (document instanceof DetailsStatus) { + documentString = documentString.substring( + documentString.indexOf("{") + 1, + documentString.lastIndexOf("}")); + } + } else if (document instanceof BandwidthStatus || + document instanceof WeightsStatus || + document instanceof ClientsStatus || + document instanceof UptimeStatus) { + documentString = document.toDocumentString(); + } else { + System.err.println("Serializing is not supported for type " + + document.getClass().getName() + "."); + return false; + } + try { + documentFile.getParentFile().mkdirs(); + File documentTempFile = new File( + documentFile.getAbsolutePath() + ".tmp"); + BufferedWriter bw = new BufferedWriter(new FileWriter( + documentTempFile)); + bw.write(documentString); + bw.close(); + documentFile.delete(); + documentTempFile.renameTo(documentFile); + this.storedFiles++; + this.storedBytes += documentString.length(); + } catch (IOException e) { + System.err.println("Could not write file '" + + documentFile.getAbsolutePath() + "'."); + e.printStackTrace(); + return false; + } + return true; + } + + public <T extends Document> T retrieve(Class<T> documentType, + boolean parse) { + return this.retrieve(documentType, parse, null); + } + + public <T extends Document> T retrieve(Class<T> documentType, + boolean parse, String fingerprint) { + if (documentType.equals(NodeStatus.class)) { + return documentType.cast(this.retrieveNodeStatus(fingerprint)); + } else if (documentType.equals(SummaryDocument.class)) { + return documentType.cast(this.retrieveSummaryDocument(fingerprint)); + } else { + return this.retrieveDocumentFile(documentType, parse, fingerprint); + } + } + + private NodeStatus retrieveNodeStatus(String fingerprint) { + if (this.cachedNodeStatuses == null) { + this.cacheNodeStatuses(); + } + return this.cachedNodeStatuses.get(fingerprint); + } + + private SummaryDocument retrieveSummaryDocument(String fingerprint) { + if (this.cachedSummaryDocuments == null) { + this.cacheSummaryDocuments(); + } + if (this.cachedSummaryDocuments.containsKey(fingerprint)) { + return this.cachedSummaryDocuments.get(fingerprint); + } + /* TODO This is an evil hack to support looking up relays or bridges + * that haven't been running for a week without having to load + * 500,000 NodeStatus instances into memory. Maybe there's a better + * way? Or do we need to switch to a real database for this? */ + DetailsDocument detailsDocument = this.retrieveDocumentFile( + DetailsDocument.class, true, fingerprint); + if (detailsDocument == null) { + return null; + } + boolean isRelay = detailsDocument.getHashedFingerprint() == null; + boolean running = false; + String nickname = detailsDocument.getNickname(); + List<String> addresses = new ArrayList<String>(); + String countryCode = null, aSNumber = null, contact = null; + for (String orAddressAndPort : detailsDocument.getOrAddresses()) { + if (!orAddressAndPort.contains(":")) { + return null; + } + String orAddress = orAddressAndPort.substring(0, + orAddressAndPort.lastIndexOf(":")); + if (!addresses.contains(orAddress)) { + addresses.add(orAddress); + } + } + if (detailsDocument.getExitAddresses() != null) { + for (String exitAddress : detailsDocument.getExitAddresses()) { + if (!addresses.contains(exitAddress)) { + addresses.add(exitAddress); + } + } + } + SortedSet<String> relayFlags = new TreeSet<String>(), family = null; + long lastSeenMillis = -1L, consensusWeight = -1L, + firstSeenMillis = -1L; + SummaryDocument summaryDocument = new SummaryDocument(isRelay, + nickname, fingerprint, addresses, lastSeenMillis, running, + relayFlags, consensusWeight, countryCode, firstSeenMillis, + aSNumber, contact, family); + return summaryDocument; + } + + private <T extends Document> T retrieveDocumentFile( + Class<T> documentType, boolean parse, String fingerprint) { + File documentFile = this.getDocumentFile(documentType, fingerprint); + if (documentFile == null || !documentFile.exists()) { + return null; + } else if (documentFile.isDirectory()) { + System.err.println("Could not read file '" + + documentFile.getAbsolutePath() + "', because it is a " + + "directory."); + return null; + } + String documentString = null; + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BufferedInputStream bis = new BufferedInputStream( + new FileInputStream(documentFile)); + int len; + byte[] data = new byte[1024]; + while ((len = bis.read(data, 0, 1024)) >= 0) { + baos.write(data, 0, len); + } + bis.close(); + byte[] allData = baos.toByteArray(); + if (allData.length == 0) { + return null; + } + documentString = new String(allData, "US-ASCII"); + this.retrievedFiles++; + this.retrievedBytes += documentString.length(); + } catch (IOException e) { + System.err.println("Could not read file '" + + documentFile.getAbsolutePath() + "'."); + e.printStackTrace(); + return null; + } + T result = null; + if (!parse) { + return this.retrieveUnparsedDocumentFile(documentType, + documentString); + } else if (documentType.equals(DetailsDocument.class) || + documentType.equals(BandwidthDocument.class) || + documentType.equals(WeightsDocument.class) || + documentType.equals(ClientsDocument.class) || + documentType.equals(UptimeDocument.class)) { + return this.retrieveParsedDocumentFile(documentType, + documentString); + } else if (documentType.equals(BandwidthStatus.class) || + documentType.equals(WeightsStatus.class) || + documentType.equals(ClientsStatus.class) || + documentType.equals(UptimeStatus.class)) { + return this.retrieveParsedStatusFile(documentType, documentString); + } else if (documentType.equals(DetailsStatus.class)) { + return this.retrieveParsedDocumentFile(documentType, "{" + + documentString + "}"); + } else { + System.err.println("Parsing is not supported for type " + + documentType.getName() + "."); + } + return result; + } + + private <T extends Document> T retrieveParsedStatusFile( + Class<T> documentType, String documentString) { + T result = null; + try { + result = documentType.newInstance(); + result.fromDocumentString(documentString); + } catch (InstantiationException e) { + /* Handle below. */ + e.printStackTrace(); + } catch (IllegalAccessException e) { + /* Handle below. */ + e.printStackTrace(); + } + if (result == null) { + System.err.println("Could not initialize parsed status file of " + + "type " + documentType.getName() + "."); + } + return result; + } + + private <T extends Document> T retrieveParsedDocumentFile( + Class<T> documentType, String documentString) { + T result = null; + Gson gson = new Gson(); + try { + result = gson.fromJson(documentString, documentType); + } catch (JsonParseException e) { + /* Handle below. */ + e.printStackTrace(); + } + if (result == null) { + System.err.println("Could not initialize parsed document of type " + + documentType.getName() + "."); + } + return result; + } + + private <T extends Document> T retrieveUnparsedDocumentFile( + Class<T> documentType, String documentString) { + T result = null; + try { + result = documentType.newInstance(); + result.setDocumentString(documentString); + } catch (InstantiationException e) { + /* Handle below. */ + e.printStackTrace(); + } catch (IllegalAccessException e) { + /* Handle below. */ + e.printStackTrace(); + } + if (result == null) { + System.err.println("Could not initialize unparsed document of type " + + documentType.getName() + "."); + } + return result; + } + + public <T extends Document> boolean remove(Class<T> documentType) { + return this.remove(documentType, null); + } + + public <T extends Document> boolean remove(Class<T> documentType, + String fingerprint) { + if (documentType.equals(NodeStatus.class)) { + return this.removeNodeStatus(fingerprint); + } else if (documentType.equals(SummaryDocument.class)) { + return this.removeSummaryDocument(fingerprint); + } else { + return this.removeDocumentFile(documentType, fingerprint); + } + } + + private boolean removeNodeStatus(String fingerprint) { + if (this.cachedNodeStatuses == null) { + this.cacheNodeStatuses(); + } + return this.cachedNodeStatuses.remove(fingerprint) != null; + } + + private boolean removeSummaryDocument(String fingerprint) { + if (this.cachedSummaryDocuments == null) { + this.cacheSummaryDocuments(); + } + return this.cachedSummaryDocuments.remove(fingerprint) != null; + } + + private <T extends Document> boolean removeDocumentFile( + Class<T> documentType, String fingerprint) { + File documentFile = this.getDocumentFile(documentType, fingerprint); + if (documentFile == null || !documentFile.delete()) { + System.err.println("Could not delete file '" + + documentFile.getAbsolutePath() + "'."); + return false; + } + this.removedFiles++; + return true; + } + + private <T extends Document> File getDocumentFile(Class<T> documentType, + String fingerprint) { + File documentFile = null; + if (fingerprint == null && !documentType.equals(UpdateStatus.class) && + !documentType.equals(UptimeStatus.class)) { + // TODO Instead of using the update file workaround, add new method + // lastModified(Class<T> documentType) that serves a similar + // purpose. + return null; + } + File directory = null; + String fileName = null; + if (documentType.equals(DetailsStatus.class)) { + directory = this.statusDir; + fileName = String.format("details/%s/%s/%s", + fingerprint.substring(0, 1), fingerprint.substring(1, 2), + fingerprint); + } else if (documentType.equals(BandwidthStatus.class)) { + directory = this.statusDir; + fileName = String.format("bandwidth/%s/%s/%s", + fingerprint.substring(0, 1), fingerprint.substring(1, 2), + fingerprint); + } else if (documentType.equals(WeightsStatus.class)) { + directory = this.statusDir; + fileName = String.format("weights/%s/%s/%s", + fingerprint.substring(0, 1), fingerprint.substring(1, 2), + fingerprint); + } else if (documentType.equals(ClientsStatus.class)) { + directory = this.statusDir; + fileName = String.format("clients/%s/%s/%s", + fingerprint.substring(0, 1), fingerprint.substring(1, 2), + fingerprint); + } else if (documentType.equals(UptimeStatus.class)) { + directory = this.statusDir; + if (fingerprint == null) { + fileName = "uptime"; + } else { + fileName = String.format("uptimes/%s/%s/%s", + fingerprint.substring(0, 1), fingerprint.substring(1, 2), + fingerprint); + } + } else if (documentType.equals(UpdateStatus.class)) { + directory = this.outDir; + fileName = "update"; + } else if (documentType.equals(DetailsDocument.class)) { + directory = this.outDir; + fileName = String.format("details/%s", fingerprint); + } else if (documentType.equals(BandwidthDocument.class)) { + directory = this.outDir; + fileName = String.format("bandwidth/%s", fingerprint); + } else if (documentType.equals(WeightsDocument.class)) { + directory = this.outDir; + fileName = String.format("weights/%s", fingerprint); + } else if (documentType.equals(ClientsDocument.class)) { + directory = this.outDir; + fileName = String.format("clients/%s", fingerprint); + } else if (documentType.equals(UptimeDocument.class)) { + directory = this.outDir; + fileName = String.format("uptimes/%s", fingerprint); + } + if (directory != null && fileName != null) { + documentFile = new File(directory, fileName); + } + return documentFile; + } + + public void flushDocumentCache() { + /* Write cached node statuses to disk, and write update file + * containing current time. It's important to write the update file + * now, not earlier, because the front-end should not read new node + * statuses until all details, bandwidths, and weights are ready. */ + if (this.cachedNodeStatuses != null || + this.cachedSummaryDocuments != null) { + if (this.cachedNodeStatuses != null) { + this.writeNodeStatuses(); + } + if (this.cachedSummaryDocuments != null) { + this.writeSummaryDocuments(); + } + this.writeUpdateStatus(); + } + } + + private void writeNodeStatuses() { + File directory = this.statusDir; + if (directory == null) { + return; + } + File summaryFile = new File(directory, "summary"); + SortedMap<String, NodeStatus> + cachedRelays = new TreeMap<String, NodeStatus>(), + cachedBridges = new TreeMap<String, NodeStatus>(); + for (Map.Entry<String, NodeStatus> e : + this.cachedNodeStatuses.entrySet()) { + if (e.getValue().isRelay()) { + cachedRelays.put(e.getKey(), e.getValue()); + } else { + cachedBridges.put(e.getKey(), e.getValue()); + } + } + StringBuilder sb = new StringBuilder(); + for (NodeStatus relay : cachedRelays.values()) { + String line = relay.toString(); + if (line != null) { + sb.append(line + "\n"); + } else { + System.err.println("Could not serialize relay node status '" + + relay.getFingerprint() + "'"); + } + } + for (NodeStatus bridge : cachedBridges.values()) { + String line = bridge.toString(); + if (line != null) { + sb.append(line + "\n"); + } else { + System.err.println("Could not serialize bridge node status '" + + bridge.getFingerprint() + "'"); + } + } + String documentString = sb.toString(); + try { + summaryFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); + bw.write(documentString); + bw.close(); + this.storedFiles++; + this.storedBytes += documentString.length(); + } catch (IOException e) { + System.err.println("Could not write file '" + + summaryFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } + } + + private void writeSummaryDocuments() { + StringBuilder sb = new StringBuilder(); + Gson gson = new Gson(); + for (SummaryDocument summaryDocument : + this.cachedSummaryDocuments.values()) { + String line = gson.toJson(summaryDocument); + if (line != null) { + sb.append(line + "\n"); + } else { + System.err.println("Could not serialize relay summary document '" + + summaryDocument.getFingerprint() + "'"); + } + } + String documentString = sb.toString(); + File summaryFile = new File(this.outDir, "summary"); + try { + summaryFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile)); + bw.write(documentString); + bw.close(); + this.storedFiles++; + this.storedBytes += documentString.length(); + } catch (IOException e) { + System.err.println("Could not write file '" + + summaryFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } + } + + private void writeUpdateStatus() { + if (this.outDir == null) { + return; + } + File updateFile = new File(this.outDir, "update"); + String documentString = String.valueOf(this.time.currentTimeMillis()); + try { + updateFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter(updateFile)); + bw.write(documentString); + bw.close(); + this.storedFiles++; + this.storedBytes += documentString.length(); + } catch (IOException e) { + System.err.println("Could not write file '" + + updateFile.getAbsolutePath() + "'."); + e.printStackTrace(); + } + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber(listOperations) + + " list operations performed\n"); + sb.append(" " + Logger.formatDecimalNumber(listedFiles) + + " files listed\n"); + sb.append(" " + Logger.formatDecimalNumber(storedFiles) + + " files stored\n"); + sb.append(" " + Logger.formatBytes(storedBytes) + " stored\n"); + sb.append(" " + Logger.formatDecimalNumber(retrievedFiles) + + " files retrieved\n"); + sb.append(" " + Logger.formatBytes(retrievedBytes) + + " retrieved\n"); + sb.append(" " + Logger.formatDecimalNumber(removedFiles) + + " files removed\n"); + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/docs/GraphHistory.java b/src/org/torproject/onionoo/docs/GraphHistory.java new file mode 100644 index 0000000..19ace31 --- /dev/null +++ b/src/org/torproject/onionoo/docs/GraphHistory.java @@ -0,0 +1,56 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.List; + +public class GraphHistory { + + private String first; + public void setFirst(String first) { + this.first = first; + } + public String getFirst() { + return this.first; + } + + private String last; + public void setLast(String last) { + this.last = last; + } + public String getLast() { + return this.last; + } + + private Integer interval; + public void setInterval(Integer interval) { + this.interval = interval; + } + public Integer getInterval() { + return this.interval; + } + + private Double factor; + public void setFactor(Double factor) { + this.factor = factor; + } + public Double getFactor() { + return this.factor; + } + + private Integer count; + public void setCount(Integer count) { + this.count = count; + } + public Integer getCount() { + return this.count; + } + + private List<Integer> values; + public void setValues(List<Integer> values) { + this.values = values; + } + public List<Integer> getValues() { + return this.values; + } +} diff --git a/src/org/torproject/onionoo/docs/NodeStatus.java b/src/org/torproject/onionoo/docs/NodeStatus.java new file mode 100644 index 0000000..41292fd --- /dev/null +++ b/src/org/torproject/onionoo/docs/NodeStatus.java @@ -0,0 +1,582 @@ +/* Copyright 2011, 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; +import org.torproject.onionoo.util.DateTimeHelper; + +/* Store search data of a single relay that was running in the past seven + * days. */ +public class NodeStatus extends Document { + + private boolean isRelay; + public boolean isRelay() { + return this.isRelay; + } + + private String fingerprint; + public String getFingerprint() { + return this.fingerprint; + } + + private String hashedFingerprint; + public String getHashedFingerprint() { + return this.hashedFingerprint; + } + + private String nickname; + public String getNickname() { + return this.nickname; + } + + private String address; + public String getAddress() { + return this.address; + } + + private SortedSet<String> orAddresses; + private SortedSet<String> orAddressesAndPorts; + public SortedSet<String> getOrAddresses() { + return new TreeSet<String>(this.orAddresses); + } + public void addOrAddressAndPort(String orAddressAndPort) { + if (orAddressAndPort.contains(":") && orAddressAndPort.length() > 0) { + String orAddress = orAddressAndPort.substring(0, + orAddressAndPort.lastIndexOf(':')); + if (this.exitAddresses.contains(orAddress)) { + this.exitAddresses.remove(orAddress); + } + this.orAddresses.add(orAddress); + this.orAddressesAndPorts.add(orAddressAndPort); + } + } + public SortedSet<String> getOrAddressesAndPorts() { + return new TreeSet<String>(this.orAddressesAndPorts); + } + + private SortedSet<String> exitAddresses; + public void addExitAddress(String exitAddress) { + if (exitAddress.length() > 0 && !this.address.equals(exitAddress) && + !this.orAddresses.contains(exitAddress)) { + this.exitAddresses.add(exitAddress); + } + } + public SortedSet<String> getExitAddresses() { + return new TreeSet<String>(this.exitAddresses); + } + + private Float latitude; + public void setLatitude(Float latitude) { + this.latitude = latitude; + } + public Float getLatitude() { + return this.latitude; + } + + private Float longitude; + public void setLongitude(Float longitude) { + this.longitude = longitude; + } + public Float getLongitude() { + return this.longitude; + } + + private String countryCode; + public void setCountryCode(String countryCode) { + this.countryCode = countryCode; + } + public String getCountryCode() { + return this.countryCode; + } + + private String countryName; + public void setCountryName(String countryName) { + this.countryName = countryName; + } + public String getCountryName() { + return this.countryName; + } + + private String regionName; + public void setRegionName(String regionName) { + this.regionName = regionName; + } + public String getRegionName() { + return this.regionName; + } + + private String cityName; + public void setCityName(String cityName) { + this.cityName = cityName; + } + public String getCityName() { + return this.cityName; + } + + private String aSName; + public void setASName(String aSName) { + this.aSName = aSName; + } + public String getASName() { + return this.aSName; + } + + private String aSNumber; + public void setASNumber(String aSNumber) { + this.aSNumber = aSNumber; + } + public String getASNumber() { + return this.aSNumber; + } + + private long firstSeenMillis; + public long getFirstSeenMillis() { + return this.firstSeenMillis; + } + + private long lastSeenMillis; + public long getLastSeenMillis() { + return this.lastSeenMillis; + } + + private int orPort; + public int getOrPort() { + return this.orPort; + } + + private int dirPort; + public int getDirPort() { + return this.dirPort; + } + + private SortedSet<String> relayFlags; + public SortedSet<String> getRelayFlags() { + return this.relayFlags; + } + + private long consensusWeight; + public long getConsensusWeight() { + return this.consensusWeight; + } + + private boolean running; + public void setRunning(boolean running) { + this.running = running; + } + public boolean getRunning() { + return this.running; + } + + private String hostName; + public void setHostName(String hostName) { + this.hostName = hostName; + } + public String getHostName() { + return this.hostName; + } + + private long lastRdnsLookup = -1L; + public void setLastRdnsLookup(long lastRdnsLookup) { + this.lastRdnsLookup = lastRdnsLookup; + } + public long getLastRdnsLookup() { + return this.lastRdnsLookup; + } + + private double advertisedBandwidthFraction = -1.0; + public void setAdvertisedBandwidthFraction( + double advertisedBandwidthFraction) { + this.advertisedBandwidthFraction = advertisedBandwidthFraction; + } + public double getAdvertisedBandwidthFraction() { + return this.advertisedBandwidthFraction; + } + + private double consensusWeightFraction = -1.0; + public void setConsensusWeightFraction(double consensusWeightFraction) { + this.consensusWeightFraction = consensusWeightFraction; + } + public double getConsensusWeightFraction() { + return this.consensusWeightFraction; + } + + private double guardProbability = -1.0; + public void setGuardProbability(double guardProbability) { + this.guardProbability = guardProbability; + } + public double getGuardProbability() { + return this.guardProbability; + } + + private double middleProbability = -1.0; + public void setMiddleProbability(double middleProbability) { + this.middleProbability = middleProbability; + } + public double getMiddleProbability() { + return this.middleProbability; + } + + private double exitProbability = -1.0; + public void setExitProbability(double exitProbability) { + this.exitProbability = exitProbability; + } + public double getExitProbability() { + return this.exitProbability; + } + + private String defaultPolicy; + public String getDefaultPolicy() { + return this.defaultPolicy; + } + + private String portList; + public String getPortList() { + return this.portList; + } + + private SortedMap<Long, Set<String>> lastAddresses; + public SortedMap<Long, Set<String>> getLastAddresses() { + return this.lastAddresses == null ? null : + new TreeMap<Long, Set<String>>(this.lastAddresses); + } + public long getLastChangedOrAddress() { + long lastChangedAddressesMillis = -1L; + if (this.lastAddresses != null) { + Set<String> lastAddresses = null; + for (Map.Entry<Long, Set<String>> e : this.lastAddresses.entrySet()) { + if (lastAddresses != null) { + for (String address : e.getValue()) { + if (!lastAddresses.contains(address)) { + return lastChangedAddressesMillis; + } + } + } + lastChangedAddressesMillis = e.getKey(); + lastAddresses = e.getValue(); + } + } + return lastChangedAddressesMillis; + } + + private String contact; + public void setContact(String contact) { + if (contact == null) { + this.contact = null; + } else { + StringBuilder sb = new StringBuilder(); + for (char c : contact.toLowerCase().toCharArray()) { + if (c >= 32 && c < 127) { + sb.append(c); + } else { + sb.append(" "); + } + } + this.contact = sb.toString(); + } + } + public String getContact() { + return this.contact; + } + + private Boolean recommendedVersion; + public Boolean getRecommendedVersion() { + return this.recommendedVersion; + } + + private SortedSet<String> familyFingerprints; + public void setFamilyFingerprints( + SortedSet<String> familyFingerprints) { + this.familyFingerprints = familyFingerprints; + } + public SortedSet<String> getFamilyFingerprints() { + return this.familyFingerprints; + } + + public NodeStatus(boolean isRelay, String nickname, String fingerprint, + String address, SortedSet<String> orAddressesAndPorts, + SortedSet<String> exitAddresses, long lastSeenMillis, int orPort, + int dirPort, SortedSet<String> relayFlags, long consensusWeight, + String countryCode, String hostName, long lastRdnsLookup, + String defaultPolicy, String portList, long firstSeenMillis, + long lastChangedAddresses, String aSNumber, String contact, + Boolean recommendedVersion, SortedSet<String> familyFingerprints) { + this.isRelay = isRelay; + this.nickname = nickname; + this.fingerprint = fingerprint; + try { + this.hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex( + this.fingerprint.toCharArray())).toUpperCase(); + } catch (DecoderException e) { + throw new IllegalArgumentException("Fingerprint '" + fingerprint + + "' is not a valid fingerprint.", e); + } + this.address = address; + this.exitAddresses = new TreeSet<String>(); + if (exitAddresses != null) { + this.exitAddresses.addAll(exitAddresses); + } + this.exitAddresses.remove(this.address); + this.orAddresses = new TreeSet<String>(); + this.orAddressesAndPorts = new TreeSet<String>(); + if (orAddressesAndPorts != null) { + for (String orAddressAndPort : orAddressesAndPorts) { + this.addOrAddressAndPort(orAddressAndPort); + } + } + this.lastSeenMillis = lastSeenMillis; + this.orPort = orPort; + this.dirPort = dirPort; + this.relayFlags = relayFlags; + this.consensusWeight = consensusWeight; + this.countryCode = countryCode; + this.hostName = hostName; + this.lastRdnsLookup = lastRdnsLookup; + this.defaultPolicy = defaultPolicy; + this.portList = portList; + this.firstSeenMillis = firstSeenMillis; + this.lastAddresses = + new TreeMap<Long, Set<String>>(Collections.reverseOrder()); + Set<String> addresses = new HashSet<String>(); + addresses.add(address + ":" + orPort); + if (dirPort > 0) { + addresses.add(address + ":" + dirPort); + } + addresses.addAll(orAddressesAndPorts); + this.lastAddresses.put(lastChangedAddresses, addresses); + this.aSNumber = aSNumber; + this.contact = contact; + this.recommendedVersion = recommendedVersion; + this.familyFingerprints = familyFingerprints; + } + + public static NodeStatus fromString(String documentString) { + boolean isRelay = false; + String nickname = null, fingerprint = null, address = null, + countryCode = null, hostName = null, defaultPolicy = null, + portList = null, aSNumber = null, contact = null; + SortedSet<String> orAddressesAndPorts = null, exitAddresses = null, + relayFlags = null, familyFingerprints = null; + long lastSeenMillis = -1L, consensusWeight = -1L, + lastRdnsLookup = -1L, firstSeenMillis = -1L, + lastChangedAddresses = -1L; + int orPort = -1, dirPort = -1; + Boolean recommendedVersion = null; + try { + String separator = documentString.contains("\t") ? "\t" : " "; + String[] parts = documentString.trim().split(separator); + isRelay = parts[0].equals("r"); + if (parts.length < 9) { + System.err.println("Too few space-separated values in line '" + + documentString.trim() + "'. Skipping."); + return null; + } + nickname = parts[1]; + fingerprint = parts[2]; + orAddressesAndPorts = new TreeSet<String>(); + exitAddresses = new TreeSet<String>(); + String addresses = parts[3]; + if (addresses.contains(";")) { + String[] addressParts = addresses.split(";", -1); + if (addressParts.length != 3) { + System.err.println("Invalid addresses entry in line '" + + documentString.trim() + "'. Skipping."); + return null; + } + address = addressParts[0]; + if (addressParts[1].length() > 0) { + orAddressesAndPorts.addAll(Arrays.asList( + addressParts[1].split("\\+"))); + } + if (addressParts[2].length() > 0) { + exitAddresses.addAll(Arrays.asList( + addressParts[2].split("\\+"))); + } + } else { + address = addresses; + } + lastSeenMillis = DateTimeHelper.parse(parts[4] + " " + parts[5]); + if (lastSeenMillis < 0L) { + System.err.println("Parse exception while parsing node status " + + "line '" + documentString + "'. Skipping."); + return null; + } + orPort = Integer.parseInt(parts[6]); + dirPort = Integer.parseInt(parts[7]); + relayFlags = new TreeSet<String>(); + if (parts[8].length() > 0) { + relayFlags.addAll(Arrays.asList(parts[8].split(","))); + } + if (parts.length > 9) { + consensusWeight = Long.parseLong(parts[9]); + } + if (parts.length > 10) { + countryCode = parts[10]; + } + if (parts.length > 12) { + hostName = parts[11].equals("null") ? null : parts[11]; + lastRdnsLookup = Long.parseLong(parts[12]); + } + if (parts.length > 14) { + if (!parts[13].equals("null")) { + defaultPolicy = parts[13]; + } + if (!parts[14].equals("null")) { + portList = parts[14]; + } + } + firstSeenMillis = lastSeenMillis; + if (parts.length > 16) { + firstSeenMillis = DateTimeHelper.parse(parts[15] + " " + + parts[16]); + if (firstSeenMillis < 0L) { + System.err.println("Parse exception while parsing node status " + + "line '" + documentString + "'. Skipping."); + return null; + } + } + lastChangedAddresses = lastSeenMillis; + if (parts.length > 18 && !parts[17].equals("null")) { + lastChangedAddresses = DateTimeHelper.parse(parts[17] + " " + + parts[18]); + if (lastChangedAddresses < 0L) { + System.err.println("Parse exception while parsing node status " + + "line '" + documentString + "'. Skipping."); + return null; + } + } + if (parts.length > 19) { + aSNumber = parts[19]; + } + if (parts.length > 20) { + contact = parts[20]; + } + if (parts.length > 21) { + recommendedVersion = parts[21].equals("null") ? null : + parts[21].equals("true"); + } + if (parts.length > 22 && !parts[22].equals("null")) { + familyFingerprints = new TreeSet<String>(Arrays.asList( + parts[22].split(";"))); + } + } catch (NumberFormatException e) { + System.err.println("Number format exception while parsing node " + + "status line '" + documentString + "': " + e.getMessage() + + ". Skipping."); + return null; + } catch (Exception e) { + /* This catch block is only here to handle yet unknown errors. It + * should go away once we're sure what kind of errors can occur. */ + System.err.println("Unknown exception while parsing node status " + + "line '" + documentString + "': " + e.getMessage() + ". " + + "Skipping."); + return null; + } + NodeStatus newNodeStatus = new NodeStatus(isRelay, nickname, + fingerprint, address, orAddressesAndPorts, exitAddresses, + lastSeenMillis, orPort, dirPort, relayFlags, consensusWeight, + countryCode, hostName, lastRdnsLookup, defaultPolicy, portList, + firstSeenMillis, lastChangedAddresses, aSNumber, contact, + recommendedVersion, familyFingerprints); + return newNodeStatus; + } + + public void update(NodeStatus newNodeStatus) { + if (newNodeStatus.lastSeenMillis > this.lastSeenMillis) { + this.nickname = newNodeStatus.nickname; + this.address = newNodeStatus.address; + this.orAddressesAndPorts = newNodeStatus.orAddressesAndPorts; + this.lastSeenMillis = newNodeStatus.lastSeenMillis; + this.orPort = newNodeStatus.orPort; + this.dirPort = newNodeStatus.dirPort; + this.relayFlags = newNodeStatus.relayFlags; + this.consensusWeight = newNodeStatus.consensusWeight; + this.countryCode = newNodeStatus.countryCode; + this.defaultPolicy = newNodeStatus.defaultPolicy; + this.portList = newNodeStatus.portList; + this.aSNumber = newNodeStatus.aSNumber; + this.recommendedVersion = newNodeStatus.recommendedVersion; + } + if (this.isRelay && newNodeStatus.isRelay) { + this.lastAddresses.putAll(newNodeStatus.lastAddresses); + } + this.firstSeenMillis = Math.min(newNodeStatus.firstSeenMillis, + this.firstSeenMillis); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(this.isRelay ? "r" : "b"); + sb.append("\t" + this.nickname); + sb.append("\t" + this.fingerprint); + sb.append("\t" + this.address + ";"); + int written = 0; + for (String orAddressAndPort : this.orAddressesAndPorts) { + sb.append((written++ > 0 ? "+" : "") + orAddressAndPort); + } + sb.append(";"); + if (this.isRelay) { + written = 0; + for (String exitAddress : this.exitAddresses) { + sb.append((written++ > 0 ? "+" : "") + + exitAddress); + } + } + sb.append("\t" + DateTimeHelper.format(this.lastSeenMillis, + DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); + sb.append("\t" + this.orPort); + sb.append("\t" + this.dirPort + "\t"); + written = 0; + for (String relayFlag : this.relayFlags) { + sb.append((written++ > 0 ? "," : "") + relayFlag); + } + if (this.isRelay) { + sb.append("\t" + String.valueOf(this.consensusWeight)); + sb.append("\t" + + (this.countryCode != null ? this.countryCode : "??")); + sb.append("\t" + (this.hostName != null ? this.hostName : "null")); + sb.append("\t" + String.valueOf(this.lastRdnsLookup)); + sb.append("\t" + (this.defaultPolicy != null ? this.defaultPolicy + : "null")); + sb.append("\t" + (this.portList != null ? this.portList : "null")); + } else { + sb.append("\t-1\t??\tnull\t-1\tnull\tnull"); + } + sb.append("\t" + DateTimeHelper.format(this.firstSeenMillis, + DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); + if (this.isRelay) { + sb.append("\t" + DateTimeHelper.format( + this.getLastChangedOrAddress(), + DateTimeHelper.ISO_DATETIME_TAB_FORMAT)); + sb.append("\t" + (this.aSNumber != null ? this.aSNumber : "null")); + } else { + sb.append("\tnull\tnull\tnull"); + } + sb.append("\t" + (this.contact != null ? this.contact : "")); + sb.append("\t" + (this.recommendedVersion == null ? "null" : + this.recommendedVersion ? "true" : "false")); + if (this.familyFingerprints == null || + this.familyFingerprints.isEmpty()) { + sb.append("\tnull"); + } else { + sb.append("\t"); + written = 0; + for (String familyFingerprint : this.familyFingerprints) { + sb.append((written++ > 0 ? ";" : "") + familyFingerprint); + } + } + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/docs/SummaryDocument.java b/src/org/torproject/onionoo/docs/SummaryDocument.java new file mode 100644 index 0000000..0c71ae2 --- /dev/null +++ b/src/org/torproject/onionoo/docs/SummaryDocument.java @@ -0,0 +1,202 @@ +/* Copyright 2013--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.regex.Pattern; + +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.codec.digest.DigestUtils; +import org.torproject.onionoo.util.DateTimeHelper; + +public class SummaryDocument extends Document { + + private boolean t; + public void setRelay(boolean isRelay) { + this.t = isRelay; + } + public boolean isRelay() { + return this.t; + } + + private String f; + public void setFingerprint(String fingerprint) { + if (fingerprint != null) { + Pattern fingerprintPattern = Pattern.compile("^[0-9a-fA-F]{40}$"); + if (!fingerprintPattern.matcher(fingerprint).matches()) { + throw new IllegalArgumentException("Fingerprint '" + fingerprint + + "' is not a valid fingerprint."); + } + } + this.f = fingerprint; + } + public String getFingerprint() { + return this.f; + } + + public String getHashedFingerprint() { + String hashedFingerprint = null; + if (this.f != null) { + try { + hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex( + this.f.toCharArray())).toUpperCase(); + } catch (DecoderException e) { + /* Format tested in setFingerprint(). */ + } + } + return hashedFingerprint; + } + + private String n; + public void setNickname(String nickname) { + if (nickname == null || nickname.equals("Unnamed")) { + this.n = null; + } else { + this.n = nickname; + } + } + public String getNickname() { + return this.n == null ? "Unnamed" : this.n; + } + + private String[] ad; + public void setAddresses(List<String> addresses) { + this.ad = this.collectionToStringArray(addresses); + } + public List<String> getAddresses() { + return this.stringArrayToList(this.ad); + } + + private String[] collectionToStringArray( + Collection<String> collection) { + String[] stringArray = null; + if (collection != null && !collection.isEmpty()) { + stringArray = new String[collection.size()]; + int i = 0; + for (String string : collection) { + stringArray[i++] = string; + } + } + return stringArray; + } + private List<String> stringArrayToList(String[] stringArray) { + List<String> list; + if (stringArray == null) { + list = new ArrayList<String>(); + } else { + list = Arrays.asList(stringArray); + } + return list; + } + private SortedSet<String> stringArrayToSortedSet(String[] stringArray) { + SortedSet<String> sortedSet = new TreeSet<String>(); + if (stringArray != null) { + sortedSet.addAll(Arrays.asList(stringArray)); + } + return sortedSet; + } + + private String cc; + public void setCountryCode(String countryCode) { + this.cc = countryCode; + } + public String getCountryCode() { + return this.cc; + } + + private String as; + public void setASNumber(String aSNumber) { + this.as = aSNumber; + } + public String getASNumber() { + return this.as; + } + + private String fs; + public void setFirstSeenMillis(long firstSeenMillis) { + this.fs = DateTimeHelper.format(firstSeenMillis); + } + public long getFirstSeenMillis() { + return DateTimeHelper.parse(this.fs); + } + + private String ls; + public void setLastSeenMillis(long lastSeenMillis) { + this.ls = DateTimeHelper.format(lastSeenMillis); + } + public long getLastSeenMillis() { + return DateTimeHelper.parse(this.ls); + } + + private String[] rf; + public void setRelayFlags(SortedSet<String> relayFlags) { + this.rf = this.collectionToStringArray(relayFlags); + } + public SortedSet<String> getRelayFlags() { + return this.stringArrayToSortedSet(this.rf); + } + + private long cw; + public void setConsensusWeight(long consensusWeight) { + this.cw = consensusWeight; + } + public long getConsensusWeight() { + return this.cw; + } + + private boolean r; + public void setRunning(boolean isRunning) { + this.r = isRunning; + } + public boolean isRunning() { + return this.r; + } + + private String c; + public void setContact(String contact) { + if (contact != null && contact.length() == 0) { + this.c = null; + } else { + this.c = contact; + } + } + public String getContact() { + return this.c; + } + + private String[] ff; + public void setFamilyFingerprints( + SortedSet<String> familyFingerprints) { + this.ff = this.collectionToStringArray(familyFingerprints); + } + public SortedSet<String> getFamilyFingerprints() { + return this.stringArrayToSortedSet(this.ff); + } + + public SummaryDocument(boolean isRelay, String nickname, + String fingerprint, List<String> addresses, long lastSeenMillis, + boolean running, SortedSet<String> relayFlags, long consensusWeight, + String countryCode, long firstSeenMillis, String aSNumber, + String contact, SortedSet<String> familyFingerprints) { + this.setRelay(isRelay); + this.setNickname(nickname); + this.setFingerprint(fingerprint); + this.setAddresses(addresses); + this.setLastSeenMillis(lastSeenMillis); + this.setRunning(running); + this.setRelayFlags(relayFlags); + this.setConsensusWeight(consensusWeight); + this.setCountryCode(countryCode); + this.setFirstSeenMillis(firstSeenMillis); + this.setASNumber(aSNumber); + this.setContact(contact); + this.setFamilyFingerprints(familyFingerprints); + } +} + diff --git a/src/org/torproject/onionoo/docs/UpdateStatus.java b/src/org/torproject/onionoo/docs/UpdateStatus.java new file mode 100644 index 0000000..7bd710b --- /dev/null +++ b/src/org/torproject/onionoo/docs/UpdateStatus.java @@ -0,0 +1,7 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +public class UpdateStatus extends Document { +} + diff --git a/src/org/torproject/onionoo/docs/UptimeDocument.java b/src/org/torproject/onionoo/docs/UptimeDocument.java new file mode 100644 index 0000000..7f0bacc --- /dev/null +++ b/src/org/torproject/onionoo/docs/UptimeDocument.java @@ -0,0 +1,23 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.Map; + +public class UptimeDocument extends Document { + + @SuppressWarnings("unused") + private String fingerprint; + public void setFingerprint(String fingerprint) { + this.fingerprint = fingerprint; + } + + private Map<String, GraphHistory> uptime; + public void setUptime(Map<String, GraphHistory> uptime) { + this.uptime = uptime; + } + public Map<String, GraphHistory> getUptime() { + return this.uptime; + } +} + diff --git a/src/org/torproject/onionoo/docs/UptimeHistory.java b/src/org/torproject/onionoo/docs/UptimeHistory.java new file mode 100644 index 0000000..f0a966b --- /dev/null +++ b/src/org/torproject/onionoo/docs/UptimeHistory.java @@ -0,0 +1,90 @@ +package org.torproject.onionoo.docs; + +import org.torproject.onionoo.util.DateTimeHelper; + +public class UptimeHistory + implements Comparable<UptimeHistory> { + + private boolean relay; + public boolean isRelay() { + return this.relay; + } + + private long startMillis; + public long getStartMillis() { + return this.startMillis; + } + + private int uptimeHours; + public int getUptimeHours() { + return this.uptimeHours; + } + + UptimeHistory(boolean relay, long startMillis, + int uptimeHours) { + this.relay = relay; + this.startMillis = startMillis; + this.uptimeHours = uptimeHours; + } + + public static UptimeHistory fromString(String uptimeHistoryString) { + String[] parts = uptimeHistoryString.split(" ", 3); + if (parts.length != 3) { + return null; + } + boolean relay = false; + if (parts[0].equals("r")) { + relay = true; + } else if (!parts[0].equals("b")) { + return null; + } + long startMillis = DateTimeHelper.parse(parts[1], + DateTimeHelper.DATEHOUR_NOSPACE_FORMAT); + if (startMillis < 0L) { + return null; + } + int uptimeHours = -1; + try { + uptimeHours = Integer.parseInt(parts[2]); + } catch (NumberFormatException e) { + return null; + } + return new UptimeHistory(relay, startMillis, uptimeHours); + } + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(this.relay ? "r" : "b"); + sb.append(" " + DateTimeHelper.format(this.startMillis, + DateTimeHelper.DATEHOUR_NOSPACE_FORMAT)); + sb.append(" " + String.format("%d", this.uptimeHours)); + return sb.toString(); + } + + public void addUptime(UptimeHistory other) { + this.uptimeHours += other.uptimeHours; + if (this.startMillis > other.startMillis) { + this.startMillis = other.startMillis; + } + } + + public int compareTo(UptimeHistory other) { + if (this.relay && !other.relay) { + return -1; + } else if (!this.relay && other.relay) { + return 1; + } + return this.startMillis < other.startMillis ? -1 : + this.startMillis > other.startMillis ? 1 : 0; + } + + public boolean equals(Object other) { + return other instanceof UptimeHistory && + this.relay == ((UptimeHistory) other).relay && + this.startMillis == ((UptimeHistory) other).startMillis; + } + + public int hashCode() { + return (int) this.startMillis + (this.relay ? 1 : 0); + } +} \ No newline at end of file diff --git a/src/org/torproject/onionoo/docs/UptimeStatus.java b/src/org/torproject/onionoo/docs/UptimeStatus.java new file mode 100644 index 0000000..1da11f0 --- /dev/null +++ b/src/org/torproject/onionoo/docs/UptimeStatus.java @@ -0,0 +1,142 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.Scanner; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; + +public class UptimeStatus extends Document { + + private transient String fingerprint; + + private transient boolean isDirty = false; + + private SortedSet<UptimeHistory> relayHistory = + new TreeSet<UptimeHistory>(); + public void setRelayHistory(SortedSet<UptimeHistory> relayHistory) { + this.relayHistory = relayHistory; + } + public SortedSet<UptimeHistory> getRelayHistory() { + return this.relayHistory; + } + + private SortedSet<UptimeHistory> bridgeHistory = + new TreeSet<UptimeHistory>(); + public void setBridgeHistory(SortedSet<UptimeHistory> bridgeHistory) { + this.bridgeHistory = bridgeHistory; + } + public SortedSet<UptimeHistory> getBridgeHistory() { + return this.bridgeHistory; + } + + public static UptimeStatus loadOrCreate(String fingerprint) { + UptimeStatus uptimeStatus = (fingerprint == null) ? + ApplicationFactory.getDocumentStore().retrieve( + UptimeStatus.class, true) : + ApplicationFactory.getDocumentStore().retrieve( + UptimeStatus.class, true, fingerprint); + if (uptimeStatus == null) { + uptimeStatus = new UptimeStatus(); + } + uptimeStatus.fingerprint = fingerprint; + return uptimeStatus; + } + + public void fromDocumentString(String documentString) { + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + UptimeHistory parsedLine = UptimeHistory.fromString(line); + if (parsedLine != null) { + if (parsedLine.isRelay()) { + this.relayHistory.add(parsedLine); + } else { + this.bridgeHistory.add(parsedLine); + } + } else { + System.err.println("Could not parse uptime history line '" + + line + "'. Skipping."); + } + } + s.close(); + } + + public void addToHistory(boolean relay, SortedSet<Long> newIntervals) { + for (long startMillis : newIntervals) { + SortedSet<UptimeHistory> history = relay ? this.relayHistory + : this.bridgeHistory; + UptimeHistory interval = new UptimeHistory(relay, startMillis, 1); + if (!history.headSet(interval).isEmpty()) { + UptimeHistory prev = history.headSet(interval).last(); + if (prev.isRelay() == interval.isRelay() && + prev.getStartMillis() + DateTimeHelper.ONE_HOUR + * prev.getUptimeHours() > interval.getStartMillis()) { + continue; + } + } + if (!history.tailSet(interval).isEmpty()) { + UptimeHistory next = history.tailSet(interval).first(); + if (next.isRelay() == interval.isRelay() && + next.getStartMillis() < interval.getStartMillis() + + DateTimeHelper.ONE_HOUR) { + continue; + } + } + history.add(interval); + this.isDirty = true; + } + } + + public void storeIfChanged() { + if (this.isDirty) { + this.compressHistory(this.relayHistory); + this.compressHistory(this.bridgeHistory); + if (fingerprint == null) { + ApplicationFactory.getDocumentStore().store(this); + } else { + ApplicationFactory.getDocumentStore().store(this, + this.fingerprint); + } + this.isDirty = false; + } + } + + private void compressHistory(SortedSet<UptimeHistory> history) { + SortedSet<UptimeHistory> uncompressedHistory = + new TreeSet<UptimeHistory>(history); + history.clear(); + UptimeHistory lastInterval = null; + for (UptimeHistory interval : uncompressedHistory) { + if (lastInterval != null && + lastInterval.getStartMillis() + DateTimeHelper.ONE_HOUR + * lastInterval.getUptimeHours() == interval.getStartMillis() && + lastInterval.isRelay() == interval.isRelay()) { + lastInterval.addUptime(interval); + } else { + if (lastInterval != null) { + history.add(lastInterval); + } + lastInterval = interval; + } + } + if (lastInterval != null) { + history.add(lastInterval); + } + } + + public String toDocumentString() { + StringBuilder sb = new StringBuilder(); + for (UptimeHistory interval : this.relayHistory) { + sb.append(interval.toString() + "\n"); + } + for (UptimeHistory interval : this.bridgeHistory) { + sb.append(interval.toString() + "\n"); + } + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/docs/WeightsDocument.java b/src/org/torproject/onionoo/docs/WeightsDocument.java new file mode 100644 index 0000000..104b661 --- /dev/null +++ b/src/org/torproject/onionoo/docs/WeightsDocument.java @@ -0,0 +1,64 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.docs; + +import java.util.Map; + +public class WeightsDocument extends Document { + + @SuppressWarnings("unused") + private String fingerprint; + public void setFingerprint(String fingerprint) { + this.fingerprint = fingerprint; + } + + @SuppressWarnings("unused") + private Map<String, GraphHistory> advertised_bandwidth_fraction; + public void setAdvertisedBandwidthFraction( + Map<String, GraphHistory> advertisedBandwidthFraction) { + this.advertised_bandwidth_fraction = advertisedBandwidthFraction; + } + + @SuppressWarnings("unused") + private Map<String, GraphHistory> consensus_weight_fraction; + public void setConsensusWeightFraction( + Map<String, GraphHistory> consensusWeightFraction) { + this.consensus_weight_fraction = consensusWeightFraction; + } + + @SuppressWarnings("unused") + private Map<String, GraphHistory> guard_probability; + public void setGuardProbability( + Map<String, GraphHistory> guardProbability) { + this.guard_probability = guardProbability; + } + + @SuppressWarnings("unused") + private Map<String, GraphHistory> middle_probability; + public void setMiddleProbability( + Map<String, GraphHistory> middleProbability) { + this.middle_probability = middleProbability; + } + + @SuppressWarnings("unused") + private Map<String, GraphHistory> exit_probability; + public void setExitProbability( + Map<String, GraphHistory> exitProbability) { + this.exit_probability = exitProbability; + } + + @SuppressWarnings("unused") + private Map<String, GraphHistory> advertised_bandwidth; + public void setAdvertisedBandwidth( + Map<String, GraphHistory> advertisedBandwidth) { + this.advertised_bandwidth = advertisedBandwidth; + } + + @SuppressWarnings("unused") + private Map<String, GraphHistory> consensus_weight; + public void setConsensusWeight( + Map<String, GraphHistory> consensusWeight) { + this.consensus_weight = consensusWeight; + } +} + diff --git a/src/org/torproject/onionoo/docs/WeightsStatus.java b/src/org/torproject/onionoo/docs/WeightsStatus.java new file mode 100644 index 0000000..678789b --- /dev/null +++ b/src/org/torproject/onionoo/docs/WeightsStatus.java @@ -0,0 +1,99 @@ +package org.torproject.onionoo.docs; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.Scanner; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.torproject.onionoo.util.DateTimeHelper; + +public class WeightsStatus extends Document { + + private SortedMap<long[], double[]> history = + new TreeMap<long[], double[]>(new Comparator<long[]>() { + public int compare(long[] a, long[] b) { + return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0; + } + }); + public void setHistory(SortedMap<long[], double[]> history) { + this.history = history; + } + public SortedMap<long[], double[]> getHistory() { + return this.history; + } + + private Map<String, Integer> advertisedBandwidths = + new HashMap<String, Integer>(); + public Map<String, Integer> getAdvertisedBandwidths() { + return this.advertisedBandwidths; + } + + public void fromDocumentString(String documentString) { + Scanner s = new Scanner(documentString); + while (s.hasNextLine()) { + String line = s.nextLine(); + String[] parts = line.split(" "); + if (parts.length == 2) { + String descriptorDigest = parts[0]; + int advertisedBandwidth = Integer.parseInt(parts[1]); + this.advertisedBandwidths.put(descriptorDigest, + advertisedBandwidth); + continue; + } + if (parts.length != 9 && parts.length != 11) { + System.err.println("Illegal line '" + line + "' in weights " + + "status file. Skipping this line."); + continue; + } + if (parts[4].equals("NaN")) { + /* Remove corrupt lines written on 2013-07-07 and the days + * after. */ + continue; + } + long validAfterMillis = DateTimeHelper.parse(parts[0] + " " + + parts[1]); + long freshUntilMillis = DateTimeHelper.parse(parts[2] + " " + + parts[3]); + if (validAfterMillis < 0L || freshUntilMillis < 0L) { + System.err.println("Could not parse timestamp while reading " + + "weights status file. Skipping."); + break; + } + long[] interval = new long[] { validAfterMillis, freshUntilMillis }; + double[] weights = new double[] { + Double.parseDouble(parts[4]), + Double.parseDouble(parts[5]), + Double.parseDouble(parts[6]), + Double.parseDouble(parts[7]), + Double.parseDouble(parts[8]), -1.0, -1.0 }; + if (parts.length == 11) { + weights[5] = Double.parseDouble(parts[9]); + weights[6] = Double.parseDouble(parts[10]); + } + this.history.put(interval, weights); + } + s.close(); + } + + public String toDocumentString() { + StringBuilder sb = new StringBuilder(); + for (Map.Entry<String, Integer> e : + this.advertisedBandwidths.entrySet()) { + sb.append(e.getKey() + " " + String.valueOf(e.getValue()) + "\n"); + } + for (Map.Entry<long[], double[]> e : history.entrySet()) { + long[] fresh = e.getKey(); + double[] weights = e.getValue(); + sb.append(DateTimeHelper.format(fresh[0]) + " " + + DateTimeHelper.format(fresh[1])); + for (double weight : weights) { + sb.append(String.format(" %.12f", weight)); + } + sb.append("\n"); + } + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/server/NodeIndexer.java b/src/org/torproject/onionoo/server/NodeIndexer.java new file mode 100644 index 0000000..b76f4c1 --- /dev/null +++ b/src/org/torproject/onionoo/server/NodeIndexer.java @@ -0,0 +1,432 @@ +package org.torproject.onionoo.server; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; + +import javax.servlet.ServletContext; +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; + +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.docs.SummaryDocument; +import org.torproject.onionoo.docs.UpdateStatus; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; +import org.torproject.onionoo.util.Time; + +class NodeIndex { + + private String relaysPublishedString; + public void setRelaysPublishedString(String relaysPublishedString) { + this.relaysPublishedString = relaysPublishedString; + } + public String getRelaysPublishedString() { + return relaysPublishedString; + } + + private String bridgesPublishedString; + public void setBridgesPublishedString(String bridgesPublishedString) { + this.bridgesPublishedString = bridgesPublishedString; + } + public String getBridgesPublishedString() { + return bridgesPublishedString; + } + + private List<String> relaysByConsensusWeight; + public void setRelaysByConsensusWeight( + List<String> relaysByConsensusWeight) { + this.relaysByConsensusWeight = relaysByConsensusWeight; + } + public List<String> getRelaysByConsensusWeight() { + return relaysByConsensusWeight; + } + + + private Map<String, SummaryDocument> relayFingerprintSummaryLines; + public void setRelayFingerprintSummaryLines( + Map<String, SummaryDocument> relayFingerprintSummaryLines) { + this.relayFingerprintSummaryLines = relayFingerprintSummaryLines; + } + public Map<String, SummaryDocument> getRelayFingerprintSummaryLines() { + return this.relayFingerprintSummaryLines; + } + + private Map<String, SummaryDocument> bridgeFingerprintSummaryLines; + public void setBridgeFingerprintSummaryLines( + Map<String, SummaryDocument> bridgeFingerprintSummaryLines) { + this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines; + } + public Map<String, SummaryDocument> getBridgeFingerprintSummaryLines() { + return this.bridgeFingerprintSummaryLines; + } + + private Map<String, Set<String>> relaysByCountryCode = null; + public void setRelaysByCountryCode( + Map<String, Set<String>> relaysByCountryCode) { + this.relaysByCountryCode = relaysByCountryCode; + } + public Map<String, Set<String>> getRelaysByCountryCode() { + return relaysByCountryCode; + } + + private Map<String, Set<String>> relaysByASNumber = null; + public void setRelaysByASNumber( + Map<String, Set<String>> relaysByASNumber) { + this.relaysByASNumber = relaysByASNumber; + } + public Map<String, Set<String>> getRelaysByASNumber() { + return relaysByASNumber; + } + + private Map<String, Set<String>> relaysByFlag = null; + public void setRelaysByFlag(Map<String, Set<String>> relaysByFlag) { + this.relaysByFlag = relaysByFlag; + } + public Map<String, Set<String>> getRelaysByFlag() { + return relaysByFlag; + } + + private Map<String, Set<String>> bridgesByFlag = null; + public void setBridgesByFlag(Map<String, Set<String>> bridgesByFlag) { + this.bridgesByFlag = bridgesByFlag; + } + public Map<String, Set<String>> getBridgesByFlag() { + return bridgesByFlag; + } + + private Map<String, Set<String>> relaysByContact = null; + public void setRelaysByContact( + Map<String, Set<String>> relaysByContact) { + this.relaysByContact = relaysByContact; + } + public Map<String, Set<String>> getRelaysByContact() { + return relaysByContact; + } + + private Map<String, Set<String>> relaysByFamily = null; + public void setRelaysByFamily(Map<String, Set<String>> relaysByFamily) { + this.relaysByFamily = relaysByFamily; + } + public Map<String, Set<String>> getRelaysByFamily() { + return this.relaysByFamily; + } + + private SortedMap<Integer, Set<String>> relaysByFirstSeenDays; + public void setRelaysByFirstSeenDays( + SortedMap<Integer, Set<String>> relaysByFirstSeenDays) { + this.relaysByFirstSeenDays = relaysByFirstSeenDays; + } + public SortedMap<Integer, Set<String>> getRelaysByFirstSeenDays() { + return relaysByFirstSeenDays; + } + + private SortedMap<Integer, Set<String>> bridgesByFirstSeenDays; + public void setBridgesByFirstSeenDays( + SortedMap<Integer, Set<String>> bridgesByFirstSeenDays) { + this.bridgesByFirstSeenDays = bridgesByFirstSeenDays; + } + public SortedMap<Integer, Set<String>> getBridgesByFirstSeenDays() { + return bridgesByFirstSeenDays; + } + + private SortedMap<Integer, Set<String>> relaysByLastSeenDays; + public void setRelaysByLastSeenDays( + SortedMap<Integer, Set<String>> relaysByLastSeenDays) { + this.relaysByLastSeenDays = relaysByLastSeenDays; + } + public SortedMap<Integer, Set<String>> getRelaysByLastSeenDays() { + return relaysByLastSeenDays; + } + + private SortedMap<Integer, Set<String>> bridgesByLastSeenDays; + public void setBridgesByLastSeenDays( + SortedMap<Integer, Set<String>> bridgesByLastSeenDays) { + this.bridgesByLastSeenDays = bridgesByLastSeenDays; + } + public SortedMap<Integer, Set<String>> getBridgesByLastSeenDays() { + return bridgesByLastSeenDays; + } +} + +public class NodeIndexer implements ServletContextListener, Runnable { + + public void contextInitialized(ServletContextEvent contextEvent) { + ServletContext servletContext = contextEvent.getServletContext(); + File outDir = new File(servletContext.getInitParameter("outDir")); + DocumentStore documentStore = ApplicationFactory.getDocumentStore(); + documentStore.setOutDir(outDir); + /* The servlet container created us, and we need to avoid that + * ApplicationFactory creates another instance of us. */ + ApplicationFactory.setNodeIndexer(this); + this.startIndexing(); + } + + public void contextDestroyed(ServletContextEvent contextEvent) { + this.stopIndexing(); + } + + private long lastIndexed = -1L; + + private NodeIndex latestNodeIndex = null; + + private Thread nodeIndexerThread = null; + + public synchronized long getLastIndexed(long timeoutMillis) { + if (this.lastIndexed == -1L && this.nodeIndexerThread != null && + timeoutMillis > 0L) { + try { + this.wait(timeoutMillis); + } catch (InterruptedException e) { + } + } + return this.lastIndexed; + } + + public synchronized NodeIndex getLatestNodeIndex(long timeoutMillis) { + if (this.latestNodeIndex == null && this.nodeIndexerThread != null && + timeoutMillis > 0L) { + try { + this.wait(timeoutMillis); + } catch (InterruptedException e) { + } + } + return this.latestNodeIndex; + } + + public synchronized void startIndexing() { + if (this.nodeIndexerThread == null) { + this.nodeIndexerThread = new Thread(this); + this.nodeIndexerThread.setDaemon(true); + this.nodeIndexerThread.start(); + } + } + + public void run() { + while (this.nodeIndexerThread != null) { + this.indexNodeStatuses(); + try { + Thread.sleep(DateTimeHelper.ONE_MINUTE); + } catch (InterruptedException e) { + } + } + } + + public synchronized void stopIndexing() { + Thread indexerThread = this.nodeIndexerThread; + this.nodeIndexerThread = null; + indexerThread.interrupt(); + } + + private void indexNodeStatuses() { + long updateStatusMillis = -1L; + DocumentStore documentStore = ApplicationFactory.getDocumentStore(); + UpdateStatus updateStatus = documentStore.retrieve(UpdateStatus.class, + false); + if (updateStatus != null && + updateStatus.getDocumentString() != null) { + String updateString = updateStatus.getDocumentString(); + try { + updateStatusMillis = Long.parseLong(updateString.trim()); + } catch (NumberFormatException e) { + /* Handle below. */ + } + } + synchronized (this) { + if (updateStatusMillis <= this.lastIndexed) { + return; + } + } + List<String> newRelaysByConsensusWeight = new ArrayList<String>(); + Map<String, SummaryDocument> + newRelayFingerprintSummaryLines = + new HashMap<String, SummaryDocument>(), + newBridgeFingerprintSummaryLines = + new HashMap<String, SummaryDocument>(); + Map<String, Set<String>> + newRelaysByCountryCode = new HashMap<String, Set<String>>(), + newRelaysByASNumber = new HashMap<String, Set<String>>(), + newRelaysByFlag = new HashMap<String, Set<String>>(), + newBridgesByFlag = new HashMap<String, Set<String>>(), + newRelaysByContact = new HashMap<String, Set<String>>(), + newRelaysByFamily = new HashMap<String, Set<String>>(); + SortedMap<Integer, Set<String>> + newRelaysByFirstSeenDays = new TreeMap<Integer, Set<String>>(), + newBridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(), + newRelaysByLastSeenDays = new TreeMap<Integer, Set<String>>(), + newBridgesByLastSeenDays = new TreeMap<Integer, Set<String>>(); + Set<SummaryDocument> currentRelays = new HashSet<SummaryDocument>(), + currentBridges = new HashSet<SummaryDocument>(); + SortedSet<String> fingerprints = documentStore.list( + SummaryDocument.class); + long relaysLastValidAfterMillis = 0L, bridgesLastPublishedMillis = 0L; + for (String fingerprint : fingerprints) { + SummaryDocument node = documentStore.retrieve(SummaryDocument.class, + true, fingerprint); + if (node.isRelay()) { + relaysLastValidAfterMillis = Math.max( + relaysLastValidAfterMillis, node.getLastSeenMillis()); + currentRelays.add(node); + } else { + bridgesLastPublishedMillis = Math.max( + bridgesLastPublishedMillis, node.getLastSeenMillis()); + currentBridges.add(node); + } + } + Time time = ApplicationFactory.getTime(); + List<String> orderRelaysByConsensusWeight = new ArrayList<String>(); + for (SummaryDocument entry : currentRelays) { + String fingerprint = entry.getFingerprint().toUpperCase(); + String hashedFingerprint = entry.getHashedFingerprint(). + toUpperCase(); + newRelayFingerprintSummaryLines.put(fingerprint, entry); + newRelayFingerprintSummaryLines.put(hashedFingerprint, entry); + long consensusWeight = entry.getConsensusWeight(); + orderRelaysByConsensusWeight.add(String.format("%020d %s", + consensusWeight, fingerprint)); + orderRelaysByConsensusWeight.add(String.format("%020d %s", + consensusWeight, hashedFingerprint)); + if (entry.getCountryCode() != null) { + String countryCode = entry.getCountryCode(); + if (!newRelaysByCountryCode.containsKey(countryCode)) { + newRelaysByCountryCode.put(countryCode, + new HashSet<String>()); + } + newRelaysByCountryCode.get(countryCode).add(fingerprint); + newRelaysByCountryCode.get(countryCode).add(hashedFingerprint); + } + if (entry.getASNumber() != null) { + String aSNumber = entry.getASNumber(); + if (!newRelaysByASNumber.containsKey(aSNumber)) { + newRelaysByASNumber.put(aSNumber, new HashSet<String>()); + } + newRelaysByASNumber.get(aSNumber).add(fingerprint); + newRelaysByASNumber.get(aSNumber).add(hashedFingerprint); + } + for (String flag : entry.getRelayFlags()) { + String flagLowerCase = flag.toLowerCase(); + if (!newRelaysByFlag.containsKey(flagLowerCase)) { + newRelaysByFlag.put(flagLowerCase, new HashSet<String>()); + } + newRelaysByFlag.get(flagLowerCase).add(fingerprint); + newRelaysByFlag.get(flagLowerCase).add(hashedFingerprint); + } + if (entry.getFamilyFingerprints() != null) { + newRelaysByFamily.put(fingerprint, entry.getFamilyFingerprints()); + } + int daysSinceFirstSeen = (int) ((time.currentTimeMillis() + - entry.getFirstSeenMillis()) / DateTimeHelper.ONE_DAY); + if (!newRelaysByFirstSeenDays.containsKey(daysSinceFirstSeen)) { + newRelaysByFirstSeenDays.put(daysSinceFirstSeen, + new HashSet<String>()); + } + newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add(fingerprint); + newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add( + hashedFingerprint); + int daysSinceLastSeen = (int) ((time.currentTimeMillis() + - entry.getLastSeenMillis()) / DateTimeHelper.ONE_DAY); + if (!newRelaysByLastSeenDays.containsKey(daysSinceLastSeen)) { + newRelaysByLastSeenDays.put(daysSinceLastSeen, + new HashSet<String>()); + } + newRelaysByLastSeenDays.get(daysSinceLastSeen).add(fingerprint); + newRelaysByLastSeenDays.get(daysSinceLastSeen).add( + hashedFingerprint); + String contact = entry.getContact(); + if (!newRelaysByContact.containsKey(contact)) { + newRelaysByContact.put(contact, new HashSet<String>()); + } + newRelaysByContact.get(contact).add(fingerprint); + newRelaysByContact.get(contact).add(hashedFingerprint); + } + Collections.sort(orderRelaysByConsensusWeight); + newRelaysByConsensusWeight = new ArrayList<String>(); + for (String relay : orderRelaysByConsensusWeight) { + newRelaysByConsensusWeight.add(relay.split(" ")[1]); + } + for (Map.Entry<String, Set<String>> e : + newRelaysByFamily.entrySet()) { + String fingerprint = e.getKey(); + Set<String> inMutualFamilyRelation = new HashSet<String>(); + for (String otherFingerprint : e.getValue()) { + if (newRelaysByFamily.containsKey(otherFingerprint) && + newRelaysByFamily.get(otherFingerprint).contains( + fingerprint)) { + inMutualFamilyRelation.add(otherFingerprint); + } + } + e.getValue().retainAll(inMutualFamilyRelation); + } + for (SummaryDocument entry : currentBridges) { + String hashedFingerprint = entry.getFingerprint().toUpperCase(); + String hashedHashedFingerprint = entry.getHashedFingerprint(). + toUpperCase(); + newBridgeFingerprintSummaryLines.put(hashedFingerprint, entry); + newBridgeFingerprintSummaryLines.put(hashedHashedFingerprint, + entry); + for (String flag : entry.getRelayFlags()) { + String flagLowerCase = flag.toLowerCase(); + if (!newBridgesByFlag.containsKey(flagLowerCase)) { + newBridgesByFlag.put(flagLowerCase, new HashSet<String>()); + } + newBridgesByFlag.get(flagLowerCase).add(hashedFingerprint); + newBridgesByFlag.get(flagLowerCase).add( + hashedHashedFingerprint); + } + int daysSinceFirstSeen = (int) ((time.currentTimeMillis() + - entry.getFirstSeenMillis()) / DateTimeHelper.ONE_DAY); + if (!newBridgesByFirstSeenDays.containsKey(daysSinceFirstSeen)) { + newBridgesByFirstSeenDays.put(daysSinceFirstSeen, + new HashSet<String>()); + } + newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add( + hashedFingerprint); + newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add( + hashedHashedFingerprint); + int daysSinceLastSeen = (int) ((time.currentTimeMillis() + - entry.getLastSeenMillis()) / DateTimeHelper.ONE_DAY); + if (!newBridgesByLastSeenDays.containsKey(daysSinceLastSeen)) { + newBridgesByLastSeenDays.put(daysSinceLastSeen, + new HashSet<String>()); + } + newBridgesByLastSeenDays.get(daysSinceLastSeen).add( + hashedFingerprint); + newBridgesByLastSeenDays.get(daysSinceLastSeen).add( + hashedHashedFingerprint); + } + NodeIndex newNodeIndex = new NodeIndex(); + newNodeIndex.setRelaysByConsensusWeight(newRelaysByConsensusWeight); + newNodeIndex.setRelayFingerprintSummaryLines( + newRelayFingerprintSummaryLines); + newNodeIndex.setBridgeFingerprintSummaryLines( + newBridgeFingerprintSummaryLines); + newNodeIndex.setRelaysByCountryCode(newRelaysByCountryCode); + newNodeIndex.setRelaysByASNumber(newRelaysByASNumber); + newNodeIndex.setRelaysByFlag(newRelaysByFlag); + newNodeIndex.setBridgesByFlag(newBridgesByFlag); + newNodeIndex.setRelaysByContact(newRelaysByContact); + newNodeIndex.setRelaysByFamily(newRelaysByFamily); + newNodeIndex.setRelaysByFirstSeenDays(newRelaysByFirstSeenDays); + newNodeIndex.setRelaysByLastSeenDays(newRelaysByLastSeenDays); + newNodeIndex.setBridgesByFirstSeenDays(newBridgesByFirstSeenDays); + newNodeIndex.setBridgesByLastSeenDays(newBridgesByLastSeenDays); + newNodeIndex.setRelaysPublishedString(DateTimeHelper.format( + relaysLastValidAfterMillis)); + newNodeIndex.setBridgesPublishedString(DateTimeHelper.format( + bridgesLastPublishedMillis)); + synchronized (this) { + this.lastIndexed = updateStatusMillis; + this.latestNodeIndex = newNodeIndex; + this.notifyAll(); + } + } +} + diff --git a/src/org/torproject/onionoo/server/RequestHandler.java b/src/org/torproject/onionoo/server/RequestHandler.java new file mode 100644 index 0000000..22e82fb --- /dev/null +++ b/src/org/torproject/onionoo/server/RequestHandler.java @@ -0,0 +1,552 @@ +/* Copyright 2011--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.server; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; + +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.docs.SummaryDocument; +import org.torproject.onionoo.util.ApplicationFactory; + +public class RequestHandler { + + private NodeIndex nodeIndex; + + private DocumentStore documentStore; + + public RequestHandler(NodeIndex nodeIndex) { + this.nodeIndex = nodeIndex; + this.documentStore = ApplicationFactory.getDocumentStore(); + } + + private String resourceType; + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + private String type; + public void setType(String type) { + this.type = type; + } + + private String running; + public void setRunning(String running) { + this.running = running; + } + + private String[] search; + public void setSearch(String[] search) { + this.search = new String[search.length]; + System.arraycopy(search, 0, this.search, 0, search.length); + } + + private String lookup; + public void setLookup(String lookup) { + this.lookup = lookup; + } + + private String fingerprint; + public void setFingerprint(String fingerprint) { + this.fingerprint = fingerprint; + } + + private String country; + public void setCountry(String country) { + this.country = country; + } + + private String as; + public void setAs(String as) { + this.as = as; + } + + private String flag; + public void setFlag(String flag) { + this.flag = flag; + } + + private String[] contact; + public void setContact(String[] contact) { + this.contact = new String[contact.length]; + System.arraycopy(contact, 0, this.contact, 0, contact.length); + } + + private String[] order; + public void setOrder(String[] order) { + this.order = new String[order.length]; + System.arraycopy(order, 0, this.order, 0, order.length); + } + + private String offset; + public void setOffset(String offset) { + this.offset = offset; + } + + private String limit; + public void setLimit(String limit) { + this.limit = limit; + } + + private int[] firstSeenDays; + public void setFirstSeenDays(int[] firstSeenDays) { + this.firstSeenDays = new int[firstSeenDays.length]; + System.arraycopy(firstSeenDays, 0, this.firstSeenDays, 0, + firstSeenDays.length); + } + + private int[] lastSeenDays; + public void setLastSeenDays(int[] lastSeenDays) { + this.lastSeenDays = new int[lastSeenDays.length]; + System.arraycopy(lastSeenDays, 0, this.lastSeenDays, 0, + lastSeenDays.length); + } + + private String family; + public void setFamily(String family) { + this.family = family; + } + + private Map<String, SummaryDocument> filteredRelays = + new HashMap<String, SummaryDocument>(); + + private Map<String, SummaryDocument> filteredBridges = + new HashMap<String, SummaryDocument>(); + + public void handleRequest() { + this.filteredRelays.putAll( + this.nodeIndex.getRelayFingerprintSummaryLines()); + this.filteredBridges.putAll( + this.nodeIndex.getBridgeFingerprintSummaryLines()); + this.filterByResourceType(); + this.filterByType(); + this.filterByRunning(); + this.filterBySearchTerms(); + this.filterByLookup(); + this.filterByFingerprint(); + this.filterByCountryCode(); + this.filterByASNumber(); + this.filterByFlag(); + this.filterNodesByFirstSeenDays(); + this.filterNodesByLastSeenDays(); + this.filterByContact(); + this.filterByFamily(); + this.order(); + this.offset(); + this.limit(); + } + + + private void filterByResourceType() { + if (this.resourceType.equals("clients")) { + this.filteredRelays.clear(); + } + if (this.resourceType.equals("weights")) { + this.filteredBridges.clear(); + } + } + + private void filterByType() { + if (this.type == null) { + return; + } else if (this.type.equals("relay")) { + this.filteredBridges.clear(); + } else { + this.filteredRelays.clear(); + } + } + + private void filterByRunning() { + if (this.running == null) { + return; + } + boolean runningRequested = this.running.equals("true"); + Set<String> removeRelays = new HashSet<String>(); + for (Map.Entry<String, SummaryDocument> e : + filteredRelays.entrySet()) { + if (e.getValue().isRunning() != runningRequested) { + removeRelays.add(e.getKey()); + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + Set<String> removeBridges = new HashSet<String>(); + for (Map.Entry<String, SummaryDocument> e : + filteredBridges.entrySet()) { + if (e.getValue().isRunning() != runningRequested) { + removeBridges.add(e.getKey()); + } + } + for (String fingerprint : removeBridges) { + this.filteredBridges.remove(fingerprint); + } + } + + private void filterBySearchTerms() { + if (this.search == null) { + return; + } + for (String searchTerm : this.search) { + filterBySearchTerm(searchTerm); + } + } + + private void filterBySearchTerm(String searchTerm) { + Set<String> removeRelays = new HashSet<String>(); + for (Map.Entry<String, SummaryDocument> e : + filteredRelays.entrySet()) { + String fingerprint = e.getKey(); + SummaryDocument entry = e.getValue(); + boolean lineMatches = false; + String nickname = entry.getNickname() != null ? + entry.getNickname().toLowerCase() : "unnamed"; + if (searchTerm.startsWith("$")) { + /* Search is for $-prefixed fingerprint. */ + if (fingerprint.startsWith( + searchTerm.substring(1).toUpperCase())) { + /* $-prefixed fingerprint matches. */ + lineMatches = true; + } + } else if (nickname.contains(searchTerm.toLowerCase())) { + /* Nickname matches. */ + lineMatches = true; + } else if (fingerprint.startsWith(searchTerm.toUpperCase())) { + /* Non-$-prefixed fingerprint matches. */ + lineMatches = true; + } else { + List<String> addresses = entry.getAddresses(); + for (String address : addresses) { + if (address.startsWith(searchTerm.toLowerCase())) { + /* Address matches. */ + lineMatches = true; + break; + } + } + } + if (!lineMatches) { + removeRelays.add(e.getKey()); + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + Set<String> removeBridges = new HashSet<String>(); + for (Map.Entry<String, SummaryDocument> e : + filteredBridges.entrySet()) { + String hashedFingerprint = e.getKey(); + SummaryDocument entry = e.getValue(); + boolean lineMatches = false; + String nickname = entry.getNickname() != null ? + entry.getNickname().toLowerCase() : "unnamed"; + if (searchTerm.startsWith("$")) { + /* Search is for $-prefixed hashed fingerprint. */ + if (hashedFingerprint.startsWith( + searchTerm.substring(1).toUpperCase())) { + /* $-prefixed hashed fingerprint matches. */ + lineMatches = true; + } + } else if (nickname.contains(searchTerm.toLowerCase())) { + /* Nickname matches. */ + lineMatches = true; + } else if (hashedFingerprint.startsWith(searchTerm.toUpperCase())) { + /* Non-$-prefixed hashed fingerprint matches. */ + lineMatches = true; + } + if (!lineMatches) { + removeBridges.add(e.getKey()); + } + } + for (String fingerprint : removeBridges) { + this.filteredBridges.remove(fingerprint); + } + } + + private void filterByLookup() { + if (this.lookup == null) { + return; + } + String fingerprint = this.lookup; + SummaryDocument relayLine = this.filteredRelays.get(fingerprint); + this.filteredRelays.clear(); + if (relayLine != null) { + this.filteredRelays.put(fingerprint, relayLine); + } + SummaryDocument bridgeLine = this.filteredBridges.get(fingerprint); + this.filteredBridges.clear(); + if (bridgeLine != null) { + this.filteredBridges.put(fingerprint, bridgeLine); + } + } + + private void filterByFingerprint() { + if (this.fingerprint == null) { + return; + } + this.filteredRelays.clear(); + this.filteredBridges.clear(); + String fingerprint = this.fingerprint; + SummaryDocument entry = this.documentStore.retrieve( + SummaryDocument.class, true, fingerprint); + if (entry != null) { + if (entry.isRelay()) { + this.filteredRelays.put(fingerprint, entry); + } else { + this.filteredBridges.put(fingerprint, entry); + } + } + } + + private void filterByCountryCode() { + if (this.country == null) { + return; + } + String countryCode = this.country.toLowerCase(); + if (!this.nodeIndex.getRelaysByCountryCode().containsKey( + countryCode)) { + this.filteredRelays.clear(); + } else { + Set<String> relaysWithCountryCode = + this.nodeIndex.getRelaysByCountryCode().get(countryCode); + Set<String> removeRelays = new HashSet<String>(); + for (String fingerprint : this.filteredRelays.keySet()) { + if (!relaysWithCountryCode.contains(fingerprint)) { + removeRelays.add(fingerprint); + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + } + this.filteredBridges.clear(); + } + + private void filterByASNumber() { + if (this.as == null) { + return; + } + String aSNumber = this.as.toUpperCase(); + if (!aSNumber.startsWith("AS")) { + aSNumber = "AS" + aSNumber; + } + if (!this.nodeIndex.getRelaysByASNumber().containsKey(aSNumber)) { + this.filteredRelays.clear(); + } else { + Set<String> relaysWithASNumber = + this.nodeIndex.getRelaysByASNumber().get(aSNumber); + Set<String> removeRelays = new HashSet<String>(); + for (String fingerprint : this.filteredRelays.keySet()) { + if (!relaysWithASNumber.contains(fingerprint)) { + removeRelays.add(fingerprint); + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + } + this.filteredBridges.clear(); + } + + private void filterByFlag() { + if (this.flag == null) { + return; + } + String flag = this.flag.toLowerCase(); + if (!this.nodeIndex.getRelaysByFlag().containsKey(flag)) { + this.filteredRelays.clear(); + } else { + Set<String> relaysWithFlag = this.nodeIndex.getRelaysByFlag().get( + flag); + Set<String> removeRelays = new HashSet<String>(); + for (String fingerprint : this.filteredRelays.keySet()) { + if (!relaysWithFlag.contains(fingerprint)) { + removeRelays.add(fingerprint); + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + } + if (!this.nodeIndex.getBridgesByFlag().containsKey(flag)) { + this.filteredBridges.clear(); + } else { + Set<String> bridgesWithFlag = this.nodeIndex.getBridgesByFlag().get( + flag); + Set<String> removeBridges = new HashSet<String>(); + for (String fingerprint : this.filteredBridges.keySet()) { + if (!bridgesWithFlag.contains(fingerprint)) { + removeBridges.add(fingerprint); + } + } + for (String fingerprint : removeBridges) { + this.filteredBridges.remove(fingerprint); + } + } + } + + private void filterNodesByFirstSeenDays() { + if (this.firstSeenDays == null) { + return; + } + filterNodesByDays(this.filteredRelays, + this.nodeIndex.getRelaysByFirstSeenDays(), this.firstSeenDays); + filterNodesByDays(this.filteredBridges, + this.nodeIndex.getBridgesByFirstSeenDays(), this.firstSeenDays); + } + + private void filterNodesByLastSeenDays() { + if (this.lastSeenDays == null) { + return; + } + filterNodesByDays(this.filteredRelays, + this.nodeIndex.getRelaysByLastSeenDays(), this.lastSeenDays); + filterNodesByDays(this.filteredBridges, + this.nodeIndex.getBridgesByLastSeenDays(), this.lastSeenDays); + } + + private void filterNodesByDays( + Map<String, SummaryDocument> filteredNodes, + SortedMap<Integer, Set<String>> nodesByDays, int[] days) { + Set<String> removeNodes = new HashSet<String>(); + for (Set<String> nodes : nodesByDays.headMap(days[0]).values()) { + removeNodes.addAll(nodes); + } + if (days[1] < Integer.MAX_VALUE) { + for (Set<String> nodes : + nodesByDays.tailMap(days[1] + 1).values()) { + removeNodes.addAll(nodes); + } + } + for (String fingerprint : removeNodes) { + filteredNodes.remove(fingerprint); + } + } + + private void filterByContact() { + if (this.contact == null) { + return; + } + Set<String> removeRelays = new HashSet<String>(); + for (Map.Entry<String, Set<String>> e : + this.nodeIndex.getRelaysByContact().entrySet()) { + String contact = e.getKey(); + for (String contactPart : this.contact) { + if (contact == null || + !contact.contains(contactPart.toLowerCase())) { + removeRelays.addAll(e.getValue()); + break; + } + } + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + this.filteredBridges.clear(); + } + + private void filterByFamily() { + if (this.family == null) { + return; + } + Set<String> removeRelays = new HashSet<String>( + this.filteredRelays.keySet()); + removeRelays.remove(this.family); + if (this.nodeIndex.getRelaysByFamily().containsKey(this.family)) { + removeRelays.removeAll(this.nodeIndex.getRelaysByFamily(). + get(this.family)); + } + for (String fingerprint : removeRelays) { + this.filteredRelays.remove(fingerprint); + } + this.filteredBridges.clear(); + } + + private void order() { + if (this.order != null && this.order.length == 1) { + List<String> orderBy = new ArrayList<String>( + this.nodeIndex.getRelaysByConsensusWeight()); + if (this.order[0].startsWith("-")) { + Collections.reverse(orderBy); + } + for (String relay : orderBy) { + if (this.filteredRelays.containsKey(relay) && + !this.orderedRelays.contains(filteredRelays.get(relay))) { + this.orderedRelays.add(this.filteredRelays.remove(relay)); + } + } + for (String relay : this.filteredRelays.keySet()) { + if (!this.orderedRelays.contains(this.filteredRelays.get(relay))) { + this.orderedRelays.add(this.filteredRelays.remove(relay)); + } + } + Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>( + this.filteredBridges.values()); + this.orderedBridges.addAll(uniqueBridges); + } else { + Set<SummaryDocument> uniqueRelays = new HashSet<SummaryDocument>( + this.filteredRelays.values()); + this.orderedRelays.addAll(uniqueRelays); + Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>( + this.filteredBridges.values()); + this.orderedBridges.addAll(uniqueBridges); + } + } + + private void offset() { + if (this.offset == null) { + return; + } + int offsetValue = Integer.parseInt(this.offset); + while (offsetValue-- > 0 && + (!this.orderedRelays.isEmpty() || + !this.orderedBridges.isEmpty())) { + if (!this.orderedRelays.isEmpty()) { + this.orderedRelays.remove(0); + } else { + this.orderedBridges.remove(0); + } + } + } + + private void limit() { + if (this.limit == null) { + return; + } + int limitValue = Integer.parseInt(this.limit); + while (!this.orderedRelays.isEmpty() && + limitValue < this.orderedRelays.size()) { + this.orderedRelays.remove(this.orderedRelays.size() - 1); + } + limitValue -= this.orderedRelays.size(); + while (!this.orderedBridges.isEmpty() && + limitValue < this.orderedBridges.size()) { + this.orderedBridges.remove(this.orderedBridges.size() - 1); + } + } + + private List<SummaryDocument> orderedRelays = + new ArrayList<SummaryDocument>(); + public List<SummaryDocument> getOrderedRelays() { + return this.orderedRelays; + } + + private List<SummaryDocument> orderedBridges = + new ArrayList<SummaryDocument>(); + public List<SummaryDocument> getOrderedBridges() { + return this.orderedBridges; + } + + public String getRelaysPublishedString() { + return this.nodeIndex.getRelaysPublishedString(); + } + + public String getBridgesPublishedString() { + return this.nodeIndex.getBridgesPublishedString(); + } +} diff --git a/src/org/torproject/onionoo/server/ResourceServlet.java b/src/org/torproject/onionoo/server/ResourceServlet.java new file mode 100644 index 0000000..1e05d12 --- /dev/null +++ b/src/org/torproject/onionoo/server/ResourceServlet.java @@ -0,0 +1,451 @@ +/* Copyright 2011, 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.server; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +import javax.servlet.ServletConfig; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; + +public class ResourceServlet extends HttpServlet { + + private static final long serialVersionUID = 7236658979947465319L; + + private boolean maintenanceMode = false; + + /* Called by servlet container, not by test class. */ + public void init(ServletConfig config) throws ServletException { + super.init(config); + this.maintenanceMode = + config.getInitParameter("maintenance") != null && + config.getInitParameter("maintenance").equals("1"); + } + + public long getLastModified(HttpServletRequest request) { + if (this.maintenanceMode) { + return super.getLastModified(request); + } else { + return ApplicationFactory.getNodeIndexer().getLastIndexed( + DateTimeHelper.TEN_SECONDS); + } + } + + public static class HttpServletRequestWrapper { + private HttpServletRequest request; + protected HttpServletRequestWrapper(HttpServletRequest request) { + this.request = request; + } + protected String getRequestURI() { + return this.request.getRequestURI(); + } + @SuppressWarnings("rawtypes") + protected Map getParameterMap() { + return this.request.getParameterMap(); + } + protected String[] getParameterValues(String parameterKey) { + return this.request.getParameterValues(parameterKey); + } + } + + public static class HttpServletResponseWrapper { + private HttpServletResponse response = null; + protected HttpServletResponseWrapper(HttpServletResponse response) { + this.response = response; + } + protected void sendError(int errorStatusCode) throws IOException { + this.response.sendError(errorStatusCode); + } + protected void setHeader(String headerName, String headerValue) { + this.response.setHeader(headerName, headerValue); + } + protected void setContentType(String contentType) { + this.response.setContentType(contentType); + } + protected void setCharacterEncoding(String characterEncoding) { + this.response.setCharacterEncoding(characterEncoding); + } + protected PrintWriter getWriter() throws IOException { + return this.response.getWriter(); + } + } + + public void doGet(HttpServletRequest request, + HttpServletResponse response) throws IOException, ServletException { + HttpServletRequestWrapper requestWrapper = + new HttpServletRequestWrapper(request); + HttpServletResponseWrapper responseWrapper = + new HttpServletResponseWrapper(response); + this.doGet(requestWrapper, responseWrapper); + } + + public void doGet(HttpServletRequestWrapper request, + HttpServletResponseWrapper response) throws IOException { + + if (this.maintenanceMode) { + response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE); + return; + } + + if (ApplicationFactory.getNodeIndexer().getLastIndexed( + DateTimeHelper.TEN_SECONDS) + DateTimeHelper.SIX_HOURS + < ApplicationFactory.getTime().currentTimeMillis()) { + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return; + } + + NodeIndex nodeIndex = ApplicationFactory.getNodeIndexer(). + getLatestNodeIndex(DateTimeHelper.TEN_SECONDS); + if (nodeIndex == null) { + response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return; + } + + String uri = request.getRequestURI(); + if (uri.startsWith("/onionoo/")) { + uri = uri.substring("/onionoo".length()); + } + String resourceType = null; + if (uri.startsWith("/summary")) { + resourceType = "summary"; + } else if (uri.startsWith("/details")) { + resourceType = "details"; + } else if (uri.startsWith("/bandwidth")) { + resourceType = "bandwidth"; + } else if (uri.startsWith("/weights")) { + resourceType = "weights"; + } else if (uri.startsWith("/clients")) { + resourceType = "clients"; + } else if (uri.startsWith("/uptime")) { + resourceType = "uptime"; + } else { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + + RequestHandler rh = new RequestHandler(nodeIndex); + rh.setResourceType(resourceType); + + /* Extract parameters either from the old-style URI or from request + * parameters. */ + Map<String, String> parameterMap = new HashMap<String, String>(); + for (Object parameterKey : request.getParameterMap().keySet()) { + String[] parameterValues = + request.getParameterValues((String) parameterKey); + parameterMap.put((String) parameterKey, parameterValues[0]); + } + + /* Make sure that the request doesn't contain any unknown + * parameters. */ + Set<String> knownParameters = new HashSet<String>(Arrays.asList(( + "type,running,search,lookup,fingerprint,country,as,flag," + + "first_seen_days,last_seen_days,contact,order,limit,offset," + + "fields,family").split(","))); + for (String parameterKey : parameterMap.keySet()) { + if (!knownParameters.contains(parameterKey)) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + } + + /* Filter relays and bridges matching the request. */ + if (parameterMap.containsKey("type")) { + String typeParameterValue = parameterMap.get("type").toLowerCase(); + boolean relaysRequested = true; + if (typeParameterValue.equals("bridge")) { + relaysRequested = false; + } else if (!typeParameterValue.equals("relay")) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setType(relaysRequested ? "relay" : "bridge"); + } + if (parameterMap.containsKey("running")) { + String runningParameterValue = + parameterMap.get("running").toLowerCase(); + boolean runningRequested = true; + if (runningParameterValue.equals("false")) { + runningRequested = false; + } else if (!runningParameterValue.equals("true")) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setRunning(runningRequested ? "true" : "false"); + } + if (parameterMap.containsKey("search")) { + String[] searchTerms = this.parseSearchParameters( + parameterMap.get("search")); + if (searchTerms == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setSearch(searchTerms); + } + if (parameterMap.containsKey("lookup")) { + String lookupParameter = this.parseFingerprintParameter( + parameterMap.get("lookup")); + if (lookupParameter == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + String fingerprint = lookupParameter.toUpperCase(); + rh.setLookup(fingerprint); + } + if (parameterMap.containsKey("fingerprint")) { + String fingerprintParameter = this.parseFingerprintParameter( + parameterMap.get("fingerprint")); + if (fingerprintParameter == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + String fingerprint = fingerprintParameter.toUpperCase(); + rh.setFingerprint(fingerprint); + } + if (parameterMap.containsKey("country")) { + String countryCodeParameter = this.parseCountryCodeParameter( + parameterMap.get("country")); + if (countryCodeParameter == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setCountry(countryCodeParameter); + } + if (parameterMap.containsKey("as")) { + String aSNumberParameter = this.parseASNumberParameter( + parameterMap.get("as")); + if (aSNumberParameter == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setAs(aSNumberParameter); + } + if (parameterMap.containsKey("flag")) { + String flagParameter = this.parseFlagParameter( + parameterMap.get("flag")); + if (flagParameter == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setFlag(flagParameter); + } + if (parameterMap.containsKey("first_seen_days")) { + int[] days = this.parseDaysParameter( + parameterMap.get("first_seen_days")); + if (days == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setFirstSeenDays(days); + } + if (parameterMap.containsKey("last_seen_days")) { + int[] days = this.parseDaysParameter( + parameterMap.get("last_seen_days")); + if (days == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setLastSeenDays(days); + } + if (parameterMap.containsKey("contact")) { + String[] contactParts = this.parseContactParameter( + parameterMap.get("contact")); + if (contactParts == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setContact(contactParts); + } + if (parameterMap.containsKey("order")) { + String orderParameter = parameterMap.get("order").toLowerCase(); + String orderByField = orderParameter; + if (orderByField.startsWith("-")) { + orderByField = orderByField.substring(1); + } + if (!orderByField.equals("consensus_weight")) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setOrder(new String[] { orderParameter }); + } + if (parameterMap.containsKey("offset")) { + String offsetParameter = parameterMap.get("offset"); + if (offsetParameter.length() > 6) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + try { + Integer.parseInt(offsetParameter); + } catch (NumberFormatException e) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setOffset(offsetParameter); + } + if (parameterMap.containsKey("limit")) { + String limitParameter = parameterMap.get("limit"); + if (limitParameter.length() > 6) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + try { + Integer.parseInt(limitParameter); + } catch (NumberFormatException e) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rh.setLimit(limitParameter); + } + if (parameterMap.containsKey("family")) { + String familyParameter = this.parseFingerprintParameter( + parameterMap.get("family")); + if (familyParameter == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + String family = familyParameter.toUpperCase(); + rh.setFamily(family); + } + rh.handleRequest(); + + ResponseBuilder rb = new ResponseBuilder(); + rb.setResourceType(resourceType); + rb.setRelaysPublishedString(rh.getRelaysPublishedString()); + rb.setBridgesPublishedString(rh.getBridgesPublishedString()); + rb.setOrderedRelays(rh.getOrderedRelays()); + rb.setOrderedBridges(rh.getOrderedBridges()); + String[] fields = null; + if (parameterMap.containsKey("fields")) { + fields = this.parseFieldsParameter(parameterMap.get("fields")); + if (fields == null) { + response.sendError(HttpServletResponse.SC_BAD_REQUEST); + return; + } + rb.setFields(fields); + } + + response.setHeader("Access-Control-Allow-Origin", "*"); + response.setContentType("application/json"); + response.setCharacterEncoding("utf-8"); + PrintWriter pw = response.getWriter(); + rb.buildResponse(pw); + pw.flush(); + pw.close(); + } + + private static Pattern searchParameterPattern = + Pattern.compile("^\\$?[0-9a-fA-F]{1,40}$|" /* Fingerprint. */ + + "^[0-9a-zA-Z\\.]{1,19}$|" /* Nickname or IPv4 address. */ + + "^\\[[0-9a-fA-F:\\.]{1,39}\\]?$"); /* IPv6 address. */ + private String[] parseSearchParameters(String parameter) { + String[] searchParameters; + if (parameter.contains(" ")) { + searchParameters = parameter.split(" "); + } else { + searchParameters = new String[] { parameter }; + } + for (String searchParameter : searchParameters) { + if (!searchParameterPattern.matcher(searchParameter).matches()) { + return null; + } + } + return searchParameters; + } + + private static Pattern fingerprintParameterPattern = + Pattern.compile("^[0-9a-zA-Z]{1,40}$"); + private String parseFingerprintParameter(String parameter) { + if (!fingerprintParameterPattern.matcher(parameter).matches()) { + return null; + } + if (parameter.length() != 40) { + return null; + } + return parameter; + } + + private static Pattern countryCodeParameterPattern = + Pattern.compile("^[0-9a-zA-Z]{2}$"); + private String parseCountryCodeParameter(String parameter) { + if (!countryCodeParameterPattern.matcher(parameter).matches()) { + return null; + } + return parameter; + } + + private static Pattern aSNumberParameterPattern = + Pattern.compile("^[asAS]{0,2}[0-9]{1,10}$"); + private String parseASNumberParameter(String parameter) { + if (!aSNumberParameterPattern.matcher(parameter).matches()) { + return null; + } + return parameter; + } + + private static Pattern flagPattern = + Pattern.compile("^[a-zA-Z0-9]{1,20}$"); + private String parseFlagParameter(String parameter) { + if (!flagPattern.matcher(parameter).matches()) { + return null; + } + return parameter; + } + + private static Pattern daysPattern = Pattern.compile("^[0-9-]{1,10}$"); + private int[] parseDaysParameter(String parameter) { + if (!daysPattern.matcher(parameter).matches()) { + return null; + } + int x = 0, y = Integer.MAX_VALUE; + try { + if (!parameter.contains("-")) { + x = Integer.parseInt(parameter); + y = x; + } else { + String[] parts = parameter.split("-", 2); + if (parts[0].length() > 0) { + x = Integer.parseInt(parts[0]); + } + if (parts.length > 1 && parts[1].length() > 0) { + y = Integer.parseInt(parts[1]); + } + } + } catch (NumberFormatException e) { + return null; + } + if (x > y) { + return null; + } + return new int[] { x, y }; + } + + private String[] parseContactParameter(String parameter) { + for (char c : parameter.toCharArray()) { + if (c < 32 || c >= 127) { + return null; + } + } + return parameter.split(" "); + } + + private static Pattern fieldsParameterPattern = + Pattern.compile("^[0-9a-zA-Z_,]*$"); + private String[] parseFieldsParameter(String parameter) { + if (!fieldsParameterPattern.matcher(parameter).matches()) { + return null; + } + return parameter.split(","); + } +} + diff --git a/src/org/torproject/onionoo/server/ResponseBuilder.java b/src/org/torproject/onionoo/server/ResponseBuilder.java new file mode 100644 index 0000000..161692c --- /dev/null +++ b/src/org/torproject/onionoo/server/ResponseBuilder.java @@ -0,0 +1,320 @@ +/* Copyright 2011--2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.server; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +import org.torproject.onionoo.docs.BandwidthDocument; +import org.torproject.onionoo.docs.ClientsDocument; +import org.torproject.onionoo.docs.DetailsDocument; +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.docs.SummaryDocument; +import org.torproject.onionoo.docs.UptimeDocument; +import org.torproject.onionoo.docs.WeightsDocument; +import org.torproject.onionoo.util.ApplicationFactory; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +public class ResponseBuilder { + + private DocumentStore documentStore; + + public ResponseBuilder() { + this.documentStore = ApplicationFactory.getDocumentStore(); + } + + private String resourceType; + public void setResourceType(String resourceType) { + this.resourceType = resourceType; + } + + private String relaysPublishedString; + public void setRelaysPublishedString(String relaysPublishedString) { + this.relaysPublishedString = relaysPublishedString; + } + + private String bridgesPublishedString; + public void setBridgesPublishedString(String bridgesPublishedString) { + this.bridgesPublishedString = bridgesPublishedString; + } + + private List<SummaryDocument> orderedRelays = + new ArrayList<SummaryDocument>(); + public void setOrderedRelays(List<SummaryDocument> orderedRelays) { + this.orderedRelays = orderedRelays; + } + + private List<SummaryDocument> orderedBridges = + new ArrayList<SummaryDocument>(); + public void setOrderedBridges(List<SummaryDocument> orderedBridges) { + this.orderedBridges = orderedBridges; + } + + private String[] fields; + public void setFields(String[] fields) { + this.fields = new String[fields.length]; + System.arraycopy(fields, 0, this.fields, 0, fields.length); + } + + public void buildResponse(PrintWriter pw) { + writeRelays(orderedRelays, pw); + writeBridges(orderedBridges, pw); + } + + private void writeRelays(List<SummaryDocument> relays, PrintWriter pw) { + pw.write("{\"relays_published\":\"" + relaysPublishedString + + "\",\n\"relays\":["); + int written = 0; + for (SummaryDocument entry : relays) { + String lines = this.formatNodeStatus(entry); + if (lines.length() > 0) { + pw.print((written++ > 0 ? ",\n" : "\n") + lines); + } + } + pw.print("\n],\n"); + } + + private void writeBridges(List<SummaryDocument> bridges, + PrintWriter pw) { + pw.write("\"bridges_published\":\"" + bridgesPublishedString + + "\",\n\"bridges\":["); + int written = 0; + for (SummaryDocument entry : bridges) { + String lines = this.formatNodeStatus(entry); + if (lines.length() > 0) { + pw.print((written++ > 0 ? ",\n" : "\n") + lines); + } + } + pw.print("\n]}\n"); + } + + private String formatNodeStatus(SummaryDocument entry) { + if (this.resourceType == null) { + return ""; + } else if (this.resourceType.equals("summary")) { + return this.writeSummaryLine(entry); + } else if (this.resourceType.equals("details")) { + return this.writeDetailsLines(entry); + } else if (this.resourceType.equals("bandwidth")) { + return this.writeBandwidthLines(entry); + } else if (this.resourceType.equals("weights")) { + return this.writeWeightsLines(entry); + } else if (this.resourceType.equals("clients")) { + return this.writeClientsLines(entry); + } else if (this.resourceType.equals("uptime")) { + return this.writeUptimeLines(entry); + } else { + return ""; + } + } + + private String writeSummaryLine(SummaryDocument entry) { + return entry.isRelay() ? writeRelaySummaryLine(entry) + : writeBridgeSummaryLine(entry); + } + + private String writeRelaySummaryLine(SummaryDocument entry) { + String nickname = !entry.getNickname().equals("Unnamed") ? + entry.getNickname() : null; + String fingerprint = entry.getFingerprint(); + String running = entry.isRunning() ? "true" : "false"; + List<String> addresses = entry.getAddresses(); + StringBuilder addressesBuilder = new StringBuilder(); + int written = 0; + for (String address : addresses) { + addressesBuilder.append((written++ > 0 ? "," : "") + "\"" + + address.toLowerCase() + "\""); + } + return String.format("{%s\"f\":\"%s\",\"a\":[%s],\"r\":%s}", + (nickname == null ? "" : "\"n\":\"" + nickname + "\","), + fingerprint, addressesBuilder.toString(), running); + } + + private String writeBridgeSummaryLine(SummaryDocument entry) { + String nickname = !entry.getNickname().equals("Unnamed") ? + entry.getNickname() : null; + String hashedFingerprint = entry.getFingerprint(); + String running = entry.isRunning() ? "true" : "false"; + return String.format("{%s\"h\":\"%s\",\"r\":%s}", + (nickname == null ? "" : "\"n\":\"" + nickname + "\","), + hashedFingerprint, running); + } + + private String writeDetailsLines(SummaryDocument entry) { + String fingerprint = entry.getFingerprint(); + if (this.fields != null) { + /* TODO Maybe there's a more elegant way (more maintainable, more + * efficient, etc.) to implement this? */ + DetailsDocument detailsDocument = documentStore.retrieve( + DetailsDocument.class, true, fingerprint); + if (detailsDocument != null) { + DetailsDocument dd = new DetailsDocument(); + for (String field : this.fields) { + if (field.equals("nickname")) { + dd.setNickname(detailsDocument.getNickname()); + } else if (field.equals("fingerprint")) { + dd.setFingerprint(detailsDocument.getFingerprint()); + } else if (field.equals("hashed_fingerprint")) { + dd.setHashedFingerprint( + detailsDocument.getHashedFingerprint()); + } else if (field.equals("or_addresses")) { + dd.setOrAddresses(detailsDocument.getOrAddresses()); + } else if (field.equals("exit_addresses")) { + dd.setExitAddresses(detailsDocument.getExitAddresses()); + } else if (field.equals("dir_address")) { + dd.setDirAddress(detailsDocument.getDirAddress()); + } else if (field.equals("last_seen")) { + dd.setLastSeen(detailsDocument.getLastSeen()); + } else if (field.equals("last_changed_address_or_port")) { + dd.setLastChangedAddressOrPort( + detailsDocument.getLastChangedAddressOrPort()); + } else if (field.equals("first_seen")) { + dd.setFirstSeen(detailsDocument.getFirstSeen()); + } else if (field.equals("running")) { + dd.setRunning(detailsDocument.getRunning()); + } else if (field.equals("flags")) { + dd.setFlags(detailsDocument.getFlags()); + } else if (field.equals("country")) { + dd.setCountry(detailsDocument.getCountry()); + } else if (field.equals("country_name")) { + dd.setCountryName(detailsDocument.getCountryName()); + } else if (field.equals("region_name")) { + dd.setRegionName(detailsDocument.getRegionName()); + } else if (field.equals("city_name")) { + dd.setCityName(detailsDocument.getCityName()); + } else if (field.equals("latitude")) { + dd.setLatitude(detailsDocument.getLatitude()); + } else if (field.equals("longitude")) { + dd.setLongitude(detailsDocument.getLongitude()); + } else if (field.equals("as_number")) { + dd.setAsNumber(detailsDocument.getAsNumber()); + } else if (field.equals("as_name")) { + dd.setAsName(detailsDocument.getAsName()); + } else if (field.equals("consensus_weight")) { + dd.setConsensusWeight(detailsDocument.getConsensusWeight()); + } else if (field.equals("host_name")) { + dd.setHostName(detailsDocument.getHostName()); + } else if (field.equals("last_restarted")) { + dd.setLastRestarted(detailsDocument.getLastRestarted()); + } else if (field.equals("bandwidth_rate")) { + dd.setBandwidthRate(detailsDocument.getBandwidthRate()); + } else if (field.equals("bandwidth_burst")) { + dd.setBandwidthBurst(detailsDocument.getBandwidthBurst()); + } else if (field.equals("observed_bandwidth")) { + dd.setObservedBandwidth( + detailsDocument.getObservedBandwidth()); + } else if (field.equals("advertised_bandwidth")) { + dd.setAdvertisedBandwidth( + detailsDocument.getAdvertisedBandwidth()); + } else if (field.equals("exit_policy")) { + dd.setExitPolicy(detailsDocument.getExitPolicy()); + } else if (field.equals("exit_policy_summary")) { + dd.setExitPolicySummary( + detailsDocument.getExitPolicySummary()); + } else if (field.equals("exit_policy_v6_summary")) { + dd.setExitPolicyV6Summary( + detailsDocument.getExitPolicyV6Summary()); + } else if (field.equals("contact")) { + dd.setContact(detailsDocument.getContact()); + } else if (field.equals("platform")) { + dd.setPlatform(detailsDocument.getPlatform()); + } else if (field.equals("family")) { + dd.setFamily(detailsDocument.getFamily()); + } else if (field.equals("advertised_bandwidth_fraction")) { + dd.setAdvertisedBandwidthFraction( + detailsDocument.getAdvertisedBandwidthFraction()); + } else if (field.equals("consensus_weight_fraction")) { + dd.setConsensusWeightFraction( + detailsDocument.getConsensusWeightFraction()); + } else if (field.equals("guard_probability")) { + dd.setGuardProbability(detailsDocument.getGuardProbability()); + } else if (field.equals("middle_probability")) { + dd.setMiddleProbability( + detailsDocument.getMiddleProbability()); + } else if (field.equals("exit_probability")) { + dd.setExitProbability(detailsDocument.getExitProbability()); + } else if (field.equals("recommended_version")) { + dd.setRecommendedVersion( + detailsDocument.getRecommendedVersion()); + } else if (field.equals("hibernating")) { + dd.setHibernating(detailsDocument.getHibernating()); + } else if (field.equals("pool_assignment")) { + dd.setPoolAssignment(detailsDocument.getPoolAssignment()); + } + } + /* Don't escape HTML characters, like < and >, contained in + * strings. */ + Gson gson = new GsonBuilder().disableHtmlEscaping().create(); + /* Whenever we provide Gson with a string containing an escaped + * non-ASCII character like \u00F2, it escapes the \ to \\, which + * we need to undo before including the string in a response. */ + return gson.toJson(dd).replaceAll("\\\\\\\\u", "\\\\u"); + } else { + // TODO We should probably log that we didn't find a details + // document that we expected to exist. + return ""; + } + } else { + DetailsDocument detailsDocument = documentStore.retrieve( + DetailsDocument.class, false, fingerprint); + if (detailsDocument != null) { + return detailsDocument.getDocumentString(); + } else { + // TODO We should probably log that we didn't find a details + // document that we expected to exist. + return ""; + } + } + } + + private String writeBandwidthLines(SummaryDocument entry) { + String fingerprint = entry.getFingerprint(); + BandwidthDocument bandwidthDocument = this.documentStore.retrieve( + BandwidthDocument.class, false, fingerprint); + if (bandwidthDocument != null && + bandwidthDocument.getDocumentString() != null) { + return bandwidthDocument.getDocumentString(); + } else { + return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}"; + } + } + + private String writeWeightsLines(SummaryDocument entry) { + String fingerprint = entry.getFingerprint(); + WeightsDocument weightsDocument = this.documentStore.retrieve( + WeightsDocument.class, false, fingerprint); + if (weightsDocument != null && + weightsDocument.getDocumentString() != null) { + return weightsDocument.getDocumentString(); + } else { + return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}"; + } + } + + private String writeClientsLines(SummaryDocument entry) { + String fingerprint = entry.getFingerprint(); + ClientsDocument clientsDocument = this.documentStore.retrieve( + ClientsDocument.class, false, fingerprint); + if (clientsDocument != null && + clientsDocument.getDocumentString() != null) { + return clientsDocument.getDocumentString(); + } else { + return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}"; + } + } + + private String writeUptimeLines(SummaryDocument entry) { + String fingerprint = entry.getFingerprint(); + UptimeDocument uptimeDocument = this.documentStore.retrieve( + UptimeDocument.class, false, fingerprint); + if (uptimeDocument != null && + uptimeDocument.getDocumentString() != null) { + return uptimeDocument.getDocumentString(); + } else { + return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}"; + } + } +} diff --git a/src/org/torproject/onionoo/updater/BandwidthStatusUpdater.java b/src/org/torproject/onionoo/updater/BandwidthStatusUpdater.java new file mode 100644 index 0000000..bc7dd74 --- /dev/null +++ b/src/org/torproject/onionoo/updater/BandwidthStatusUpdater.java @@ -0,0 +1,149 @@ +/* Copyright 2011--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +import java.util.SortedMap; +import java.util.TreeMap; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.ExtraInfoDescriptor; +import org.torproject.onionoo.docs.BandwidthStatus; +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; + +public class BandwidthStatusUpdater implements DescriptorListener, + StatusUpdater { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public BandwidthStatusUpdater() { + this.descriptorSource = ApplicationFactory.getDescriptorSource(); + this.documentStore = ApplicationFactory.getDocumentStore(); + this.now = ApplicationFactory.getTime().currentTimeMillis(); + this.registerDescriptorListeners(); + } + + private void registerDescriptorListeners() { + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_EXTRA_INFOS); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_EXTRA_INFOS); + } + + public void processDescriptor(Descriptor descriptor, boolean relay) { + if (descriptor instanceof ExtraInfoDescriptor) { + this.parseDescriptor((ExtraInfoDescriptor) descriptor); + } + } + + public void updateStatuses() { + /* Status files are already updated while processing descriptors. */ + } + + private void parseDescriptor(ExtraInfoDescriptor descriptor) { + String fingerprint = descriptor.getFingerprint(); + BandwidthStatus bandwidthStatus = this.documentStore.retrieve( + BandwidthStatus.class, true, fingerprint); + if (bandwidthStatus == null) { + bandwidthStatus = new BandwidthStatus(); + } + if (descriptor.getWriteHistory() != null) { + parseHistoryLine(descriptor.getWriteHistory().getLine(), + bandwidthStatus.getWriteHistory()); + } + if (descriptor.getReadHistory() != null) { + parseHistoryLine(descriptor.getReadHistory().getLine(), + bandwidthStatus.getReadHistory()); + } + this.compressHistory(bandwidthStatus.getWriteHistory()); + this.compressHistory(bandwidthStatus.getReadHistory()); + this.documentStore.store(bandwidthStatus, fingerprint); + } + + private void parseHistoryLine(String line, + SortedMap<Long, long[]> history) { + String[] parts = line.split(" "); + if (parts.length < 6) { + return; + } + long endMillis = DateTimeHelper.parse(parts[1] + " " + parts[2]); + if (endMillis < 0L) { + System.err.println("Could not parse timestamp in line '" + line + + "'. Skipping."); + return; + } + long intervalMillis = Long.parseLong(parts[3].substring(1)) + * DateTimeHelper.ONE_SECOND; + String[] values = parts[5].split(","); + for (int i = values.length - 1; i >= 0; i--) { + long bandwidthValue = Long.parseLong(values[i]); + long startMillis = endMillis - intervalMillis; + /* TODO Should we first check whether an interval is already + * contained in history? */ + history.put(startMillis, new long[] { startMillis, endMillis, + bandwidthValue }); + endMillis -= intervalMillis; + } + } + + private void compressHistory(SortedMap<Long, long[]> history) { + SortedMap<Long, long[]> uncompressedHistory = + new TreeMap<Long, long[]>(history); + history.clear(); + long lastStartMillis = 0L, lastEndMillis = 0L, lastBandwidth = 0L; + String lastMonthString = "1970-01"; + for (long[] v : uncompressedHistory.values()) { + long startMillis = v[0], endMillis = v[1], bandwidth = v[2]; + long intervalLengthMillis; + if (this.now - endMillis <= DateTimeHelper.THREE_DAYS) { + intervalLengthMillis = DateTimeHelper.FIFTEEN_MINUTES; + } else if (this.now - endMillis <= DateTimeHelper.ONE_WEEK) { + intervalLengthMillis = DateTimeHelper.ONE_HOUR; + } else if (this.now - endMillis <= + DateTimeHelper.ROUGHLY_ONE_MONTH) { + intervalLengthMillis = DateTimeHelper.FOUR_HOURS; + } else if (this.now - endMillis <= + DateTimeHelper.ROUGHLY_THREE_MONTHS) { + intervalLengthMillis = DateTimeHelper.TWELVE_HOURS; + } else if (this.now - endMillis <= + DateTimeHelper.ROUGHLY_ONE_YEAR) { + intervalLengthMillis = DateTimeHelper.TWO_DAYS; + } else { + intervalLengthMillis = DateTimeHelper.TEN_DAYS; + } + String monthString = DateTimeHelper.format(startMillis, + DateTimeHelper.ISO_YEARMONTH_FORMAT); + if (lastEndMillis == startMillis && + ((lastEndMillis - 1L) / intervalLengthMillis) == + ((endMillis - 1L) / intervalLengthMillis) && + lastMonthString.equals(monthString)) { + lastEndMillis = endMillis; + lastBandwidth += bandwidth; + } else { + if (lastStartMillis > 0L) { + history.put(lastStartMillis, new long[] { lastStartMillis, + lastEndMillis, lastBandwidth }); + } + lastStartMillis = startMillis; + lastEndMillis = endMillis; + lastBandwidth = bandwidth; + } + lastMonthString = monthString; + } + if (lastStartMillis > 0L) { + history.put(lastStartMillis, new long[] { lastStartMillis, + lastEndMillis, lastBandwidth }); + } + } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } +} + diff --git a/src/org/torproject/onionoo/updater/ClientsStatusUpdater.java b/src/org/torproject/onionoo/updater/ClientsStatusUpdater.java new file mode 100644 index 0000000..79c1060 --- /dev/null +++ b/src/org/torproject/onionoo/updater/ClientsStatusUpdater.java @@ -0,0 +1,230 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.ExtraInfoDescriptor; +import org.torproject.onionoo.docs.ClientsHistory; +import org.torproject.onionoo.docs.ClientsStatus; +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; +import org.torproject.onionoo.util.Logger; + +/* + * Example extra-info descriptor used as input: + * + * extra-info ndnop2 DE6397A047ABE5F78B4C87AF725047831B221AAB + * dirreq-stats-end 2014-02-16 16:42:11 (86400 s) + * dirreq-v3-resp ok=856,not-enough-sigs=0,unavailable=0,not-found=0, + * not-modified=40,busy=0 + * bridge-stats-end 2014-02-16 16:42:17 (86400 s) + * bridge-ips ??=8,in=8,se=8 + * bridge-ip-versions v4=8,v6=0 + * + * Clients status file produced as intermediate output: + * + * 2014-02-15 16:42:11 2014-02-16 00:00:00 + * 259.042 in=86.347,se=86.347 v4=259.042 + * 2014-02-16 00:00:00 2014-02-16 16:42:11 + * 592.958 in=197.653,se=197.653 v4=592.958 + */ +public class ClientsStatusUpdater implements DescriptorListener, + StatusUpdater { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public ClientsStatusUpdater() { + this.descriptorSource = ApplicationFactory.getDescriptorSource(); + this.documentStore = ApplicationFactory.getDocumentStore(); + this.now = ApplicationFactory.getTime().currentTimeMillis(); + this.registerDescriptorListeners(); + } + + private void registerDescriptorListeners() { + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_EXTRA_INFOS); + } + + public void processDescriptor(Descriptor descriptor, boolean relay) { + if (descriptor instanceof ExtraInfoDescriptor && !relay) { + this.processBridgeExtraInfoDescriptor( + (ExtraInfoDescriptor) descriptor); + } + } + + private SortedMap<String, SortedSet<ClientsHistory>> newResponses = + new TreeMap<String, SortedSet<ClientsHistory>>(); + + private void processBridgeExtraInfoDescriptor( + ExtraInfoDescriptor descriptor) { + long dirreqStatsEndMillis = descriptor.getDirreqStatsEndMillis(); + long dirreqStatsIntervalLengthMillis = + descriptor.getDirreqStatsIntervalLength() + * DateTimeHelper.ONE_SECOND; + SortedMap<String, Integer> responses = descriptor.getDirreqV3Resp(); + if (dirreqStatsEndMillis < 0L || + dirreqStatsIntervalLengthMillis != DateTimeHelper.ONE_DAY || + responses == null || !responses.containsKey("ok")) { + return; + } + double okResponses = (double) (responses.get("ok") - 4); + if (okResponses < 0.0) { + return; + } + String hashedFingerprint = descriptor.getFingerprint().toUpperCase(); + long dirreqStatsStartMillis = dirreqStatsEndMillis + - dirreqStatsIntervalLengthMillis; + long utcBreakMillis = (dirreqStatsEndMillis / DateTimeHelper.ONE_DAY) + * DateTimeHelper.ONE_DAY; + for (int i = 0; i < 2; i++) { + long startMillis = i == 0 ? dirreqStatsStartMillis : utcBreakMillis; + long endMillis = i == 0 ? utcBreakMillis : dirreqStatsEndMillis; + if (startMillis >= endMillis) { + continue; + } + double totalResponses = okResponses + * ((double) (endMillis - startMillis)) + / ((double) DateTimeHelper.ONE_DAY); + SortedMap<String, Double> responsesByCountry = + this.weightResponsesWithUniqueIps(totalResponses, + descriptor.getBridgeIps(), "??"); + SortedMap<String, Double> responsesByTransport = + this.weightResponsesWithUniqueIps(totalResponses, + descriptor.getBridgeIpTransports(), "<??>"); + SortedMap<String, Double> responsesByVersion = + this.weightResponsesWithUniqueIps(totalResponses, + descriptor.getBridgeIpVersions(), ""); + ClientsHistory newResponseHistory = new ClientsHistory( + startMillis, endMillis, totalResponses, responsesByCountry, + responsesByTransport, responsesByVersion); + if (!this.newResponses.containsKey(hashedFingerprint)) { + this.newResponses.put(hashedFingerprint, + new TreeSet<ClientsHistory>()); + } + this.newResponses.get(hashedFingerprint).add( + newResponseHistory); + } + } + + private SortedMap<String, Double> weightResponsesWithUniqueIps( + double totalResponses, SortedMap<String, Integer> uniqueIps, + String omitString) { + SortedMap<String, Double> weightedResponses = + new TreeMap<String, Double>(); + int totalUniqueIps = 0; + if (uniqueIps != null) { + for (Map.Entry<String, Integer> e : uniqueIps.entrySet()) { + if (e.getValue() > 4) { + totalUniqueIps += e.getValue() - 4; + } + } + } + if (totalUniqueIps > 0) { + for (Map.Entry<String, Integer> e : uniqueIps.entrySet()) { + if (!e.getKey().equals(omitString) && e.getValue() > 4) { + weightedResponses.put(e.getKey(), + (((double) (e.getValue() - 4)) * totalResponses) + / ((double) totalUniqueIps)); + } + } + } + return weightedResponses; + } + + public void updateStatuses() { + for (Map.Entry<String, SortedSet<ClientsHistory>> e : + this.newResponses.entrySet()) { + String hashedFingerprint = e.getKey(); + ClientsStatus clientsStatus = this.documentStore.retrieve( + ClientsStatus.class, true, hashedFingerprint); + if (clientsStatus == null) { + clientsStatus = new ClientsStatus(); + } + this.addToHistory(clientsStatus, e.getValue()); + this.compressHistory(clientsStatus); + this.documentStore.store(clientsStatus, hashedFingerprint); + } + } + + private void addToHistory(ClientsStatus clientsStatus, + SortedSet<ClientsHistory> newIntervals) { + SortedSet<ClientsHistory> history = clientsStatus.getHistory(); + for (ClientsHistory interval : newIntervals) { + if ((history.headSet(interval).isEmpty() || + history.headSet(interval).last().getEndMillis() <= + interval.getStartMillis()) && + (history.tailSet(interval).isEmpty() || + history.tailSet(interval).first().getStartMillis() >= + interval.getEndMillis())) { + history.add(interval); + } + } + } + + private void compressHistory(ClientsStatus clientsStatus) { + SortedSet<ClientsHistory> history = clientsStatus.getHistory(); + SortedSet<ClientsHistory> compressedHistory = + new TreeSet<ClientsHistory>(); + ClientsHistory lastResponses = null; + String lastMonthString = "1970-01"; + for (ClientsHistory responses : history) { + long intervalLengthMillis; + if (this.now - responses.getEndMillis() <= + DateTimeHelper.ROUGHLY_THREE_MONTHS) { + intervalLengthMillis = DateTimeHelper.ONE_DAY; + } else if (this.now - responses.getEndMillis() <= + DateTimeHelper.ROUGHLY_ONE_YEAR) { + intervalLengthMillis = DateTimeHelper.TWO_DAYS; + } else { + intervalLengthMillis = DateTimeHelper.TEN_DAYS; + } + String monthString = DateTimeHelper.format( + responses.getStartMillis(), + DateTimeHelper.ISO_YEARMONTH_FORMAT); + if (lastResponses != null && + lastResponses.getEndMillis() == responses.getStartMillis() && + ((lastResponses.getEndMillis() - 1L) / intervalLengthMillis) == + ((responses.getEndMillis() - 1L) / intervalLengthMillis) && + lastMonthString.equals(monthString)) { + lastResponses.addResponses(responses); + } else { + if (lastResponses != null) { + compressedHistory.add(lastResponses); + } + lastResponses = responses; + } + lastMonthString = monthString; + } + if (lastResponses != null) { + compressedHistory.add(lastResponses); + } + clientsStatus.setHistory(compressedHistory); + } + + public String getStatsString() { + int newIntervals = 0; + for (SortedSet<ClientsHistory> hist : this.newResponses.values()) { + newIntervals += hist.size(); + } + StringBuilder sb = new StringBuilder(); + sb.append(" " + + Logger.formatDecimalNumber(newIntervals / 2) + + " client statistics processed from extra-info descriptors\n"); + sb.append(" " + + Logger.formatDecimalNumber(this.newResponses.size()) + + " client status files updated\n"); + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/updater/DescriptorListener.java b/src/org/torproject/onionoo/updater/DescriptorListener.java new file mode 100644 index 0000000..3613879 --- /dev/null +++ b/src/org/torproject/onionoo/updater/DescriptorListener.java @@ -0,0 +1,7 @@ +package org.torproject.onionoo.updater; + +import org.torproject.descriptor.Descriptor; + +public interface DescriptorListener { + abstract void processDescriptor(Descriptor descriptor, boolean relay); +} \ No newline at end of file diff --git a/src/org/torproject/onionoo/updater/DescriptorSource.java b/src/org/torproject/onionoo/updater/DescriptorSource.java new file mode 100644 index 0000000..32fbd2a --- /dev/null +++ b/src/org/torproject/onionoo/updater/DescriptorSource.java @@ -0,0 +1,646 @@ +/* Copyright 2013, 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.zip.GZIPInputStream; + +import org.torproject.descriptor.BridgeNetworkStatus; +import org.torproject.descriptor.BridgePoolAssignment; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorFile; +import org.torproject.descriptor.DescriptorReader; +import org.torproject.descriptor.DescriptorSourceFactory; +import org.torproject.descriptor.ExitList; +import org.torproject.descriptor.ExitListEntry; +import org.torproject.descriptor.ExtraInfoDescriptor; +import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.ServerDescriptor; +import org.torproject.onionoo.util.Logger; + +enum DescriptorHistory { + RELAY_CONSENSUS_HISTORY, + RELAY_SERVER_HISTORY, + RELAY_EXTRAINFO_HISTORY, + EXIT_LIST_HISTORY, + BRIDGE_STATUS_HISTORY, + BRIDGE_SERVER_HISTORY, + BRIDGE_EXTRAINFO_HISTORY, + BRIDGE_POOLASSIGN_HISTORY, +} + +class DescriptorDownloader { + + private final String protocolHostNameResourcePrefix = + "https://collector.torproject.org/recent/"; + + private String directory; + + private final File inDir = new File("in/recent"); + + public DescriptorDownloader(DescriptorType descriptorType) { + switch (descriptorType) { + case RELAY_CONSENSUSES: + this.directory = "relay-descriptors/consensuses/"; + break; + case RELAY_SERVER_DESCRIPTORS: + this.directory = "relay-descriptors/server-descriptors/"; + break; + case RELAY_EXTRA_INFOS: + this.directory = "relay-descriptors/extra-infos/"; + break; + case EXIT_LISTS: + this.directory = "exit-lists/"; + break; + case BRIDGE_STATUSES: + this.directory = "bridge-descriptors/statuses/"; + break; + case BRIDGE_SERVER_DESCRIPTORS: + this.directory = "bridge-descriptors/server-descriptors/"; + break; + case BRIDGE_EXTRA_INFOS: + this.directory = "bridge-descriptors/extra-infos/"; + break; + case BRIDGE_POOL_ASSIGNMENTS: + this.directory = "bridge-pool-assignments/"; + break; + default: + System.err.println("Unknown descriptor type."); + return; + } + } + + private SortedSet<String> localFiles = new TreeSet<String>(); + + public int statLocalFiles() { + File localDirectory = new File(this.inDir, this.directory); + if (localDirectory.exists()) { + for (File file : localDirectory.listFiles()) { + this.localFiles.add(file.getName()); + } + } + return this.localFiles.size(); + } + + private SortedSet<String> remoteFiles = new TreeSet<String>(); + + public int fetchRemoteDirectory() { + String directoryUrl = this.protocolHostNameResourcePrefix + + this.directory; + try { + URL u = new URL(directoryUrl); + HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + huc.setRequestMethod("GET"); + huc.connect(); + if (huc.getResponseCode() != 200) { + System.err.println("Could not fetch " + directoryUrl + + ": " + huc.getResponseCode() + " " + + huc.getResponseMessage() + ". Skipping."); + return 0; + } + BufferedReader br = new BufferedReader(new InputStreamReader( + huc.getInputStream())); + String line; + while ((line = br.readLine()) != null) { + if (!line.trim().startsWith("<tr>") || + !line.contains("<a href="")) { + continue; + } + String linePart = line.substring( + line.indexOf("<a href="") + "<a href="".length()); + if (!linePart.contains(""")) { + continue; + } + linePart = linePart.substring(0, linePart.indexOf(""")); + if (linePart.endsWith("/")) { + continue; + } + this.remoteFiles.add(linePart); + } + br.close(); + } catch (IOException e) { + System.err.println("Could not fetch or parse " + directoryUrl + + ". Skipping."); + } + return this.remoteFiles.size(); + } + + public int fetchRemoteFiles() { + int fetchedFiles = 0; + for (String remoteFile : this.remoteFiles) { + if (this.localFiles.contains(remoteFile)) { + continue; + } + String fileUrl = this.protocolHostNameResourcePrefix + + this.directory + remoteFile; + File localTempFile = new File(this.inDir, this.directory + + remoteFile + ".tmp"); + File localFile = new File(this.inDir, this.directory + remoteFile); + try { + localFile.getParentFile().mkdirs(); + URL u = new URL(fileUrl); + HttpURLConnection huc = (HttpURLConnection) u.openConnection(); + huc.setRequestMethod("GET"); + huc.addRequestProperty("Accept-Encoding", "gzip"); + huc.connect(); + if (huc.getResponseCode() != 200) { + System.err.println("Could not fetch " + fileUrl + + ": " + huc.getResponseCode() + " " + + huc.getResponseMessage() + ". Skipping."); + continue; + } + long lastModified = huc.getHeaderFieldDate("Last-Modified", -1L); + InputStream is; + if (huc.getContentEncoding() != null && + huc.getContentEncoding().equalsIgnoreCase("gzip")) { + is = new GZIPInputStream(huc.getInputStream()); + } else { + is = huc.getInputStream(); + } + BufferedInputStream bis = new BufferedInputStream(is); + BufferedOutputStream bos = new BufferedOutputStream( + new FileOutputStream(localTempFile)); + int len; + byte[] data = new byte[1024]; + while ((len = bis.read(data, 0, 1024)) >= 0) { + bos.write(data, 0, len); + } + bis.close(); + bos.close(); + localTempFile.renameTo(localFile); + if (lastModified >= 0) { + localFile.setLastModified(lastModified); + } + fetchedFiles++; + } catch (IOException e) { + System.err.println("Could not fetch or store " + fileUrl + + ". Skipping."); + } + } + return fetchedFiles; + } + + public int deleteOldLocalFiles() { + int deletedFiles = 0; + for (String localFile : this.localFiles) { + if (!this.remoteFiles.contains(localFile)) { + new File(this.inDir, this.directory + localFile).delete(); + deletedFiles++; + } + } + return deletedFiles; + } +} + +class DescriptorQueue { + + private File inDir; + + private File statusDir; + + private DescriptorReader descriptorReader; + + private File historyFile; + + private Iterator<DescriptorFile> descriptorFiles; + + private List<Descriptor> descriptors; + + private int historySizeBefore; + public int getHistorySizeBefore() { + return this.historySizeBefore; + } + + private int historySizeAfter; + public int getHistorySizeAfter() { + return this.historySizeAfter; + } + + private long returnedDescriptors = 0L; + public long getReturnedDescriptors() { + return this.returnedDescriptors; + } + + private long returnedBytes = 0L; + public long getReturnedBytes() { + return this.returnedBytes; + } + + public DescriptorQueue(File inDir, File statusDir) { + this.inDir = inDir; + this.statusDir = statusDir; + this.descriptorReader = + DescriptorSourceFactory.createDescriptorReader(); + } + + public void addDirectory(DescriptorType descriptorType) { + String directoryName = null; + switch (descriptorType) { + case RELAY_CONSENSUSES: + directoryName = "relay-descriptors/consensuses"; + break; + case RELAY_SERVER_DESCRIPTORS: + directoryName = "relay-descriptors/server-descriptors"; + break; + case RELAY_EXTRA_INFOS: + directoryName = "relay-descriptors/extra-infos"; + break; + case BRIDGE_STATUSES: + directoryName = "bridge-descriptors/statuses"; + break; + case BRIDGE_SERVER_DESCRIPTORS: + directoryName = "bridge-descriptors/server-descriptors"; + break; + case BRIDGE_EXTRA_INFOS: + directoryName = "bridge-descriptors/extra-infos"; + break; + case BRIDGE_POOL_ASSIGNMENTS: + directoryName = "bridge-pool-assignments"; + break; + case EXIT_LISTS: + directoryName = "exit-lists"; + break; + default: + System.err.println("Unknown descriptor type. Not adding directory " + + "to descriptor reader."); + return; + } + File directory = new File(this.inDir, directoryName); + if (directory.exists() && directory.isDirectory()) { + this.descriptorReader.addDirectory(directory); + this.descriptorReader.setMaxDescriptorFilesInQueue(1); + } else { + System.err.println("Directory " + directory.getAbsolutePath() + + " either does not exist or is not a directory. Not adding " + + "to descriptor reader."); + } + } + + public void readHistoryFile(DescriptorHistory descriptorHistory) { + String historyFileName = null; + switch (descriptorHistory) { + case RELAY_EXTRAINFO_HISTORY: + historyFileName = "relay-extrainfo-history"; + break; + case BRIDGE_EXTRAINFO_HISTORY: + historyFileName = "bridge-extrainfo-history"; + break; + case EXIT_LIST_HISTORY: + historyFileName = "exit-list-history"; + break; + case BRIDGE_POOLASSIGN_HISTORY: + historyFileName = "bridge-poolassign-history"; + break; + case RELAY_CONSENSUS_HISTORY: + historyFileName = "relay-consensus-history"; + break; + case BRIDGE_STATUS_HISTORY: + historyFileName = "bridge-status-history"; + break; + case RELAY_SERVER_HISTORY: + historyFileName = "relay-server-history"; + break; + case BRIDGE_SERVER_HISTORY: + historyFileName = "bridge-server-history"; + break; + default: + System.err.println("Unknown descriptor history. Not excluding " + + "files."); + return; + } + this.historyFile = new File(this.statusDir, historyFileName); + if (this.historyFile.exists() && this.historyFile.isFile()) { + SortedMap<String, Long> excludedFiles = new TreeMap<String, Long>(); + try { + BufferedReader br = new BufferedReader(new FileReader( + this.historyFile)); + String line; + while ((line = br.readLine()) != null) { + try { + String[] parts = line.split(" ", 2); + excludedFiles.put(parts[1], Long.parseLong(parts[0])); + } catch (NumberFormatException e) { + System.err.println("Illegal line '" + line + "' in parse " + + "history. Skipping line."); + } + } + br.close(); + } catch (IOException e) { + System.err.println("Could not read history file '" + + this.historyFile.getAbsolutePath() + "'. Not excluding " + + "descriptors in this execution."); + e.printStackTrace(); + return; + } + this.historySizeBefore = excludedFiles.size(); + this.descriptorReader.setExcludedFiles(excludedFiles); + } + } + + public void writeHistoryFile() { + if (this.historyFile == null) { + return; + } + SortedMap<String, Long> excludedAndParsedFiles = + new TreeMap<String, Long>(); + excludedAndParsedFiles.putAll( + this.descriptorReader.getExcludedFiles()); + excludedAndParsedFiles.putAll(this.descriptorReader.getParsedFiles()); + this.historySizeAfter = excludedAndParsedFiles.size(); + try { + this.historyFile.getParentFile().mkdirs(); + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.historyFile)); + for (Map.Entry<String, Long> e : excludedAndParsedFiles.entrySet()) { + String absolutePath = e.getKey(); + long lastModifiedMillis = e.getValue(); + bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath + + "\n"); + } + bw.close(); + } catch (IOException e) { + System.err.println("Could not write history file '" + + this.historyFile.getAbsolutePath() + "'. Not excluding " + + "descriptors in next execution."); + return; + } + } + + public Descriptor nextDescriptor() { + Descriptor nextDescriptor = null; + if (this.descriptorFiles == null) { + this.descriptorFiles = this.descriptorReader.readDescriptors(); + } + while (this.descriptors == null && this.descriptorFiles.hasNext()) { + DescriptorFile descriptorFile = this.descriptorFiles.next(); + if (descriptorFile.getException() != null) { + System.err.println("Could not parse " + + descriptorFile.getFileName()); + descriptorFile.getException().printStackTrace(); + } + if (descriptorFile.getDescriptors() != null && + !descriptorFile.getDescriptors().isEmpty()) { + this.descriptors = descriptorFile.getDescriptors(); + } + } + if (this.descriptors != null) { + nextDescriptor = this.descriptors.remove(0); + this.returnedDescriptors++; + this.returnedBytes += nextDescriptor.getRawDescriptorBytes().length; + if (this.descriptors.isEmpty()) { + this.descriptors = null; + } + } + return nextDescriptor; + } +} + +public class DescriptorSource { + + private final File inDir = new File("in/recent"); + + private final File statusDir = new File("status"); + + private List<DescriptorQueue> descriptorQueues; + + public DescriptorSource() { + this.descriptorQueues = new ArrayList<DescriptorQueue>(); + this.descriptorListeners = + new HashMap<DescriptorType, Set<DescriptorListener>>(); + this.fingerprintListeners = + new HashMap<DescriptorType, Set<FingerprintListener>>(); + } + + private DescriptorQueue getDescriptorQueue( + DescriptorType descriptorType, + DescriptorHistory descriptorHistory) { + DescriptorQueue descriptorQueue = new DescriptorQueue(this.inDir, + this.statusDir); + descriptorQueue.addDirectory(descriptorType); + if (descriptorHistory != null) { + descriptorQueue.readHistoryFile(descriptorHistory); + } + this.descriptorQueues.add(descriptorQueue); + return descriptorQueue; + } + + private Map<DescriptorType, Set<DescriptorListener>> + descriptorListeners; + + private Map<DescriptorType, Set<FingerprintListener>> + fingerprintListeners; + + public void registerDescriptorListener(DescriptorListener listener, + DescriptorType descriptorType) { + if (!this.descriptorListeners.containsKey(descriptorType)) { + this.descriptorListeners.put(descriptorType, + new HashSet<DescriptorListener>()); + } + this.descriptorListeners.get(descriptorType).add(listener); + } + + public void registerFingerprintListener(FingerprintListener listener, + DescriptorType descriptorType) { + if (!this.fingerprintListeners.containsKey(descriptorType)) { + this.fingerprintListeners.put(descriptorType, + new HashSet<FingerprintListener>()); + } + this.fingerprintListeners.get(descriptorType).add(listener); + } + + public void downloadDescriptors() { + for (DescriptorType descriptorType : DescriptorType.values()) { + this.downloadDescriptors(descriptorType); + } + } + + private int localFilesBefore = 0, foundRemoteFiles = 0, + downloadedFiles = 0, deletedLocalFiles = 0; + + private void downloadDescriptors(DescriptorType descriptorType) { + if (!this.descriptorListeners.containsKey(descriptorType) && + !this.fingerprintListeners.containsKey(descriptorType)) { + return; + } + DescriptorDownloader descriptorDownloader = + new DescriptorDownloader(descriptorType); + this.localFilesBefore += descriptorDownloader.statLocalFiles(); + this.foundRemoteFiles += + descriptorDownloader.fetchRemoteDirectory(); + this.downloadedFiles += descriptorDownloader.fetchRemoteFiles(); + this.deletedLocalFiles += descriptorDownloader.deleteOldLocalFiles(); + } + + public void readDescriptors() { + /* Careful when changing the order of parsing descriptor types! The + * various status updaters may base assumptions on this order. */ + this.readDescriptors(DescriptorType.RELAY_SERVER_DESCRIPTORS, + DescriptorHistory.RELAY_SERVER_HISTORY, true); + this.readDescriptors(DescriptorType.RELAY_EXTRA_INFOS, + DescriptorHistory.RELAY_EXTRAINFO_HISTORY, true); + this.readDescriptors(DescriptorType.EXIT_LISTS, + DescriptorHistory.EXIT_LIST_HISTORY, true); + this.readDescriptors(DescriptorType.RELAY_CONSENSUSES, + DescriptorHistory.RELAY_CONSENSUS_HISTORY, true); + this.readDescriptors(DescriptorType.BRIDGE_SERVER_DESCRIPTORS, + DescriptorHistory.BRIDGE_SERVER_HISTORY, false); + this.readDescriptors(DescriptorType.BRIDGE_EXTRA_INFOS, + DescriptorHistory.BRIDGE_EXTRAINFO_HISTORY, false); + this.readDescriptors(DescriptorType.BRIDGE_POOL_ASSIGNMENTS, + DescriptorHistory.BRIDGE_POOLASSIGN_HISTORY, false); + this.readDescriptors(DescriptorType.BRIDGE_STATUSES, + DescriptorHistory.BRIDGE_STATUS_HISTORY, false); + } + + private void readDescriptors(DescriptorType descriptorType, + DescriptorHistory descriptorHistory, boolean relay) { + if (!this.descriptorListeners.containsKey(descriptorType) && + !this.fingerprintListeners.containsKey(descriptorType)) { + return; + } + Set<DescriptorListener> descriptorListeners = + this.descriptorListeners.get(descriptorType); + Set<FingerprintListener> fingerprintListeners = + this.fingerprintListeners.get(descriptorType); + DescriptorQueue descriptorQueue = this.getDescriptorQueue( + descriptorType, descriptorHistory); + Descriptor descriptor; + while ((descriptor = descriptorQueue.nextDescriptor()) != null) { + for (DescriptorListener descriptorListener : descriptorListeners) { + descriptorListener.processDescriptor(descriptor, relay); + } + if (fingerprintListeners == null) { + continue; + } + SortedSet<String> fingerprints = new TreeSet<String>(); + if (descriptorType == DescriptorType.RELAY_CONSENSUSES && + descriptor instanceof RelayNetworkStatusConsensus) { + fingerprints.addAll(((RelayNetworkStatusConsensus) descriptor). + getStatusEntries().keySet()); + } else if (descriptorType + == DescriptorType.RELAY_SERVER_DESCRIPTORS && + descriptor instanceof ServerDescriptor) { + fingerprints.add(((ServerDescriptor) descriptor). + getFingerprint()); + } else if (descriptorType == DescriptorType.RELAY_EXTRA_INFOS && + descriptor instanceof ExtraInfoDescriptor) { + fingerprints.add(((ExtraInfoDescriptor) descriptor). + getFingerprint()); + } else if (descriptorType == DescriptorType.EXIT_LISTS && + descriptor instanceof ExitList) { + for (ExitListEntry entry : + ((ExitList) descriptor).getExitListEntries()) { + fingerprints.add(entry.getFingerprint()); + } + } else if (descriptorType == DescriptorType.BRIDGE_STATUSES && + descriptor instanceof BridgeNetworkStatus) { + fingerprints.addAll(((BridgeNetworkStatus) descriptor). + getStatusEntries().keySet()); + } else if (descriptorType == + DescriptorType.BRIDGE_SERVER_DESCRIPTORS && + descriptor instanceof ServerDescriptor) { + fingerprints.add(((ServerDescriptor) descriptor). + getFingerprint()); + } else if (descriptorType == DescriptorType.BRIDGE_EXTRA_INFOS && + descriptor instanceof ExtraInfoDescriptor) { + fingerprints.add(((ExtraInfoDescriptor) descriptor). + getFingerprint()); + } else if (descriptorType == + DescriptorType.BRIDGE_POOL_ASSIGNMENTS && + descriptor instanceof BridgePoolAssignment) { + fingerprints.addAll(((BridgePoolAssignment) descriptor). + getEntries().keySet()); + } + for (FingerprintListener fingerprintListener : + fingerprintListeners) { + fingerprintListener.processFingerprints(fingerprints, relay); + } + } + switch (descriptorType) { + case RELAY_CONSENSUSES: + Logger.printStatusTime("Read relay network consensuses"); + break; + case RELAY_SERVER_DESCRIPTORS: + Logger.printStatusTime("Read relay server descriptors"); + break; + case RELAY_EXTRA_INFOS: + Logger.printStatusTime("Read relay extra-info descriptors"); + break; + case EXIT_LISTS: + Logger.printStatusTime("Read exit lists"); + break; + case BRIDGE_STATUSES: + Logger.printStatusTime("Read bridge network statuses"); + break; + case BRIDGE_SERVER_DESCRIPTORS: + Logger.printStatusTime("Read bridge server descriptors"); + break; + case BRIDGE_EXTRA_INFOS: + Logger.printStatusTime("Read bridge extra-info descriptors"); + break; + case BRIDGE_POOL_ASSIGNMENTS: + Logger.printStatusTime("Read bridge-pool assignments"); + break; + } + } + + public void writeHistoryFiles() { + for (DescriptorQueue descriptorQueue : this.descriptorQueues) { + descriptorQueue.writeHistoryFile(); + } + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + this.localFilesBefore + " descriptor files found " + + "locally\n"); + sb.append(" " + this.foundRemoteFiles + " descriptor files found " + + "remotely\n"); + sb.append(" " + this.downloadedFiles + " descriptor files " + + "downloaded from remote\n"); + sb.append(" " + this.deletedLocalFiles + " descriptor files " + + "deleted locally\n"); + sb.append(" " + this.descriptorQueues.size() + " descriptor " + + "queues created\n"); + int historySizeBefore = 0, historySizeAfter = 0; + long descriptors = 0L, bytes = 0L; + for (DescriptorQueue descriptorQueue : descriptorQueues) { + historySizeBefore += descriptorQueue.getHistorySizeBefore(); + historySizeAfter += descriptorQueue.getHistorySizeAfter(); + descriptors += descriptorQueue.getReturnedDescriptors(); + bytes += descriptorQueue.getReturnedBytes(); + } + sb.append(" " + Logger.formatDecimalNumber(historySizeBefore) + + " descriptors excluded from this execution\n"); + sb.append(" " + Logger.formatDecimalNumber(descriptors) + + " descriptors provided\n"); + sb.append(" " + Logger.formatBytes(bytes) + " provided\n"); + sb.append(" " + Logger.formatDecimalNumber(historySizeAfter) + + " descriptors excluded from next execution\n"); + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/updater/DescriptorType.java b/src/org/torproject/onionoo/updater/DescriptorType.java new file mode 100644 index 0000000..41956da --- /dev/null +++ b/src/org/torproject/onionoo/updater/DescriptorType.java @@ -0,0 +1,15 @@ +/* Copyright 2013, 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +public enum DescriptorType { + RELAY_CONSENSUSES, + RELAY_SERVER_DESCRIPTORS, + RELAY_EXTRA_INFOS, + EXIT_LISTS, + BRIDGE_STATUSES, + BRIDGE_SERVER_DESCRIPTORS, + BRIDGE_EXTRA_INFOS, + BRIDGE_POOL_ASSIGNMENTS, +} + diff --git a/src/org/torproject/onionoo/updater/FingerprintListener.java b/src/org/torproject/onionoo/updater/FingerprintListener.java new file mode 100644 index 0000000..5e16eae --- /dev/null +++ b/src/org/torproject/onionoo/updater/FingerprintListener.java @@ -0,0 +1,10 @@ +/* Copyright 2013, 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +import java.util.SortedSet; + +public interface FingerprintListener { + abstract void processFingerprints(SortedSet<String> fingerprints, + boolean relay); +} \ No newline at end of file diff --git a/src/org/torproject/onionoo/updater/LookupResult.java b/src/org/torproject/onionoo/updater/LookupResult.java new file mode 100644 index 0000000..dcf3a2a --- /dev/null +++ b/src/org/torproject/onionoo/updater/LookupResult.java @@ -0,0 +1,70 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +public class LookupResult { + + private String countryCode; + public void setCountryCode(String countryCode) { + this.countryCode = countryCode; + } + public String getCountryCode() { + return this.countryCode; + } + + private String countryName; + public void setCountryName(String countryName) { + this.countryName = countryName; + } + public String getCountryName() { + return this.countryName; + } + + private String regionName; + public void setRegionName(String regionName) { + this.regionName = regionName; + } + public String getRegionName() { + return this.regionName; + } + + private String cityName; + public void setCityName(String cityName) { + this.cityName = cityName; + } + public String getCityName() { + return this.cityName; + } + + private Float latitude; + public void setLatitude(Float latitude) { + this.latitude = latitude; + } + public Float getLatitude() { + return this.latitude; + } + + private Float longitude; + public void setLongitude(Float longitude) { + this.longitude = longitude; + } + public Float getLongitude() { + return this.longitude; + } + + private String asNumber; + public void setAsNumber(String asNumber) { + this.asNumber = asNumber; + } + public String getAsNumber() { + return this.asNumber; + } + + private String asName; + public void setAsName(String asName) { + this.asName = asName; + } + public String getAsName() { + return this.asName; + } +} \ No newline at end of file diff --git a/src/org/torproject/onionoo/updater/LookupService.java b/src/org/torproject/onionoo/updater/LookupService.java new file mode 100644 index 0000000..b816091 --- /dev/null +++ b/src/org/torproject/onionoo/updater/LookupService.java @@ -0,0 +1,343 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.regex.Pattern; + +import org.torproject.onionoo.util.Logger; + +public class LookupService { + + private File geoipDir; + private File geoLite2CityBlocksCsvFile; + private File geoLite2CityLocationsCsvFile; + private File geoIPASNum2CsvFile; + private boolean hasAllFiles = false; + public LookupService(File geoipDir) { + this.geoipDir = geoipDir; + this.findRequiredCsvFiles(); + } + + /* Make sure we have all required .csv files. */ + private void findRequiredCsvFiles() { + this.geoLite2CityBlocksCsvFile = new File(this.geoipDir, + "GeoLite2-City-Blocks.csv"); + if (!this.geoLite2CityBlocksCsvFile.exists()) { + System.err.println("No GeoLite2-City-Blocks.csv file in geoip/."); + return; + } + this.geoLite2CityLocationsCsvFile = new File(this.geoipDir, + "GeoLite2-City-Locations.csv"); + if (!this.geoLite2CityLocationsCsvFile.exists()) { + System.err.println("No GeoLite2-City-Locations.csv file in " + + "geoip/."); + return; + } + this.geoIPASNum2CsvFile = new File(this.geoipDir, "GeoIPASNum2.csv"); + if (!this.geoIPASNum2CsvFile.exists()) { + System.err.println("No GeoIPASNum2.csv file in geoip/."); + return; + } + this.hasAllFiles = true; + } + + private Pattern ipv4Pattern = Pattern.compile("^[0-9\.]{7,15}$"); + private long parseAddressString(String addressString) { + long addressNumber = -1L; + if (ipv4Pattern.matcher(addressString).matches()) { + String[] parts = addressString.split("\.", 4); + if (parts.length == 4) { + addressNumber = 0L; + for (int i = 0; i < 4; i++) { + addressNumber *= 256L; + int octetValue = -1; + try { + octetValue = Integer.parseInt(parts[i]); + } catch (NumberFormatException e) { + } + if (octetValue < 0 || octetValue > 255) { + addressNumber = -1L; + break; + } + addressNumber += octetValue; + } + } + } + return addressNumber; + } + + public SortedMap<String, LookupResult> lookup( + SortedSet<String> addressStrings) { + + SortedMap<String, LookupResult> lookupResults = + new TreeMap<String, LookupResult>(); + + if (!this.hasAllFiles) { + return lookupResults; + } + + /* Obtain a map from relay IP address strings to numbers. */ + Map<String, Long> addressStringNumbers = new HashMap<String, Long>(); + for (String addressString : addressStrings) { + long addressNumber = this.parseAddressString(addressString); + if (addressNumber >= 0L) { + addressStringNumbers.put(addressString, addressNumber); + } + } + if (addressStringNumbers.isEmpty()) { + return lookupResults; + } + + /* Obtain a map from IP address numbers to blocks and to latitudes and + longitudes. */ + Map<Long, Long> addressNumberBlocks = new HashMap<Long, Long>(); + Map<Long, Float[]> addressNumberLatLong = + new HashMap<Long, Float[]>(); + try { + SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>( + addressStringNumbers.values()); + BufferedReader br = new BufferedReader(new InputStreamReader( + new FileInputStream(geoLite2CityBlocksCsvFile), "ISO-8859-1")); + String line = br.readLine(); + while ((line = br.readLine()) != null) { + if (!line.startsWith("::ffff:")) { + /* TODO Make this less hacky and IPv6-ready at some point. */ + continue; + } + String[] parts = line.replaceAll(""", "").split(",", 10); + if (parts.length != 10) { + System.err.println("Illegal line '" + line + "' in " + + geoLite2CityBlocksCsvFile.getAbsolutePath() + "."); + br.close(); + return lookupResults; + } + try { + String startAddressString = parts[0].substring(7); /* ::ffff: */ + long startIpNum = this.parseAddressString(startAddressString); + if (startIpNum < 0L) { + System.err.println("Illegal IP address in '" + line + + "' in " + geoLite2CityBlocksCsvFile.getAbsolutePath() + + "."); + br.close(); + return lookupResults; + } + int networkMaskLength = Integer.parseInt(parts[1]); + if (networkMaskLength < 96 || networkMaskLength > 128) { + System.err.println("Illegal network mask in '" + line + + "' in " + geoLite2CityBlocksCsvFile.getAbsolutePath() + + "."); + br.close(); + return lookupResults; + } + if (parts[2].length() == 0 && parts[3].length() == 0) { + continue; + } + long endIpNum = startIpNum + (1 << (128 - networkMaskLength)) + - 1; + for (long addressNumber : sortedAddressNumbers. + tailSet(startIpNum).headSet(endIpNum + 1L)) { + String blockString = parts[2].length() > 0 ? parts[2] : + parts[3]; + long blockNumber = Long.parseLong(blockString); + addressNumberBlocks.put(addressNumber, blockNumber); + if (parts[6].length() > 0 && parts[7].length() > 0) { + addressNumberLatLong.put(addressNumber, + new Float[] { Float.parseFloat(parts[6]), + Float.parseFloat(parts[7]) }); + } + } + } catch (NumberFormatException e) { + System.err.println("Number format exception while parsing line " + + "'" + line + "' in " + + geoLite2CityBlocksCsvFile.getAbsolutePath() + "."); + br.close(); + return lookupResults; + } + } + br.close(); + } catch (IOException e) { + System.err.println("I/O exception while reading " + + geoLite2CityBlocksCsvFile.getAbsolutePath() + "."); + return lookupResults; + } + + /* Obtain a map from relevant blocks to location lines. */ + Map<Long, String> blockLocations = new HashMap<Long, String>(); + try { + Set<Long> blockNumbers = new HashSet<Long>( + addressNumberBlocks.values()); + BufferedReader br = new BufferedReader(new InputStreamReader( + new FileInputStream(geoLite2CityLocationsCsvFile), + "ISO-8859-1")); + String line = br.readLine(); + while ((line = br.readLine()) != null) { + String[] parts = line.replaceAll(""", "").split(",", 10); + if (parts.length != 10) { + System.err.println("Illegal line '" + line + "' in " + + geoLite2CityLocationsCsvFile.getAbsolutePath() + "."); + br.close(); + return lookupResults; + } + try { + long locId = Long.parseLong(parts[0]); + if (blockNumbers.contains(locId)) { + blockLocations.put(locId, line); + } + } catch (NumberFormatException e) { + System.err.println("Number format exception while parsing line " + + "'" + line + "' in " + + geoLite2CityLocationsCsvFile.getAbsolutePath() + "."); + br.close(); + return lookupResults; + } + } + br.close(); + } catch (IOException e) { + System.err.println("I/O exception while reading " + + geoLite2CityLocationsCsvFile.getAbsolutePath() + "."); + return lookupResults; + } + + /* Obtain a map from IP address numbers to ASN. */ + Map<Long, String> addressNumberASN = new HashMap<Long, String>(); + try { + SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>( + addressStringNumbers.values()); + long firstAddressNumber = sortedAddressNumbers.first(); + BufferedReader br = new BufferedReader(new InputStreamReader( + new FileInputStream(geoIPASNum2CsvFile), "ISO-8859-1")); + String line; + long previousStartIpNum = -1L; + while ((line = br.readLine()) != null) { + String[] parts = line.replaceAll(""", "").split(",", 3); + if (parts.length != 3) { + System.err.println("Illegal line '" + line + "' in " + + geoIPASNum2CsvFile.getAbsolutePath() + "."); + br.close(); + return lookupResults; + } + try { + long startIpNum = Long.parseLong(parts[0]); + if (startIpNum <= previousStartIpNum) { + System.err.println("Line '" + line + "' not sorted in " + + geoIPASNum2CsvFile.getAbsolutePath() + "."); + br.close(); + return lookupResults; + } + previousStartIpNum = startIpNum; + while (firstAddressNumber < startIpNum && + firstAddressNumber != -1L) { + sortedAddressNumbers.remove(firstAddressNumber); + if (sortedAddressNumbers.isEmpty()) { + firstAddressNumber = -1L; + } else { + firstAddressNumber = sortedAddressNumbers.first(); + } + } + long endIpNum = Long.parseLong(parts[1]); + while (firstAddressNumber <= endIpNum && + firstAddressNumber != -1L) { + if (parts[2].startsWith("AS") && + parts[2].split(" ", 2).length == 2) { + addressNumberASN.put(firstAddressNumber, parts[2]); + } + sortedAddressNumbers.remove(firstAddressNumber); + if (sortedAddressNumbers.isEmpty()) { + firstAddressNumber = -1L; + } else { + firstAddressNumber = sortedAddressNumbers.first(); + } + } + if (firstAddressNumber == -1L) { + break; + } + } + catch (NumberFormatException e) { + System.err.println("Number format exception while parsing line " + + "'" + line + "' in " + + geoIPASNum2CsvFile.getAbsolutePath() + "."); + br.close(); + return lookupResults; + } + } + br.close(); + } catch (IOException e) { + System.err.println("I/O exception while reading " + + geoIPASNum2CsvFile.getAbsolutePath() + "."); + return lookupResults; + } + + /* Finally, put together lookup results. */ + for (String addressString : addressStrings) { + if (!addressStringNumbers.containsKey(addressString)) { + continue; + } + long addressNumber = addressStringNumbers.get(addressString); + if (!addressNumberBlocks.containsKey(addressNumber) && + !addressNumberLatLong.containsKey(addressNumber) && + !addressNumberASN.containsKey(addressNumber)) { + continue; + } + LookupResult lookupResult = new LookupResult(); + if (addressNumberBlocks.containsKey(addressNumber)) { + long blockNumber = addressNumberBlocks.get(addressNumber); + if (blockLocations.containsKey(blockNumber)) { + String[] parts = blockLocations.get(blockNumber). + replaceAll(""", "").split(",", -1); + lookupResult.setCountryCode(parts[3].toLowerCase()); + if (parts[4].length() > 0) { + lookupResult.setCountryName(parts[4]); + } + if (parts[6].length() > 0) { + lookupResult.setRegionName(parts[6]); + } + if (parts[7].length() > 0) { + lookupResult.setCityName(parts[7]); + } + } + } + if (addressNumberLatLong.containsKey(addressNumber)) { + Float[] latLong = addressNumberLatLong.get(addressNumber); + lookupResult.setLatitude(latLong[0]); + lookupResult.setLongitude(latLong[1]); + } + if (addressNumberASN.containsKey(addressNumber)) { + String[] parts = addressNumberASN.get(addressNumber).split(" ", + 2); + lookupResult.setAsNumber(parts[0]); + lookupResult.setAsName(parts[1]); + } + lookupResults.put(addressString, lookupResult); + } + + /* Keep statistics. */ + this.addressesLookedUp += addressStrings.size(); + this.addressesResolved += lookupResults.size(); + + return lookupResults; + } + + private int addressesLookedUp = 0, addressesResolved = 0; + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber(addressesLookedUp) + + " addresses looked up\n"); + sb.append(" " + Logger.formatDecimalNumber(addressesResolved) + + " addresses resolved\n"); + return sb.toString(); + } +} diff --git a/src/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java new file mode 100644 index 0000000..c687704 --- /dev/null +++ b/src/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java @@ -0,0 +1,626 @@ +/* Copyright 2011--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.torproject.descriptor.BridgeNetworkStatus; +import org.torproject.descriptor.BridgePoolAssignment; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.ExitList; +import org.torproject.descriptor.ExitListEntry; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.ServerDescriptor; +import org.torproject.onionoo.docs.DetailsStatus; +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.docs.NodeStatus; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; +import org.torproject.onionoo.util.Logger; + +public class NodeDetailsStatusUpdater implements DescriptorListener, + StatusUpdater { + + private DescriptorSource descriptorSource; + + private ReverseDomainNameResolver reverseDomainNameResolver; + + private LookupService lookupService; + + private DocumentStore documentStore; + + private long now; + + private SortedMap<String, NodeStatus> knownNodes = + new TreeMap<String, NodeStatus>(); + + private SortedMap<String, NodeStatus> relays; + + private SortedMap<String, NodeStatus> bridges; + + private long relaysLastValidAfterMillis = -1L; + + private long bridgesLastPublishedMillis = -1L; + + private SortedMap<String, Integer> lastBandwidthWeights = null; + + private int relayConsensusesProcessed = 0, bridgeStatusesProcessed = 0; + + public NodeDetailsStatusUpdater( + ReverseDomainNameResolver reverseDomainNameResolver, + LookupService lookupService) { + this.descriptorSource = ApplicationFactory.getDescriptorSource(); + this.reverseDomainNameResolver = reverseDomainNameResolver; + this.lookupService = lookupService; + this.documentStore = ApplicationFactory.getDocumentStore(); + this.now = ApplicationFactory.getTime().currentTimeMillis(); + this.registerDescriptorListeners(); + } + + private void registerDescriptorListeners() { + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_SERVER_DESCRIPTORS); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_STATUSES); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_SERVER_DESCRIPTORS); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_POOL_ASSIGNMENTS); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.EXIT_LISTS); + } + + public void processDescriptor(Descriptor descriptor, boolean relay) { + if (descriptor instanceof ServerDescriptor && relay) { + this.processRelayServerDescriptor((ServerDescriptor) descriptor); + } else if (descriptor instanceof ExitList) { + this.processExitList((ExitList) descriptor); + } else if (descriptor instanceof RelayNetworkStatusConsensus) { + this.processRelayNetworkStatusConsensus( + (RelayNetworkStatusConsensus) descriptor); + } else if (descriptor instanceof ServerDescriptor && !relay) { + this.processBridgeServerDescriptor((ServerDescriptor) descriptor); + } else if (descriptor instanceof BridgePoolAssignment) { + this.processBridgePoolAssignment((BridgePoolAssignment) descriptor); + } else if (descriptor instanceof BridgeNetworkStatus) { + this.processBridgeNetworkStatus((BridgeNetworkStatus) descriptor); + } + } + + private void processRelayServerDescriptor( + ServerDescriptor descriptor) { + String fingerprint = descriptor.getFingerprint(); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + String publishedDateTime = + DateTimeHelper.format(descriptor.getPublishedMillis()); + if (detailsStatus == null) { + detailsStatus = new DetailsStatus(); + } else if (detailsStatus.getDescPublished() != null && + publishedDateTime.compareTo( + detailsStatus.getDescPublished()) < 0) { + return; + } + String lastRestartedString = DateTimeHelper.format( + descriptor.getPublishedMillis() - descriptor.getUptime() + * DateTimeHelper.ONE_SECOND); + int bandwidthRate = descriptor.getBandwidthRate(); + int bandwidthBurst = descriptor.getBandwidthBurst(); + int observedBandwidth = descriptor.getBandwidthObserved(); + int advertisedBandwidth = Math.min(bandwidthRate, + Math.min(bandwidthBurst, observedBandwidth)); + detailsStatus.setDescPublished(publishedDateTime); + detailsStatus.setLastRestarted(lastRestartedString); + detailsStatus.setBandwidthRate(bandwidthRate); + detailsStatus.setBandwidthBurst(bandwidthBurst); + detailsStatus.setObservedBandwidth(observedBandwidth); + detailsStatus.setAdvertisedBandwidth(advertisedBandwidth); + detailsStatus.setExitPolicy(descriptor.getExitPolicyLines()); + detailsStatus.setContact(descriptor.getContact()); + detailsStatus.setPlatform(descriptor.getPlatform()); + detailsStatus.setFamily(descriptor.getFamilyEntries()); + if (descriptor.getIpv6DefaultPolicy() != null && + (descriptor.getIpv6DefaultPolicy().equals("accept") || + descriptor.getIpv6DefaultPolicy().equals("reject")) && + descriptor.getIpv6PortList() != null) { + Map<String, List<String>> exitPolicyV6Summary = + new HashMap<String, List<String>>(); + List<String> portsOrPortRanges = Arrays.asList( + descriptor.getIpv6PortList().split(",")); + exitPolicyV6Summary.put(descriptor.getIpv6DefaultPolicy(), + portsOrPortRanges); + detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary); + } + if (descriptor.isHibernating()) { + detailsStatus.setHibernating(true); + } + this.documentStore.store(detailsStatus, fingerprint); + } + + private Map<String, Map<String, Long>> exitListEntries = + new HashMap<String, Map<String, Long>>(); + + private void processExitList(ExitList exitList) { + for (ExitListEntry exitListEntry : exitList.getExitListEntries()) { + String fingerprint = exitListEntry.getFingerprint(); + if (exitListEntry.getScanMillis() < + this.now - DateTimeHelper.ONE_DAY) { + continue; + } + if (!this.exitListEntries.containsKey(fingerprint)) { + this.exitListEntries.put(fingerprint, + new HashMap<String, Long>()); + } + String exitAddress = exitListEntry.getExitAddress(); + long scanMillis = exitListEntry.getScanMillis(); + if (!this.exitListEntries.get(fingerprint).containsKey(exitAddress) + || this.exitListEntries.get(fingerprint).get(exitAddress) + < scanMillis) { + this.exitListEntries.get(fingerprint).put(exitAddress, + scanMillis); + } + } + } + + private void processRelayNetworkStatusConsensus( + RelayNetworkStatusConsensus consensus) { + long validAfterMillis = consensus.getValidAfterMillis(); + if (validAfterMillis > this.relaysLastValidAfterMillis) { + this.relaysLastValidAfterMillis = validAfterMillis; + } + Set<String> recommendedVersions = null; + if (consensus.getRecommendedServerVersions() != null) { + recommendedVersions = new HashSet<String>(); + for (String recommendedVersion : + consensus.getRecommendedServerVersions()) { + recommendedVersions.add("Tor " + recommendedVersion); + } + } + for (NetworkStatusEntry entry : + consensus.getStatusEntries().values()) { + String nickname = entry.getNickname(); + String fingerprint = entry.getFingerprint(); + String address = entry.getAddress(); + SortedSet<String> orAddressesAndPorts = new TreeSet<String>( + entry.getOrAddresses()); + int orPort = entry.getOrPort(); + int dirPort = entry.getDirPort(); + SortedSet<String> relayFlags = entry.getFlags(); + long consensusWeight = entry.getBandwidth(); + String defaultPolicy = entry.getDefaultPolicy(); + String portList = entry.getPortList(); + Boolean recommendedVersion = (recommendedVersions == null || + entry.getVersion() == null) ? null : + recommendedVersions.contains(entry.getVersion()); + NodeStatus newNodeStatus = new NodeStatus(true, nickname, + fingerprint, address, orAddressesAndPorts, null, + validAfterMillis, orPort, dirPort, relayFlags, consensusWeight, + null, null, -1L, defaultPolicy, portList, validAfterMillis, + validAfterMillis, null, null, recommendedVersion, null); + if (this.knownNodes.containsKey(fingerprint)) { + this.knownNodes.get(fingerprint).update(newNodeStatus); + } else { + this.knownNodes.put(fingerprint, newNodeStatus); + } + } + this.relayConsensusesProcessed++; + if (this.relaysLastValidAfterMillis == validAfterMillis) { + this.lastBandwidthWeights = consensus.getBandwidthWeights(); + } + } + + private void processBridgeServerDescriptor( + ServerDescriptor descriptor) { + String fingerprint = descriptor.getFingerprint(); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + String publishedDateTime = + DateTimeHelper.format(descriptor.getPublishedMillis()); + if (detailsStatus == null) { + detailsStatus = new DetailsStatus(); + } else if (detailsStatus.getDescPublished() != null && + publishedDateTime.compareTo( + detailsStatus.getDescPublished()) < 0) { + return; + } + String lastRestartedString = DateTimeHelper.format( + descriptor.getPublishedMillis() - descriptor.getUptime() + * DateTimeHelper.ONE_SECOND); + int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(), + Math.min(descriptor.getBandwidthBurst(), + descriptor.getBandwidthObserved())); + detailsStatus.setDescPublished(publishedDateTime); + detailsStatus.setLastRestarted(lastRestartedString); + detailsStatus.setAdvertisedBandwidth(advertisedBandwidth); + detailsStatus.setPlatform(descriptor.getPlatform()); + this.documentStore.store(detailsStatus, fingerprint); + } + + private void processBridgePoolAssignment( + BridgePoolAssignment bridgePoolAssignment) { + for (Map.Entry<String, String> e : + bridgePoolAssignment.getEntries().entrySet()) { + String fingerprint = e.getKey(); + String details = e.getValue(); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + if (detailsStatus == null) { + detailsStatus = new DetailsStatus(); + } else if (details.equals(detailsStatus.getPoolAssignment())) { + continue; + } + detailsStatus.setPoolAssignment(details); + this.documentStore.store(detailsStatus, fingerprint); + } + } + + private void processBridgeNetworkStatus(BridgeNetworkStatus status) { + long publishedMillis = status.getPublishedMillis(); + if (publishedMillis > this.bridgesLastPublishedMillis) { + this.bridgesLastPublishedMillis = publishedMillis; + } + for (NetworkStatusEntry entry : status.getStatusEntries().values()) { + String nickname = entry.getNickname(); + String fingerprint = entry.getFingerprint(); + String address = entry.getAddress(); + SortedSet<String> orAddressesAndPorts = new TreeSet<String>( + entry.getOrAddresses()); + int orPort = entry.getOrPort(); + int dirPort = entry.getDirPort(); + SortedSet<String> relayFlags = entry.getFlags(); + NodeStatus newNodeStatus = new NodeStatus(false, nickname, + fingerprint, address, orAddressesAndPorts, null, + publishedMillis, orPort, dirPort, relayFlags, -1L, "??", null, + -1L, null, null, publishedMillis, -1L, null, null, null, null); + if (this.knownNodes.containsKey(fingerprint)) { + this.knownNodes.get(fingerprint).update(newNodeStatus); + } else { + this.knownNodes.put(fingerprint, newNodeStatus); + } + } + this.bridgeStatusesProcessed++; + } + + public void updateStatuses() { + this.readStatusSummary(); + Logger.printStatusTime("Read status summary"); + this.setCurrentNodes(); + Logger.printStatusTime("Set current node fingerprints"); + this.startReverseDomainNameLookups(); + Logger.printStatusTime("Started reverse domain name lookups"); + this.lookUpCitiesAndASes(); + Logger.printStatusTime("Looked up cities and ASes"); + this.setDescriptorPartsOfNodeStatus(); + Logger.printStatusTime("Set descriptor parts of node statuses."); + this.calculatePathSelectionProbabilities(); + Logger.printStatusTime("Calculated path selection probabilities"); + this.finishReverseDomainNameLookups(); + Logger.printStatusTime("Finished reverse domain name lookups"); + this.writeStatusSummary(); + Logger.printStatusTime("Wrote status summary"); + this.updateDetailsStatuses(); + Logger.printStatusTime("Updated exit addresses in details statuses"); + } + + private void readStatusSummary() { + SortedSet<String> fingerprints = this.documentStore.list( + NodeStatus.class); + for (String fingerprint : fingerprints) { + NodeStatus node = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (node.isRelay()) { + this.relaysLastValidAfterMillis = Math.max( + this.relaysLastValidAfterMillis, node.getLastSeenMillis()); + } else { + this.bridgesLastPublishedMillis = Math.max( + this.bridgesLastPublishedMillis, node.getLastSeenMillis()); + } + if (this.knownNodes.containsKey(fingerprint)) { + this.knownNodes.get(fingerprint).update(node); + } else { + this.knownNodes.put(fingerprint, node); + } + } + } + + private void setCurrentNodes() { + long cutoff = Math.max(this.relaysLastValidAfterMillis, + this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L; + SortedMap<String, NodeStatus> currentNodes = + new TreeMap<String, NodeStatus>(); + for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { + if (e.getValue().getLastSeenMillis() >= cutoff) { + currentNodes.put(e.getKey(), e.getValue()); + } + } + this.relays = new TreeMap<String, NodeStatus>(); + this.bridges = new TreeMap<String, NodeStatus>(); + for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) { + if (e.getValue().isRelay()) { + this.relays.put(e.getKey(), e.getValue()); + } else { + this.bridges.put(e.getKey(), e.getValue()); + } + } + } + + private void startReverseDomainNameLookups() { + Map<String, Long> addressLastLookupTimes = + new HashMap<String, Long>(); + for (NodeStatus relay : relays.values()) { + addressLastLookupTimes.put(relay.getAddress(), + relay.getLastRdnsLookup()); + } + this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes); + this.reverseDomainNameResolver.startReverseDomainNameLookups(); + } + + private void lookUpCitiesAndASes() { + SortedSet<String> addressStrings = new TreeSet<String>(); + for (NodeStatus node : this.knownNodes.values()) { + if (node.isRelay()) { + addressStrings.add(node.getAddress()); + } + } + if (addressStrings.isEmpty()) { + System.err.println("No relay IP addresses to resolve to cities or " + + "ASN."); + return; + } + SortedMap<String, LookupResult> lookupResults = + this.lookupService.lookup(addressStrings); + for (NodeStatus node : knownNodes.values()) { + if (!node.isRelay()) { + continue; + } + String addressString = node.getAddress(); + if (lookupResults.containsKey(addressString)) { + LookupResult lookupResult = lookupResults.get(addressString); + node.setCountryCode(lookupResult.getCountryCode()); + node.setCountryName(lookupResult.getCountryName()); + node.setRegionName(lookupResult.getRegionName()); + node.setCityName(lookupResult.getCityName()); + node.setLatitude(lookupResult.getLatitude()); + node.setLongitude(lookupResult.getLongitude()); + node.setASNumber(lookupResult.getAsNumber()); + node.setASName(lookupResult.getAsName()); + } + } + } + + private void setDescriptorPartsOfNodeStatus() { + for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { + String fingerprint = e.getKey(); + NodeStatus node = e.getValue(); + if (node.isRelay()) { + if (node.getRelayFlags().contains("Running") && + node.getLastSeenMillis() == this.relaysLastValidAfterMillis) { + node.setRunning(true); + } + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + if (detailsStatus != null) { + node.setContact(detailsStatus.getContact()); + if (detailsStatus.getExitAddresses() != null) { + for (Map.Entry<String, Long> ea : + detailsStatus.getExitAddresses().entrySet()) { + if (ea.getValue() >= this.now - DateTimeHelper.ONE_DAY) { + node.addExitAddress(ea.getKey()); + } + } + } + if (detailsStatus.getFamily() != null && + !detailsStatus.getFamily().isEmpty()) { + SortedSet<String> familyFingerprints = new TreeSet<String>(); + for (String familyMember : detailsStatus.getFamily()) { + if (familyMember.startsWith("$") && + familyMember.length() == 41) { + familyFingerprints.add(familyMember.substring(1)); + } + } + if (!familyFingerprints.isEmpty()) { + node.setFamilyFingerprints(familyFingerprints); + } + } + } + } + if (!node.isRelay() && node.getRelayFlags().contains("Running") && + node.getLastSeenMillis() == this.bridgesLastPublishedMillis) { + node.setRunning(true); + } + } + } + + private void calculatePathSelectionProbabilities() { + boolean consensusContainsBandwidthWeights = false; + double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0, + wmd = 0.0, wee = 0.0, wed = 0.0; + if (this.lastBandwidthWeights != null) { + SortedSet<String> weightKeys = new TreeSet<String>(Arrays.asList( + "Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(","))); + weightKeys.removeAll(this.lastBandwidthWeights.keySet()); + if (weightKeys.isEmpty()) { + consensusContainsBandwidthWeights = true; + wgg = ((double) this.lastBandwidthWeights.get("Wgg")) / 10000.0; + wgd = ((double) this.lastBandwidthWeights.get("Wgd")) / 10000.0; + wmg = ((double) this.lastBandwidthWeights.get("Wmg")) / 10000.0; + wmm = ((double) this.lastBandwidthWeights.get("Wmm")) / 10000.0; + wme = ((double) this.lastBandwidthWeights.get("Wme")) / 10000.0; + wmd = ((double) this.lastBandwidthWeights.get("Wmd")) / 10000.0; + wee = ((double) this.lastBandwidthWeights.get("Wee")) / 10000.0; + wed = ((double) this.lastBandwidthWeights.get("Wed")) / 10000.0; + } + } else { + System.err.println("Could not determine most recent Wxx parameter " + + "values, probably because we didn't parse a consensus in " + + "this execution. All relays' guard/middle/exit weights are " + + "going to be 0.0."); + } + SortedMap<String, Double> + advertisedBandwidths = new TreeMap<String, Double>(), + consensusWeights = new TreeMap<String, Double>(), + guardWeights = new TreeMap<String, Double>(), + middleWeights = new TreeMap<String, Double>(), + exitWeights = new TreeMap<String, Double>(); + double totalAdvertisedBandwidth = 0.0; + double totalConsensusWeight = 0.0; + double totalGuardWeight = 0.0; + double totalMiddleWeight = 0.0; + double totalExitWeight = 0.0; + for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { + String fingerprint = e.getKey(); + NodeStatus relay = e.getValue(); + if (!relay.getRunning()) { + continue; + } + boolean isExit = relay.getRelayFlags().contains("Exit") && + !relay.getRelayFlags().contains("BadExit"); + boolean isGuard = relay.getRelayFlags().contains("Guard"); + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + if (detailsStatus != null) { + double advertisedBandwidth = + detailsStatus.getAdvertisedBandwidth(); + if (advertisedBandwidth >= 0.0) { + advertisedBandwidths.put(fingerprint, advertisedBandwidth); + totalAdvertisedBandwidth += advertisedBandwidth; + } + } + double consensusWeight = (double) relay.getConsensusWeight(); + consensusWeights.put(fingerprint, consensusWeight); + totalConsensusWeight += consensusWeight; + if (consensusContainsBandwidthWeights) { + double guardWeight = consensusWeight, + middleWeight = consensusWeight, + exitWeight = consensusWeight; + if (isGuard && isExit) { + guardWeight *= wgd; + middleWeight *= wmd; + exitWeight *= wed; + } else if (isGuard) { + guardWeight *= wgg; + middleWeight *= wmg; + exitWeight = 0.0; + } else if (isExit) { + guardWeight = 0.0; + middleWeight *= wme; + exitWeight *= wee; + } else { + guardWeight = 0.0; + middleWeight *= wmm; + exitWeight = 0.0; + } + guardWeights.put(fingerprint, guardWeight); + middleWeights.put(fingerprint, middleWeight); + exitWeights.put(fingerprint, exitWeight); + totalGuardWeight += guardWeight; + totalMiddleWeight += middleWeight; + totalExitWeight += exitWeight; + } + } + for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) { + String fingerprint = e.getKey(); + NodeStatus relay = e.getValue(); + if (advertisedBandwidths.containsKey(fingerprint)) { + relay.setAdvertisedBandwidthFraction(advertisedBandwidths.get( + fingerprint) / totalAdvertisedBandwidth); + } + if (consensusWeights.containsKey(fingerprint)) { + relay.setConsensusWeightFraction(consensusWeights.get(fingerprint) + / totalConsensusWeight); + } + if (guardWeights.containsKey(fingerprint)) { + relay.setGuardProbability(guardWeights.get(fingerprint) + / totalGuardWeight); + } + if (middleWeights.containsKey(fingerprint)) { + relay.setMiddleProbability(middleWeights.get(fingerprint) + / totalMiddleWeight); + } + if (exitWeights.containsKey(fingerprint)) { + relay.setExitProbability(exitWeights.get(fingerprint) + / totalExitWeight); + } + } + } + + private void finishReverseDomainNameLookups() { + this.reverseDomainNameResolver.finishReverseDomainNameLookups(); + Map<String, String> lookupResults = + this.reverseDomainNameResolver.getLookupResults(); + long startedRdnsLookups = + this.reverseDomainNameResolver.getLookupStartMillis(); + for (NodeStatus relay : relays.values()) { + if (lookupResults.containsKey(relay.getAddress())) { + relay.setHostName(lookupResults.get(relay.getAddress())); + relay.setLastRdnsLookup(startedRdnsLookups); + } + } + } + + private void writeStatusSummary() { + for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) { + this.documentStore.store(e.getValue(), e.getKey()); + } + } + + private void updateDetailsStatuses() { + SortedSet<String> fingerprints = new TreeSet<String>(); + fingerprints.addAll(this.exitListEntries.keySet()); + for (String fingerprint : fingerprints) { + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + if (detailsStatus == null) { + detailsStatus = new DetailsStatus(); + } + Map<String, Long> exitAddresses = new HashMap<String, Long>(); + if (detailsStatus.getExitAddresses() != null) { + for (Map.Entry<String, Long> e : + detailsStatus.getExitAddresses().entrySet()) { + if (e.getValue() >= this.now - DateTimeHelper.ONE_DAY) { + exitAddresses.put(e.getKey(), e.getValue()); + } + } + } + if (this.exitListEntries.containsKey(fingerprint)) { + for (Map.Entry<String, Long> e : + this.exitListEntries.get(fingerprint).entrySet()) { + if (!exitAddresses.containsKey(e.getKey()) || + exitAddresses.get(e.getKey()) < e.getValue()) { + exitAddresses.put(e.getKey(), e.getValue()); + } + } + } + if (this.knownNodes.containsKey(fingerprint)) { + for (String orAddress : + this.knownNodes.get(fingerprint).getOrAddresses()) { + this.exitListEntries.remove(orAddress); + } + } + detailsStatus.setExitAddresses(exitAddresses); + this.documentStore.store(detailsStatus, fingerprint); + } + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber( + relayConsensusesProcessed) + " relay consensuses processed\n"); + sb.append(" " + Logger.formatDecimalNumber(bridgeStatusesProcessed) + + " bridge statuses processed\n"); + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java b/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java new file mode 100644 index 0000000..8ca7eb4 --- /dev/null +++ b/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java @@ -0,0 +1,179 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; +import org.torproject.onionoo.util.Logger; +import org.torproject.onionoo.util.Time; + +public class ReverseDomainNameResolver { + + private class RdnsLookupWorker extends Thread { + public void run() { + while (time.currentTimeMillis() - RDNS_LOOKUP_MAX_DURATION_MILLIS + <= startedRdnsLookups) { + String rdnsLookupJob = null; + synchronized (rdnsLookupJobs) { + for (String job : rdnsLookupJobs) { + rdnsLookupJob = job; + rdnsLookupJobs.remove(job); + break; + } + } + if (rdnsLookupJob == null) { + break; + } + RdnsLookupRequest request = new RdnsLookupRequest(this, + rdnsLookupJob); + request.setDaemon(true); + request.start(); + try { + Thread.sleep(RDNS_LOOKUP_MAX_REQUEST_MILLIS); + } catch (InterruptedException e) { + /* Getting interrupted should be the default case. */ + } + String hostName = request.getHostName(); + if (hostName != null) { + synchronized (rdnsLookupResults) { + rdnsLookupResults.put(rdnsLookupJob, hostName); + } + } + long lookupMillis = request.getLookupMillis(); + if (lookupMillis >= 0L) { + synchronized (rdnsLookupMillis) { + rdnsLookupMillis.add(lookupMillis); + } + } + } + } + } + + private class RdnsLookupRequest extends Thread { + private RdnsLookupWorker parent; + private String address, hostName; + private long lookupStartedMillis = -1L, lookupCompletedMillis = -1L; + public RdnsLookupRequest(RdnsLookupWorker parent, String address) { + this.parent = parent; + this.address = address; + } + public void run() { + this.lookupStartedMillis = time.currentTimeMillis(); + try { + String result = InetAddress.getByName(this.address).getHostName(); + synchronized (this) { + this.hostName = result; + } + } catch (UnknownHostException e) { + /* We'll try again the next time. */ + } + this.lookupCompletedMillis = time.currentTimeMillis(); + this.parent.interrupt(); + } + public synchronized String getHostName() { + return hostName; + } + public synchronized long getLookupMillis() { + return this.lookupCompletedMillis - this.lookupStartedMillis; + } + } + + private Time time; + + public ReverseDomainNameResolver() { + this.time = ApplicationFactory.getTime(); + } + + private static final long RDNS_LOOKUP_MAX_REQUEST_MILLIS = + DateTimeHelper.TEN_SECONDS; + private static final long RDNS_LOOKUP_MAX_DURATION_MILLIS = + DateTimeHelper.FIVE_MINUTES; + private static final long RDNS_LOOKUP_MAX_AGE_MILLIS = + DateTimeHelper.TWELVE_HOURS; + private static final int RDNS_LOOKUP_WORKERS_NUM = 5; + + private Map<String, Long> addressLastLookupTimes; + + private Set<String> rdnsLookupJobs; + + private Map<String, String> rdnsLookupResults; + + private List<Long> rdnsLookupMillis; + + private long startedRdnsLookups; + + private List<RdnsLookupWorker> rdnsLookupWorkers; + + public void setAddresses(Map<String, Long> addressLastLookupTimes) { + this.addressLastLookupTimes = addressLastLookupTimes; + } + + public void startReverseDomainNameLookups() { + this.startedRdnsLookups = this.time.currentTimeMillis(); + this.rdnsLookupJobs = new HashSet<String>(); + for (Map.Entry<String, Long> e : + this.addressLastLookupTimes.entrySet()) { + if (e.getValue() < this.startedRdnsLookups + - RDNS_LOOKUP_MAX_AGE_MILLIS) { + this.rdnsLookupJobs.add(e.getKey()); + } + } + this.rdnsLookupResults = new HashMap<String, String>(); + this.rdnsLookupMillis = new ArrayList<Long>(); + this.rdnsLookupWorkers = new ArrayList<RdnsLookupWorker>(); + for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) { + RdnsLookupWorker rdnsLookupWorker = new RdnsLookupWorker(); + this.rdnsLookupWorkers.add(rdnsLookupWorker); + rdnsLookupWorker.setDaemon(true); + rdnsLookupWorker.start(); + } + } + + public void finishReverseDomainNameLookups() { + for (RdnsLookupWorker rdnsLookupWorker : this.rdnsLookupWorkers) { + try { + rdnsLookupWorker.join(); + } catch (InterruptedException e) { + /* This is not something that we can take care of. Just leave the + * worker thread alone. */ + } + } + } + + public Map<String, String> getLookupResults() { + synchronized (this.rdnsLookupResults) { + return new HashMap<String, String>(this.rdnsLookupResults); + } + } + + public long getLookupStartMillis() { + return this.startedRdnsLookups; + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber(rdnsLookupMillis.size()) + + " lookups performed\n"); + if (rdnsLookupMillis.size() > 0) { + Collections.sort(rdnsLookupMillis); + sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get(0)) + + " minimum lookup time\n"); + sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get( + rdnsLookupMillis.size() / 2)) + " median lookup time\n"); + sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get( + rdnsLookupMillis.size() - 1)) + " maximum lookup time\n"); + } + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/updater/StatusUpdater.java b/src/org/torproject/onionoo/updater/StatusUpdater.java new file mode 100644 index 0000000..9fc34d3 --- /dev/null +++ b/src/org/torproject/onionoo/updater/StatusUpdater.java @@ -0,0 +1,11 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +public interface StatusUpdater { + + public abstract void updateStatuses(); + + public abstract String getStatsString(); +} + diff --git a/src/org/torproject/onionoo/updater/UptimeStatusUpdater.java b/src/org/torproject/onionoo/updater/UptimeStatusUpdater.java new file mode 100644 index 0000000..dd71e74 --- /dev/null +++ b/src/org/torproject/onionoo/updater/UptimeStatusUpdater.java @@ -0,0 +1,130 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.torproject.descriptor.BridgeNetworkStatus; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.onionoo.docs.UptimeStatus; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; +import org.torproject.onionoo.util.Logger; + +public class UptimeStatusUpdater implements DescriptorListener, + StatusUpdater { + + private DescriptorSource descriptorSource; + + public UptimeStatusUpdater() { + this.descriptorSource = ApplicationFactory.getDescriptorSource(); + this.registerDescriptorListeners(); + } + + private void registerDescriptorListeners() { + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.BRIDGE_STATUSES); + } + + public void processDescriptor(Descriptor descriptor, boolean relay) { + if (descriptor instanceof RelayNetworkStatusConsensus) { + this.processRelayNetworkStatusConsensus( + (RelayNetworkStatusConsensus) descriptor); + } else if (descriptor instanceof BridgeNetworkStatus) { + this.processBridgeNetworkStatus( + (BridgeNetworkStatus) descriptor); + } + } + + private SortedSet<Long> newRelayStatuses = new TreeSet<Long>(), + newBridgeStatuses = new TreeSet<Long>(); + private SortedMap<String, SortedSet<Long>> + newRunningRelays = new TreeMap<String, SortedSet<Long>>(), + newRunningBridges = new TreeMap<String, SortedSet<Long>>(); + + private void processRelayNetworkStatusConsensus( + RelayNetworkStatusConsensus consensus) { + SortedSet<String> fingerprints = new TreeSet<String>(); + for (NetworkStatusEntry entry : + consensus.getStatusEntries().values()) { + if (entry.getFlags().contains("Running")) { + fingerprints.add(entry.getFingerprint()); + } + } + if (!fingerprints.isEmpty()) { + long dateHourMillis = (consensus.getValidAfterMillis() + / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR; + for (String fingerprint : fingerprints) { + if (!this.newRunningRelays.containsKey(fingerprint)) { + this.newRunningRelays.put(fingerprint, new TreeSet<Long>()); + } + this.newRunningRelays.get(fingerprint).add(dateHourMillis); + } + this.newRelayStatuses.add(dateHourMillis); + } + } + + private void processBridgeNetworkStatus(BridgeNetworkStatus status) { + SortedSet<String> fingerprints = new TreeSet<String>(); + for (NetworkStatusEntry entry : + status.getStatusEntries().values()) { + if (entry.getFlags().contains("Running")) { + fingerprints.add(entry.getFingerprint()); + } + } + if (!fingerprints.isEmpty()) { + long dateHourMillis = (status.getPublishedMillis() + / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR; + for (String fingerprint : fingerprints) { + if (!this.newRunningBridges.containsKey(fingerprint)) { + this.newRunningBridges.put(fingerprint, new TreeSet<Long>()); + } + this.newRunningBridges.get(fingerprint).add(dateHourMillis); + } + this.newBridgeStatuses.add(dateHourMillis); + } + } + + public void updateStatuses() { + for (Map.Entry<String, SortedSet<Long>> e : + this.newRunningRelays.entrySet()) { + this.updateStatus(true, e.getKey(), e.getValue()); + } + this.updateStatus(true, null, this.newRelayStatuses); + for (Map.Entry<String, SortedSet<Long>> e : + this.newRunningBridges.entrySet()) { + this.updateStatus(false, e.getKey(), e.getValue()); + } + this.updateStatus(false, null, this.newBridgeStatuses); + } + + private void updateStatus(boolean relay, String fingerprint, + SortedSet<Long> newUptimeHours) { + UptimeStatus uptimeStatus = UptimeStatus.loadOrCreate(fingerprint); + uptimeStatus.addToHistory(relay, newUptimeHours); + uptimeStatus.storeIfChanged(); + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber( + this.newRelayStatuses.size()) + " hours of relay uptimes " + + "processed\n"); + sb.append(" " + Logger.formatDecimalNumber( + this.newBridgeStatuses.size()) + " hours of bridge uptimes " + + "processed\n"); + sb.append(" " + Logger.formatDecimalNumber( + this.newRunningRelays.size() + this.newRunningBridges.size()) + + " uptime status files updated\n"); + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/updater/WeightsStatusUpdater.java b/src/org/torproject/onionoo/updater/WeightsStatusUpdater.java new file mode 100644 index 0000000..333afcc --- /dev/null +++ b/src/org/torproject/onionoo/updater/WeightsStatusUpdater.java @@ -0,0 +1,332 @@ +/* Copyright 2012--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.updater; + +import java.util.Arrays; +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.NetworkStatusEntry; +import org.torproject.descriptor.RelayNetworkStatusConsensus; +import org.torproject.descriptor.ServerDescriptor; +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.docs.WeightsStatus; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; + +public class WeightsStatusUpdater implements DescriptorListener, + StatusUpdater { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public WeightsStatusUpdater() { + this.descriptorSource = ApplicationFactory.getDescriptorSource(); + this.documentStore = ApplicationFactory.getDocumentStore(); + this.now = ApplicationFactory.getTime().currentTimeMillis(); + this.registerDescriptorListeners(); + } + + private void registerDescriptorListeners() { + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerDescriptorListener(this, + DescriptorType.RELAY_SERVER_DESCRIPTORS); + } + + public void processDescriptor(Descriptor descriptor, boolean relay) { + if (descriptor instanceof ServerDescriptor) { + this.processRelayServerDescriptor((ServerDescriptor) descriptor); + } else if (descriptor instanceof RelayNetworkStatusConsensus) { + this.processRelayNetworkConsensus( + (RelayNetworkStatusConsensus) descriptor); + } + } + + public void updateStatuses() { + /* Nothing to do. */ + } + + private void processRelayNetworkConsensus( + RelayNetworkStatusConsensus consensus) { + long validAfterMillis = consensus.getValidAfterMillis(), + freshUntilMillis = consensus.getFreshUntilMillis(); + SortedMap<String, double[]> pathSelectionWeights = + this.calculatePathSelectionProbabilities(consensus); + this.updateWeightsHistory(validAfterMillis, freshUntilMillis, + pathSelectionWeights); + } + + private void processRelayServerDescriptor( + ServerDescriptor serverDescriptor) { + String digest = serverDescriptor.getServerDescriptorDigest(). + toUpperCase(); + int advertisedBandwidth = Math.min(Math.min( + serverDescriptor.getBandwidthBurst(), + serverDescriptor.getBandwidthObserved()), + serverDescriptor.getBandwidthRate()); + String fingerprint = serverDescriptor.getFingerprint(); + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus == null) { + weightsStatus = new WeightsStatus(); + } + weightsStatus.getAdvertisedBandwidths().put(digest, + advertisedBandwidth); + this.documentStore.store(weightsStatus, fingerprint); +} + + private void updateWeightsHistory(long validAfterMillis, + long freshUntilMillis, + SortedMap<String, double[]> pathSelectionWeights) { + String fingerprint = null; + double[] weights = null; + do { + fingerprint = null; + synchronized (pathSelectionWeights) { + if (!pathSelectionWeights.isEmpty()) { + fingerprint = pathSelectionWeights.firstKey(); + weights = pathSelectionWeights.remove(fingerprint); + } + } + if (fingerprint != null) { + this.addToHistory(fingerprint, validAfterMillis, + freshUntilMillis, weights); + } + } while (fingerprint != null); + } + + private SortedMap<String, double[]> calculatePathSelectionProbabilities( + RelayNetworkStatusConsensus consensus) { + boolean containsBandwidthWeights = false; + double wgg = 1.0, wgd = 1.0, wmg = 1.0, wmm = 1.0, wme = 1.0, + wmd = 1.0, wee = 1.0, wed = 1.0; + SortedMap<String, Integer> bandwidthWeights = + consensus.getBandwidthWeights(); + if (bandwidthWeights != null) { + SortedSet<String> missingWeightKeys = new TreeSet<String>( + Arrays.asList("Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(","))); + missingWeightKeys.removeAll(bandwidthWeights.keySet()); + if (missingWeightKeys.isEmpty()) { + wgg = ((double) bandwidthWeights.get("Wgg")) / 10000.0; + wgd = ((double) bandwidthWeights.get("Wgd")) / 10000.0; + wmg = ((double) bandwidthWeights.get("Wmg")) / 10000.0; + wmm = ((double) bandwidthWeights.get("Wmm")) / 10000.0; + wme = ((double) bandwidthWeights.get("Wme")) / 10000.0; + wmd = ((double) bandwidthWeights.get("Wmd")) / 10000.0; + wee = ((double) bandwidthWeights.get("Wee")) / 10000.0; + wed = ((double) bandwidthWeights.get("Wed")) / 10000.0; + containsBandwidthWeights = true; + } + } + SortedMap<String, Double> + advertisedBandwidths = new TreeMap<String, Double>(), + consensusWeights = new TreeMap<String, Double>(), + guardWeights = new TreeMap<String, Double>(), + middleWeights = new TreeMap<String, Double>(), + exitWeights = new TreeMap<String, Double>(); + double totalAdvertisedBandwidth = 0.0; + double totalConsensusWeight = 0.0; + double totalGuardWeight = 0.0; + double totalMiddleWeight = 0.0; + double totalExitWeight = 0.0; + for (NetworkStatusEntry relay : + consensus.getStatusEntries().values()) { + String fingerprint = relay.getFingerprint(); + if (!relay.getFlags().contains("Running")) { + continue; + } + String digest = relay.getDescriptor().toUpperCase(); + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus != null && + weightsStatus.getAdvertisedBandwidths() != null && + weightsStatus.getAdvertisedBandwidths().containsKey(digest)) { + /* Read advertised bandwidth from weights status file. Server + * descriptors are parsed before consensuses, so we're sure that + * if there's a server descriptor for this relay, it'll be + * contained in the weights status file by now. */ + double advertisedBandwidth = + (double) weightsStatus.getAdvertisedBandwidths().get(digest); + advertisedBandwidths.put(fingerprint, advertisedBandwidth); + totalAdvertisedBandwidth += advertisedBandwidth; + } + if (relay.getBandwidth() >= 0L) { + double consensusWeight = (double) relay.getBandwidth(); + consensusWeights.put(fingerprint, consensusWeight); + totalConsensusWeight += consensusWeight; + if (containsBandwidthWeights) { + double guardWeight = (double) relay.getBandwidth(); + double middleWeight = (double) relay.getBandwidth(); + double exitWeight = (double) relay.getBandwidth(); + boolean isExit = relay.getFlags().contains("Exit") && + !relay.getFlags().contains("BadExit"); + boolean isGuard = relay.getFlags().contains("Guard"); + if (isGuard && isExit) { + guardWeight *= wgd; + middleWeight *= wmd; + exitWeight *= wed; + } else if (isGuard) { + guardWeight *= wgg; + middleWeight *= wmg; + exitWeight = 0.0; + } else if (isExit) { + guardWeight = 0.0; + middleWeight *= wme; + exitWeight *= wee; + } else { + guardWeight = 0.0; + middleWeight *= wmm; + exitWeight = 0.0; + } + guardWeights.put(fingerprint, guardWeight); + middleWeights.put(fingerprint, middleWeight); + exitWeights.put(fingerprint, exitWeight); + totalGuardWeight += guardWeight; + totalMiddleWeight += middleWeight; + totalExitWeight += exitWeight; + } + } + } + SortedMap<String, double[]> pathSelectionProbabilities = + new TreeMap<String, double[]>(); + SortedSet<String> fingerprints = new TreeSet<String>(); + fingerprints.addAll(consensusWeights.keySet()); + fingerprints.addAll(advertisedBandwidths.keySet()); + for (String fingerprint : fingerprints) { + double[] probabilities = new double[] { -1.0, -1.0, -1.0, -1.0, + -1.0, -1.0, -1.0 }; + if (consensusWeights.containsKey(fingerprint) && + totalConsensusWeight > 0.0) { + probabilities[1] = consensusWeights.get(fingerprint) / + totalConsensusWeight; + probabilities[6] = consensusWeights.get(fingerprint); + } + if (guardWeights.containsKey(fingerprint) && + totalGuardWeight > 0.0) { + probabilities[2] = guardWeights.get(fingerprint) / + totalGuardWeight; + } + if (middleWeights.containsKey(fingerprint) && + totalMiddleWeight > 0.0) { + probabilities[3] = middleWeights.get(fingerprint) / + totalMiddleWeight; + } + if (exitWeights.containsKey(fingerprint) && + totalExitWeight > 0.0) { + probabilities[4] = exitWeights.get(fingerprint) / + totalExitWeight; + } + if (advertisedBandwidths.containsKey(fingerprint) && + totalAdvertisedBandwidth > 0.0) { + probabilities[0] = advertisedBandwidths.get(fingerprint) + / totalAdvertisedBandwidth; + probabilities[5] = advertisedBandwidths.get(fingerprint); + } + pathSelectionProbabilities.put(fingerprint, probabilities); + } + return pathSelectionProbabilities; + } + + private void addToHistory(String fingerprint, long validAfterMillis, + long freshUntilMillis, double[] weights) { + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus == null) { + weightsStatus = new WeightsStatus(); + } + SortedMap<long[], double[]> history = weightsStatus.getHistory(); + long[] interval = new long[] { validAfterMillis, freshUntilMillis }; + if ((history.headMap(interval).isEmpty() || + history.headMap(interval).lastKey()[1] <= validAfterMillis) && + (history.tailMap(interval).isEmpty() || + history.tailMap(interval).firstKey()[0] >= freshUntilMillis)) { + history.put(interval, weights); + this.compressHistory(weightsStatus); + this.documentStore.store(weightsStatus, fingerprint); + } + } + + private void compressHistory(WeightsStatus weightsStatus) { + SortedMap<long[], double[]> history = weightsStatus.getHistory(); + SortedMap<long[], double[]> compressedHistory = + new TreeMap<long[], double[]>(history.comparator()); + long lastStartMillis = 0L, lastEndMillis = 0L; + double[] lastWeights = null; + String lastMonthString = "1970-01"; + int lastMissingValues = -1; + for (Map.Entry<long[], double[]> e : history.entrySet()) { + long startMillis = e.getKey()[0], endMillis = e.getKey()[1]; + double[] weights = e.getValue(); + long intervalLengthMillis; + if (this.now - endMillis <= DateTimeHelper.ONE_WEEK) { + intervalLengthMillis = DateTimeHelper.ONE_HOUR; + } else if (this.now - endMillis <= + DateTimeHelper.ROUGHLY_ONE_MONTH) { + intervalLengthMillis = DateTimeHelper.FOUR_HOURS; + } else if (this.now - endMillis <= + DateTimeHelper.ROUGHLY_THREE_MONTHS) { + intervalLengthMillis = DateTimeHelper.TWELVE_HOURS; + } else if (this.now - endMillis <= + DateTimeHelper.ROUGHLY_ONE_YEAR) { + intervalLengthMillis = DateTimeHelper.TWO_DAYS; + } else { + intervalLengthMillis = DateTimeHelper.TEN_DAYS; + } + String monthString = DateTimeHelper.format(startMillis, + DateTimeHelper.ISO_YEARMONTH_FORMAT); + int missingValues = 0; + for (int i = 0; i < weights.length; i++) { + if (weights[i] < -0.5) { + missingValues += 1 << i; + } + } + if (lastEndMillis == startMillis && + ((lastEndMillis - 1L) / intervalLengthMillis) == + ((endMillis - 1L) / intervalLengthMillis) && + lastMonthString.equals(monthString) && + lastMissingValues == missingValues) { + double lastIntervalInHours = (double) ((lastEndMillis + - lastStartMillis) / DateTimeHelper.ONE_HOUR); + double currentIntervalInHours = (double) ((endMillis + - startMillis) / DateTimeHelper.ONE_HOUR); + double newIntervalInHours = (double) ((endMillis + - lastStartMillis) / DateTimeHelper.ONE_HOUR); + for (int i = 0; i < lastWeights.length; i++) { + lastWeights[i] *= lastIntervalInHours; + lastWeights[i] += weights[i] * currentIntervalInHours; + lastWeights[i] /= newIntervalInHours; + } + lastEndMillis = endMillis; + } else { + if (lastStartMillis > 0L) { + compressedHistory.put(new long[] { lastStartMillis, + lastEndMillis }, lastWeights); + } + lastStartMillis = startMillis; + lastEndMillis = endMillis; + lastWeights = weights; + } + lastMonthString = monthString; + lastMissingValues = missingValues; + } + if (lastStartMillis > 0L) { + compressedHistory.put(new long[] { lastStartMillis, lastEndMillis }, + lastWeights); + } + weightsStatus.setHistory(compressedHistory); + } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } +} + diff --git a/src/org/torproject/onionoo/util/ApplicationFactory.java b/src/org/torproject/onionoo/util/ApplicationFactory.java new file mode 100644 index 0000000..8eafca9 --- /dev/null +++ b/src/org/torproject/onionoo/util/ApplicationFactory.java @@ -0,0 +1,55 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.util; + +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.server.NodeIndexer; +import org.torproject.onionoo.updater.DescriptorSource; + +public class ApplicationFactory { + + private static Time timeInstance; + public static void setTime(Time time) { + timeInstance = time; + } + public static Time getTime() { + if (timeInstance == null) { + timeInstance = new Time(); + } + return timeInstance; + } + + private static DescriptorSource descriptorSourceInstance; + public static void setDescriptorSource( + DescriptorSource descriptorSource) { + descriptorSourceInstance = descriptorSource; + } + public static DescriptorSource getDescriptorSource() { + if (descriptorSourceInstance == null) { + descriptorSourceInstance = new DescriptorSource(); + } + return descriptorSourceInstance; + } + + private static DocumentStore documentStoreInstance; + public static void setDocumentStore(DocumentStore documentStore) { + documentStoreInstance = documentStore; + } + public static DocumentStore getDocumentStore() { + if (documentStoreInstance == null) { + documentStoreInstance = new DocumentStore(); + } + return documentStoreInstance; + } + + private static NodeIndexer nodeIndexerInstance; + public static void setNodeIndexer(NodeIndexer nodeIndexer) { + nodeIndexerInstance = nodeIndexer; + } + public static NodeIndexer getNodeIndexer() { + if (nodeIndexerInstance == null) { + nodeIndexerInstance = new NodeIndexer(); + } + return nodeIndexerInstance; + } +} diff --git a/src/org/torproject/onionoo/util/DateTimeHelper.java b/src/org/torproject/onionoo/util/DateTimeHelper.java new file mode 100644 index 0000000..1fcf6e1 --- /dev/null +++ b/src/org/torproject/onionoo/util/DateTimeHelper.java @@ -0,0 +1,92 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.util; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.HashMap; +import java.util.Map; +import java.util.TimeZone; + +public class DateTimeHelper { + + private DateTimeHelper() { + } + + public static final long ONE_SECOND = 1000L, + TEN_SECONDS = 10L * ONE_SECOND, + ONE_MINUTE = 60L * ONE_SECOND, + FIVE_MINUTES = 5L * ONE_MINUTE, + FIFTEEN_MINUTES = 15L * ONE_MINUTE, + ONE_HOUR = 60L * ONE_MINUTE, + FOUR_HOURS = 4L * ONE_HOUR, + SIX_HOURS = 6L * ONE_HOUR, + TWELVE_HOURS = 12L * ONE_HOUR, + ONE_DAY = 24L * ONE_HOUR, + TWO_DAYS = 2L * ONE_DAY, + THREE_DAYS = 3L * ONE_DAY, + ONE_WEEK = 7L * ONE_DAY, + TEN_DAYS = 10L * ONE_DAY, + ROUGHLY_ONE_MONTH = 31L * ONE_DAY, + ROUGHLY_THREE_MONTHS = 92L * ONE_DAY, + ROUGHLY_ONE_YEAR = 366L * ONE_DAY, + ROUGHLY_FIVE_YEARS = 5L * ROUGHLY_ONE_YEAR; + + public static final String ISO_DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; + + public static final String ISO_DATETIME_TAB_FORMAT = + "yyyy-MM-dd\tHH:mm:ss"; + + public static final String ISO_YEARMONTH_FORMAT = "yyyy-MM"; + + public static final String DATEHOUR_NOSPACE_FORMAT = "yyyy-MM-dd-HH"; + + private static ThreadLocal<Map<String, DateFormat>> dateFormats = + new ThreadLocal<Map<String, DateFormat>> () { + public Map<String, DateFormat> get() { + return super.get(); + } + protected Map<String, DateFormat> initialValue() { + return new HashMap<String, DateFormat>(); + } + public void remove() { + super.remove(); + } + public void set(Map<String, DateFormat> value) { + super.set(value); + } + }; + + private static DateFormat getDateFormat(String format) { + Map<String, DateFormat> threadDateFormats = dateFormats.get(); + if (!threadDateFormats.containsKey(format)) { + DateFormat dateFormat = new SimpleDateFormat(format); + dateFormat.setLenient(false); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + threadDateFormats.put(format, dateFormat); + } + return threadDateFormats.get(format); + } + + public static String format(long millis, String format) { + return getDateFormat(format).format(millis); + } + + public static String format(long millis) { + return format(millis, ISO_DATETIME_FORMAT); + } + + public static long parse(String string, String format) { + try { + return getDateFormat(format).parse(string).getTime(); + } catch (ParseException e) { + return -1L; + } + } + + public static long parse(String string) { + return parse(string, ISO_DATETIME_FORMAT); + } +} + diff --git a/src/org/torproject/onionoo/util/LockFile.java b/src/org/torproject/onionoo/util/LockFile.java new file mode 100644 index 0000000..01c4dcb --- /dev/null +++ b/src/org/torproject/onionoo/util/LockFile.java @@ -0,0 +1,43 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.util; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; + +public class LockFile { + + private final File lockFile = new File("lock"); + + public boolean acquireLock() { + Time time = ApplicationFactory.getTime(); + try { + if (this.lockFile.exists()) { + return false; + } + if (this.lockFile.getParentFile() != null) { + this.lockFile.getParentFile().mkdirs(); + } + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.lockFile)); + bw.append("" + time.currentTimeMillis() + "\n"); + bw.close(); + return true; + } catch (IOException e) { + System.err.println("Caught exception while trying to acquire " + + "lock!"); + e.printStackTrace(); + return false; + } + } + + public boolean releaseLock() { + if (this.lockFile.exists()) { + this.lockFile.delete(); + } + return !this.lockFile.exists(); + } +} + diff --git a/src/org/torproject/onionoo/util/Logger.java b/src/org/torproject/onionoo/util/Logger.java new file mode 100644 index 0000000..443c1ca --- /dev/null +++ b/src/org/torproject/onionoo/util/Logger.java @@ -0,0 +1,81 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.util; + +import java.util.Date; + +public class Logger { + + private Logger() { + } + + private static Time time; + + public static void setTime() { + time = ApplicationFactory.getTime(); + } + + private static long currentTimeMillis() { + if (time == null) { + return System.currentTimeMillis(); + } else { + return time.currentTimeMillis(); + } + } + + public static String formatDecimalNumber(long decimalNumber) { + return String.format("%,d", decimalNumber); + } + + public static String formatMillis(long millis) { + return String.format("%02d:%02d.%03d minutes", + millis / DateTimeHelper.ONE_MINUTE, + (millis % DateTimeHelper.ONE_MINUTE) / DateTimeHelper.ONE_SECOND, + millis % DateTimeHelper.ONE_SECOND); + } + + public static String formatBytes(long bytes) { + if (bytes < 1024) { + return bytes + " B"; + } else { + int exp = (int) (Math.log(bytes) / Math.log(1024)); + return String.format("%.1f %siB", bytes / Math.pow(1024, exp), + "KMGTPE".charAt(exp-1)); + } + } + + private static long printedLastStatusMessage = -1L; + + public static void printStatus(String message) { + System.out.println(new Date() + ": " + message); + printedLastStatusMessage = currentTimeMillis(); + } + + public static void printStatistics(String component, String message) { + System.out.print(" " + component + " statistics:\n" + message); + } + + public static void printStatusTime(String message) { + printStatusOrErrorTime(message, false); + } + + public static void printErrorTime(String message) { + printStatusOrErrorTime(message, true); + } + + private static void printStatusOrErrorTime(String message, + boolean printToSystemErr) { + long now = currentTimeMillis(); + long millis = printedLastStatusMessage < 0 ? 0 : + now - printedLastStatusMessage; + String line = " " + message + " (" + Logger.formatMillis(millis) + + ")."; + if (printToSystemErr) { + System.err.println(line); + } else { + System.out.println(line); + } + printedLastStatusMessage = now; + } +} + diff --git a/src/org/torproject/onionoo/util/Time.java b/src/org/torproject/onionoo/util/Time.java new file mode 100644 index 0000000..126a910 --- /dev/null +++ b/src/org/torproject/onionoo/util/Time.java @@ -0,0 +1,14 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.util; + +/* + * Wrapper for System.currentTimeMillis() that can be replaced with a + * custom time source for testing. + */ +public class Time { + public long currentTimeMillis() { + return System.currentTimeMillis(); + } +} + diff --git a/src/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/org/torproject/onionoo/writer/BandwidthDocumentWriter.java new file mode 100644 index 0000000..908ec7c --- /dev/null +++ b/src/org/torproject/onionoo/writer/BandwidthDocumentWriter.java @@ -0,0 +1,201 @@ +/* Copyright 2011--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.writer; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; + +import org.torproject.onionoo.docs.BandwidthDocument; +import org.torproject.onionoo.docs.BandwidthStatus; +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.updater.DescriptorSource; +import org.torproject.onionoo.updater.DescriptorType; +import org.torproject.onionoo.updater.FingerprintListener; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; +import org.torproject.onionoo.util.Logger; + +public class BandwidthDocumentWriter implements FingerprintListener, + DocumentWriter{ + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public BandwidthDocumentWriter() { + this.descriptorSource = ApplicationFactory.getDescriptorSource(); + this.documentStore = ApplicationFactory.getDocumentStore(); + this.now = ApplicationFactory.getTime().currentTimeMillis(); + this.registerFingerprintListeners(); + } + + private void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_EXTRA_INFOS); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_EXTRA_INFOS); + } + + private Set<String> updateBandwidthDocuments = new HashSet<String>(); + + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + this.updateBandwidthDocuments.addAll(fingerprints); + } + + public void writeDocuments() { + for (String fingerprint : this.updateBandwidthDocuments) { + BandwidthStatus bandwidthStatus = this.documentStore.retrieve( + BandwidthStatus.class, true, fingerprint); + if (bandwidthStatus == null) { + continue; + } + BandwidthDocument bandwidthDocument = this.compileBandwidthDocument( + fingerprint, bandwidthStatus); + this.documentStore.store(bandwidthDocument, fingerprint); + } + Logger.printStatusTime("Wrote bandwidth document files"); + } + + + private BandwidthDocument compileBandwidthDocument(String fingerprint, + BandwidthStatus bandwidthStatus) { + BandwidthDocument bandwidthDocument = new BandwidthDocument(); + bandwidthDocument.setFingerprint(fingerprint); + bandwidthDocument.setWriteHistory(this.compileGraphType( + bandwidthStatus.getWriteHistory())); + bandwidthDocument.setReadHistory(this.compileGraphType( + bandwidthStatus.getReadHistory())); + return bandwidthDocument; + } + + private String[] graphNames = new String[] { + "3_days", + "1_week", + "1_month", + "3_months", + "1_year", + "5_years" }; + + private long[] graphIntervals = new long[] { + DateTimeHelper.THREE_DAYS, + DateTimeHelper.ONE_WEEK, + DateTimeHelper.ROUGHLY_ONE_MONTH, + DateTimeHelper.ROUGHLY_THREE_MONTHS, + DateTimeHelper.ROUGHLY_ONE_YEAR, + DateTimeHelper.ROUGHLY_FIVE_YEARS }; + + private long[] dataPointIntervals = new long[] { + DateTimeHelper.FIFTEEN_MINUTES, + DateTimeHelper.ONE_HOUR, + DateTimeHelper.FOUR_HOURS, + DateTimeHelper.TWELVE_HOURS, + DateTimeHelper.TWO_DAYS, + DateTimeHelper.TEN_DAYS }; + + private Map<String, GraphHistory> compileGraphType( + SortedMap<Long, long[]> history) { + Map<String, GraphHistory> graphs = + new LinkedHashMap<String, GraphHistory>(); + for (int i = 0; i < this.graphIntervals.length; i++) { + String graphName = this.graphNames[i]; + long graphInterval = this.graphIntervals[i]; + long dataPointInterval = this.dataPointIntervals[i]; + List<Long> dataPoints = new ArrayList<Long>(); + long intervalStartMillis = ((this.now - graphInterval) + / dataPointInterval) * dataPointInterval; + long totalMillis = 0L, totalBandwidth = 0L; + for (long[] v : history.values()) { + long startMillis = v[0], endMillis = v[1], bandwidth = v[2]; + if (endMillis < intervalStartMillis) { + continue; + } + while ((intervalStartMillis / dataPointInterval) != + (endMillis / dataPointInterval)) { + dataPoints.add(totalMillis * 5L < dataPointInterval + ? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND) + / totalMillis); + totalBandwidth = 0L; + totalMillis = 0L; + intervalStartMillis += dataPointInterval; + } + totalBandwidth += bandwidth; + totalMillis += (endMillis - startMillis); + } + dataPoints.add(totalMillis * 5L < dataPointInterval + ? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND) + / totalMillis); + long maxValue = 1L; + int firstNonNullIndex = -1, lastNonNullIndex = -1; + for (int j = 0; j < dataPoints.size(); j++) { + long dataPoint = dataPoints.get(j); + if (dataPoint >= 0L) { + if (firstNonNullIndex < 0) { + firstNonNullIndex = j; + } + lastNonNullIndex = j; + if (dataPoint > maxValue) { + maxValue = dataPoint; + } + } + } + if (firstNonNullIndex < 0) { + continue; + } + long firstDataPointMillis = (((this.now - graphInterval) + / dataPointInterval) + firstNonNullIndex) * dataPointInterval + + dataPointInterval / 2L; + if (i > 0 && + firstDataPointMillis >= this.now - graphIntervals[i - 1]) { + /* Skip bandwidth history object, because it doesn't contain + * anything new that wasn't already contained in the last + * bandwidth history object(s). */ + continue; + } + long lastDataPointMillis = firstDataPointMillis + + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; + double factor = ((double) maxValue) / 999.0; + int count = lastNonNullIndex - firstNonNullIndex + 1; + GraphHistory graphHistory = new GraphHistory(); + graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis)); + graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis)); + graphHistory.setInterval((int) (dataPointInterval + / DateTimeHelper.ONE_SECOND)); + graphHistory.setFactor(factor); + graphHistory.setCount(count); + int previousNonNullIndex = -2; + boolean foundTwoAdjacentDataPoints = false; + List<Integer> values = new ArrayList<Integer>(); + for (int j = firstNonNullIndex; j <= lastNonNullIndex; j++) { + long dataPoint = dataPoints.get(j); + if (dataPoint >= 0L) { + if (j - previousNonNullIndex == 1) { + foundTwoAdjacentDataPoints = true; + } + previousNonNullIndex = j; + } + values.add(dataPoint < 0L ? null : + (int) ((dataPoint * 999L) / maxValue)); + } + graphHistory.setValues(values); + if (foundTwoAdjacentDataPoints) { + graphs.put(graphName, graphHistory); + } + } + return graphs; + } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } +} diff --git a/src/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/org/torproject/onionoo/writer/ClientsDocumentWriter.java new file mode 100644 index 0000000..976804c --- /dev/null +++ b/src/org/torproject/onionoo/writer/ClientsDocumentWriter.java @@ -0,0 +1,296 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.writer; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +import org.torproject.onionoo.docs.ClientsDocument; +import org.torproject.onionoo.docs.ClientsGraphHistory; +import org.torproject.onionoo.docs.ClientsHistory; +import org.torproject.onionoo.docs.ClientsStatus; +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.updater.DescriptorSource; +import org.torproject.onionoo.updater.DescriptorType; +import org.torproject.onionoo.updater.FingerprintListener; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; +import org.torproject.onionoo.util.Logger; + +/* + * Clients status file produced as intermediate output: + * + * 2014-02-15 16:42:11 2014-02-16 00:00:00 + * 259.042 in=86.347,se=86.347 v4=259.042 + * 2014-02-16 00:00:00 2014-02-16 16:42:11 + * 592.958 in=197.653,se=197.653 v4=592.958 + * + * Clients document file produced as output: + * + * "1_month":{ + * "first":"2014-02-03 12:00:00", + * "last":"2014-02-28 12:00:00", + * "interval":86400, + * "factor":0.139049349, + * "count":26, + * "values":[371,354,349,374,432,null,485,458,493,536,null,null,524,576, + * 607,622,null,635,null,566,774,999,945,690,656,681], + * "countries":{"cn":0.0192,"in":0.1768,"ir":0.2487,"ru":0.0104, + * "se":0.1698,"sy":0.0325,"us":0.0406}, + * "transports":{"obfs2":0.4581}, + * "versions":{"v4":1.0000}} + */ +public class ClientsDocumentWriter implements FingerprintListener, + DocumentWriter { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public ClientsDocumentWriter() { + this.descriptorSource = ApplicationFactory.getDescriptorSource(); + this.documentStore = ApplicationFactory.getDocumentStore(); + this.now = ApplicationFactory.getTime().currentTimeMillis(); + this.registerFingerprintListeners(); + } + + private void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_EXTRA_INFOS); + } + + private SortedSet<String> updateDocuments = new TreeSet<String>(); + + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + if (!relay) { + this.updateDocuments.addAll(fingerprints); + } + } + + private int writtenDocuments = 0; + + public void writeDocuments() { + for (String hashedFingerprint : this.updateDocuments) { + ClientsStatus clientsStatus = this.documentStore.retrieve( + ClientsStatus.class, true, hashedFingerprint); + if (clientsStatus == null) { + continue; + } + SortedSet<ClientsHistory> history = clientsStatus.getHistory(); + ClientsDocument clientsDocument = this.compileClientsDocument( + hashedFingerprint, history); + this.documentStore.store(clientsDocument, hashedFingerprint); + this.writtenDocuments++; + } + Logger.printStatusTime("Wrote clients document files"); + } + + private String[] graphNames = new String[] { + "1_week", + "1_month", + "3_months", + "1_year", + "5_years" }; + + private long[] graphIntervals = new long[] { + DateTimeHelper.ONE_WEEK, + DateTimeHelper.ROUGHLY_ONE_MONTH, + DateTimeHelper.ROUGHLY_THREE_MONTHS, + DateTimeHelper.ROUGHLY_ONE_YEAR, + DateTimeHelper.ROUGHLY_FIVE_YEARS }; + + private long[] dataPointIntervals = new long[] { + DateTimeHelper.ONE_DAY, + DateTimeHelper.ONE_DAY, + DateTimeHelper.ONE_DAY, + DateTimeHelper.TWO_DAYS, + DateTimeHelper.TEN_DAYS }; + + private ClientsDocument compileClientsDocument(String hashedFingerprint, + SortedSet<ClientsHistory> history) { + ClientsDocument clientsDocument = new ClientsDocument(); + clientsDocument.setFingerprint(hashedFingerprint); + Map<String, ClientsGraphHistory> averageClients = + new LinkedHashMap<String, ClientsGraphHistory>(); + for (int graphIntervalIndex = 0; graphIntervalIndex < + this.graphIntervals.length; graphIntervalIndex++) { + String graphName = this.graphNames[graphIntervalIndex]; + ClientsGraphHistory graphHistory = this.compileClientsHistory( + graphIntervalIndex, history); + if (graphHistory != null) { + averageClients.put(graphName, graphHistory); + } + } + clientsDocument.setAverageClients(averageClients); + return clientsDocument; + } + + private ClientsGraphHistory compileClientsHistory( + int graphIntervalIndex, SortedSet<ClientsHistory> history) { + long graphInterval = this.graphIntervals[graphIntervalIndex]; + long dataPointInterval = + this.dataPointIntervals[graphIntervalIndex]; + List<Double> dataPoints = new ArrayList<Double>(); + long intervalStartMillis = ((this.now - graphInterval) + / dataPointInterval) * dataPointInterval; + long millis = 0L; + double responses = 0.0, totalResponses = 0.0; + SortedMap<String, Double> + totalResponsesByCountry = new TreeMap<String, Double>(), + totalResponsesByTransport = new TreeMap<String, Double>(), + totalResponsesByVersion = new TreeMap<String, Double>(); + for (ClientsHistory hist : history) { + if (hist.getEndMillis() < intervalStartMillis) { + continue; + } + while ((intervalStartMillis / dataPointInterval) != + (hist.getEndMillis() / dataPointInterval)) { + dataPoints.add(millis * 2L < dataPointInterval + ? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY) + / (((double) millis) * 10.0)); + responses = 0.0; + millis = 0L; + intervalStartMillis += dataPointInterval; + } + responses += hist.getTotalResponses(); + totalResponses += hist.getTotalResponses(); + for (Map.Entry<String, Double> e : + hist.getResponsesByCountry().entrySet()) { + if (!totalResponsesByCountry.containsKey(e.getKey())) { + totalResponsesByCountry.put(e.getKey(), 0.0); + } + totalResponsesByCountry.put(e.getKey(), e.getValue() + + totalResponsesByCountry.get(e.getKey())); + } + for (Map.Entry<String, Double> e : + hist.getResponsesByTransport().entrySet()) { + if (!totalResponsesByTransport.containsKey(e.getKey())) { + totalResponsesByTransport.put(e.getKey(), 0.0); + } + totalResponsesByTransport.put(e.getKey(), e.getValue() + + totalResponsesByTransport.get(e.getKey())); + } + for (Map.Entry<String, Double> e : + hist.getResponsesByVersion().entrySet()) { + if (!totalResponsesByVersion.containsKey(e.getKey())) { + totalResponsesByVersion.put(e.getKey(), 0.0); + } + totalResponsesByVersion.put(e.getKey(), e.getValue() + + totalResponsesByVersion.get(e.getKey())); + } + millis += (hist.getEndMillis() - hist.getStartMillis()); + } + dataPoints.add(millis * 2L < dataPointInterval + ? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY) + / (((double) millis) * 10.0)); + double maxValue = 0.0; + int firstNonNullIndex = -1, lastNonNullIndex = -1; + for (int dataPointIndex = 0; dataPointIndex < dataPoints.size(); + dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (firstNonNullIndex < 0) { + firstNonNullIndex = dataPointIndex; + } + lastNonNullIndex = dataPointIndex; + if (dataPoint > maxValue) { + maxValue = dataPoint; + } + } + } + if (firstNonNullIndex < 0) { + return null; + } + long firstDataPointMillis = (((this.now - graphInterval) + / dataPointInterval) + firstNonNullIndex) * dataPointInterval + + dataPointInterval / 2L; + if (graphIntervalIndex > 0 && firstDataPointMillis >= + this.now - graphIntervals[graphIntervalIndex - 1]) { + /* Skip clients history object, because it doesn't contain + * anything new that wasn't already contained in the last + * clients history object(s). */ + return null; + } + long lastDataPointMillis = firstDataPointMillis + + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; + double factor = ((double) maxValue) / 999.0; + int count = lastNonNullIndex - firstNonNullIndex + 1; + ClientsGraphHistory graphHistory = new ClientsGraphHistory(); + graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis)); + graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis)); + graphHistory.setInterval((int) (dataPointInterval + / DateTimeHelper.ONE_SECOND)); + graphHistory.setFactor(factor); + graphHistory.setCount(count); + int previousNonNullIndex = -2; + boolean foundTwoAdjacentDataPoints = false; + List<Integer> values = new ArrayList<Integer>(); + for (int dataPointIndex = firstNonNullIndex; dataPointIndex <= + lastNonNullIndex; dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (dataPointIndex - previousNonNullIndex == 1) { + foundTwoAdjacentDataPoints = true; + } + previousNonNullIndex = dataPointIndex; + } + values.add(dataPoint < 0.0 ? null : + (int) ((dataPoint * 999.0) / maxValue)); + } + graphHistory.setValues(values); + if (!totalResponsesByCountry.isEmpty()) { + SortedMap<String, Float> countries = new TreeMap<String, Float>(); + for (Map.Entry<String, Double> e : + totalResponsesByCountry.entrySet()) { + if (e.getValue() > totalResponses / 100.0) { + countries.put(e.getKey(), + (float) (e.getValue() / totalResponses)); + } + } + graphHistory.setCountries(countries); + } + if (!totalResponsesByTransport.isEmpty()) { + SortedMap<String, Float> transports = new TreeMap<String, Float>(); + for (Map.Entry<String, Double> e : + totalResponsesByTransport.entrySet()) { + if (e.getValue() > totalResponses / 100.0) { + transports.put(e.getKey(), + (float) (e.getValue() / totalResponses)); + } + } + graphHistory.setTransports(transports); + } + if (!totalResponsesByVersion.isEmpty()) { + SortedMap<String, Float> versions = new TreeMap<String, Float>(); + for (Map.Entry<String, Double> e : + totalResponsesByVersion.entrySet()) { + if (e.getValue() > totalResponses / 100.0) { + versions.put(e.getKey(), + (float) (e.getValue() / totalResponses)); + } + } + graphHistory.setVersions(versions); + } + if (foundTwoAdjacentDataPoints) { + return graphHistory; + } else { + return null; + } + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments) + + " clients document files updated\n"); + return sb.toString(); + } +} diff --git a/src/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/org/torproject/onionoo/writer/DetailsDocumentWriter.java new file mode 100644 index 0000000..03f7024 --- /dev/null +++ b/src/org/torproject/onionoo/writer/DetailsDocumentWriter.java @@ -0,0 +1,233 @@ +package org.torproject.onionoo.writer; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.torproject.onionoo.docs.DetailsDocument; +import org.torproject.onionoo.docs.DetailsStatus; +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.docs.NodeStatus; +import org.torproject.onionoo.updater.DescriptorSource; +import org.torproject.onionoo.updater.DescriptorType; +import org.torproject.onionoo.updater.FingerprintListener; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; +import org.torproject.onionoo.util.Logger; + +public class DetailsDocumentWriter implements FingerprintListener, + DocumentWriter { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public DetailsDocumentWriter() { + this.descriptorSource = ApplicationFactory.getDescriptorSource(); + this.documentStore = ApplicationFactory.getDocumentStore(); + this.now = ApplicationFactory.getTime().currentTimeMillis(); + this.registerFingerprintListeners(); + } + + private void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_SERVER_DESCRIPTORS); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_STATUSES); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_SERVER_DESCRIPTORS); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_POOL_ASSIGNMENTS); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.EXIT_LISTS); + } + + private SortedSet<String> newRelays = new TreeSet<String>(), + newBridges = new TreeSet<String>(); + + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + if (relay) { + this.newRelays.addAll(fingerprints); + } else { + this.newBridges.addAll(fingerprints); + } + } + + public void writeDocuments() { + this.updateRelayDetailsFiles(); + this.updateBridgeDetailsFiles(); + Logger.printStatusTime("Wrote details document files"); + } + + private void updateRelayDetailsFiles() { + for (String fingerprint : this.newRelays) { + + /* Generate network-status-specific part. */ + NodeStatus entry = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (entry == null) { + continue; + } + DetailsDocument detailsDocument = new DetailsDocument(); + detailsDocument.setNickname(entry.getNickname()); + detailsDocument.setFingerprint(fingerprint); + List<String> orAddresses = new ArrayList<String>(); + orAddresses.add(entry.getAddress() + ":" + entry.getOrPort()); + for (String orAddress : entry.getOrAddressesAndPorts()) { + orAddresses.add(orAddress.toLowerCase()); + } + detailsDocument.setOrAddresses(orAddresses); + if (entry.getDirPort() != 0) { + detailsDocument.setDirAddress(entry.getAddress() + ":" + + entry.getDirPort()); + } + detailsDocument.setLastSeen(DateTimeHelper.format( + entry.getLastSeenMillis())); + detailsDocument.setFirstSeen(DateTimeHelper.format( + entry.getFirstSeenMillis())); + detailsDocument.setLastChangedAddressOrPort( + DateTimeHelper.format(entry.getLastChangedOrAddress())); + detailsDocument.setRunning(entry.getRunning()); + if (!entry.getRelayFlags().isEmpty()) { + detailsDocument.setFlags(new ArrayList<String>( + entry.getRelayFlags())); + } + detailsDocument.setCountry(entry.getCountryCode()); + detailsDocument.setLatitude(entry.getLatitude()); + detailsDocument.setLongitude(entry.getLongitude()); + detailsDocument.setCountryName(entry.getCountryName()); + detailsDocument.setRegionName(entry.getRegionName()); + detailsDocument.setCityName(entry.getCityName()); + detailsDocument.setAsNumber(entry.getASNumber()); + detailsDocument.setAsName(entry.getASName()); + detailsDocument.setConsensusWeight(entry.getConsensusWeight()); + detailsDocument.setHostName(entry.getHostName()); + detailsDocument.setAdvertisedBandwidthFraction( + (float) entry.getAdvertisedBandwidthFraction()); + detailsDocument.setConsensusWeightFraction( + (float) entry.getConsensusWeightFraction()); + detailsDocument.setGuardProbability( + (float) entry.getGuardProbability()); + detailsDocument.setMiddleProbability( + (float) entry.getMiddleProbability()); + detailsDocument.setExitProbability( + (float) entry.getExitProbability()); + String defaultPolicy = entry.getDefaultPolicy(); + String portList = entry.getPortList(); + if (defaultPolicy != null && (defaultPolicy.equals("accept") || + defaultPolicy.equals("reject")) && portList != null) { + Map<String, List<String>> exitPolicySummary = + new HashMap<String, List<String>>(); + List<String> portsOrPortRanges = Arrays.asList( + portList.split(",")); + exitPolicySummary.put(defaultPolicy, portsOrPortRanges); + detailsDocument.setExitPolicySummary(exitPolicySummary); + } + detailsDocument.setRecommendedVersion( + entry.getRecommendedVersion()); + + /* Append descriptor-specific part and exit addresses from details + * status file. */ + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + if (detailsStatus != null) { + detailsDocument.setLastRestarted( + detailsStatus.getLastRestarted()); + detailsDocument.setBandwidthRate( + detailsStatus.getBandwidthRate()); + detailsDocument.setBandwidthBurst( + detailsStatus.getBandwidthBurst()); + detailsDocument.setObservedBandwidth( + detailsStatus.getObservedBandwidth()); + detailsDocument.setAdvertisedBandwidth( + detailsStatus.getAdvertisedBandwidth()); + detailsDocument.setExitPolicy(detailsStatus.getExitPolicy()); + detailsDocument.setContact(detailsStatus.getContact()); + detailsDocument.setPlatform(detailsStatus.getPlatform()); + detailsDocument.setFamily(detailsStatus.getFamily()); + detailsDocument.setExitPolicyV6Summary( + detailsStatus.getExitPolicyV6Summary()); + detailsDocument.setHibernating(detailsStatus.getHibernating()); + if (detailsStatus.getExitAddresses() != null) { + SortedSet<String> exitAddresses = new TreeSet<String>(); + for (Map.Entry<String, Long> e : + detailsStatus.getExitAddresses().entrySet()) { + String exitAddress = e.getKey().toLowerCase(); + long scanMillis = e.getValue(); + if (!entry.getAddress().equals(exitAddress) && + !entry.getOrAddresses().contains(exitAddress) && + scanMillis >= this.now - DateTimeHelper.ONE_DAY) { + exitAddresses.add(exitAddress); + } + } + if (!exitAddresses.isEmpty()) { + detailsDocument.setExitAddresses(new ArrayList<String>( + exitAddresses)); + } + } + } + + /* Write details file to disk. */ + this.documentStore.store(detailsDocument, fingerprint); + } + } + + private void updateBridgeDetailsFiles() { + for (String fingerprint : this.newBridges) { + + /* Generate network-status-specific part. */ + NodeStatus entry = this.documentStore.retrieve(NodeStatus.class, + true, fingerprint); + if (entry == null) { + continue; + } + DetailsDocument detailsDocument = new DetailsDocument(); + detailsDocument.setNickname(entry.getNickname()); + detailsDocument.setHashedFingerprint(fingerprint); + String address = entry.getAddress(); + List<String> orAddresses = new ArrayList<String>(); + orAddresses.add(address + ":" + entry.getOrPort()); + for (String orAddress : entry.getOrAddressesAndPorts()) { + orAddresses.add(orAddress.toLowerCase()); + } + detailsDocument.setOrAddresses(orAddresses); + detailsDocument.setLastSeen(DateTimeHelper.format( + entry.getLastSeenMillis())); + detailsDocument.setFirstSeen(DateTimeHelper.format( + entry.getFirstSeenMillis())); + detailsDocument.setRunning(entry.getRunning()); + detailsDocument.setFlags(new ArrayList<String>( + entry.getRelayFlags())); + + /* Append descriptor-specific part from details status file. */ + DetailsStatus detailsStatus = this.documentStore.retrieve( + DetailsStatus.class, true, fingerprint); + if (detailsStatus != null) { + detailsDocument.setLastRestarted( + detailsStatus.getLastRestarted()); + detailsDocument.setAdvertisedBandwidth( + detailsStatus.getAdvertisedBandwidth()); + detailsDocument.setPlatform(detailsStatus.getPlatform()); + detailsDocument.setPoolAssignment( + detailsStatus.getPoolAssignment()); + } + + /* Write details file to disk. */ + this.documentStore.store(detailsDocument, fingerprint); + } + } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } +} diff --git a/src/org/torproject/onionoo/writer/DocumentWriter.java b/src/org/torproject/onionoo/writer/DocumentWriter.java new file mode 100644 index 0000000..c238170 --- /dev/null +++ b/src/org/torproject/onionoo/writer/DocumentWriter.java @@ -0,0 +1,11 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.writer; + +public interface DocumentWriter { + + public abstract void writeDocuments(); + + public abstract String getStatsString(); +} + diff --git a/src/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/org/torproject/onionoo/writer/SummaryDocumentWriter.java new file mode 100644 index 0000000..1b4630e --- /dev/null +++ b/src/org/torproject/onionoo/writer/SummaryDocumentWriter.java @@ -0,0 +1,94 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.writer; + +import java.util.ArrayList; +import java.util.List; +import java.util.SortedSet; + +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.docs.NodeStatus; +import org.torproject.onionoo.docs.SummaryDocument; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; +import org.torproject.onionoo.util.Logger; + +public class SummaryDocumentWriter implements DocumentWriter { + + private DocumentStore documentStore; + + public SummaryDocumentWriter() { + this.documentStore = ApplicationFactory.getDocumentStore(); + } + + private int writtenDocuments = 0, deletedDocuments = 0; + + public void writeDocuments() { + long maxLastSeenMillis = 0L; + for (String fingerprint : this.documentStore.list(NodeStatus.class)) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, true, fingerprint); + if (nodeStatus != null && + nodeStatus.getLastSeenMillis() > maxLastSeenMillis) { + maxLastSeenMillis = nodeStatus.getLastSeenMillis(); + } + } + long cutoff = maxLastSeenMillis - DateTimeHelper.ONE_WEEK; + for (String fingerprint : this.documentStore.list(NodeStatus.class)) { + NodeStatus nodeStatus = this.documentStore.retrieve( + NodeStatus.class, + true, fingerprint); + if (nodeStatus == null) { + continue; + } + if (nodeStatus.getLastSeenMillis() < cutoff) { + if (this.documentStore.remove(SummaryDocument.class, + fingerprint)) { + this.deletedDocuments++; + } + continue; + } + boolean isRelay = nodeStatus.isRelay(); + String nickname = nodeStatus.getNickname(); + List<String> addresses = new ArrayList<String>(); + addresses.add(nodeStatus.getAddress()); + for (String orAddress : nodeStatus.getOrAddresses()) { + if (!addresses.contains(orAddress)) { + addresses.add(orAddress); + } + } + for (String exitAddress : nodeStatus.getExitAddresses()) { + if (!addresses.contains(exitAddress)) { + addresses.add(exitAddress); + } + } + long lastSeenMillis = nodeStatus.getLastSeenMillis(); + boolean running = nodeStatus.getRunning(); + SortedSet<String> relayFlags = nodeStatus.getRelayFlags(); + long consensusWeight = nodeStatus.getConsensusWeight(); + String countryCode = nodeStatus.getCountryCode(); + long firstSeenMillis = nodeStatus.getFirstSeenMillis(); + String aSNumber = nodeStatus.getASNumber(); + String contact = nodeStatus.getContact(); + SortedSet<String> familyFingerprints = + nodeStatus.getFamilyFingerprints(); + SummaryDocument summaryDocument = new SummaryDocument(isRelay, + nickname, fingerprint, addresses, lastSeenMillis, running, + relayFlags, consensusWeight, countryCode, firstSeenMillis, + aSNumber, contact, familyFingerprints); + if (this.documentStore.store(summaryDocument, fingerprint)) { + this.writtenDocuments++; + }; + } + Logger.printStatusTime("Wrote summary document files"); + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments) + + " summary document files written\n"); + sb.append(" " + Logger.formatDecimalNumber(this.deletedDocuments) + + " summary document files deleted\n"); + return sb.toString(); + } +} diff --git a/src/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/org/torproject/onionoo/writer/UptimeDocumentWriter.java new file mode 100644 index 0000000..3e04abb --- /dev/null +++ b/src/org/torproject/onionoo/writer/UptimeDocumentWriter.java @@ -0,0 +1,303 @@ +/* Copyright 2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.writer; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; + +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.UptimeDocument; +import org.torproject.onionoo.docs.UptimeHistory; +import org.torproject.onionoo.docs.UptimeStatus; +import org.torproject.onionoo.updater.DescriptorSource; +import org.torproject.onionoo.updater.DescriptorType; +import org.torproject.onionoo.updater.FingerprintListener; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; +import org.torproject.onionoo.util.Logger; + +public class UptimeDocumentWriter implements FingerprintListener, + DocumentWriter { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public UptimeDocumentWriter() { + this.descriptorSource = ApplicationFactory.getDescriptorSource(); + this.documentStore = ApplicationFactory.getDocumentStore(); + this.now = ApplicationFactory.getTime().currentTimeMillis(); + this.registerFingerprintListeners(); + } + + private void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.BRIDGE_STATUSES); + } + + private SortedSet<String> newRelayFingerprints = new TreeSet<String>(), + newBridgeFingerprints = new TreeSet<String>(); + + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + if (relay) { + this.newRelayFingerprints.addAll(fingerprints); + } else { + this.newBridgeFingerprints.addAll(fingerprints); + } + } + + public void writeDocuments() { + UptimeStatus uptimeStatus = this.documentStore.retrieve( + UptimeStatus.class, true); + if (uptimeStatus == null) { + return; + } + for (String fingerprint : this.newRelayFingerprints) { + this.updateDocument(true, fingerprint, + uptimeStatus.getRelayHistory()); + } + for (String fingerprint : this.newBridgeFingerprints) { + this.updateDocument(false, fingerprint, + uptimeStatus.getBridgeHistory()); + } + Logger.printStatusTime("Wrote uptime document files"); + } + + private int writtenDocuments = 0; + + private void updateDocument(boolean relay, String fingerprint, + SortedSet<UptimeHistory> knownStatuses) { + UptimeStatus uptimeStatus = this.documentStore.retrieve( + UptimeStatus.class, true, fingerprint); + if (uptimeStatus != null) { + SortedSet<UptimeHistory> history = relay + ? uptimeStatus.getRelayHistory() + : uptimeStatus.getBridgeHistory(); + UptimeDocument uptimeDocument = this.compileUptimeDocument(relay, + fingerprint, history, knownStatuses); + this.documentStore.store(uptimeDocument, fingerprint); + this.writtenDocuments++; + } + } + + private String[] graphNames = new String[] { + "1_week", + "1_month", + "3_months", + "1_year", + "5_years" }; + + private long[] graphIntervals = new long[] { + DateTimeHelper.ONE_WEEK, + DateTimeHelper.ROUGHLY_ONE_MONTH, + DateTimeHelper.ROUGHLY_THREE_MONTHS, + DateTimeHelper.ROUGHLY_ONE_YEAR, + DateTimeHelper.ROUGHLY_FIVE_YEARS }; + + private long[] dataPointIntervals = new long[] { + DateTimeHelper.ONE_HOUR, + DateTimeHelper.FOUR_HOURS, + DateTimeHelper.TWELVE_HOURS, + DateTimeHelper.TWO_DAYS, + DateTimeHelper.TEN_DAYS }; + + private UptimeDocument compileUptimeDocument(boolean relay, + String fingerprint, SortedSet<UptimeHistory> history, + SortedSet<UptimeHistory> knownStatuses) { + UptimeDocument uptimeDocument = new UptimeDocument(); + uptimeDocument.setFingerprint(fingerprint); + Map<String, GraphHistory> uptime = + new LinkedHashMap<String, GraphHistory>(); + for (int graphIntervalIndex = 0; graphIntervalIndex < + this.graphIntervals.length; graphIntervalIndex++) { + String graphName = this.graphNames[graphIntervalIndex]; + GraphHistory graphHistory = this.compileUptimeHistory( + graphIntervalIndex, relay, history, knownStatuses); + if (graphHistory != null) { + uptime.put(graphName, graphHistory); + } + } + uptimeDocument.setUptime(uptime); + return uptimeDocument; + } + + private GraphHistory compileUptimeHistory(int graphIntervalIndex, + boolean relay, SortedSet<UptimeHistory> history, + SortedSet<UptimeHistory> knownStatuses) { + long graphInterval = this.graphIntervals[graphIntervalIndex]; + long dataPointInterval = + this.dataPointIntervals[graphIntervalIndex]; + int dataPointIntervalHours = (int) (dataPointInterval + / DateTimeHelper.ONE_HOUR); + List<Integer> uptimeDataPoints = new ArrayList<Integer>(); + long intervalStartMillis = ((this.now - graphInterval) + / dataPointInterval) * dataPointInterval; + int uptimeHours = 0; + long firstStatusStartMillis = -1L; + for (UptimeHistory hist : history) { + if (hist.isRelay() != relay) { + continue; + } + if (firstStatusStartMillis < 0L) { + firstStatusStartMillis = hist.getStartMillis(); + } + long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR + * hist.getUptimeHours(); + if (histEndMillis < intervalStartMillis) { + continue; + } + while (hist.getStartMillis() >= intervalStartMillis + + dataPointInterval) { + if (firstStatusStartMillis < intervalStartMillis + + dataPointInterval) { + uptimeDataPoints.add(uptimeHours); + } else { + uptimeDataPoints.add(-1); + } + uptimeHours = 0; + intervalStartMillis += dataPointInterval; + } + while (histEndMillis >= intervalStartMillis + dataPointInterval) { + uptimeHours += (int) ((intervalStartMillis + dataPointInterval + - Math.max(hist.getStartMillis(), intervalStartMillis)) + / DateTimeHelper.ONE_HOUR); + uptimeDataPoints.add(uptimeHours); + uptimeHours = 0; + intervalStartMillis += dataPointInterval; + } + uptimeHours += (int) ((histEndMillis - Math.max( + hist.getStartMillis(), intervalStartMillis)) + / DateTimeHelper.ONE_HOUR); + } + uptimeDataPoints.add(uptimeHours); + List<Integer> statusDataPoints = new ArrayList<Integer>(); + intervalStartMillis = ((this.now - graphInterval) + / dataPointInterval) * dataPointInterval; + int statusHours = -1; + for (UptimeHistory hist : knownStatuses) { + if (hist.isRelay() != relay) { + continue; + } + long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR + * hist.getUptimeHours(); + if (histEndMillis < intervalStartMillis) { + continue; + } + while (hist.getStartMillis() >= intervalStartMillis + + dataPointInterval) { + statusDataPoints.add(statusHours * 5 > dataPointIntervalHours + ? statusHours : -1); + statusHours = -1; + intervalStartMillis += dataPointInterval; + } + while (histEndMillis >= intervalStartMillis + dataPointInterval) { + if (statusHours < 0) { + statusHours = 0; + } + statusHours += (int) ((intervalStartMillis + dataPointInterval + - Math.max(Math.max(hist.getStartMillis(), + firstStatusStartMillis), intervalStartMillis)) + / DateTimeHelper.ONE_HOUR); + statusDataPoints.add(statusHours * 5 > dataPointIntervalHours + ? statusHours : -1); + statusHours = -1; + intervalStartMillis += dataPointInterval; + } + if (statusHours < 0) { + statusHours = 0; + } + statusHours += (int) ((histEndMillis - Math.max(Math.max( + hist.getStartMillis(), firstStatusStartMillis), + intervalStartMillis)) / DateTimeHelper.ONE_HOUR); + } + if (statusHours > 0) { + statusDataPoints.add(statusHours * 5 > dataPointIntervalHours + ? statusHours : -1); + } + List<Double> dataPoints = new ArrayList<Double>(); + for (int dataPointIndex = 0; dataPointIndex < statusDataPoints.size(); + dataPointIndex++) { + if (dataPointIndex >= uptimeDataPoints.size()) { + dataPoints.add(0.0); + } else if (uptimeDataPoints.get(dataPointIndex) >= 0 && + statusDataPoints.get(dataPointIndex) > 0) { + dataPoints.add(((double) uptimeDataPoints.get(dataPointIndex)) + / ((double) statusDataPoints.get(dataPointIndex))); + } else { + dataPoints.add(-1.0); + } + } + int firstNonNullIndex = -1, lastNonNullIndex = -1; + for (int dataPointIndex = 0; dataPointIndex < dataPoints.size(); + dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (firstNonNullIndex < 0) { + firstNonNullIndex = dataPointIndex; + } + lastNonNullIndex = dataPointIndex; + } + } + if (firstNonNullIndex < 0) { + return null; + } + long firstDataPointMillis = (((this.now - graphInterval) + / dataPointInterval) + firstNonNullIndex) + * dataPointInterval + dataPointInterval / 2L; + if (graphIntervalIndex > 0 && firstDataPointMillis >= + this.now - graphIntervals[graphIntervalIndex - 1]) { + /* Skip uptime history object, because it doesn't contain + * anything new that wasn't already contained in the last + * uptime history object(s). */ + return null; + } + long lastDataPointMillis = firstDataPointMillis + + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; + int count = lastNonNullIndex - firstNonNullIndex + 1; + GraphHistory graphHistory = new GraphHistory(); + graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis)); + graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis)); + graphHistory.setInterval((int) (dataPointInterval + / DateTimeHelper.ONE_SECOND)); + graphHistory.setFactor(1.0 / 999.0); + graphHistory.setCount(count); + int previousNonNullIndex = -2; + boolean foundTwoAdjacentDataPoints = false; + List<Integer> values = new ArrayList<Integer>(); + for (int dataPointIndex = firstNonNullIndex; dataPointIndex <= + lastNonNullIndex; dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (dataPointIndex - previousNonNullIndex == 1) { + foundTwoAdjacentDataPoints = true; + } + previousNonNullIndex = dataPointIndex; + } + values.add(dataPoint < -0.5 ? null : ((int) (dataPoint * 999.0))); + } + graphHistory.setValues(values); + if (foundTwoAdjacentDataPoints) { + return graphHistory; + } else { + return null; + } + } + + public String getStatsString() { + StringBuilder sb = new StringBuilder(); + sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments) + + " uptime document files written\n"); + return sb.toString(); + } +} + diff --git a/src/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/org/torproject/onionoo/writer/WeightsDocumentWriter.java new file mode 100644 index 0000000..ddd2774 --- /dev/null +++ b/src/org/torproject/onionoo/writer/WeightsDocumentWriter.java @@ -0,0 +1,233 @@ +/* Copyright 2012--2014 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo.writer; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.SortedSet; + +import org.torproject.onionoo.docs.DocumentStore; +import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.WeightsDocument; +import org.torproject.onionoo.docs.WeightsStatus; +import org.torproject.onionoo.updater.DescriptorSource; +import org.torproject.onionoo.updater.DescriptorType; +import org.torproject.onionoo.updater.FingerprintListener; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; +import org.torproject.onionoo.util.Logger; + +public class WeightsDocumentWriter implements FingerprintListener, + DocumentWriter { + + private DescriptorSource descriptorSource; + + private DocumentStore documentStore; + + private long now; + + public WeightsDocumentWriter() { + this.descriptorSource = ApplicationFactory.getDescriptorSource(); + this.documentStore = ApplicationFactory.getDocumentStore(); + this.now = ApplicationFactory.getTime().currentTimeMillis(); + this.registerFingerprintListeners(); + } + + private void registerFingerprintListeners() { + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_CONSENSUSES); + this.descriptorSource.registerFingerprintListener(this, + DescriptorType.RELAY_SERVER_DESCRIPTORS); + } + + private Set<String> updateWeightsDocuments = new HashSet<String>(); + + public void processFingerprints(SortedSet<String> fingerprints, + boolean relay) { + if (relay) { + this.updateWeightsDocuments.addAll(fingerprints); + } + } + + public void writeDocuments() { + this.writeWeightsDataFiles(); + Logger.printStatusTime("Wrote weights document files"); + } + + private void writeWeightsDataFiles() { + for (String fingerprint : this.updateWeightsDocuments) { + WeightsStatus weightsStatus = this.documentStore.retrieve( + WeightsStatus.class, true, fingerprint); + if (weightsStatus == null) { + continue; + } + SortedMap<long[], double[]> history = weightsStatus.getHistory(); + WeightsDocument weightsDocument = this.compileWeightsDocument( + fingerprint, history); + this.documentStore.store(weightsDocument, fingerprint); + } + Logger.printStatusTime("Wrote weights document files"); + } + + private String[] graphNames = new String[] { + "1_week", + "1_month", + "3_months", + "1_year", + "5_years" }; + + private long[] graphIntervals = new long[] { + DateTimeHelper.ONE_WEEK, + DateTimeHelper.ROUGHLY_ONE_MONTH, + DateTimeHelper.ROUGHLY_THREE_MONTHS, + DateTimeHelper.ROUGHLY_ONE_YEAR, + DateTimeHelper.ROUGHLY_FIVE_YEARS }; + + private long[] dataPointIntervals = new long[] { + DateTimeHelper.ONE_HOUR, + DateTimeHelper.FOUR_HOURS, + DateTimeHelper.TWELVE_HOURS, + DateTimeHelper.TWO_DAYS, + DateTimeHelper.TEN_DAYS }; + + private WeightsDocument compileWeightsDocument(String fingerprint, + SortedMap<long[], double[]> history) { + WeightsDocument weightsDocument = new WeightsDocument(); + weightsDocument.setFingerprint(fingerprint); + weightsDocument.setAdvertisedBandwidthFraction( + this.compileGraphType(history, 0)); + weightsDocument.setConsensusWeightFraction( + this.compileGraphType(history, 1)); + weightsDocument.setGuardProbability( + this.compileGraphType(history, 2)); + weightsDocument.setMiddleProbability( + this.compileGraphType(history, 3)); + weightsDocument.setExitProbability( + this.compileGraphType(history, 4)); + weightsDocument.setAdvertisedBandwidth( + this.compileGraphType(history, 5)); + weightsDocument.setConsensusWeight( + this.compileGraphType(history, 6)); + return weightsDocument; + } + + private Map<String, GraphHistory> compileGraphType( + SortedMap<long[], double[]> history, int graphTypeIndex) { + Map<String, GraphHistory> graphs = + new LinkedHashMap<String, GraphHistory>(); + for (int graphIntervalIndex = 0; graphIntervalIndex < + this.graphIntervals.length; graphIntervalIndex++) { + String graphName = this.graphNames[graphIntervalIndex]; + GraphHistory graphHistory = this.compileWeightsHistory( + graphTypeIndex, graphIntervalIndex, history); + if (graphHistory != null) { + graphs.put(graphName, graphHistory); + } + } + return graphs; + } + + private GraphHistory compileWeightsHistory(int graphTypeIndex, + int graphIntervalIndex, SortedMap<long[], double[]> history) { + long graphInterval = this.graphIntervals[graphIntervalIndex]; + long dataPointInterval = + this.dataPointIntervals[graphIntervalIndex]; + List<Double> dataPoints = new ArrayList<Double>(); + long intervalStartMillis = ((this.now - graphInterval) + / dataPointInterval) * dataPointInterval; + long totalMillis = 0L; + double totalWeightTimesMillis = 0.0; + for (Map.Entry<long[], double[]> e : history.entrySet()) { + long startMillis = e.getKey()[0], endMillis = e.getKey()[1]; + double weight = e.getValue()[graphTypeIndex]; + if (endMillis < intervalStartMillis) { + continue; + } + while ((intervalStartMillis / dataPointInterval) != + (endMillis / dataPointInterval)) { + dataPoints.add(totalMillis * 5L < dataPointInterval + ? -1.0 : totalWeightTimesMillis / (double) totalMillis); + totalWeightTimesMillis = 0.0; + totalMillis = 0L; + intervalStartMillis += dataPointInterval; + } + if (weight >= 0.0) { + totalWeightTimesMillis += weight + * ((double) (endMillis - startMillis)); + totalMillis += (endMillis - startMillis); + } + } + dataPoints.add(totalMillis * 5L < dataPointInterval + ? -1.0 : totalWeightTimesMillis / (double) totalMillis); + double maxValue = 0.0; + int firstNonNullIndex = -1, lastNonNullIndex = -1; + for (int dataPointIndex = 0; dataPointIndex < dataPoints.size(); + dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (firstNonNullIndex < 0) { + firstNonNullIndex = dataPointIndex; + } + lastNonNullIndex = dataPointIndex; + if (dataPoint > maxValue) { + maxValue = dataPoint; + } + } + } + if (firstNonNullIndex < 0) { + return null; + } + long firstDataPointMillis = (((this.now - graphInterval) + / dataPointInterval) + firstNonNullIndex) * dataPointInterval + + dataPointInterval / 2L; + if (graphIntervalIndex > 0 && firstDataPointMillis >= + this.now - graphIntervals[graphIntervalIndex - 1]) { + /* Skip weights history object, because it doesn't contain + * anything new that wasn't already contained in the last + * weights history object(s). */ + return null; + } + long lastDataPointMillis = firstDataPointMillis + + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval; + double factor = ((double) maxValue) / 999.0; + int count = lastNonNullIndex - firstNonNullIndex + 1; + GraphHistory graphHistory = new GraphHistory(); + graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis)); + graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis)); + graphHistory.setInterval((int) (dataPointInterval + / DateTimeHelper.ONE_SECOND)); + graphHistory.setFactor(factor); + graphHistory.setCount(count); + int previousNonNullIndex = -2; + boolean foundTwoAdjacentDataPoints = false; + List<Integer> values = new ArrayList<Integer>(); + for (int dataPointIndex = firstNonNullIndex; dataPointIndex <= + lastNonNullIndex; dataPointIndex++) { + double dataPoint = dataPoints.get(dataPointIndex); + if (dataPoint >= 0.0) { + if (dataPointIndex - previousNonNullIndex == 1) { + foundTwoAdjacentDataPoints = true; + } + previousNonNullIndex = dataPointIndex; + } + values.add(dataPoint < 0.0 ? null : + (int) ((dataPoint * 999.0) / maxValue)); + } + graphHistory.setValues(values); + if (foundTwoAdjacentDataPoints) { + return graphHistory; + } else { + return null; + } + } + + public String getStatsString() { + /* TODO Add statistics string. */ + return null; + } +} diff --git a/test/org/torproject/onionoo/DummyDescriptorSource.java b/test/org/torproject/onionoo/DummyDescriptorSource.java index 06ec499..e93b063 100644 --- a/test/org/torproject/onionoo/DummyDescriptorSource.java +++ b/test/org/torproject/onionoo/DummyDescriptorSource.java @@ -9,6 +9,10 @@ import java.util.SortedSet; import java.util.TreeSet;

import org.torproject.descriptor.Descriptor; +import org.torproject.onionoo.updater.DescriptorListener; +import org.torproject.onionoo.updater.DescriptorSource; +import org.torproject.onionoo.updater.DescriptorType; +import org.torproject.onionoo.updater.FingerprintListener;

public class DummyDescriptorSource extends DescriptorSource {

diff --git a/test/org/torproject/onionoo/DummyDocumentStore.java b/test/org/torproject/onionoo/DummyDocumentStore.java index 5a5905b..54311aa 100644 --- a/test/org/torproject/onionoo/DummyDocumentStore.java +++ b/test/org/torproject/onionoo/DummyDocumentStore.java @@ -7,6 +7,9 @@ import java.util.SortedSet; import java.util.TreeMap; import java.util.TreeSet;

+import org.torproject.onionoo.docs.Document; +import org.torproject.onionoo.docs.DocumentStore; + public class DummyDocumentStore extends DocumentStore {

private Map<Class<? extends Document>, SortedMap<String, Document>> diff --git a/test/org/torproject/onionoo/DummyTime.java b/test/org/torproject/onionoo/DummyTime.java index 7178ed1..ffbd6e3 100644 --- a/test/org/torproject/onionoo/DummyTime.java +++ b/test/org/torproject/onionoo/DummyTime.java @@ -2,6 +2,8 @@ * See LICENSE for licensing information */ package org.torproject.onionoo;

+import org.torproject.onionoo.util.Time; + public class DummyTime extends Time { private long currentTimeMillis; public DummyTime(long currentTimeMillis) { diff --git a/test/org/torproject/onionoo/LookupServiceTest.java b/test/org/torproject/onionoo/LookupServiceTest.java index 23b5a91..052b4c0 100644 --- a/test/org/torproject/onionoo/LookupServiceTest.java +++ b/test/org/torproject/onionoo/LookupServiceTest.java @@ -22,6 +22,8 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.torproject.onionoo.updater.LookupResult; +import org.torproject.onionoo.updater.LookupService;

public class LookupServiceTest {

diff --git a/test/org/torproject/onionoo/ResourceServletTest.java b/test/org/torproject/onionoo/ResourceServletTest.java index f2ace5d..8cd3752 100644 --- a/test/org/torproject/onionoo/ResourceServletTest.java +++ b/test/org/torproject/onionoo/ResourceServletTest.java @@ -21,8 +21,14 @@ import java.util.TreeSet;

import org.junit.Before; import org.junit.Test; -import org.torproject.onionoo.ResourceServlet.HttpServletRequestWrapper; -import org.torproject.onionoo.ResourceServlet.HttpServletResponseWrapper; +import org.torproject.onionoo.docs.UpdateStatus; +import org.torproject.onionoo.server.NodeIndexer; +import org.torproject.onionoo.server.ResourceServlet; +import org.torproject.onionoo.server.ResourceServlet.HttpServletRequestWrapper; +import org.torproject.onionoo.server.ResourceServlet.HttpServletResponseWrapper; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; +import org.torproject.onionoo.util.Time;

import com.google.gson.Gson;

@@ -31,7 +37,7 @@ import com.google.gson.Gson; * which tests servlet specifics. */ public class ResourceServletTest {

- private SortedMap<String, org.torproject.onionoo.SummaryDocument> + private SortedMap<String, org.torproject.onionoo.docs.SummaryDocument> relays, bridges;

private long currentTimeMillis = DateTimeHelper.parse( @@ -102,8 +108,8 @@ public class ResourceServletTest {

@Before public void createSampleRelaysAndBridges() { - org.torproject.onionoo.SummaryDocument relayTorkaZ = - new org.torproject.onionoo.SummaryDocument(true, "TorkaZ", + org.torproject.onionoo.docs.SummaryDocument relayTorkaZ = + new org.torproject.onionoo.docs.SummaryDocument(true, "TorkaZ", "000C5F55BD4814B917CC474BD537F1A3B33CCE2A", Arrays.asList( new String[] { "62.216.201.221", "62.216.201.222", "62.216.201.223" }), DateTimeHelper.parse("2013-04-19 05:00:00"), @@ -114,8 +120,8 @@ public class ResourceServletTest { + "fb-token:np5_g_83jmf=", new TreeSet<String>(Arrays.asList( new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC", "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" }))); - org.torproject.onionoo.SummaryDocument relayFerrari458 = - new org.torproject.onionoo.SummaryDocument(true, "Ferrari458", + org.torproject.onionoo.docs.SummaryDocument relayFerrari458 = + new org.torproject.onionoo.docs.SummaryDocument(true, "Ferrari458", "001C13B3A55A71B977CA65EC85539D79C653A3FC", Arrays.asList( new String[] { "68.38.171.200", "[2001:4f8:3:2e::51]" }), DateTimeHelper.parse("2013-04-24 12:00:00"), true, @@ -124,8 +130,8 @@ public class ResourceServletTest { DateTimeHelper.parse("2013-02-12 16:00:00"), "AS7922", null, new TreeSet<String>(Arrays.asList(new String[] { "000C5F55BD4814B917CC474BD537F1A3B33CCE2A" }))); - org.torproject.onionoo.SummaryDocument relayTimMayTribute = - new org.torproject.onionoo.SummaryDocument(true, "TimMayTribute", + org.torproject.onionoo.docs.SummaryDocument relayTimMayTribute = + new org.torproject.onionoo.docs.SummaryDocument(true, "TimMayTribute", "0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", Arrays.asList( new String[] { "89.69.68.246" }), DateTimeHelper.parse("2013-04-22 20:00:00"), false, @@ -135,24 +141,24 @@ public class ResourceServletTest { "1024D/51E2A1C7 steven j. murdoch " + "tor+steven.murdoch@cl.cam.ac.uk fb-token:5sr_k_zs2wm=", new TreeSet<String>()); - org.torproject.onionoo.SummaryDocument bridgeec2bridgercc7f31fe = - new org.torproject.onionoo.SummaryDocument(false, + org.torproject.onionoo.docs.SummaryDocument bridgeec2bridgercc7f31fe = + new org.torproject.onionoo.docs.SummaryDocument(false, "ec2bridgercc7f31fe", "0000831B236DFF73D409AD17B40E2A728A53994F", Arrays.asList(new String[] { "10.199.7.176" }), DateTimeHelper.parse("2013-04-21 18:07:03"), false, new TreeSet<String>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null, null); - org.torproject.onionoo.SummaryDocument bridgeUnnamed = - new org.torproject.onionoo.SummaryDocument(false, "Unnamed", + org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed = + new org.torproject.onionoo.docs.SummaryDocument(false, "Unnamed", "0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", Arrays.asList( new String[] { "10.0.52.84" }), DateTimeHelper.parse("2013-04-20 17:37:04"), false, new TreeSet<String>(Arrays.asList(new String[] { "Valid" })), -1L, null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null, null); - org.torproject.onionoo.SummaryDocument bridgegummy = - new org.torproject.onionoo.SummaryDocument(false, "gummy", + org.torproject.onionoo.docs.SummaryDocument bridgegummy = + new org.torproject.onionoo.docs.SummaryDocument(false, "gummy", "1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", Arrays.asList( new String[] { "10.63.169.98" }), DateTimeHelper.parse("2013-04-24 01:07:04"), true, @@ -160,7 +166,7 @@ public class ResourceServletTest { "Valid" })), -1L, null, DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null); this.relays = - new TreeMap<String, org.torproject.onionoo.SummaryDocument>(); + new TreeMap<String, org.torproject.onionoo.docs.SummaryDocument>(); this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A", relayTorkaZ); this.relays.put("001C13B3A55A71B977CA65EC85539D79C653A3FC", @@ -168,7 +174,7 @@ public class ResourceServletTest { this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", relayTimMayTribute); this.bridges = - new TreeMap<String, org.torproject.onionoo.SummaryDocument>(); + new TreeMap<String, org.torproject.onionoo.docs.SummaryDocument>(); this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F", bridgeec2bridgercc7f31fe); this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", @@ -201,11 +207,11 @@ public class ResourceServletTest { updateStatus.setDocumentString(String.valueOf( this.currentTimeMillis)); documentStore.addDocument(updateStatus, null); - for (Map.Entry<String, org.torproject.onionoo.SummaryDocument> e : + for (Map.Entry<String, org.torproject.onionoo.docs.SummaryDocument> e : this.relays.entrySet()) { documentStore.addDocument(e.getValue(), e.getKey()); } - for (Map.Entry<String, org.torproject.onionoo.SummaryDocument> e : + for (Map.Entry<String, org.torproject.onionoo.docs.SummaryDocument> e : this.bridges.entrySet()) { documentStore.addDocument(e.getValue(), e.getKey()); } diff --git a/test/org/torproject/onionoo/UptimeDocumentWriterTest.java b/test/org/torproject/onionoo/UptimeDocumentWriterTest.java index 5065e4d..5a77514 100644 --- a/test/org/torproject/onionoo/UptimeDocumentWriterTest.java +++ b/test/org/torproject/onionoo/UptimeDocumentWriterTest.java @@ -10,6 +10,13 @@ import java.util.List;

import org.junit.Before; import org.junit.Test; +import org.torproject.onionoo.docs.GraphHistory; +import org.torproject.onionoo.docs.UptimeDocument; +import org.torproject.onionoo.docs.UptimeStatus; +import org.torproject.onionoo.updater.DescriptorType; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper; +import org.torproject.onionoo.writer.UptimeDocumentWriter;

public class UptimeDocumentWriterTest {

diff --git a/test/org/torproject/onionoo/UptimeStatusTest.java b/test/org/torproject/onionoo/UptimeStatusTest.java index 884ccc5..671ffa3 100644 --- a/test/org/torproject/onionoo/UptimeStatusTest.java +++ b/test/org/torproject/onionoo/UptimeStatusTest.java @@ -9,6 +9,10 @@ import java.util.TreeSet;

import org.junit.Before; import org.junit.Test; +import org.torproject.onionoo.docs.UptimeHistory; +import org.torproject.onionoo.docs.UptimeStatus; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper;

public class UptimeStatusTest {

diff --git a/test/org/torproject/onionoo/UptimeStatusUpdaterTest.java b/test/org/torproject/onionoo/UptimeStatusUpdaterTest.java index a34292b..8070ae4 100644 --- a/test/org/torproject/onionoo/UptimeStatusUpdaterTest.java +++ b/test/org/torproject/onionoo/UptimeStatusUpdaterTest.java @@ -6,6 +6,12 @@ import static org.junit.Assert.assertEquals;

import org.junit.Before; import org.junit.Test; +import org.torproject.onionoo.docs.UptimeHistory; +import org.torproject.onionoo.docs.UptimeStatus; +import org.torproject.onionoo.updater.DescriptorType; +import org.torproject.onionoo.updater.UptimeStatusUpdater; +import org.torproject.onionoo.util.ApplicationFactory; +import org.torproject.onionoo.util.DateTimeHelper;

public class UptimeStatusUpdaterTest {