commit 680b43922f756b92b6603d492b81f1e19259195f Author: Karsten Loesing karsten.loesing@gmx.net Date: Sat Nov 23 17:54:25 2013 +0100

Move logging-related stuff to new Logger class. --- src/org/torproject/onionoo/DescriptorSource.java | 19 +-- src/org/torproject/onionoo/DocumentStore.java | 31 ++--- src/org/torproject/onionoo/Logger.java | 47 +++++++ src/org/torproject/onionoo/LookupService.java | 9 +- src/org/torproject/onionoo/Main.java | 131 ++++++++------------ src/org/torproject/onionoo/NodeDataWriter.java | 11 +- .../onionoo/ReverseDomainNameResolver.java | 19 +-- 7 files changed, 120 insertions(+), 147 deletions(-)

diff --git a/src/org/torproject/onionoo/DescriptorSource.java b/src/org/torproject/onionoo/DescriptorSource.java index 2bac013..bdf9be8 100644 --- a/src/org/torproject/onionoo/DescriptorSource.java +++ b/src/org/torproject/onionoo/DescriptorSource.java @@ -355,25 +355,14 @@ public class DescriptorSource { descriptors += descriptorQueue.returnedDescriptors; bytes += descriptorQueue.returnedBytes; } - sb.append(" " + String.format("%,d", historySizeBefore) + sb.append(" " + Logger.formatDecimalNumber(historySizeBefore) + " descriptors excluded from this execution\n"); - sb.append(" " + String.format("%,d", descriptors) + sb.append(" " + Logger.formatDecimalNumber(descriptors) + " descriptors provided\n"); - sb.append(" " + formatBytes(bytes) + " provided\n"); - sb.append(" " + String.format("%,d", historySizeAfter) + sb.append(" " + Logger.formatBytes(bytes) + " provided\n"); + sb.append(" " + Logger.formatDecimalNumber(historySizeAfter) + " descriptors excluded from next execution\n"); return sb.toString(); } - - // TODO This method should go into a utility class. - private static String formatBytes(long bytes) { - if (bytes < 1024) { - return bytes + " B"; - } else { - int exp = (int) (Math.log(bytes) / Math.log(1024)); - return String.format("%.1f %siB", bytes / Math.pow(1024, exp), - "KMGTPE".charAt(exp-1)); - } - } }

diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java index 28fc0e6..d96d1c5 100644 --- a/src/org/torproject/onionoo/DocumentStore.java +++ b/src/org/torproject/onionoo/DocumentStore.java @@ -490,35 +490,20 @@ public class DocumentStore {

public String getStatsString() { StringBuilder sb = new StringBuilder(); - sb.append(" " + formatDecimalNumber(listOperations) + sb.append(" " + Logger.formatDecimalNumber(listOperations) + " list operations performed\n"); - sb.append(" " + formatDecimalNumber(listedFiles) + sb.append(" " + Logger.formatDecimalNumber(listedFiles) + " files listed\n"); - sb.append(" " + formatDecimalNumber(storedFiles) + sb.append(" " + Logger.formatDecimalNumber(storedFiles) + " files stored\n"); - sb.append(" " + formatBytes(storedBytes) + " stored\n"); - sb.append(" " + formatDecimalNumber(retrievedFiles) + sb.append(" " + Logger.formatBytes(storedBytes) + " stored\n"); + sb.append(" " + Logger.formatDecimalNumber(retrievedFiles) + " files retrieved\n"); - sb.append(" " + formatBytes(retrievedBytes) + " retrieved\n"); - sb.append(" " + formatDecimalNumber(removedFiles) + sb.append(" " + Logger.formatBytes(retrievedBytes) + + " retrieved\n"); + sb.append(" " + Logger.formatDecimalNumber(removedFiles) + " files removed\n"); return sb.toString(); } - - //TODO This method should go into a utility class. - private static String formatDecimalNumber(long decimalNumber) { - return String.format("%,d", decimalNumber); - } - - // TODO This method should go into a utility class. - private static String formatBytes(long bytes) { - if (bytes < 1024) { - return bytes + " B"; - } else { - int exp = (int) (Math.log(bytes) / Math.log(1024)); - return String.format("%.1f %siB", bytes / Math.pow(1024, exp), - "KMGTPE".charAt(exp-1)); - } - } }

diff --git a/src/org/torproject/onionoo/Logger.java b/src/org/torproject/onionoo/Logger.java new file mode 100644 index 0000000..7248ee6 --- /dev/null +++ b/src/org/torproject/onionoo/Logger.java @@ -0,0 +1,47 @@ +/* Copyright 2013 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.onionoo; + +import java.util.Date; + +public class Logger { + public static String formatDecimalNumber(long decimalNumber) { + return String.format("%,d", decimalNumber); + } + + public static String formatMillis(long millis) { + return String.format("%02d:%02d.%03d minutes", + millis / (1000L * 60L), (millis / 1000L) % 60L, millis % 1000L); + } + + public static String formatBytes(long bytes) { + if (bytes < 1024) { + return bytes + " B"; + } else { + int exp = (int) (Math.log(bytes) / Math.log(1024)); + return String.format("%.1f %siB", bytes / Math.pow(1024, exp), + "KMGTPE".charAt(exp-1)); + } + } + + private static long printedLastStatusMessage = + System.currentTimeMillis(); + + public static void printStatus(String message) { + System.out.println(new Date() + ": " + message); + printedLastStatusMessage = System.currentTimeMillis(); + } + + public static void printStatistics(String component, String message) { + System.out.print(" " + component + " statistics:\n" + message); + } + + public static void printStatusTime(String message) { + long now = System.currentTimeMillis(); + long millis = now - printedLastStatusMessage; + System.out.println(" " + message + " (" + Logger.formatMillis(millis) + + ")."); + printedLastStatusMessage = now; + } +} + diff --git a/src/org/torproject/onionoo/LookupService.java b/src/org/torproject/onionoo/LookupService.java index 6fc2bf6..43971b5 100644 --- a/src/org/torproject/onionoo/LookupService.java +++ b/src/org/torproject/onionoo/LookupService.java @@ -402,15 +402,10 @@ public class LookupService {

public String getStatsString() { StringBuilder sb = new StringBuilder(); - sb.append(" " + formatDecimalNumber(addressesLookedUp) + sb.append(" " + Logger.formatDecimalNumber(addressesLookedUp) + " addresses looked up\n"); - sb.append(" " + formatDecimalNumber(addressesResolved) + sb.append(" " + Logger.formatDecimalNumber(addressesResolved) + " addresses resolved\n"); return sb.toString(); } - - //TODO This method should go into a utility class. - private static String formatDecimalNumber(long decimalNumber) { - return String.format("%,d", decimalNumber); - } } diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java index e81a26a..a03a9fe 100644 --- a/src/org/torproject/onionoo/Main.java +++ b/src/org/torproject/onionoo/Main.java @@ -3,155 +3,128 @@ package org.torproject.onionoo;

import java.io.File; -import java.util.Date; import java.util.SortedMap;

/* Update search data and status data files. */ public class Main { public static void main(String[] args) {

- printStatus("Initializing."); + Logger.printStatus("Initializing.");

LockFile lf = new LockFile(new File("lock")); if (lf.acquireLock()) { - printStatusTime("Acquired lock"); + Logger.printStatusTime("Acquired lock"); } else { - printStatusTime("Could not acquire lock. Is Onionoo already " - + "running? Terminating"); + Logger.printStatusTime("Could not acquire lock. Is Onionoo " + + "already running? Terminating"); return; }

DescriptorSource dso = new DescriptorSource(new File("in"), new File("status")); - printStatusTime("Initialized descriptor source"); + Logger.printStatusTime("Initialized descriptor source"); DocumentStore ds = new DocumentStore(new File("status"), new File("out")); - printStatusTime("Initialized document store"); + Logger.printStatusTime("Initialized document store"); LookupService ls = new LookupService(new File("geoip")); - printStatusTime("Initialized Geoip lookup service"); + Logger.printStatusTime("Initialized Geoip lookup service"); ReverseDomainNameResolver rdnr = new ReverseDomainNameResolver(); - printStatusTime("Initialized reverse domain name resolver"); + Logger.printStatusTime("Initialized reverse domain name resolver"); NodeDataWriter ndw = new NodeDataWriter(dso, ls, ds); - printStatusTime("Initialized node data writer"); + Logger.printStatusTime("Initialized node data writer"); DetailsDataWriter ddw = new DetailsDataWriter(dso, rdnr, ds); - printStatusTime("Initialized details data writer"); + Logger.printStatusTime("Initialized details data writer"); BandwidthDataWriter bdw = new BandwidthDataWriter(dso, ds); - printStatusTime("Initialized bandwidth data writer"); + Logger.printStatusTime("Initialized bandwidth data writer"); WeightsDataWriter wdw = new WeightsDataWriter(dso, ds); - printStatusTime("Initialized weights data writer"); + Logger.printStatusTime("Initialized weights data writer");

- printStatus("Reading descriptors."); + Logger.printStatus("Reading descriptors."); dso.readRelayNetworkConsensuses(); - printStatusTime("Read relay network consensuses"); + Logger.printStatusTime("Read relay network consensuses"); dso.readRelayServerDescriptors(); - printStatusTime("Read relay server descriptors"); + Logger.printStatusTime("Read relay server descriptors"); dso.readRelayExtraInfos(); - printStatusTime("Read relay extra-info descriptors"); + Logger.printStatusTime("Read relay extra-info descriptors"); dso.readExitLists(); - printStatusTime("Read exit lists"); + Logger.printStatusTime("Read exit lists"); dso.readBridgeNetworkStatuses(); - printStatusTime("Read bridge network statuses"); + Logger.printStatusTime("Read bridge network statuses"); dso.readBridgeServerDescriptors(); - printStatusTime("Read bridge server descriptors"); + Logger.printStatusTime("Read bridge server descriptors"); dso.readBridgeExtraInfos(); - printStatusTime("Read bridge extra-info descriptors"); + Logger.printStatusTime("Read bridge extra-info descriptors"); dso.readBridgePoolAssignments(); - printStatusTime("Read bridge-pool assignments"); + Logger.printStatusTime("Read bridge-pool assignments");

- printStatus("Updating internal node list."); + Logger.printStatus("Updating internal node list."); ndw.readStatusSummary(); - printStatusTime("Read status summary"); + Logger.printStatusTime("Read status summary"); ndw.lookUpCitiesAndASes(); - printStatusTime("Looked up cities and ASes"); + Logger.printStatusTime("Looked up cities and ASes"); ndw.setRunningBits(); - printStatusTime("Set running bits"); + Logger.printStatusTime("Set running bits"); ndw.writeStatusSummary(); - printStatusTime("Wrote status summary"); + Logger.printStatusTime("Wrote status summary"); ndw.writeOutSummary(); - printStatusTime("Wrote out summary"); + Logger.printStatusTime("Wrote out summary"); SortedMap<String, NodeStatus> currentNodes = ndw.getCurrentNodes(); SortedMap<String, Integer> lastBandwidthWeights = ndw.getLastBandwidthWeights();

- printStatus("Updating detail data."); + Logger.printStatus("Updating detail data."); // TODO Instead of using ndw's currentNodes and lastBandwidthWeights, // parse statuses once again, keeping separate parse history. Allows // us to run ndw and ddw in parallel in the future. Alternatively, // merge ndw and ddw, because they're doing similar things anyway. ddw.setCurrentNodes(currentNodes); - printStatusTime("Set current node fingerprints"); + Logger.printStatusTime("Set current node fingerprints"); ddw.startReverseDomainNameLookups(); - printStatusTime("Started reverse domain name lookups"); + Logger.printStatusTime("Started reverse domain name lookups"); ddw.calculatePathSelectionProbabilities(lastBandwidthWeights); - printStatusTime("Calculated path selection probabilities"); + Logger.printStatusTime("Calculated path selection probabilities"); ddw.finishReverseDomainNameLookups(); - printStatusTime("Finished reverse domain name lookups"); + Logger.printStatusTime("Finished reverse domain name lookups"); ddw.writeOutDetails(); - printStatusTime("Wrote detail data files"); + Logger.printStatusTime("Wrote detail data files");

- printStatus("Updating bandwidth data."); + Logger.printStatus("Updating bandwidth data.");

- printStatus("Updating weights data."); + Logger.printStatus("Updating weights data."); wdw.updateWeightsHistories(); - printStatusTime("Updated weights histories"); + Logger.printStatusTime("Updated weights histories"); wdw.updateWeightsStatuses(); - printStatusTime("Updated weights status files"); + Logger.printStatusTime("Updated weights status files"); wdw.writeWeightsDataFiles(); - printStatusTime("Wrote weights document files"); + Logger.printStatusTime("Wrote weights document files");

- printStatus("Shutting down."); + Logger.printStatus("Shutting down."); dso.writeHistoryFiles(); - printStatusTime("Wrote parse histories"); + Logger.printStatusTime("Wrote parse histories"); ds.flushDocumentCache(); - printStatusTime("Flushed document cache"); + Logger.printStatusTime("Flushed document cache");

- printStatus("Gathering statistics."); - printStatistics("Node data writer", ndw.getStatsString()); + Logger.printStatus("Gathering statistics."); + Logger.printStatistics("Node data writer", ndw.getStatsString()); /* TODO Add statistics to remaining *Writers. */ //printStatistics("Details data writer", ddw.getStatsString()); //printStatistics("Bandwidth data writer", bdw.getStatsString()); //printStatistics("Weights data writer", wdw.getStatsString()); - printStatistics("Descriptor source", dso.getStatsString()); - printStatistics("Document store", ds.getStatsString()); - printStatistics("GeoIP lookup service", ls.getStatsString()); - printStatistics("Reverse domain name resolver", + Logger.printStatistics("Descriptor source", dso.getStatsString()); + Logger.printStatistics("Document store", ds.getStatsString()); + Logger.printStatistics("GeoIP lookup service", ls.getStatsString()); + Logger.printStatistics("Reverse domain name resolver", rdnr.getStatsString());

- printStatus("Releasing lock."); + Logger.printStatus("Releasing lock."); if (lf.releaseLock()) { - printStatusTime("Released lock"); + Logger.printStatusTime("Released lock"); } else { - printStatusTime("Could not release lock. The next execution " - + "may not start as expected"); + Logger.printStatusTime("Could not release lock. The next " + + "execution may not start as expected"); }

- printStatus("Terminating."); - } - - private static long printedLastStatusMessage = - System.currentTimeMillis(); - - private static void printStatus(String message) { - System.out.println(new Date() + ": " + message); - printedLastStatusMessage = System.currentTimeMillis(); - } - - private static void printStatistics(String component, String message) { - System.out.print(" " + component + " statistics:\n" + message); - } - - private static void printStatusTime(String message) { - long now = System.currentTimeMillis(); - long millis = now - printedLastStatusMessage; - System.out.println(" " + message + " (" + formatMillis(millis) - + ")."); - printedLastStatusMessage = now; - } - - // TODO This method should go into a utility class. - private static String formatMillis(long millis) { - return String.format("%02d:%02d.%03d minutes", - millis / (1000L * 60L), (millis / 1000L) % 60L, millis % 1000L); + Logger.printStatus("Terminating."); } }

diff --git a/src/org/torproject/onionoo/NodeDataWriter.java b/src/org/torproject/onionoo/NodeDataWriter.java index b1e9b5c..56071f4 100644 --- a/src/org/torproject/onionoo/NodeDataWriter.java +++ b/src/org/torproject/onionoo/NodeDataWriter.java @@ -224,16 +224,11 @@ public class NodeDataWriter implements DescriptorListener {

public String getStatsString() { StringBuilder sb = new StringBuilder(); - sb.append(" " + formatDecimalNumber(relayConsensusesProcessed) - + " relay consensuses processed\n"); - sb.append(" " + formatDecimalNumber(bridgeStatusesProcessed) + sb.append(" " + Logger.formatDecimalNumber( + relayConsensusesProcessed) + " relay consensuses processed\n"); + sb.append(" " + Logger.formatDecimalNumber(bridgeStatusesProcessed) + " bridge statuses processed\n"); return sb.toString(); } - - //TODO This method should go into a utility class. - private static String formatDecimalNumber(long decimalNumber) { - return String.format("%,d", decimalNumber); - } }

diff --git a/src/org/torproject/onionoo/ReverseDomainNameResolver.java b/src/org/torproject/onionoo/ReverseDomainNameResolver.java index b04ad0a..bb10747 100644 --- a/src/org/torproject/onionoo/ReverseDomainNameResolver.java +++ b/src/org/torproject/onionoo/ReverseDomainNameResolver.java @@ -150,29 +150,18 @@ public class ReverseDomainNameResolver {

public String getStatsString() { StringBuilder sb = new StringBuilder(); - sb.append(" " + formatDecimalNumber(rdnsLookupMillis.size()) + sb.append(" " + Logger.formatDecimalNumber(rdnsLookupMillis.size()) + " lookups performed\n"); if (rdnsLookupMillis.size() > 0) { Collections.sort(rdnsLookupMillis); - sb.append(" " + formatMillis(rdnsLookupMillis.get(0)) + sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get(0)) + " minimum lookup time\n"); - sb.append(" " + formatMillis(rdnsLookupMillis.get( + sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get( rdnsLookupMillis.size() / 2)) + " median lookup time\n"); - sb.append(" " + formatMillis(rdnsLookupMillis.get( + sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get( rdnsLookupMillis.size() - 1)) + " maximum lookup time\n"); } return sb.toString(); } - - //TODO This method should go into a utility class. - private static String formatDecimalNumber(long decimalNumber) { - return String.format("%,d", decimalNumber); - } - - // TODO This method should go into a utility class. - private static String formatMillis(long millis) { - return String.format("%02d:%02d.%03d minutes", - millis / (1000L * 60L), (millis / 1000L) % 60L, millis % 1000L); - } }