commit 00494ff1f4ea85c8501cb56a0adba56fd3305616 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Sun Apr 26 19:10:34 2015 +0200 Support parsing archived descriptors from in/archive/. Implements part of #13600. --- .../onionoo/updater/DescriptorQueue.java | 29 +++-- .../onionoo/updater/DescriptorSource.java | 137 ++++++++++++++------ 2 files changed, 120 insertions(+), 46 deletions(-) diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java b/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java index 93a291c..b86d585 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java @@ -24,8 +24,6 @@ class DescriptorQueue { private final static Logger log = LoggerFactory.getLogger( DescriptorQueue.class); - private File inDir; - private File statusDir; private DescriptorReader descriptorReader; @@ -56,14 +54,21 @@ class DescriptorQueue { return this.returnedBytes; } - public DescriptorQueue(File inDir, File statusDir) { - this.inDir = inDir; + public DescriptorQueue(File inDir, DescriptorType descriptorType, + File statusDir) { + File directory = inDir; + if (descriptorType != null) { + directory = this.getDirectoryForDescriptorType(inDir, + descriptorType); + } this.statusDir = statusDir; this.descriptorReader = DescriptorSourceFactory.createDescriptorReader(); + this.addDirectory(directory); } - public void addDirectory(DescriptorType descriptorType) { + public File getDirectoryForDescriptorType(File inDir, + DescriptorType descriptorType) { String directoryName = null; switch (descriptorType) { case RELAY_CONSENSUSES: @@ -93,9 +98,16 @@ class DescriptorQueue { default: log.error("Unknown descriptor type. Not adding directory " + "to descriptor reader."); + return null; + } + File directory = new File(inDir, directoryName); + return directory; + } + + private void addDirectory(File directory) { + if (directory == null) { return; } - File directory = new File(this.inDir, directoryName); if (directory.exists() && directory.isDirectory()) { this.descriptorReader.addDirectory(directory); this.descriptorReader.setMaxDescriptorFilesInQueue(1); @@ -107,6 +119,9 @@ class DescriptorQueue { } public void readHistoryFile(DescriptorHistory descriptorHistory) { + if (this.statusDir == null) { + return; + } String historyFileName = null; switch (descriptorHistory) { case RELAY_EXTRAINFO_HISTORY: @@ -168,8 +183,6 @@ class DescriptorQueue { public void writeHistoryFile() { if (this.historyFile == null) { - log.error("Unable to write history file, possibly because of an " - + "unknown descriptor type. Skipping."); return; } SortedMap<String, Long> excludedAndParsedFiles = diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java index 52e00c9..b26a692 100644 --- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java +++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java @@ -20,12 +20,15 @@ public class DescriptorSource { private static final Logger log = LoggerFactory.getLogger( DescriptorSource.class); - private final File inDir = new File("in/recent"); + private final File inRecentDir = new File("in/recent"), + inArchiveDir = new File("in/archive"); private final File statusDir = new File("status"); private List<DescriptorQueue> descriptorQueues; + private DescriptorQueue archiveDescriptorQueue; + public DescriptorSource() { this.descriptorQueues = new ArrayList<DescriptorQueue>(); this.descriptorListeners = @@ -35,9 +38,8 @@ public class DescriptorSource { private DescriptorQueue getDescriptorQueue( DescriptorType descriptorType, DescriptorHistory descriptorHistory) { - DescriptorQueue descriptorQueue = new DescriptorQueue(this.inDir, - this.statusDir); - descriptorQueue.addDirectory(descriptorType); + DescriptorQueue descriptorQueue = new DescriptorQueue( + this.inRecentDir, descriptorType, this.statusDir); if (descriptorHistory != null) { descriptorQueue.readHistoryFile(descriptorHistory); } @@ -78,35 +80,32 @@ public class DescriptorSource { } public void readDescriptors() { - /* Careful when changing the order of parsing descriptor types! The - * various status updaters may base assumptions on this order. With - * #13674 being implemented, this may not be the case anymore. */ - - log.debug("Reading " + DescriptorType.RELAY_SERVER_DESCRIPTORS + this.readArchivedDescriptors(); + log.debug("Reading recent " + DescriptorType.RELAY_SERVER_DESCRIPTORS + " ..."); this.readDescriptors(DescriptorType.RELAY_SERVER_DESCRIPTORS, DescriptorHistory.RELAY_SERVER_HISTORY, true); - log.debug("Reading " + DescriptorType.RELAY_EXTRA_INFOS + " ..."); + log.debug("Reading recent " + DescriptorType.RELAY_EXTRA_INFOS + " ..."); this.readDescriptors(DescriptorType.RELAY_EXTRA_INFOS, DescriptorHistory.RELAY_EXTRAINFO_HISTORY, true); - log.debug("Reading " + DescriptorType.EXIT_LISTS + " ..."); + log.debug("Reading recent " + DescriptorType.EXIT_LISTS + " ..."); this.readDescriptors(DescriptorType.EXIT_LISTS, DescriptorHistory.EXIT_LIST_HISTORY, true); - log.debug("Reading " + DescriptorType.RELAY_CONSENSUSES + " ..."); + log.debug("Reading recent " + DescriptorType.RELAY_CONSENSUSES + " ..."); this.readDescriptors(DescriptorType.RELAY_CONSENSUSES, DescriptorHistory.RELAY_CONSENSUS_HISTORY, true); - log.debug("Reading " + DescriptorType.BRIDGE_SERVER_DESCRIPTORS + log.debug("Reading recent " + DescriptorType.BRIDGE_SERVER_DESCRIPTORS + " ..."); this.readDescriptors(DescriptorType.BRIDGE_SERVER_DESCRIPTORS, DescriptorHistory.BRIDGE_SERVER_HISTORY, false); - log.debug("Reading " + DescriptorType.BRIDGE_EXTRA_INFOS + " ..."); + log.debug("Reading recent " + DescriptorType.BRIDGE_EXTRA_INFOS + " ..."); this.readDescriptors(DescriptorType.BRIDGE_EXTRA_INFOS, DescriptorHistory.BRIDGE_EXTRAINFO_HISTORY, false); - log.debug("Reading " + DescriptorType.BRIDGE_POOL_ASSIGNMENTS + log.debug("Reading recent " + DescriptorType.BRIDGE_POOL_ASSIGNMENTS + " ..."); this.readDescriptors(DescriptorType.BRIDGE_POOL_ASSIGNMENTS, DescriptorHistory.BRIDGE_POOLASSIGN_HISTORY, false); - log.debug("Reading " + DescriptorType.BRIDGE_STATUSES + " ..."); + log.debug("Reading recent " + DescriptorType.BRIDGE_STATUSES + " ..."); this.readDescriptors(DescriptorType.BRIDGE_STATUSES, DescriptorHistory.BRIDGE_STATUS_HISTORY, false); } @@ -128,34 +127,88 @@ public class DescriptorSource { } switch (descriptorType) { case RELAY_CONSENSUSES: - log.info("Read relay network consensuses"); + log.info("Read recent relay network consensuses"); break; case RELAY_SERVER_DESCRIPTORS: - log.info("Read relay server descriptors"); + log.info("Read recent relay server descriptors"); break; case RELAY_EXTRA_INFOS: - log.info("Read relay extra-info descriptors"); + log.info("Read recent relay extra-info descriptors"); break; case EXIT_LISTS: - log.info("Read exit lists"); + log.info("Read recent exit lists"); break; case BRIDGE_STATUSES: - log.info("Read bridge network statuses"); + log.info("Read recent bridge network statuses"); break; case BRIDGE_SERVER_DESCRIPTORS: - log.info("Read bridge server descriptors"); + log.info("Read recent bridge server descriptors"); break; case BRIDGE_EXTRA_INFOS: - log.info("Read bridge extra-info descriptors"); + log.info("Read recent bridge extra-info descriptors"); break; case BRIDGE_POOL_ASSIGNMENTS: - log.info("Read bridge-pool assignments"); + log.info("Read recent bridge-pool assignments"); break; } } + public void readArchivedDescriptors() { + if (!this.inArchiveDir.exists()) { + return; + } + log.info("Reading archived descriptors..."); + this.archiveDescriptorQueue = new DescriptorQueue(this.inArchiveDir, + null, null); + Descriptor descriptor; + while ((descriptor = this.archiveDescriptorQueue.nextDescriptor()) + != null) { + DescriptorType descriptorType = null; + boolean relay = false; + for (String annotation : descriptor.getAnnotations()) { + if (annotation.startsWith( + "@type network-status-consensus-3 1.")) { + descriptorType = DescriptorType.RELAY_CONSENSUSES; + relay = true; + } else if (annotation.startsWith("@type server-descriptor 1.")) { + descriptorType = DescriptorType.RELAY_SERVER_DESCRIPTORS; + relay = true; + } else if (annotation.startsWith("@type extra-info 1.")) { + descriptorType = DescriptorType.RELAY_EXTRA_INFOS; + relay = true; + } else if (annotation.startsWith("@type tordnsel 1.")) { + descriptorType = DescriptorType.EXIT_LISTS; + relay = true; + } else if (annotation.startsWith( + "@type bridge-network-status 1.")) { + descriptorType = DescriptorType.BRIDGE_STATUSES; + relay = false; + } else if (annotation.startsWith( + "@type bridge-server-descriptor 1.")) { + descriptorType = DescriptorType.BRIDGE_SERVER_DESCRIPTORS; + relay = false; + } else if (annotation.startsWith("@type bridge-extra-info 1.")) { + descriptorType = DescriptorType.BRIDGE_EXTRA_INFOS; + relay = false; + } + } + if (descriptorType == null) { + log.warn("Unrecognized descriptor in " + + this.inArchiveDir.getAbsolutePath() + " with annotations " + + descriptor.getAnnotations() + ". Not reading any further" + + "archived descriptors."); + break; + } + for (DescriptorListener descriptorListener : + this.descriptorListeners.get(descriptorType)) { + descriptorListener.processDescriptor(descriptor, relay); + } + } + log.info("Read archived descriptors"); + } + public void writeHistoryFiles() { - log.debug("Writing history "); + log.debug("Writing parse histories for recent descriptors..."); for (DescriptorQueue descriptorQueue : this.descriptorQueues) { descriptorQueue.writeHistoryFile(); } @@ -163,34 +216,42 @@ public class DescriptorSource { public String getStatsString() { StringBuilder sb = new StringBuilder(); - sb.append(" " + this.localFilesBefore + " descriptor files found " - + "locally\n"); - sb.append(" " + this.foundRemoteFiles + " descriptor files found " - + "remotely\n"); - sb.append(" " + this.downloadedFiles + " descriptor files " + sb.append(" " + this.localFilesBefore + " recent descriptor files " + + "found locally\n"); + sb.append(" " + this.foundRemoteFiles + " recent descriptor files " + + "found remotely\n"); + sb.append(" " + this.downloadedFiles + " recent descriptor files " + "downloaded from remote\n"); - sb.append(" " + this.deletedLocalFiles + " descriptor files " - + "deleted locally\n"); + sb.append(" " + this.deletedLocalFiles + " recent descriptor " + + "files deleted locally\n"); sb.append(" " + this.descriptorQueues.size() + " descriptor " - + "queues created\n"); + + "queues created for recent descriptors\n"); int historySizeBefore = 0, historySizeAfter = 0; long descriptors = 0L, bytes = 0L; - for (DescriptorQueue descriptorQueue : descriptorQueues) { + for (DescriptorQueue descriptorQueue : this.descriptorQueues) { historySizeBefore += descriptorQueue.getHistorySizeBefore(); historySizeAfter += descriptorQueue.getHistorySizeAfter(); descriptors += descriptorQueue.getReturnedDescriptors(); bytes += descriptorQueue.getReturnedBytes(); } sb.append(" " + FormattingUtils.formatDecimalNumber( - historySizeBefore) + " descriptors excluded from this " + historySizeBefore) + " recent descriptors excluded from this " + "execution\n"); sb.append(" " + FormattingUtils.formatDecimalNumber(descriptors) - + " descriptors provided\n"); + + " recent descriptors provided\n"); sb.append(" " + FormattingUtils.formatBytes(bytes) - + " provided\n"); + + " of recent descriptors provided\n"); sb.append(" " + FormattingUtils.formatDecimalNumber( - historySizeAfter) + " descriptors excluded from next " + historySizeAfter) + " recent descriptors excluded from next " + "execution\n"); + if (this.archiveDescriptorQueue != null) { + sb.append(" " + FormattingUtils.formatDecimalNumber( + this.archiveDescriptorQueue.getReturnedDescriptors()) + + " archived descriptors provided\n"); + sb.append(" " + FormattingUtils.formatBytes( + this.archiveDescriptorQueue.getReturnedBytes()) + " of " + + "archived descriptors provided\n"); + } return sb.toString(); } }