[tor-commits] [onionoo/master] Support parsing archived descriptors from in/archive/.
karsten at torproject.org
karsten at torproject.org
Fri Jun 19 11:22:02 UTC 2015
commit 00494ff1f4ea85c8501cb56a0adba56fd3305616
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Sun Apr 26 19:10:34 2015 +0200
Support parsing archived descriptors from in/archive/.
Implements part of #13600.
---
.../onionoo/updater/DescriptorQueue.java | 29 +++--
.../onionoo/updater/DescriptorSource.java | 137 ++++++++++++++------
2 files changed, 120 insertions(+), 46 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java b/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java
index 93a291c..b86d585 100644
--- a/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java
+++ b/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java
@@ -24,8 +24,6 @@ class DescriptorQueue {
private final static Logger log = LoggerFactory.getLogger(
DescriptorQueue.class);
- private File inDir;
-
private File statusDir;
private DescriptorReader descriptorReader;
@@ -56,14 +54,21 @@ class DescriptorQueue {
return this.returnedBytes;
}
- public DescriptorQueue(File inDir, File statusDir) {
- this.inDir = inDir;
+ public DescriptorQueue(File inDir, DescriptorType descriptorType,
+ File statusDir) {
+ File directory = inDir;
+ if (descriptorType != null) {
+ directory = this.getDirectoryForDescriptorType(inDir,
+ descriptorType);
+ }
this.statusDir = statusDir;
this.descriptorReader =
DescriptorSourceFactory.createDescriptorReader();
+ this.addDirectory(directory);
}
- public void addDirectory(DescriptorType descriptorType) {
+ public File getDirectoryForDescriptorType(File inDir,
+ DescriptorType descriptorType) {
String directoryName = null;
switch (descriptorType) {
case RELAY_CONSENSUSES:
@@ -93,9 +98,16 @@ class DescriptorQueue {
default:
log.error("Unknown descriptor type. Not adding directory "
+ "to descriptor reader.");
+ return null;
+ }
+ File directory = new File(inDir, directoryName);
+ return directory;
+ }
+
+ private void addDirectory(File directory) {
+ if (directory == null) {
return;
}
- File directory = new File(this.inDir, directoryName);
if (directory.exists() && directory.isDirectory()) {
this.descriptorReader.addDirectory(directory);
this.descriptorReader.setMaxDescriptorFilesInQueue(1);
@@ -107,6 +119,9 @@ class DescriptorQueue {
}
public void readHistoryFile(DescriptorHistory descriptorHistory) {
+ if (this.statusDir == null) {
+ return;
+ }
String historyFileName = null;
switch (descriptorHistory) {
case RELAY_EXTRAINFO_HISTORY:
@@ -168,8 +183,6 @@ class DescriptorQueue {
public void writeHistoryFile() {
if (this.historyFile == null) {
- log.error("Unable to write history file, possibly because of an "
- + "unknown descriptor type. Skipping.");
return;
}
SortedMap<String, Long> excludedAndParsedFiles =
diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java
index 52e00c9..b26a692 100644
--- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java
+++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java
@@ -20,12 +20,15 @@ public class DescriptorSource {
private static final Logger log = LoggerFactory.getLogger(
DescriptorSource.class);
- private final File inDir = new File("in/recent");
+ private final File inRecentDir = new File("in/recent"),
+ inArchiveDir = new File("in/archive");
private final File statusDir = new File("status");
private List<DescriptorQueue> descriptorQueues;
+ private DescriptorQueue archiveDescriptorQueue;
+
public DescriptorSource() {
this.descriptorQueues = new ArrayList<DescriptorQueue>();
this.descriptorListeners =
@@ -35,9 +38,8 @@ public class DescriptorSource {
private DescriptorQueue getDescriptorQueue(
DescriptorType descriptorType,
DescriptorHistory descriptorHistory) {
- DescriptorQueue descriptorQueue = new DescriptorQueue(this.inDir,
- this.statusDir);
- descriptorQueue.addDirectory(descriptorType);
+ DescriptorQueue descriptorQueue = new DescriptorQueue(
+ this.inRecentDir, descriptorType, this.statusDir);
if (descriptorHistory != null) {
descriptorQueue.readHistoryFile(descriptorHistory);
}
@@ -78,35 +80,32 @@ public class DescriptorSource {
}
public void readDescriptors() {
- /* Careful when changing the order of parsing descriptor types! The
- * various status updaters may base assumptions on this order. With
- * #13674 being implemented, this may not be the case anymore. */
-
- log.debug("Reading " + DescriptorType.RELAY_SERVER_DESCRIPTORS
+ this.readArchivedDescriptors();
+ log.debug("Reading recent " + DescriptorType.RELAY_SERVER_DESCRIPTORS
+ " ...");
this.readDescriptors(DescriptorType.RELAY_SERVER_DESCRIPTORS,
DescriptorHistory.RELAY_SERVER_HISTORY, true);
- log.debug("Reading " + DescriptorType.RELAY_EXTRA_INFOS + " ...");
+ log.debug("Reading recent " + DescriptorType.RELAY_EXTRA_INFOS + " ...");
this.readDescriptors(DescriptorType.RELAY_EXTRA_INFOS,
DescriptorHistory.RELAY_EXTRAINFO_HISTORY, true);
- log.debug("Reading " + DescriptorType.EXIT_LISTS + " ...");
+ log.debug("Reading recent " + DescriptorType.EXIT_LISTS + " ...");
this.readDescriptors(DescriptorType.EXIT_LISTS,
DescriptorHistory.EXIT_LIST_HISTORY, true);
- log.debug("Reading " + DescriptorType.RELAY_CONSENSUSES + " ...");
+ log.debug("Reading recent " + DescriptorType.RELAY_CONSENSUSES + " ...");
this.readDescriptors(DescriptorType.RELAY_CONSENSUSES,
DescriptorHistory.RELAY_CONSENSUS_HISTORY, true);
- log.debug("Reading " + DescriptorType.BRIDGE_SERVER_DESCRIPTORS
+ log.debug("Reading recent " + DescriptorType.BRIDGE_SERVER_DESCRIPTORS
+ " ...");
this.readDescriptors(DescriptorType.BRIDGE_SERVER_DESCRIPTORS,
DescriptorHistory.BRIDGE_SERVER_HISTORY, false);
- log.debug("Reading " + DescriptorType.BRIDGE_EXTRA_INFOS + " ...");
+ log.debug("Reading recent " + DescriptorType.BRIDGE_EXTRA_INFOS + " ...");
this.readDescriptors(DescriptorType.BRIDGE_EXTRA_INFOS,
DescriptorHistory.BRIDGE_EXTRAINFO_HISTORY, false);
- log.debug("Reading " + DescriptorType.BRIDGE_POOL_ASSIGNMENTS
+ log.debug("Reading recent " + DescriptorType.BRIDGE_POOL_ASSIGNMENTS
+ " ...");
this.readDescriptors(DescriptorType.BRIDGE_POOL_ASSIGNMENTS,
DescriptorHistory.BRIDGE_POOLASSIGN_HISTORY, false);
- log.debug("Reading " + DescriptorType.BRIDGE_STATUSES + " ...");
+ log.debug("Reading recent " + DescriptorType.BRIDGE_STATUSES + " ...");
this.readDescriptors(DescriptorType.BRIDGE_STATUSES,
DescriptorHistory.BRIDGE_STATUS_HISTORY, false);
}
@@ -128,34 +127,88 @@ public class DescriptorSource {
}
switch (descriptorType) {
case RELAY_CONSENSUSES:
- log.info("Read relay network consensuses");
+ log.info("Read recent relay network consensuses");
break;
case RELAY_SERVER_DESCRIPTORS:
- log.info("Read relay server descriptors");
+ log.info("Read recent relay server descriptors");
break;
case RELAY_EXTRA_INFOS:
- log.info("Read relay extra-info descriptors");
+ log.info("Read recent relay extra-info descriptors");
break;
case EXIT_LISTS:
- log.info("Read exit lists");
+ log.info("Read recent exit lists");
break;
case BRIDGE_STATUSES:
- log.info("Read bridge network statuses");
+ log.info("Read recent bridge network statuses");
break;
case BRIDGE_SERVER_DESCRIPTORS:
- log.info("Read bridge server descriptors");
+ log.info("Read recent bridge server descriptors");
break;
case BRIDGE_EXTRA_INFOS:
- log.info("Read bridge extra-info descriptors");
+ log.info("Read recent bridge extra-info descriptors");
break;
case BRIDGE_POOL_ASSIGNMENTS:
- log.info("Read bridge-pool assignments");
+ log.info("Read recent bridge-pool assignments");
break;
}
}
+ public void readArchivedDescriptors() {
+ if (!this.inArchiveDir.exists()) {
+ return;
+ }
+ log.info("Reading archived descriptors...");
+ this.archiveDescriptorQueue = new DescriptorQueue(this.inArchiveDir,
+ null, null);
+ Descriptor descriptor;
+ while ((descriptor = this.archiveDescriptorQueue.nextDescriptor())
+ != null) {
+ DescriptorType descriptorType = null;
+ boolean relay = false;
+ for (String annotation : descriptor.getAnnotations()) {
+ if (annotation.startsWith(
+ "@type network-status-consensus-3 1.")) {
+ descriptorType = DescriptorType.RELAY_CONSENSUSES;
+ relay = true;
+ } else if (annotation.startsWith("@type server-descriptor 1.")) {
+ descriptorType = DescriptorType.RELAY_SERVER_DESCRIPTORS;
+ relay = true;
+ } else if (annotation.startsWith("@type extra-info 1.")) {
+ descriptorType = DescriptorType.RELAY_EXTRA_INFOS;
+ relay = true;
+ } else if (annotation.startsWith("@type tordnsel 1.")) {
+ descriptorType = DescriptorType.EXIT_LISTS;
+ relay = true;
+ } else if (annotation.startsWith(
+ "@type bridge-network-status 1.")) {
+ descriptorType = DescriptorType.BRIDGE_STATUSES;
+ relay = false;
+ } else if (annotation.startsWith(
+ "@type bridge-server-descriptor 1.")) {
+ descriptorType = DescriptorType.BRIDGE_SERVER_DESCRIPTORS;
+ relay = false;
+ } else if (annotation.startsWith("@type bridge-extra-info 1.")) {
+ descriptorType = DescriptorType.BRIDGE_EXTRA_INFOS;
+ relay = false;
+ }
+ }
+ if (descriptorType == null) {
+ log.warn("Unrecognized descriptor in "
+ + this.inArchiveDir.getAbsolutePath() + " with annotations "
+ + descriptor.getAnnotations() + ". Not reading any further"
+ + "archived descriptors.");
+ break;
+ }
+ for (DescriptorListener descriptorListener :
+ this.descriptorListeners.get(descriptorType)) {
+ descriptorListener.processDescriptor(descriptor, relay);
+ }
+ }
+ log.info("Read archived descriptors");
+ }
+
public void writeHistoryFiles() {
- log.debug("Writing history ");
+ log.debug("Writing parse histories for recent descriptors...");
for (DescriptorQueue descriptorQueue : this.descriptorQueues) {
descriptorQueue.writeHistoryFile();
}
@@ -163,34 +216,42 @@ public class DescriptorSource {
public String getStatsString() {
StringBuilder sb = new StringBuilder();
- sb.append(" " + this.localFilesBefore + " descriptor files found "
- + "locally\n");
- sb.append(" " + this.foundRemoteFiles + " descriptor files found "
- + "remotely\n");
- sb.append(" " + this.downloadedFiles + " descriptor files "
+ sb.append(" " + this.localFilesBefore + " recent descriptor files "
+ + "found locally\n");
+ sb.append(" " + this.foundRemoteFiles + " recent descriptor files "
+ + "found remotely\n");
+ sb.append(" " + this.downloadedFiles + " recent descriptor files "
+ "downloaded from remote\n");
- sb.append(" " + this.deletedLocalFiles + " descriptor files "
- + "deleted locally\n");
+ sb.append(" " + this.deletedLocalFiles + " recent descriptor "
+ + "files deleted locally\n");
sb.append(" " + this.descriptorQueues.size() + " descriptor "
- + "queues created\n");
+ + "queues created for recent descriptors\n");
int historySizeBefore = 0, historySizeAfter = 0;
long descriptors = 0L, bytes = 0L;
- for (DescriptorQueue descriptorQueue : descriptorQueues) {
+ for (DescriptorQueue descriptorQueue : this.descriptorQueues) {
historySizeBefore += descriptorQueue.getHistorySizeBefore();
historySizeAfter += descriptorQueue.getHistorySizeAfter();
descriptors += descriptorQueue.getReturnedDescriptors();
bytes += descriptorQueue.getReturnedBytes();
}
sb.append(" " + FormattingUtils.formatDecimalNumber(
- historySizeBefore) + " descriptors excluded from this "
+ historySizeBefore) + " recent descriptors excluded from this "
+ "execution\n");
sb.append(" " + FormattingUtils.formatDecimalNumber(descriptors)
- + " descriptors provided\n");
+ + " recent descriptors provided\n");
sb.append(" " + FormattingUtils.formatBytes(bytes)
- + " provided\n");
+ + " of recent descriptors provided\n");
sb.append(" " + FormattingUtils.formatDecimalNumber(
- historySizeAfter) + " descriptors excluded from next "
+ historySizeAfter) + " recent descriptors excluded from next "
+ "execution\n");
+ if (this.archiveDescriptorQueue != null) {
+ sb.append(" " + FormattingUtils.formatDecimalNumber(
+ this.archiveDescriptorQueue.getReturnedDescriptors()) +
+ " archived descriptors provided\n");
+ sb.append(" " + FormattingUtils.formatBytes(
+ this.archiveDescriptorQueue.getReturnedBytes()) + " of "
+ + "archived descriptors provided\n");
+ }
return sb.toString();
}
}
More information about the tor-commits
mailing list