commit 6054dddf2c6f658a0c3ab9d7a11d404d5bb71c37 Author: Karsten Loesing karsten.loesing@gmx.net Date: Wed Jan 6 14:19:00 2016 +0100
Skip lines that we don't care about.
The documents we're reading to memory can be huge, and in some cases we only care about a small portion of it. Rather than parsing all lines, we can skip lines we don't care about and only parse the remaining lines. --- .../org/torproject/metrics/hidserv/DocumentStore.java | 14 ++++++++++++++ .../src/org/torproject/metrics/hidserv/Extrapolator.java | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/modules/hidserv/src/org/torproject/metrics/hidserv/DocumentStore.java b/modules/hidserv/src/org/torproject/metrics/hidserv/DocumentStore.java index 3266df5..e7ef0aa 100644 --- a/modules/hidserv/src/org/torproject/metrics/hidserv/DocumentStore.java +++ b/modules/hidserv/src/org/torproject/metrics/hidserv/DocumentStore.java @@ -97,6 +97,12 @@ public class DocumentStore<T extends Document> {
/* Retrieve all previously stored documents from the given file. */ public Set<T> retrieve(File documentFile) { + return this.retrieve(documentFile, ""); + } + + /* Retrieve previously stored documents from the given file that start + * with the given prefix. */ + public Set<T> retrieve(File documentFile, String prefix) {
/* Check if the document file exists, and if not, return an empty set. * This is not an error case. */ @@ -120,6 +126,14 @@ public class DocumentStore<T extends Document> { + "documents.%n", documentFile.getAbsolutePath()); lnr.close(); return null; + } else if (prefix.length() > formattedString0.length() && + !(formattedString0 + line.substring(1)).startsWith(prefix)) { + /* Skip combined line not starting with prefix. */ + continue; + } else if (prefix.length() > 0 && + !formattedString0.startsWith(prefix)) { + /* Skip line not starting with prefix. */ + continue; } else { T document = this.clazz.newInstance(); if (!document.parse(new String[] { formattedString0, diff --git a/modules/hidserv/src/org/torproject/metrics/hidserv/Extrapolator.java b/modules/hidserv/src/org/torproject/metrics/hidserv/Extrapolator.java index a1ff075..e926154 100644 --- a/modules/hidserv/src/org/torproject/metrics/hidserv/Extrapolator.java +++ b/modules/hidserv/src/org/torproject/metrics/hidserv/Extrapolator.java @@ -153,7 +153,8 @@ public class Extrapolator { File documentFile = new File( this.computedNetworkFractionsDirectory, date); Set<ComputedNetworkFractions> fractions - = this.computedNetworkFractionsStore.retrieve(documentFile); + = this.computedNetworkFractionsStore.retrieve(documentFile, + fingerprint); for (ComputedNetworkFractions fraction : fractions) { knownConsensuses.add(fraction.getValidAfterMillis()); if (fraction.getFingerprint().equals(fingerprint)) {