[tor-commits] [metrics-lib/master] Implement a history file to avoid reading files twice.
karsten at torproject.org
karsten at torproject.org
Sun Jan 15 06:36:26 UTC 2012
commit fa07b8d59739c2f1d4813c2d315ab3e11a116d0d
Author: Karsten Loesing <karsten.loesing at gmx.net>
Date: Sun Jan 15 07:32:38 2012 +0100
Implement a history file to avoid reading files twice.
---
.../descriptor/RelayDescriptorReader.java | 20 +---
.../descriptor/impl/RelayDescriptorReaderImpl.java | 99 +++++++++++++++-----
2 files changed, 83 insertions(+), 36 deletions(-)
diff --git a/src/org/torproject/descriptor/RelayDescriptorReader.java b/src/org/torproject/descriptor/RelayDescriptorReader.java
index 11735c9..90063c3 100644
--- a/src/org/torproject/descriptor/RelayDescriptorReader.java
+++ b/src/org/torproject/descriptor/RelayDescriptorReader.java
@@ -13,20 +13,12 @@ public interface RelayDescriptorReader {
/* Add a local directory to read relay descriptors from. */
public void addDirectory(File directory);
- /* Exclude the given file from the results. */
- public void setExcludeFile(File fileToExclude);
-
- /* Exclude the given files from the results. */
- public void setExcludeFiles(Set<File> filesToExclude);
-
- /* Exclude the given file from the results if it wasn't modified since
- * the given timestamp. */
- public void setExcludeFile(File fileToExclude, long lastModifiedMillis);
-
- /* Exclude the given files from the results if they were not modified
- * since the given timestamps. Map keys are files to exclude and map
- * values are last modified timestamps. */
- public void setExcludeFiles(Map<File, Long> filesToExclude);
+ /* Exclude files that are contained in the given history file and that
+ * haven't changed since they were last read. Add reads from the
+ * current run to the history file. Remove files that don't exist
+ * anymore from the history file. Lines in the history file contain the
+ * last modified timestamp and the absolute path of a file. */
+ public void setExcludeFiles(File historyFile);
/* Read the previously configured relay descriptors and make them
* available via the returned blocking iterator. Whenever the reader
diff --git a/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java
index b0e2b81..81435e0 100644
--- a/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java
+++ b/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java
@@ -3,9 +3,13 @@
package org.torproject.descriptor.impl;
import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
+import java.io.FileReader;
import java.io.FileInputStream;
+import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -13,7 +17,9 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.SortedMap;
import java.util.Stack;
+import java.util.TreeMap;
import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorFile;
import org.torproject.descriptor.RelayDescriptorReader;
@@ -27,32 +33,16 @@ public class RelayDescriptorReaderImpl implements RelayDescriptorReader {
this.directories.add(directory);
}
- public void setExcludeFile(File fileToExclude) {
- throw new UnsupportedOperationException("Not implemented yet.");
- /* TODO Implement me. */
- }
-
- public void setExcludeFiles(Set<File> filesToExclude) {
- throw new UnsupportedOperationException("Not implemented yet.");
- /* TODO Implement me. */
- }
-
- public void setExcludeFile(File fileToExclude,
- long lastModifiedMillis) {
- throw new UnsupportedOperationException("Not implemented yet.");
- /* TODO Implement me. */
- }
-
- public void setExcludeFiles(Map<File, Long> filesToExclude) {
- throw new UnsupportedOperationException("Not implemented yet.");
- /* TODO Implement me. */
+ private File historyFile;
+ public void setExcludeFiles(File historyFile) {
+ this.historyFile = historyFile;
}
public Iterator<DescriptorFile> readDescriptors() {
BlockingIteratorImpl<DescriptorFile> descriptorQueue =
new BlockingIteratorImpl<DescriptorFile>();
DescriptorReader reader = new DescriptorReader(this.directories,
- descriptorQueue);
+ descriptorQueue, this.historyFile);
new Thread(reader).start();
return descriptorQueue;
}
@@ -60,12 +50,69 @@ public class RelayDescriptorReaderImpl implements RelayDescriptorReader {
private static class DescriptorReader implements Runnable {
private List<File> directories;
private BlockingIteratorImpl<DescriptorFile> descriptorQueue;
+ private File historyFile;
private DescriptorReader(List<File> directories,
- BlockingIteratorImpl<DescriptorFile> descriptorQueue) {
+ BlockingIteratorImpl<DescriptorFile> descriptorQueue,
+ File historyFile) {
this.directories = directories;
this.descriptorQueue = descriptorQueue;
+ this.historyFile = historyFile;
}
public void run() {
+ this.readOldHistory();
+ this.readDescriptors();
+ this.writeNewHistory();
+ }
+ private SortedMap<String, Long>
+ oldHistory = new TreeMap<String, Long>(),
+ newHistory = new TreeMap<String, Long>();
+ private void readOldHistory() {
+ if (this.historyFile == null) {
+ return;
+ }
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.historyFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (!line.contains(" ")) {
+ /* TODO Handle this problem? */
+ continue;
+ }
+ long lastModifiedMillis = Long.parseLong(line.substring(0,
+ line.indexOf(" ")));
+ String absolutePath = line.substring(line.indexOf(" ") + 1);
+ this.oldHistory.put(absolutePath, lastModifiedMillis);
+ }
+ br.close();
+ } catch (IOException e) {
+ /* TODO Handle this exception. */
+ } catch (NumberFormatException e) {
+ /* TODO Handle this exception. */
+ }
+ }
+ private void writeNewHistory() {
+ if (this.historyFile == null) {
+ return;
+ }
+ try {
+ if (this.historyFile.getParentFile() != null) {
+ this.historyFile.getParentFile().mkdirs();
+ }
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.historyFile));
+ for (Map.Entry<String, Long> e : this.newHistory.entrySet()) {
+ String absolutePath = e.getKey();
+ long lastModifiedMillis = e.getValue();
+ bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath
+ + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ /* TODO Handle this exception. */
+ }
+ }
+ private void readDescriptors() {
for (File directory : this.directories) {
try {
Stack<File> files = new Stack<File>();
@@ -75,13 +122,21 @@ public class RelayDescriptorReaderImpl implements RelayDescriptorReader {
if (file.isDirectory()) {
files.addAll(Arrays.asList(file.listFiles()));
} else {
+ String absolutePath = file.getAbsolutePath();
+ long lastModifiedMillis = file.lastModified();
+ this.newHistory.put(absolutePath, lastModifiedMillis);
+ if (this.oldHistory.containsKey(absolutePath) &&
+ this.oldHistory.get(absolutePath) ==
+ lastModifiedMillis) {
+ continue;
+ }
try {
List<Descriptor> parsedDescriptors = this.readFile(file);
DescriptorFileImpl descriptorFile =
new DescriptorFileImpl();
descriptorFile.setDirectory(directory);
descriptorFile.setFile(file);
- descriptorFile.setLastModified(file.lastModified());
+ descriptorFile.setLastModified(lastModifiedMillis);
descriptorFile.setDescriptors(parsedDescriptors);
this.descriptorQueue.add(descriptorFile);
} catch (DescriptorParseException e) {
More information about the tor-commits
mailing list