commit fa07b8d59739c2f1d4813c2d315ab3e11a116d0d Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Sun Jan 15 07:32:38 2012 +0100 Implement a history file to avoid reading files twice. --- .../descriptor/RelayDescriptorReader.java | 20 +--- .../descriptor/impl/RelayDescriptorReaderImpl.java | 99 +++++++++++++++----- 2 files changed, 83 insertions(+), 36 deletions(-) diff --git a/src/org/torproject/descriptor/RelayDescriptorReader.java b/src/org/torproject/descriptor/RelayDescriptorReader.java index 11735c9..90063c3 100644 --- a/src/org/torproject/descriptor/RelayDescriptorReader.java +++ b/src/org/torproject/descriptor/RelayDescriptorReader.java @@ -13,20 +13,12 @@ public interface RelayDescriptorReader { /* Add a local directory to read relay descriptors from. */ public void addDirectory(File directory); - /* Exclude the given file from the results. */ - public void setExcludeFile(File fileToExclude); - - /* Exclude the given files from the results. */ - public void setExcludeFiles(Set<File> filesToExclude); - - /* Exclude the given file from the results if it wasn't modified since - * the given timestamp. */ - public void setExcludeFile(File fileToExclude, long lastModifiedMillis); - - /* Exclude the given files from the results if they were not modified - * since the given timestamps. Map keys are files to exclude and map - * values are last modified timestamps. */ - public void setExcludeFiles(Map<File, Long> filesToExclude); + /* Exclude files that are contained in the given history file and that + * haven't changed since they were last read. Add reads from the + * current run to the history file. Remove files that don't exist + * anymore from the history file. Lines in the history file contain the + * last modified timestamp and the absolute path of a file. */ + public void setExcludeFiles(File historyFile); /* Read the previously configured relay descriptors and make them * available via the returned blocking iterator. Whenever the reader diff --git a/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java index b0e2b81..81435e0 100644 --- a/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java +++ b/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java @@ -3,9 +3,13 @@ package org.torproject.descriptor.impl; import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileReader; import java.io.FileInputStream; +import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -13,7 +17,9 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.SortedMap; import java.util.Stack; +import java.util.TreeMap; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorFile; import org.torproject.descriptor.RelayDescriptorReader; @@ -27,32 +33,16 @@ public class RelayDescriptorReaderImpl implements RelayDescriptorReader { this.directories.add(directory); } - public void setExcludeFile(File fileToExclude) { - throw new UnsupportedOperationException("Not implemented yet."); - /* TODO Implement me. */ - } - - public void setExcludeFiles(Set<File> filesToExclude) { - throw new UnsupportedOperationException("Not implemented yet."); - /* TODO Implement me. */ - } - - public void setExcludeFile(File fileToExclude, - long lastModifiedMillis) { - throw new UnsupportedOperationException("Not implemented yet."); - /* TODO Implement me. */ - } - - public void setExcludeFiles(Map<File, Long> filesToExclude) { - throw new UnsupportedOperationException("Not implemented yet."); - /* TODO Implement me. */ + private File historyFile; + public void setExcludeFiles(File historyFile) { + this.historyFile = historyFile; } public Iterator<DescriptorFile> readDescriptors() { BlockingIteratorImpl<DescriptorFile> descriptorQueue = new BlockingIteratorImpl<DescriptorFile>(); DescriptorReader reader = new DescriptorReader(this.directories, - descriptorQueue); + descriptorQueue, this.historyFile); new Thread(reader).start(); return descriptorQueue; } @@ -60,12 +50,69 @@ public class RelayDescriptorReaderImpl implements RelayDescriptorReader { private static class DescriptorReader implements Runnable { private List<File> directories; private BlockingIteratorImpl<DescriptorFile> descriptorQueue; + private File historyFile; private DescriptorReader(List<File> directories, - BlockingIteratorImpl<DescriptorFile> descriptorQueue) { + BlockingIteratorImpl<DescriptorFile> descriptorQueue, + File historyFile) { this.directories = directories; this.descriptorQueue = descriptorQueue; + this.historyFile = historyFile; } public void run() { + this.readOldHistory(); + this.readDescriptors(); + this.writeNewHistory(); + } + private SortedMap<String, Long> + oldHistory = new TreeMap<String, Long>(), + newHistory = new TreeMap<String, Long>(); + private void readOldHistory() { + if (this.historyFile == null) { + return; + } + try { + BufferedReader br = new BufferedReader(new FileReader( + this.historyFile)); + String line; + while ((line = br.readLine()) != null) { + if (!line.contains(" ")) { + /* TODO Handle this problem? */ + continue; + } + long lastModifiedMillis = Long.parseLong(line.substring(0, + line.indexOf(" "))); + String absolutePath = line.substring(line.indexOf(" ") + 1); + this.oldHistory.put(absolutePath, lastModifiedMillis); + } + br.close(); + } catch (IOException e) { + /* TODO Handle this exception. */ + } catch (NumberFormatException e) { + /* TODO Handle this exception. */ + } + } + private void writeNewHistory() { + if (this.historyFile == null) { + return; + } + try { + if (this.historyFile.getParentFile() != null) { + this.historyFile.getParentFile().mkdirs(); + } + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.historyFile)); + for (Map.Entry<String, Long> e : this.newHistory.entrySet()) { + String absolutePath = e.getKey(); + long lastModifiedMillis = e.getValue(); + bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath + + "\n"); + } + bw.close(); + } catch (IOException e) { + /* TODO Handle this exception. */ + } + } + private void readDescriptors() { for (File directory : this.directories) { try { Stack<File> files = new Stack<File>(); @@ -75,13 +122,21 @@ public class RelayDescriptorReaderImpl implements RelayDescriptorReader { if (file.isDirectory()) { files.addAll(Arrays.asList(file.listFiles())); } else { + String absolutePath = file.getAbsolutePath(); + long lastModifiedMillis = file.lastModified(); + this.newHistory.put(absolutePath, lastModifiedMillis); + if (this.oldHistory.containsKey(absolutePath) && + this.oldHistory.get(absolutePath) == + lastModifiedMillis) { + continue; + } try { List<Descriptor> parsedDescriptors = this.readFile(file); DescriptorFileImpl descriptorFile = new DescriptorFileImpl(); descriptorFile.setDirectory(directory); descriptorFile.setFile(file); - descriptorFile.setLastModified(file.lastModified()); + descriptorFile.setLastModified(lastModifiedMillis); descriptorFile.setDescriptors(parsedDescriptors); this.descriptorQueue.add(descriptorFile); } catch (DescriptorParseException e) {