commit 09f9be613307fca70f0eaef8e862981e42fee134
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Sun Jan 15 08:12:44 2012 +0100
Merge relay and bridge descriptor readers.
---
.../descriptor/BridgeDescriptorReader.java | 11 +-
.../descriptor/DescriptorSourceFactory.java | 7 +-
.../example/MetricsRelayDescriptorAggregator.java | 4 +-
.../example/TorStatusDatabaseUpdater.java | 4 +-
.../impl/BridgeDescriptorReaderImpl.java | 106 ------------
.../torproject/descriptor/impl/DescriptorImpl.java | 38 +----
.../descriptor/impl/RelayDescriptorReaderImpl.java | 172 -------------------
.../impl/RelayOrBridgeDescriptorReaderImpl.java | 173 ++++++++++++++++++++
8 files changed, 193 insertions(+), 322 deletions(-)
diff --git a/src/org/torproject/descriptor/BridgeDescriptorReader.java b/src/org/torproject/descriptor/BridgeDescriptorReader.java
index fdb30d0..c055557 100644
--- a/src/org/torproject/descriptor/BridgeDescriptorReader.java
+++ b/src/org/torproject/descriptor/BridgeDescriptorReader.java
@@ -13,11 +13,12 @@ public interface BridgeDescriptorReader {
/* Add a local directory to read bridge descriptors from. */
public void addDirectory(File directory);
- /* Exclude the given file from the results. */
- public void setExcludeFile(File fileToExclude);
-
- /* Exclude the given files from the results. */
- public void setExcludeFiles(Set<File> filesToExclude);
+ /* Exclude files that are contained in the given history file and that
+ * haven't changed since they were last read. Add reads from the
+ * current run to the history file. Remove files that don't exist
+ * anymore from the history file. Lines in the history file contain the
+ * last modified timestamp and the absolute path of a file. */
+ public void setExcludeFiles(File historyFile);
/* Read the previously configured bridge descriptors and make them
* available via the returned blocking iterator. Whenever the reader
diff --git a/src/org/torproject/descriptor/DescriptorSourceFactory.java b/src/org/torproject/descriptor/DescriptorSourceFactory.java
index ec7479b..9dcba61 100644
--- a/src/org/torproject/descriptor/DescriptorSourceFactory.java
+++ b/src/org/torproject/descriptor/DescriptorSourceFactory.java
@@ -2,16 +2,15 @@
* See LICENSE for licensing information */
package org.torproject.descriptor;
-import org.torproject.descriptor.impl.RelayDescriptorReaderImpl;
+import org.torproject.descriptor.impl.RelayOrBridgeDescriptorReaderImpl;
import org.torproject.descriptor.impl.RelayDescriptorDownloaderImpl;
-import org.torproject.descriptor.impl.BridgeDescriptorReaderImpl;
/* Create descriptor source instances. */
public class DescriptorSourceFactory {
/* Create a relay descriptor reader. */
public static RelayDescriptorReader createRelayDescriptorReader() {
- return new RelayDescriptorReaderImpl();
+ return new RelayOrBridgeDescriptorReaderImpl();
}
/* Create a relay descriptor downloader. */
@@ -22,7 +21,7 @@ public class DescriptorSourceFactory {
/* Create a bridge descriptor reader. */
public static BridgeDescriptorReader createBridgeDescriptorReader() {
- return new BridgeDescriptorReaderImpl();
+ return new RelayOrBridgeDescriptorReaderImpl();
}
}
diff --git a/src/org/torproject/descriptor/example/MetricsRelayDescriptorAggregator.java b/src/org/torproject/descriptor/example/MetricsRelayDescriptorAggregator.java
index 9aa2600..0d9da8d 100644
--- a/src/org/torproject/descriptor/example/MetricsRelayDescriptorAggregator.java
+++ b/src/org/torproject/descriptor/example/MetricsRelayDescriptorAggregator.java
@@ -52,8 +52,8 @@ public class MetricsRelayDescriptorAggregator {
/* Exclude cached descriptor files that haven't changed since we last
* ran this application. */
- reader.setExcludeFile(new File("tor-data-dir/cached-descriptors"),
- 1234567890000L);
+ //reader.setExcludeFile(new File("tor-data-dir/cached-descriptors"),
+ // 1234567890000L);
/* Read descriptors and process them. */
Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors();
diff --git a/src/org/torproject/descriptor/example/TorStatusDatabaseUpdater.java b/src/org/torproject/descriptor/example/TorStatusDatabaseUpdater.java
index 428f3b5..f3ee341 100644
--- a/src/org/torproject/descriptor/example/TorStatusDatabaseUpdater.java
+++ b/src/org/torproject/descriptor/example/TorStatusDatabaseUpdater.java
@@ -35,8 +35,8 @@ public class TorStatusDatabaseUpdater {
* ran this application. This may save some execution time. The
* application needs to store the information when files were last
* modified, because the API is supposed to be stateless. */
- reader.setExcludeFile(new File("tor-data-dir/cached-descriptors"),
- 1234567890000L);
+ //reader.setExcludeFile(new File("tor-data-dir/cached-descriptors"),
+ // 1234567890000L);
/* Read all descriptors in the given directory and import them into
* the database. Also go through the list of parsed files and store
diff --git a/src/org/torproject/descriptor/impl/BridgeDescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/BridgeDescriptorReaderImpl.java
deleted file mode 100644
index 04e5d2a..0000000
--- a/src/org/torproject/descriptor/impl/BridgeDescriptorReaderImpl.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.descriptor.impl;
-
-import java.io.BufferedInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.Stack;
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
-import org.torproject.descriptor.BridgeDescriptorReader;
-
-public class BridgeDescriptorReaderImpl
- implements BridgeDescriptorReader {
-
- private List<File> directories = new ArrayList<File>();
- public void addDirectory(File directory) {
- this.directories.add(directory);
- }
-
- public void setExcludeFile(File fileToExclude) {
- throw new UnsupportedOperationException("Not implemented yet.");
- /* TODO Implement me. */
- }
-
- public void setExcludeFiles(Set<File> filesToExclude) {
- throw new UnsupportedOperationException("Not implemented yet.");
- /* TODO Implement me. */
- }
-
- public Iterator<DescriptorFile> readDescriptors() {
- BlockingIteratorImpl<DescriptorFile> descriptorQueue =
- new BlockingIteratorImpl<DescriptorFile>();
- DescriptorReader reader = new DescriptorReader(this.directories,
- descriptorQueue);
- new Thread(reader).start();
- return descriptorQueue;
- }
-
- private static class DescriptorReader implements Runnable {
- private List<File> directories;
- private BlockingIteratorImpl<DescriptorFile> descriptorQueue;
- private DescriptorReader(List<File> directories,
- BlockingIteratorImpl<DescriptorFile> descriptorQueue) {
- this.directories = directories;
- this.descriptorQueue = descriptorQueue;
- }
- public void run() {
- for (File directory : this.directories) {
- try {
- Stack<File> files = new Stack<File>();
- files.add(directory);
- while (!files.isEmpty()) {
- File file = files.pop();
- if (file.isDirectory()) {
- files.addAll(Arrays.asList(file.listFiles()));
- } else {
- try {
- List<Descriptor> parsedDescriptors = this.readFile(file);
- DescriptorFileImpl descriptorFile =
- new DescriptorFileImpl();
- descriptorFile.setDirectory(directory);
- descriptorFile.setFile(file);
- descriptorFile.setLastModified(file.lastModified());
- descriptorFile.setDescriptors(parsedDescriptors);
- this.descriptorQueue.add(descriptorFile);
- } catch (DescriptorParseException e) {
- /* TODO Handle me. */
- }
- }
- }
- } catch (IOException e) {
- System.err.println("Error while reading descriptors in '"
- + directory.getAbsolutePath() + "'.");
- /* TODO Handle this exception somehow. */
- } finally {
- this.descriptorQueue.setOutOfDescriptors();
- }
- }
- }
- private List<Descriptor> readFile(File file) throws IOException,
- DescriptorParseException {
- FileInputStream fis = new FileInputStream(file);
- BufferedInputStream bis = new BufferedInputStream(fis);
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- bis.close();
- byte[] rawDescriptorBytes = baos.toByteArray();
- return DescriptorImpl.parseBridgeDescriptors(rawDescriptorBytes,
- file.getName());
- }
- }
-}
-
diff --git a/src/org/torproject/descriptor/impl/DescriptorImpl.java b/src/org/torproject/descriptor/impl/DescriptorImpl.java
index 233b2df..68b70e1 100644
--- a/src/org/torproject/descriptor/impl/DescriptorImpl.java
+++ b/src/org/torproject/descriptor/impl/DescriptorImpl.java
@@ -14,8 +14,9 @@ import org.torproject.descriptor.Descriptor;
public abstract class DescriptorImpl implements Descriptor {
- protected static List<Descriptor> parseRelayDescriptors(
- byte[] rawDescriptorBytes) throws DescriptorParseException {
+ protected static List<Descriptor> parseRelayOrBridgeDescriptors(
+ byte[] rawDescriptorBytes, String fileName)
+ throws DescriptorParseException {
List<Descriptor> parsedDescriptors = new ArrayList<Descriptor>();
byte[] first100Chars = new byte[Math.min(100,
rawDescriptorBytes.length)];
@@ -35,32 +36,7 @@ public abstract class DescriptorImpl implements Descriptor {
+ "network status type in descriptor starting with '"
+ firstLines + "'.");
}
- } else if (firstLines.startsWith("router ") ||
- firstLines.contains("\nrouter ")) {
- parsedDescriptors.addAll(ServerDescriptorImpl.
- parseDescriptors(rawDescriptorBytes));
- } else if (firstLines.startsWith("extra-info ") ||
- firstLines.contains("\nextra-info ")) {
- parsedDescriptors.addAll(ExtraInfoDescriptorImpl.
- parseDescriptors(rawDescriptorBytes));
- } else {
- throw new DescriptorParseException("Could not detect relay "
- + "descriptor type in descriptor starting with '" + firstLines
- + "'.");
- }
- return parsedDescriptors;
- }
-
- protected static List<Descriptor> parseBridgeDescriptors(
- byte[] rawDescriptorBytes, String fileName)
- throws DescriptorParseException {
- List<Descriptor> parsedDescriptors = new ArrayList<Descriptor>();
- byte[] first100Chars = new byte[Math.min(100,
- rawDescriptorBytes.length)];
- System.arraycopy(rawDescriptorBytes, 0, first100Chars, 0,
- first100Chars.length);
- String firstLines = new String(first100Chars);
- if (firstLines.startsWith("r ")) {
+ } else if (firstLines.startsWith("r ")) {
parsedDescriptors.add(new BridgeNetworkStatusImpl(
rawDescriptorBytes, fileName));
} else if (firstLines.startsWith("router ") ||
@@ -72,9 +48,9 @@ public abstract class DescriptorImpl implements Descriptor {
parsedDescriptors.addAll(ExtraInfoDescriptorImpl.
parseDescriptors(rawDescriptorBytes));
} else {
- throw new DescriptorParseException("Could not detect bridge "
- + "descriptor type in descriptor starting with '" + firstLines
- + "'.");
+ throw new DescriptorParseException("Could not detect relay or "
+ + "bridge descriptor type in descriptor starting with '"
+ + firstLines + "'.");
}
return parsedDescriptors;
}
diff --git a/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java
deleted file mode 100644
index 81435e0..0000000
--- a/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.descriptor.impl;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileInputStream;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.Stack;
-import java.util.TreeMap;
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
-import org.torproject.descriptor.RelayDescriptorReader;
-import org.torproject.descriptor.RelayNetworkStatusConsensus;
-import org.torproject.descriptor.RelayNetworkStatusVote;
-
-public class RelayDescriptorReaderImpl implements RelayDescriptorReader {
-
- private List<File> directories = new ArrayList<File>();
- public void addDirectory(File directory) {
- this.directories.add(directory);
- }
-
- private File historyFile;
- public void setExcludeFiles(File historyFile) {
- this.historyFile = historyFile;
- }
-
- public Iterator<DescriptorFile> readDescriptors() {
- BlockingIteratorImpl<DescriptorFile> descriptorQueue =
- new BlockingIteratorImpl<DescriptorFile>();
- DescriptorReader reader = new DescriptorReader(this.directories,
- descriptorQueue, this.historyFile);
- new Thread(reader).start();
- return descriptorQueue;
- }
-
- private static class DescriptorReader implements Runnable {
- private List<File> directories;
- private BlockingIteratorImpl<DescriptorFile> descriptorQueue;
- private File historyFile;
- private DescriptorReader(List<File> directories,
- BlockingIteratorImpl<DescriptorFile> descriptorQueue,
- File historyFile) {
- this.directories = directories;
- this.descriptorQueue = descriptorQueue;
- this.historyFile = historyFile;
- }
- public void run() {
- this.readOldHistory();
- this.readDescriptors();
- this.writeNewHistory();
- }
- private SortedMap<String, Long>
- oldHistory = new TreeMap<String, Long>(),
- newHistory = new TreeMap<String, Long>();
- private void readOldHistory() {
- if (this.historyFile == null) {
- return;
- }
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- this.historyFile));
- String line;
- while ((line = br.readLine()) != null) {
- if (!line.contains(" ")) {
- /* TODO Handle this problem? */
- continue;
- }
- long lastModifiedMillis = Long.parseLong(line.substring(0,
- line.indexOf(" ")));
- String absolutePath = line.substring(line.indexOf(" ") + 1);
- this.oldHistory.put(absolutePath, lastModifiedMillis);
- }
- br.close();
- } catch (IOException e) {
- /* TODO Handle this exception. */
- } catch (NumberFormatException e) {
- /* TODO Handle this exception. */
- }
- }
- private void writeNewHistory() {
- if (this.historyFile == null) {
- return;
- }
- try {
- if (this.historyFile.getParentFile() != null) {
- this.historyFile.getParentFile().mkdirs();
- }
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.historyFile));
- for (Map.Entry<String, Long> e : this.newHistory.entrySet()) {
- String absolutePath = e.getKey();
- long lastModifiedMillis = e.getValue();
- bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath
- + "\n");
- }
- bw.close();
- } catch (IOException e) {
- /* TODO Handle this exception. */
- }
- }
- private void readDescriptors() {
- for (File directory : this.directories) {
- try {
- Stack<File> files = new Stack<File>();
- files.add(directory);
- while (!files.isEmpty()) {
- File file = files.pop();
- if (file.isDirectory()) {
- files.addAll(Arrays.asList(file.listFiles()));
- } else {
- String absolutePath = file.getAbsolutePath();
- long lastModifiedMillis = file.lastModified();
- this.newHistory.put(absolutePath, lastModifiedMillis);
- if (this.oldHistory.containsKey(absolutePath) &&
- this.oldHistory.get(absolutePath) ==
- lastModifiedMillis) {
- continue;
- }
- try {
- List<Descriptor> parsedDescriptors = this.readFile(file);
- DescriptorFileImpl descriptorFile =
- new DescriptorFileImpl();
- descriptorFile.setDirectory(directory);
- descriptorFile.setFile(file);
- descriptorFile.setLastModified(lastModifiedMillis);
- descriptorFile.setDescriptors(parsedDescriptors);
- this.descriptorQueue.add(descriptorFile);
- } catch (DescriptorParseException e) {
- /* TODO Handle me. */
- }
- }
- }
- } catch (IOException e) {
- System.err.println("Error while reading descriptors in '"
- + directory.getAbsolutePath() + "'.");
- /* TODO Handle this exception somehow. */
- } finally {
- this.descriptorQueue.setOutOfDescriptors();
- }
- }
- }
- private List<Descriptor> readFile(File file) throws IOException,
- DescriptorParseException {
- FileInputStream fis = new FileInputStream(file);
- BufferedInputStream bis = new BufferedInputStream(fis);
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- bis.close();
- byte[] rawDescriptorBytes = baos.toByteArray();
- return DescriptorImpl.parseRelayDescriptors(rawDescriptorBytes);
- }
- }
-}
-
diff --git a/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java
new file mode 100644
index 0000000..8e1c3bc
--- /dev/null
+++ b/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java
@@ -0,0 +1,173 @@
+/* Copyright 2011, 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.descriptor.impl;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.Stack;
+import java.util.TreeMap;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorFile;
+import org.torproject.descriptor.RelayDescriptorReader;
+import org.torproject.descriptor.BridgeDescriptorReader;
+
+public class RelayOrBridgeDescriptorReaderImpl
+ implements RelayDescriptorReader, BridgeDescriptorReader {
+
+ private List<File> directories = new ArrayList<File>();
+ public void addDirectory(File directory) {
+ this.directories.add(directory);
+ }
+
+ private File historyFile;
+ public void setExcludeFiles(File historyFile) {
+ this.historyFile = historyFile;
+ }
+
+ public Iterator<DescriptorFile> readDescriptors() {
+ BlockingIteratorImpl<DescriptorFile> descriptorQueue =
+ new BlockingIteratorImpl<DescriptorFile>();
+ DescriptorReader reader = new DescriptorReader(this.directories,
+ descriptorQueue, this.historyFile);
+ new Thread(reader).start();
+ return descriptorQueue;
+ }
+
+ private static class DescriptorReader implements Runnable {
+ private List<File> directories;
+ private BlockingIteratorImpl<DescriptorFile> descriptorQueue;
+ private File historyFile;
+ private DescriptorReader(List<File> directories,
+ BlockingIteratorImpl<DescriptorFile> descriptorQueue,
+ File historyFile) {
+ this.directories = directories;
+ this.descriptorQueue = descriptorQueue;
+ this.historyFile = historyFile;
+ }
+ public void run() {
+ this.readOldHistory();
+ this.readDescriptors();
+ this.writeNewHistory();
+ }
+ private SortedMap<String, Long>
+ oldHistory = new TreeMap<String, Long>(),
+ newHistory = new TreeMap<String, Long>();
+ private void readOldHistory() {
+ if (this.historyFile == null) {
+ return;
+ }
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.historyFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (!line.contains(" ")) {
+ /* TODO Handle this problem? */
+ continue;
+ }
+ long lastModifiedMillis = Long.parseLong(line.substring(0,
+ line.indexOf(" ")));
+ String absolutePath = line.substring(line.indexOf(" ") + 1);
+ this.oldHistory.put(absolutePath, lastModifiedMillis);
+ }
+ br.close();
+ } catch (IOException e) {
+ /* TODO Handle this exception. */
+ } catch (NumberFormatException e) {
+ /* TODO Handle this exception. */
+ }
+ }
+ private void writeNewHistory() {
+ if (this.historyFile == null) {
+ return;
+ }
+ try {
+ if (this.historyFile.getParentFile() != null) {
+ this.historyFile.getParentFile().mkdirs();
+ }
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.historyFile));
+ for (Map.Entry<String, Long> e : this.newHistory.entrySet()) {
+ String absolutePath = e.getKey();
+ long lastModifiedMillis = e.getValue();
+ bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath
+ + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ /* TODO Handle this exception. */
+ }
+ }
+ private void readDescriptors() {
+ for (File directory : this.directories) {
+ try {
+ Stack<File> files = new Stack<File>();
+ files.add(directory);
+ while (!files.isEmpty()) {
+ File file = files.pop();
+ if (file.isDirectory()) {
+ files.addAll(Arrays.asList(file.listFiles()));
+ } else {
+ String absolutePath = file.getAbsolutePath();
+ long lastModifiedMillis = file.lastModified();
+ this.newHistory.put(absolutePath, lastModifiedMillis);
+ if (this.oldHistory.containsKey(absolutePath) &&
+ this.oldHistory.get(absolutePath) ==
+ lastModifiedMillis) {
+ continue;
+ }
+ try {
+ List<Descriptor> parsedDescriptors = this.readFile(file);
+ DescriptorFileImpl descriptorFile =
+ new DescriptorFileImpl();
+ descriptorFile.setDirectory(directory);
+ descriptorFile.setFile(file);
+ descriptorFile.setLastModified(lastModifiedMillis);
+ descriptorFile.setDescriptors(parsedDescriptors);
+ this.descriptorQueue.add(descriptorFile);
+ } catch (DescriptorParseException e) {
+ /* TODO Handle me. */
+ }
+ }
+ }
+ } catch (IOException e) {
+ System.err.println("Error while reading descriptors in '"
+ + directory.getAbsolutePath() + "'.");
+ /* TODO Handle this exception somehow. */
+ } finally {
+ this.descriptorQueue.setOutOfDescriptors();
+ }
+ }
+ }
+ private List<Descriptor> readFile(File file) throws IOException,
+ DescriptorParseException {
+ FileInputStream fis = new FileInputStream(file);
+ BufferedInputStream bis = new BufferedInputStream(fis);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ bis.close();
+ byte[] rawDescriptorBytes = baos.toByteArray();
+ return DescriptorImpl.parseRelayOrBridgeDescriptors(
+ rawDescriptorBytes, file.getName());
+ }
+ }
+}
+
