commit 09f9be613307fca70f0eaef8e862981e42fee134 Author: Karsten Loesing karsten.loesing@gmx.net Date: Sun Jan 15 08:12:44 2012 +0100

Merge relay and bridge descriptor readers. --- .../descriptor/BridgeDescriptorReader.java | 11 +- .../descriptor/DescriptorSourceFactory.java | 7 +- .../example/MetricsRelayDescriptorAggregator.java | 4 +- .../example/TorStatusDatabaseUpdater.java | 4 +- .../impl/BridgeDescriptorReaderImpl.java | 106 ------------ .../torproject/descriptor/impl/DescriptorImpl.java | 38 +---- .../descriptor/impl/RelayDescriptorReaderImpl.java | 172 ------------------- .../impl/RelayOrBridgeDescriptorReaderImpl.java | 173 ++++++++++++++++++++ 8 files changed, 193 insertions(+), 322 deletions(-)

diff --git a/src/org/torproject/descriptor/BridgeDescriptorReader.java b/src/org/torproject/descriptor/BridgeDescriptorReader.java index fdb30d0..c055557 100644 --- a/src/org/torproject/descriptor/BridgeDescriptorReader.java +++ b/src/org/torproject/descriptor/BridgeDescriptorReader.java @@ -13,11 +13,12 @@ public interface BridgeDescriptorReader { /* Add a local directory to read bridge descriptors from. */ public void addDirectory(File directory);

- /* Exclude the given file from the results. */ - public void setExcludeFile(File fileToExclude); - - /* Exclude the given files from the results. */ - public void setExcludeFiles(Set<File> filesToExclude); + /* Exclude files that are contained in the given history file and that + * haven't changed since they were last read. Add reads from the + * current run to the history file. Remove files that don't exist + * anymore from the history file. Lines in the history file contain the + * last modified timestamp and the absolute path of a file. */ + public void setExcludeFiles(File historyFile);

/* Read the previously configured bridge descriptors and make them * available via the returned blocking iterator. Whenever the reader diff --git a/src/org/torproject/descriptor/DescriptorSourceFactory.java b/src/org/torproject/descriptor/DescriptorSourceFactory.java index ec7479b..9dcba61 100644 --- a/src/org/torproject/descriptor/DescriptorSourceFactory.java +++ b/src/org/torproject/descriptor/DescriptorSourceFactory.java @@ -2,16 +2,15 @@ * See LICENSE for licensing information */ package org.torproject.descriptor;

-import org.torproject.descriptor.impl.RelayDescriptorReaderImpl; +import org.torproject.descriptor.impl.RelayOrBridgeDescriptorReaderImpl; import org.torproject.descriptor.impl.RelayDescriptorDownloaderImpl; -import org.torproject.descriptor.impl.BridgeDescriptorReaderImpl;

/* Create descriptor source instances. */ public class DescriptorSourceFactory {

/* Create a relay descriptor reader. */ public static RelayDescriptorReader createRelayDescriptorReader() { - return new RelayDescriptorReaderImpl(); + return new RelayOrBridgeDescriptorReaderImpl(); }

/* Create a relay descriptor downloader. */ @@ -22,7 +21,7 @@ public class DescriptorSourceFactory {

/* Create a bridge descriptor reader. */ public static BridgeDescriptorReader createBridgeDescriptorReader() { - return new BridgeDescriptorReaderImpl(); + return new RelayOrBridgeDescriptorReaderImpl(); } }

diff --git a/src/org/torproject/descriptor/example/MetricsRelayDescriptorAggregator.java b/src/org/torproject/descriptor/example/MetricsRelayDescriptorAggregator.java index 9aa2600..0d9da8d 100644 --- a/src/org/torproject/descriptor/example/MetricsRelayDescriptorAggregator.java +++ b/src/org/torproject/descriptor/example/MetricsRelayDescriptorAggregator.java @@ -52,8 +52,8 @@ public class MetricsRelayDescriptorAggregator {

/* Exclude cached descriptor files that haven't changed since we last * ran this application. */ - reader.setExcludeFile(new File("tor-data-dir/cached-descriptors"), - 1234567890000L); + //reader.setExcludeFile(new File("tor-data-dir/cached-descriptors"), + // 1234567890000L);

/* Read descriptors and process them. */ Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors(); diff --git a/src/org/torproject/descriptor/example/TorStatusDatabaseUpdater.java b/src/org/torproject/descriptor/example/TorStatusDatabaseUpdater.java index 428f3b5..f3ee341 100644 --- a/src/org/torproject/descriptor/example/TorStatusDatabaseUpdater.java +++ b/src/org/torproject/descriptor/example/TorStatusDatabaseUpdater.java @@ -35,8 +35,8 @@ public class TorStatusDatabaseUpdater { * ran this application. This may save some execution time. The * application needs to store the information when files were last * modified, because the API is supposed to be stateless. */ - reader.setExcludeFile(new File("tor-data-dir/cached-descriptors"), - 1234567890000L); + //reader.setExcludeFile(new File("tor-data-dir/cached-descriptors"), + // 1234567890000L);

/* Read all descriptors in the given directory and import them into * the database. Also go through the list of parsed files and store diff --git a/src/org/torproject/descriptor/impl/BridgeDescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/BridgeDescriptorReaderImpl.java deleted file mode 100644 index 04e5d2a..0000000 --- a/src/org/torproject/descriptor/impl/BridgeDescriptorReaderImpl.java +++ /dev/null @@ -1,106 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.descriptor.impl; - -import java.io.BufferedInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.Stack; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; -import org.torproject.descriptor.BridgeDescriptorReader; - -public class BridgeDescriptorReaderImpl - implements BridgeDescriptorReader { - - private List<File> directories = new ArrayList<File>(); - public void addDirectory(File directory) { - this.directories.add(directory); - } - - public void setExcludeFile(File fileToExclude) { - throw new UnsupportedOperationException("Not implemented yet."); - /* TODO Implement me. */ - } - - public void setExcludeFiles(Set<File> filesToExclude) { - throw new UnsupportedOperationException("Not implemented yet."); - /* TODO Implement me. */ - } - - public Iterator<DescriptorFile> readDescriptors() { - BlockingIteratorImpl<DescriptorFile> descriptorQueue = - new BlockingIteratorImpl<DescriptorFile>(); - DescriptorReader reader = new DescriptorReader(this.directories, - descriptorQueue); - new Thread(reader).start(); - return descriptorQueue; - } - - private static class DescriptorReader implements Runnable { - private List<File> directories; - private BlockingIteratorImpl<DescriptorFile> descriptorQueue; - private DescriptorReader(List<File> directories, - BlockingIteratorImpl<DescriptorFile> descriptorQueue) { - this.directories = directories; - this.descriptorQueue = descriptorQueue; - } - public void run() { - for (File directory : this.directories) { - try { - Stack<File> files = new Stack<File>(); - files.add(directory); - while (!files.isEmpty()) { - File file = files.pop(); - if (file.isDirectory()) { - files.addAll(Arrays.asList(file.listFiles())); - } else { - try { - List<Descriptor> parsedDescriptors = this.readFile(file); - DescriptorFileImpl descriptorFile = - new DescriptorFileImpl(); - descriptorFile.setDirectory(directory); - descriptorFile.setFile(file); - descriptorFile.setLastModified(file.lastModified()); - descriptorFile.setDescriptors(parsedDescriptors); - this.descriptorQueue.add(descriptorFile); - } catch (DescriptorParseException e) { - /* TODO Handle me. */ - } - } - } - } catch (IOException e) { - System.err.println("Error while reading descriptors in '" - + directory.getAbsolutePath() + "'."); - /* TODO Handle this exception somehow. */ - } finally { - this.descriptorQueue.setOutOfDescriptors(); - } - } - } - private List<Descriptor> readFile(File file) throws IOException, - DescriptorParseException { - FileInputStream fis = new FileInputStream(file); - BufferedInputStream bis = new BufferedInputStream(fis); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] data = new byte[1024]; - while ((len = bis.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - bis.close(); - byte[] rawDescriptorBytes = baos.toByteArray(); - return DescriptorImpl.parseBridgeDescriptors(rawDescriptorBytes, - file.getName()); - } - } -} - diff --git a/src/org/torproject/descriptor/impl/DescriptorImpl.java b/src/org/torproject/descriptor/impl/DescriptorImpl.java index 233b2df..68b70e1 100644 --- a/src/org/torproject/descriptor/impl/DescriptorImpl.java +++ b/src/org/torproject/descriptor/impl/DescriptorImpl.java @@ -14,8 +14,9 @@ import org.torproject.descriptor.Descriptor;

public abstract class DescriptorImpl implements Descriptor {

- protected static List<Descriptor> parseRelayDescriptors( - byte[] rawDescriptorBytes) throws DescriptorParseException { + protected static List<Descriptor> parseRelayOrBridgeDescriptors( + byte[] rawDescriptorBytes, String fileName) + throws DescriptorParseException { List<Descriptor> parsedDescriptors = new ArrayList<Descriptor>(); byte[] first100Chars = new byte[Math.min(100, rawDescriptorBytes.length)]; @@ -35,32 +36,7 @@ public abstract class DescriptorImpl implements Descriptor { + "network status type in descriptor starting with '" + firstLines + "'."); } - } else if (firstLines.startsWith("router ") || - firstLines.contains("\nrouter ")) { - parsedDescriptors.addAll(ServerDescriptorImpl. - parseDescriptors(rawDescriptorBytes)); - } else if (firstLines.startsWith("extra-info ") || - firstLines.contains("\nextra-info ")) { - parsedDescriptors.addAll(ExtraInfoDescriptorImpl. - parseDescriptors(rawDescriptorBytes)); - } else { - throw new DescriptorParseException("Could not detect relay " - + "descriptor type in descriptor starting with '" + firstLines - + "'."); - } - return parsedDescriptors; - } - - protected static List<Descriptor> parseBridgeDescriptors( - byte[] rawDescriptorBytes, String fileName) - throws DescriptorParseException { - List<Descriptor> parsedDescriptors = new ArrayList<Descriptor>(); - byte[] first100Chars = new byte[Math.min(100, - rawDescriptorBytes.length)]; - System.arraycopy(rawDescriptorBytes, 0, first100Chars, 0, - first100Chars.length); - String firstLines = new String(first100Chars); - if (firstLines.startsWith("r ")) { + } else if (firstLines.startsWith("r ")) { parsedDescriptors.add(new BridgeNetworkStatusImpl( rawDescriptorBytes, fileName)); } else if (firstLines.startsWith("router ") || @@ -72,9 +48,9 @@ public abstract class DescriptorImpl implements Descriptor { parsedDescriptors.addAll(ExtraInfoDescriptorImpl. parseDescriptors(rawDescriptorBytes)); } else { - throw new DescriptorParseException("Could not detect bridge " - + "descriptor type in descriptor starting with '" + firstLines - + "'."); + throw new DescriptorParseException("Could not detect relay or " + + "bridge descriptor type in descriptor starting with '" + + firstLines + "'."); } return parsedDescriptors; } diff --git a/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java deleted file mode 100644 index 81435e0..0000000 --- a/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java +++ /dev/null @@ -1,172 +0,0 @@ -/* Copyright 2011, 2012 The Tor Project - * See LICENSE for licensing information */ -package org.torproject.descriptor.impl; - -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileReader; -import java.io.FileInputStream; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.SortedMap; -import java.util.Stack; -import java.util.TreeMap; -import org.torproject.descriptor.Descriptor; -import org.torproject.descriptor.DescriptorFile; -import org.torproject.descriptor.RelayDescriptorReader; -import org.torproject.descriptor.RelayNetworkStatusConsensus; -import org.torproject.descriptor.RelayNetworkStatusVote; - -public class RelayDescriptorReaderImpl implements RelayDescriptorReader { - - private List<File> directories = new ArrayList<File>(); - public void addDirectory(File directory) { - this.directories.add(directory); - } - - private File historyFile; - public void setExcludeFiles(File historyFile) { - this.historyFile = historyFile; - } - - public Iterator<DescriptorFile> readDescriptors() { - BlockingIteratorImpl<DescriptorFile> descriptorQueue = - new BlockingIteratorImpl<DescriptorFile>(); - DescriptorReader reader = new DescriptorReader(this.directories, - descriptorQueue, this.historyFile); - new Thread(reader).start(); - return descriptorQueue; - } - - private static class DescriptorReader implements Runnable { - private List<File> directories; - private BlockingIteratorImpl<DescriptorFile> descriptorQueue; - private File historyFile; - private DescriptorReader(List<File> directories, - BlockingIteratorImpl<DescriptorFile> descriptorQueue, - File historyFile) { - this.directories = directories; - this.descriptorQueue = descriptorQueue; - this.historyFile = historyFile; - } - public void run() { - this.readOldHistory(); - this.readDescriptors(); - this.writeNewHistory(); - } - private SortedMap<String, Long> - oldHistory = new TreeMap<String, Long>(), - newHistory = new TreeMap<String, Long>(); - private void readOldHistory() { - if (this.historyFile == null) { - return; - } - try { - BufferedReader br = new BufferedReader(new FileReader( - this.historyFile)); - String line; - while ((line = br.readLine()) != null) { - if (!line.contains(" ")) { - /* TODO Handle this problem? */ - continue; - } - long lastModifiedMillis = Long.parseLong(line.substring(0, - line.indexOf(" "))); - String absolutePath = line.substring(line.indexOf(" ") + 1); - this.oldHistory.put(absolutePath, lastModifiedMillis); - } - br.close(); - } catch (IOException e) { - /* TODO Handle this exception. */ - } catch (NumberFormatException e) { - /* TODO Handle this exception. */ - } - } - private void writeNewHistory() { - if (this.historyFile == null) { - return; - } - try { - if (this.historyFile.getParentFile() != null) { - this.historyFile.getParentFile().mkdirs(); - } - BufferedWriter bw = new BufferedWriter(new FileWriter( - this.historyFile)); - for (Map.Entry<String, Long> e : this.newHistory.entrySet()) { - String absolutePath = e.getKey(); - long lastModifiedMillis = e.getValue(); - bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath - + "\n"); - } - bw.close(); - } catch (IOException e) { - /* TODO Handle this exception. */ - } - } - private void readDescriptors() { - for (File directory : this.directories) { - try { - Stack<File> files = new Stack<File>(); - files.add(directory); - while (!files.isEmpty()) { - File file = files.pop(); - if (file.isDirectory()) { - files.addAll(Arrays.asList(file.listFiles())); - } else { - String absolutePath = file.getAbsolutePath(); - long lastModifiedMillis = file.lastModified(); - this.newHistory.put(absolutePath, lastModifiedMillis); - if (this.oldHistory.containsKey(absolutePath) && - this.oldHistory.get(absolutePath) == - lastModifiedMillis) { - continue; - } - try { - List<Descriptor> parsedDescriptors = this.readFile(file); - DescriptorFileImpl descriptorFile = - new DescriptorFileImpl(); - descriptorFile.setDirectory(directory); - descriptorFile.setFile(file); - descriptorFile.setLastModified(lastModifiedMillis); - descriptorFile.setDescriptors(parsedDescriptors); - this.descriptorQueue.add(descriptorFile); - } catch (DescriptorParseException e) { - /* TODO Handle me. */ - } - } - } - } catch (IOException e) { - System.err.println("Error while reading descriptors in '" - + directory.getAbsolutePath() + "'."); - /* TODO Handle this exception somehow. */ - } finally { - this.descriptorQueue.setOutOfDescriptors(); - } - } - } - private List<Descriptor> readFile(File file) throws IOException, - DescriptorParseException { - FileInputStream fis = new FileInputStream(file); - BufferedInputStream bis = new BufferedInputStream(fis); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - int len; - byte[] data = new byte[1024]; - while ((len = bis.read(data, 0, 1024)) >= 0) { - baos.write(data, 0, len); - } - bis.close(); - byte[] rawDescriptorBytes = baos.toByteArray(); - return DescriptorImpl.parseRelayDescriptors(rawDescriptorBytes); - } - } -} - diff --git a/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java new file mode 100644 index 0000000..8e1c3bc --- /dev/null +++ b/src/org/torproject/descriptor/impl/RelayOrBridgeDescriptorReaderImpl.java @@ -0,0 +1,173 @@ +/* Copyright 2011, 2012 The Tor Project + * See LICENSE for licensing information */ +package org.torproject.descriptor.impl; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileReader; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.Stack; +import java.util.TreeMap; +import org.torproject.descriptor.Descriptor; +import org.torproject.descriptor.DescriptorFile; +import org.torproject.descriptor.RelayDescriptorReader; +import org.torproject.descriptor.BridgeDescriptorReader; + +public class RelayOrBridgeDescriptorReaderImpl + implements RelayDescriptorReader, BridgeDescriptorReader { + + private List<File> directories = new ArrayList<File>(); + public void addDirectory(File directory) { + this.directories.add(directory); + } + + private File historyFile; + public void setExcludeFiles(File historyFile) { + this.historyFile = historyFile; + } + + public Iterator<DescriptorFile> readDescriptors() { + BlockingIteratorImpl<DescriptorFile> descriptorQueue = + new BlockingIteratorImpl<DescriptorFile>(); + DescriptorReader reader = new DescriptorReader(this.directories, + descriptorQueue, this.historyFile); + new Thread(reader).start(); + return descriptorQueue; + } + + private static class DescriptorReader implements Runnable { + private List<File> directories; + private BlockingIteratorImpl<DescriptorFile> descriptorQueue; + private File historyFile; + private DescriptorReader(List<File> directories, + BlockingIteratorImpl<DescriptorFile> descriptorQueue, + File historyFile) { + this.directories = directories; + this.descriptorQueue = descriptorQueue; + this.historyFile = historyFile; + } + public void run() { + this.readOldHistory(); + this.readDescriptors(); + this.writeNewHistory(); + } + private SortedMap<String, Long> + oldHistory = new TreeMap<String, Long>(), + newHistory = new TreeMap<String, Long>(); + private void readOldHistory() { + if (this.historyFile == null) { + return; + } + try { + BufferedReader br = new BufferedReader(new FileReader( + this.historyFile)); + String line; + while ((line = br.readLine()) != null) { + if (!line.contains(" ")) { + /* TODO Handle this problem? */ + continue; + } + long lastModifiedMillis = Long.parseLong(line.substring(0, + line.indexOf(" "))); + String absolutePath = line.substring(line.indexOf(" ") + 1); + this.oldHistory.put(absolutePath, lastModifiedMillis); + } + br.close(); + } catch (IOException e) { + /* TODO Handle this exception. */ + } catch (NumberFormatException e) { + /* TODO Handle this exception. */ + } + } + private void writeNewHistory() { + if (this.historyFile == null) { + return; + } + try { + if (this.historyFile.getParentFile() != null) { + this.historyFile.getParentFile().mkdirs(); + } + BufferedWriter bw = new BufferedWriter(new FileWriter( + this.historyFile)); + for (Map.Entry<String, Long> e : this.newHistory.entrySet()) { + String absolutePath = e.getKey(); + long lastModifiedMillis = e.getValue(); + bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath + + "\n"); + } + bw.close(); + } catch (IOException e) { + /* TODO Handle this exception. */ + } + } + private void readDescriptors() { + for (File directory : this.directories) { + try { + Stack<File> files = new Stack<File>(); + files.add(directory); + while (!files.isEmpty()) { + File file = files.pop(); + if (file.isDirectory()) { + files.addAll(Arrays.asList(file.listFiles())); + } else { + String absolutePath = file.getAbsolutePath(); + long lastModifiedMillis = file.lastModified(); + this.newHistory.put(absolutePath, lastModifiedMillis); + if (this.oldHistory.containsKey(absolutePath) && + this.oldHistory.get(absolutePath) == + lastModifiedMillis) { + continue; + } + try { + List<Descriptor> parsedDescriptors = this.readFile(file); + DescriptorFileImpl descriptorFile = + new DescriptorFileImpl(); + descriptorFile.setDirectory(directory); + descriptorFile.setFile(file); + descriptorFile.setLastModified(lastModifiedMillis); + descriptorFile.setDescriptors(parsedDescriptors); + this.descriptorQueue.add(descriptorFile); + } catch (DescriptorParseException e) { + /* TODO Handle me. */ + } + } + } + } catch (IOException e) { + System.err.println("Error while reading descriptors in '" + + directory.getAbsolutePath() + "'."); + /* TODO Handle this exception somehow. */ + } finally { + this.descriptorQueue.setOutOfDescriptors(); + } + } + } + private List<Descriptor> readFile(File file) throws IOException, + DescriptorParseException { + FileInputStream fis = new FileInputStream(file); + BufferedInputStream bis = new BufferedInputStream(fis); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int len; + byte[] data = new byte[1024]; + while ((len = bis.read(data, 0, 1024)) >= 0) { + baos.write(data, 0, len); + } + bis.close(); + byte[] rawDescriptorBytes = baos.toByteArray(); + return DescriptorImpl.parseRelayOrBridgeDescriptors( + rawDescriptorBytes, file.getName()); + } + } +} +