commit afca9db71c6fb0704553a1b4a0d2baef507458c5 Author: Karsten Loesing <karsten.loesing@gmx.net> Date: Fri Mar 30 15:09:45 2012 +0200 Support reading descriptor tarballs. --- .../torproject/descriptor/DescriptorReader.java | 7 ++- .../descriptor/impl/DescriptorReaderImpl.java | 90 +++++++++++++++++++- 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/src/org/torproject/descriptor/DescriptorReader.java b/src/org/torproject/descriptor/DescriptorReader.java index 1d46a79..167f8b6 100644 --- a/src/org/torproject/descriptor/DescriptorReader.java +++ b/src/org/torproject/descriptor/DescriptorReader.java @@ -8,9 +8,14 @@ import java.util.Iterator; /* Read descriptors from one or more local directories. */ public interface DescriptorReader { - /* Add a local directory to read descriptors from. */ + /* Add a local directory to read descriptor files or tarballs containing + * descriptor files from. */ public void addDirectory(File directory); + /* Add an uncompressed or bz2-compressed tarball to read descriptors + * from. */ + public void addTarball(File tarball); + /* Exclude files that are contained in the given history file and that * haven't changed since they were last read. Add reads from the * current run to the history file. Remove files that don't exist diff --git a/src/org/torproject/descriptor/impl/DescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/DescriptorReaderImpl.java index c22cbe8..ee45fa1 100644 --- a/src/org/torproject/descriptor/impl/DescriptorReaderImpl.java +++ b/src/org/torproject/descriptor/impl/DescriptorReaderImpl.java @@ -20,6 +20,9 @@ import java.util.SortedMap; import java.util.Stack; import java.util.TreeMap; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorFile; import org.torproject.descriptor.DescriptorReader; @@ -37,6 +40,15 @@ public class DescriptorReaderImpl implements DescriptorReader { this.directories.add(directory); } + private List<File> tarballs = new ArrayList<File>(); + public void addTarball(File tarball) { + if (this.hasStartedReading) { + throw new IllegalStateException("Reconfiguration is not permitted " + + "after starting to read."); + } + this.tarballs.add(tarball); + } + private File historyFile; public void setExcludeFiles(File historyFile) { if (this.hasStartedReading) { @@ -64,21 +76,24 @@ public class DescriptorReaderImpl implements DescriptorReader { BlockingIteratorImpl<DescriptorFile> descriptorQueue = new BlockingIteratorImpl<DescriptorFile>(); DescriptorReaderRunnable reader = new DescriptorReaderRunnable( - this.directories, descriptorQueue, this.historyFile, - this.failUnrecognizedDescriptorLines); + this.directories, this.tarballs, descriptorQueue, + this.historyFile, this.failUnrecognizedDescriptorLines); new Thread(reader).start(); return descriptorQueue; } private static class DescriptorReaderRunnable implements Runnable { private List<File> directories; + private List<File> tarballs; private BlockingIteratorImpl<DescriptorFile> descriptorQueue; private File historyFile; private boolean failUnrecognizedDescriptorLines; private DescriptorReaderRunnable(List<File> directories, + List<File> tarballs, BlockingIteratorImpl<DescriptorFile> descriptorQueue, File historyFile, boolean failUnrecognizedDescriptorLines) { this.directories = directories; + this.tarballs = tarballs; this.descriptorQueue = descriptorQueue; this.historyFile = historyFile; this.failUnrecognizedDescriptorLines = @@ -87,6 +102,8 @@ public class DescriptorReaderImpl implements DescriptorReader { public void run() { this.readOldHistory(); this.readDescriptors(); + this.readTarballs(); + this.descriptorQueue.setOutOfDescriptors(); this.writeNewHistory(); } private SortedMap<String, Long> @@ -147,6 +164,9 @@ public class DescriptorReaderImpl implements DescriptorReader { File file = files.pop(); if (file.isDirectory()) { files.addAll(Arrays.asList(file.listFiles())); + } else if (file.getName().endsWith(".tar") || + file.getName().endsWith(".tar.bz2")) { + this.tarballs.add(file); } else { String absolutePath = file.getAbsolutePath(); long lastModifiedMillis = file.lastModified(); @@ -171,7 +191,71 @@ public class DescriptorReaderImpl implements DescriptorReader { } } } - this.descriptorQueue.setOutOfDescriptors(); + } + private void readTarballs() { + List<File> files = new ArrayList<File>(this.tarballs); + boolean abortReading = false; + while (!abortReading && !files.isEmpty()) { + File tarball = files.remove(0); + if (!tarball.getName().endsWith(".tar") && + !tarball.getName().endsWith(".tar.bz2")) { + continue; + } + String absolutePath = tarball.getAbsolutePath(); + long lastModifiedMillis = tarball.lastModified(); + this.newHistory.put(absolutePath, lastModifiedMillis); + if (this.oldHistory.containsKey(absolutePath) && + this.oldHistory.get(absolutePath) == lastModifiedMillis) { + continue; + } + try { + FileInputStream in = new FileInputStream(tarball); + if (in.available() > 0) { + TarArchiveInputStream tais = null; + if (tarball.getName().endsWith(".tar.bz2")) { + tais = new TarArchiveInputStream( + new BZip2CompressorInputStream(in)); + } else if (tarball.getName().endsWith(".tar")) { + tais = new TarArchiveInputStream(in); + } + BufferedInputStream bis = new BufferedInputStream(tais); + TarArchiveEntry tae = null; + while ((tae = tais.getNextTarEntry()) != null) { + DescriptorFileImpl descriptorFile = + new DescriptorFileImpl(); + /* TODO Is it correct to set these values for files + * contained in a tarball? */ + descriptorFile.setDirectory(tarball); + descriptorFile.setFile(null); + descriptorFile.setLastModified(lastModifiedMillis); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int len; + byte[] data = new byte[1024]; + while ((len = bis.read(data, 0, 1024)) >= 0) { + baos.write(data, 0, len); + } + byte[] rawDescriptorBytes = baos.toByteArray(); + if (rawDescriptorBytes.length < 1) { + continue; + } + try { + String fileName = tae.getName().substring( + tae.getName().lastIndexOf("/") + 1); + List<Descriptor> parsedDescriptors = + DescriptorImpl.parseRelayOrBridgeDescriptors( + rawDescriptorBytes, fileName, + this.failUnrecognizedDescriptorLines); + descriptorFile.setDescriptors(parsedDescriptors); + } catch (DescriptorParseException e) { + descriptorFile.setException(e); + } + this.descriptorQueue.add(descriptorFile); + } + } + } catch (IOException e) { + abortReading = true; + } + } } private List<Descriptor> readFile(File file) throws IOException, DescriptorParseException {