commit 8722de70447434ebb0d92551c8622566efb6d800 Author: Karsten Loesing karsten.loesing@gmx.net Date: Tue May 27 20:53:04 2014 +0200

Make queue size of descriptor reader configurable.

By default, the descriptor reader puts up to 100 parsed descriptor files in a queue in order to hand them out as quickly as possible. But if descriptor files contain hundreds or even thousands of descriptors, that default may be too high. Add a new method to make it configurable. --- src/org/torproject/descriptor/DescriptorReader.java | 5 +++++ .../torproject/descriptor/impl/BlockingIteratorImpl.java | 11 +++++++++-- .../torproject/descriptor/impl/DescriptorReaderImpl.java | 14 +++++++++++++- 3 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/src/org/torproject/descriptor/DescriptorReader.java b/src/org/torproject/descriptor/DescriptorReader.java index 2783bd4..5fe8197 100644 --- a/src/org/torproject/descriptor/DescriptorReader.java +++ b/src/org/torproject/descriptor/DescriptorReader.java @@ -49,6 +49,11 @@ public interface DescriptorReader { * explicitly. */ public void setFailUnrecognizedDescriptorLines();

+ /* Don't keep more than this number of parsed descriptor files in the + * queue. The default is 100, but if descriptor files contain hundreds + * or even thousands of descriptors, that default may be too high. */ + public void setMaxDescriptorFilesInQueue(int max); + /* Read the previously configured descriptors and make them available * via the returned blocking iterator. Whenever the reader runs out of * descriptors and expects to provide more shortly after, it blocks the diff --git a/src/org/torproject/descriptor/impl/BlockingIteratorImpl.java b/src/org/torproject/descriptor/impl/BlockingIteratorImpl.java index e23034a..477e6f1 100644 --- a/src/org/torproject/descriptor/impl/BlockingIteratorImpl.java +++ b/src/org/torproject/descriptor/impl/BlockingIteratorImpl.java @@ -15,19 +15,26 @@ public class BlockingIteratorImpl<T> implements Iterator<T> { /* Queue containing produced elemnts waiting for consumers. */ private Queue<T> queue = new LinkedList<T>();

+ /* Maximum number of elements in queue. */ + private int maxQueueSize = 100; + /* Restrict object construction to the impl package. */ protected BlockingIteratorImpl() { }

+ /* Create instance with maximum queue size. */ + protected BlockingIteratorImpl(int maxQueueSize) { + this.maxQueueSize = maxQueueSize; + } + /* Add an object to the queue if there's still room. */ - final int MAX_DESCRIPTORS = 100; protected synchronized void add(T object) { if (this.outOfDescriptors) { throw new IllegalStateException("Internal error: Adding results to " + "descriptor queue not allowed after sending end-of-stream " + "object."); } - while (this.queue.size() >= this.MAX_DESCRIPTORS) { + while (this.queue.size() >= this.maxQueueSize) { try { wait(); } catch (InterruptedException e) { diff --git a/src/org/torproject/descriptor/impl/DescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/DescriptorReaderImpl.java index e3fb908..d679154 100644 --- a/src/org/torproject/descriptor/impl/DescriptorReaderImpl.java +++ b/src/org/torproject/descriptor/impl/DescriptorReaderImpl.java @@ -93,6 +93,15 @@ public class DescriptorReaderImpl implements DescriptorReader { this.failUnrecognizedDescriptorLines = true; }

+ private Integer maxDescriptorFilesInQueue = null; + public void setMaxDescriptorFilesInQueue(int max) { + if (this.hasStartedReading) { + throw new IllegalStateException("Reconfiguration is not permitted " + + "after starting to read."); + } + this.maxDescriptorFilesInQueue = max; + } + private DescriptorReaderRunnable reader; public Iterator<DescriptorFile> readDescriptors() { if (this.hasStartedReading) { @@ -101,7 +110,10 @@ public class DescriptorReaderImpl implements DescriptorReader { } this.hasStartedReading = true; BlockingIteratorImpl<DescriptorFile> descriptorQueue = - new BlockingIteratorImpl<DescriptorFile>(); + this.maxDescriptorFilesInQueue == null + ? new BlockingIteratorImpl<DescriptorFile>() + : new BlockingIteratorImpl<DescriptorFile>( + this.maxDescriptorFilesInQueue); this.reader = new DescriptorReaderRunnable(this.directories, this.tarballs, descriptorQueue, this.historyFile, this.excludedFiles, this.failUnrecognizedDescriptorLines);