June 2017 - tor-commits - lists.torproject.org

[stem/master] Python3 support for server descriptor signing
by atagar＠torproject.org 21 Jun '17

21 Jun '17

commit 78eaa445c9344b878e55c3e114f73358faa8d008 Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Jun 21 08:51:24 2017 -0700 Python3 support for server descriptor signing While writing it I consciously didn't invest effort into making it compatible. Now that we've got it working fixing that. Most interesting issue was... ====================================================================== ERROR: test_descriptor_signing ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/atagar/Desktop/stem/test/require.py", line 58, in wrapped return func(self, *args, **kwargs) File "/home/atagar/Desktop/stem/test/unit/descriptor/server_descriptor.py", line 260, in test_descriptor_signing RelayDescriptor.create(sign = True) File "/home/atagar/Desktop/stem/stem/descriptor/server_descriptor.py", line 873, in create return cls(cls.content(attr, exclude, sign, private_signing_key), validate = validate, skip_crypto_validation = not sign) File "/home/atagar/Desktop/stem/stem/descriptor/server_descriptor.py", line 865, in content signature = base64.b64encode(private_signing_key.sign(content, padding.PKCS1v15(), hashes.SHA1())) File "/usr/local/lib/python3.2/dist-packages/cryptography/hazmat/backends/openssl/rsa.py", line 463, in sign algorithm, self, data File "/usr/local/lib/python3.2/dist-packages/cryptography/hazmat/backends/openssl/rsa.py", line 199, in _rsa_sig_sign backend.openssl_assert(res > 0) TypeError: unorderable types: NoneType() > int() ---------------------------------------------------------------------- This was due to one of our no_op mocks. --- stem/descriptor/__init__.py | 2 +- stem/descriptor/server_descriptor.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py index 96cf0b4..bf336fb 100644 --- a/stem/descriptor/__init__.py +++ b/stem/descriptor/__init__.py @@ -391,7 +391,7 @@ def _descriptor_content(attr = None, exclude = (), sign = False, header_template if keyword in exclude: continue - value = attr.pop(keyword, value) + value = stem.util.str_tools._to_unicode(attr.pop(keyword, value)) if value is None: continue diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py index 4e55ee2..2c7258a 100644 --- a/stem/descriptor/server_descriptor.py +++ b/stem/descriptor/server_descriptor.py @@ -847,7 +847,7 @@ class RelayDescriptor(ServerDescriptor): # ewww. def no_op(*args, **kwargs): - pass + return 1 private_signing_key._backend._lib.EVP_PKEY_CTX_set_signature_md = no_op private_signing_key._backend.openssl_assert = no_op @@ -855,15 +855,15 @@ class RelayDescriptor(ServerDescriptor): # create descriptor content without the router-signature, then # appending the content signature - attr['signing-key'] = '\n' + private_signing_key.public_key().public_bytes( + attr['signing-key'] = b'\n' + private_signing_key.public_key().public_bytes( encoding = serialization.Encoding.PEM, format = serialization.PublicFormat.PKCS1, ).strip() - content = _descriptor_content(attr, exclude, sign, RELAY_SERVER_HEADER) + '\nrouter-signature\n' + content = _descriptor_content(attr, exclude, sign, RELAY_SERVER_HEADER) + b'\nrouter-signature\n' signature = base64.b64encode(private_signing_key.sign(content, padding.PKCS1v15(), hashes.SHA1())) - return content + '\n'.join(['-----BEGIN SIGNATURE-----'] + stem.util.str_tools._split_by_length(signature, 64) + ['-----END SIGNATURE-----\n']) + return content + b'\n'.join([b'-----BEGIN SIGNATURE-----'] + stem.util.str_tools._split_by_length(signature, 64) + [b'-----END SIGNATURE-----\n']) else: return _descriptor_content(attr, exclude, sign, RELAY_SERVER_HEADER, RELAY_SERVER_FOOTER)

1 0

[tor/master] Merge branch 'Ryman-bug5847-squashed'
by nickm＠torproject.org 21 Jun '17

21 Jun '17

commit f0a5416f95e06ebfd27eb95436eb85cd66494b9a Merge: 884c0ff 0a96d11 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Jun 21 12:19:55 2017 -0400 Merge branch 'Ryman-bug5847-squashed' changes/bug5847 | 5 +++++ src/or/control.c | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 8 deletions(-)

1 0

[tor/master] Better error message for GETINFO desc/(id|name) whenever microdescriptors are in use. Fixes #5847.
by nickm＠torproject.org 21 Jun '17

21 Jun '17

commit 0a96d11539e27f8ef0577cc8ee4e503f046e637b Author: Kevin Butler <haqkrs(a)gmail.com> Date: Wed Sep 4 02:10:18 2013 +0100 Better error message for GETINFO desc/(id|name) whenever microdescriptors are in use. Fixes #5847. --- changes/bug5847 | 5 +++++ src/or/control.c | 36 ++++++++++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/changes/bug5847 b/changes/bug5847 new file mode 100644 index 0000000..782fc7b --- /dev/null +++ b/changes/bug5847 @@ -0,0 +1,5 @@ + o Minor features (control port): + - Provide better error message for GETINFO desc/(id|name) when + microdescriptors are in use and router descriptors are not fetched. + Closes ticket 5847. Patch by Kevin Butler. + diff --git a/src/or/control.c b/src/or/control.c index 9454a7a..b971725 100644 --- a/src/or/control.c +++ b/src/or/control.c @@ -60,6 +60,7 @@ #include "hibernate.h" #include "hs_common.h" #include "main.h" +#include "microdesc.h" #include "networkstatus.h" #include "nodelist.h" #include "policies.h" @@ -1892,6 +1893,12 @@ getinfo_helper_dir(control_connection_t *control_conn, const char *body = signed_descriptor_get_body(&ri->cache_info); if (body) *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len); + } else if (! we_fetch_router_descriptors(get_options())) { + /* Descriptors won't be available, provide proper error */ + *errmsg = "We fetch microdescriptors, not router " + "descriptors. You'll need to use md/id/* " + "instead of desc/id/*."; + return 0; } } else if (!strcmpstart(question, "desc/name/")) { const routerinfo_t *ri = NULL; @@ -1905,6 +1912,12 @@ getinfo_helper_dir(control_connection_t *control_conn, const char *body = signed_descriptor_get_body(&ri->cache_info); if (body) *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len); + } else if (! we_fetch_router_descriptors(get_options())) { + /* Descriptors won't be available, provide proper error */ + *errmsg = "We fetch microdescriptors, not router " + "descriptors. You'll need to use md/name/* " + "instead of desc/name/*."; + return 0; } } else if (!strcmp(question, "desc/all-recent")) { routerlist_t *routerlist = router_get_routerlist(); @@ -2907,7 +2920,8 @@ getinfo_helper_sr(control_connection_t *control_conn, * *a. If an internal error occurs, return -1 and optionally set * *error_out to point to an error message to be delivered to the * controller. On success, _or if the key is not recognized_, return 0. Do not - * set a if the key is not recognized. + * set a if the key is not recognized but you may set error_out + * to improve the error message. */ typedef int (*getinfo_helper_t)(control_connection_t *, const char *q, char **a, @@ -3162,7 +3176,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, smartlist_t *questions = smartlist_new(); smartlist_t *answers = smartlist_new(); smartlist_t *unrecognized = smartlist_new(); - char *msg = NULL, *ans = NULL; + char *ans = NULL; int i; (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */ @@ -3177,20 +3191,26 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, goto done; } if (!ans) { - smartlist_add(unrecognized, (char*)q); + if (errmsg) /* use provided error message */ + smartlist_add_strdup(unrecognized, errmsg); + else /* use default error message */ + smartlist_add_asprintf(unrecognized, "Unrecognized key \"%s\"", q); } else { smartlist_add_strdup(answers, q); smartlist_add(answers, ans); } } SMARTLIST_FOREACH_END(q); + if (smartlist_len(unrecognized)) { + /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */ for (i=0; i < smartlist_len(unrecognized)-1; ++i) connection_printf_to_buf(conn, - "552-Unrecognized key \"%s\"\r\n", - (char*)smartlist_get(unrecognized, i)); + "552-%s\r\n", + (char *)smartlist_get(unrecognized, i)); + connection_printf_to_buf(conn, - "552 Unrecognized key \"%s\"\r\n", - (char*)smartlist_get(unrecognized, i)); + "552 %s\r\n", + (char *)smartlist_get(unrecognized, i)); goto done; } @@ -3217,8 +3237,8 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len, smartlist_free(answers); SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp)); smartlist_free(questions); + SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp)); smartlist_free(unrecognized); - tor_free(msg); return 0; }

1 0

[metrics-lib/master] Bump version to 1.9.0-dev.
by karsten＠torproject.org 21 Jun '17

21 Jun '17

commit cad14ead4c72744698ef013404f94dfe3842df0b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jun 21 14:21:19 2017 +0200 Bump version to 1.9.0-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index 18ba365..0de365c 100644 --- a/build.xml +++ b/build.xml @@ -6,7 +6,7 @@ <project default="usage" name="descriptor" basedir="."> - <property name="release.version" value="1.9.0" /> + <property name="release.version" value="1.9.0-dev" /> <property name="javadoc-title" value="DescripTor API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/**" /> <property name="implementation-title" value="DescripTor" />

1 0

[metrics-lib/release] Prepare for 1.9.0 release.
by karsten＠torproject.org 21 Jun '17

21 Jun '17

commit 3c98457777aa4d96b2e5da6cd640c2704dd42e9f Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Tue Jun 20 22:36:58 2017 +0200 Prepare for 1.9.0 release. --- CHANGELOG.md | 2 +- build.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95cd851..98728b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# Changes in version 1.9.0 - 2017-06-?? +# Changes in version 1.9.0 - 2017-06-21 * Major changes - Simplify DescriptorReader by returning Descriptor instances diff --git a/build.xml b/build.xml index d5aaa8f..18ba365 100644 --- a/build.xml +++ b/build.xml @@ -6,7 +6,7 @@ <project default="usage" name="descriptor" basedir="."> - <property name="release.version" value="1.8.2-dev" /> + <property name="release.version" value="1.9.0" /> <property name="javadoc-title" value="DescripTor API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/**" /> <property name="implementation-title" value="DescripTor" />

1 0

[metrics-lib/release] Make DescriptorFile obsolete.
by karsten＠torproject.org 21 Jun '17

21 Jun '17

commit 1062da637f6f55313f52191314f7fb939355af23 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Wed Jun 7 17:52:02 2017 +0200 Make DescriptorFile obsolete. This change simplifies the DescriptorReader interface and allows for shorter and more concise application code. The result is that DescriptorReader returns Descriptor instances rather than DescriptorFile instances containing Descriptors and accepts a maximum queue size in Descriptors, DescriptorFile is deprecated, and Descriptor contains a File reference to the descriptor file. Implements #22141. Related to this change, this commit introduces a new UnparseableDescriptor to be returned by DescriptorParser and DescriptorReader if a descriptor cannot be parsed, as opposed to throwing a DescriptorParseException or skipping the entire descriptor file (fixes #22139), respectively. Also related to this change, DescriptorParser now returns an Iterable instead of a List, which prepares parsing large descriptor files descriptor by descriptor (will be tackled in #20395). --- CHANGELOG.md | 13 ++ .../java/org/torproject/descriptor/Descriptor.java | 19 +++ .../org/torproject/descriptor/DescriptorFile.java | 7 + .../torproject/descriptor/DescriptorParser.java | 19 +++ .../torproject/descriptor/DescriptorReader.java | 56 +++++-- .../descriptor/UnparseableDescriptor.java | 44 ++++++ .../descriptor/impl/BlockingIteratorImpl.java | 13 +- .../impl/BridgeExtraInfoDescriptorImpl.java | 9 +- .../descriptor/impl/BridgeNetworkStatusImpl.java | 7 +- .../descriptor/impl/BridgePoolAssignmentImpl.java | 9 +- .../impl/BridgeServerDescriptorImpl.java | 9 +- .../torproject/descriptor/impl/DescriptorImpl.java | 23 ++- .../descriptor/impl/DescriptorParserImpl.java | 130 ++++++++++------ .../descriptor/impl/DescriptorReaderImpl.java | 164 ++++++++++++++++++++- .../impl/DirectoryKeyCertificateImpl.java | 9 +- .../torproject/descriptor/impl/ExitListImpl.java | 7 +- .../descriptor/impl/ExtraInfoDescriptorImpl.java | 9 +- .../descriptor/impl/MicrodescriptorImpl.java | 7 +- .../descriptor/impl/NetworkStatusImpl.java | 7 +- .../descriptor/impl/RelayDirectoryImpl.java | 10 +- .../impl/RelayExtraInfoDescriptorImpl.java | 9 +- .../impl/RelayNetworkStatusConsensusImpl.java | 9 +- .../descriptor/impl/RelayNetworkStatusImpl.java | 7 +- .../impl/RelayNetworkStatusVoteImpl.java | 7 +- .../descriptor/impl/RelayServerDescriptorImpl.java | 9 +- .../descriptor/impl/ServerDescriptorImpl.java | 7 +- .../descriptor/impl/TorperfResultImpl.java | 11 +- .../descriptor/impl/UnparseableDescriptorImpl.java | 40 +++++ .../descriptor/impl/BridgeNetworkStatusTest.java | 2 +- .../descriptor/impl/ConsensusBuilder.java | 2 +- .../descriptor/impl/DescriptorImplTest.java | 1 - .../descriptor/impl/DescriptorParserImplTest.java | 9 +- .../descriptor/impl/ExitListImplTest.java | 10 +- .../impl/ExtraInfoDescriptorImplTest.java | 8 +- .../descriptor/impl/MicrodescriptorImplTest.java | 2 +- .../impl/RelayNetworkStatusConsensusImplTest.java | 4 +- .../impl/RelayNetworkStatusImplTest.java | 4 +- .../impl/RelayNetworkStatusVoteImplTest.java | 3 +- .../descriptor/impl/ServerDescriptorImplTest.java | 6 +- .../torproject/descriptor/impl/TestDescriptor.java | 4 +- .../descriptor/impl/TestServerDescriptor.java | 8 +- .../descriptor/impl/TorperfResultImplTest.java | 10 +- 42 files changed, 586 insertions(+), 157 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e79811a..44f1130 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changes in version 1.9.0 - 2017-06-?? + * Major changes + - Simplify DescriptorReader by returning Descriptor instances + rather than DescriptorFile instances containing Descriptors, + deprecate DescriptorFile, and add a File reference to Descriptor. + - Introduce a new UnparseableDescriptor to be returned by + DescriptorParser and DescriptorReader if a descriptor cannot be + parsed, as opposed to throwing a DescriptorParseException or + skipping the entire descriptor file, respectively. + + * Medium changes + - Let DescriptorParser return an Iterable instead of a List, which + prepares parsing large descriptor files descriptor by descriptor. + * Minor changes - Fix a bug where NetworkStatusEntry's getMicrodescriptorDigests() and getMicrodescriptorDigestsSha256Base64() return hex strings diff --git a/src/main/java/org/torproject/descriptor/Descriptor.java b/src/main/java/org/torproject/descriptor/Descriptor.java index be6e02f..fb511bf 100644 --- a/src/main/java/org/torproject/descriptor/Descriptor.java +++ b/src/main/java/org/torproject/descriptor/Descriptor.java @@ -3,6 +3,7 @@ package org.torproject.descriptor; +import java.io.File; import java.util.List; /** @@ -24,6 +25,10 @@ public interface Descriptor { * Return the (possibly empty) list of annotations in the format * {@code "@key( value)*"}. * + * Some implementations might not support this operation and will throw an + * {@code UnsupportedOperationException}, e.g., + * {@link UnparseableDescriptor}. + * * @since 1.0.0 */ public List<String> getAnnotations(); @@ -32,8 +37,22 @@ public interface Descriptor { * Return any unrecognized lines when parsing this descriptor, or an * empty list if there were no unrecognized lines. * + * Some implementations might not support this operation and will throw an + * {@code UnsupportedOperationException}, * e.g., + * {@link UnparseableDescriptor}. + * * @since 1.0.0 */ public List<String> getUnrecognizedLines(); + + /** + * Return the file, tarball or plain file, that contained this descriptor, or + * {@code null} if this descriptor was not read from a file. + * + * @return Descriptor file that contained this descriptor. + * + * @since 1.9.0 + */ + public File getDescriptorFile(); } diff --git a/src/main/java/org/torproject/descriptor/DescriptorFile.java b/src/main/java/org/torproject/descriptor/DescriptorFile.java index b501186..e8c0d21 100644 --- a/src/main/java/org/torproject/descriptor/DescriptorFile.java +++ b/src/main/java/org/torproject/descriptor/DescriptorFile.java @@ -15,6 +15,13 @@ import java.util.List; * This container also stores potentially useful meta-data about the * descriptor file. * + * @deprecated Irrelevant after introducing + * {@link DescriptorReader#readDescriptors(File...)}, which returns + * {@link Descriptor}s rather than {@link DescriptorFile}s, and including + * the descriptor file reference in {@link Descriptor#getDescriptorFile()} + * and the first thrown {@link DescriptorParseException} in + * {@link UnparseableDescriptor#getDescriptorParseException()}. + * * @since 1.0.0 */ public interface DescriptorFile { diff --git a/src/main/java/org/torproject/descriptor/DescriptorParser.java b/src/main/java/org/torproject/descriptor/DescriptorParser.java index 537a259..61f69f4 100644 --- a/src/main/java/org/torproject/descriptor/DescriptorParser.java +++ b/src/main/java/org/torproject/descriptor/DescriptorParser.java @@ -3,6 +3,7 @@ package org.torproject.descriptor; +import java.io.File; import java.util.List; /** @@ -48,4 +49,22 @@ public interface DescriptorParser { */ public List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes, String fileName) throws DescriptorParseException; + + /** + * Parse descriptors in the given byte array and return the parsed/unparseable + * descriptors. + * + * @param rawDescriptorBytes Raw descriptor bytes containing one or more + * descriptors + * @param descriptorFile Optional descriptor file reference included in + * parsed/unparseable descriptors + * @param fileName Descriptor file name used for parsing the descriptor + * publication time of some descriptor types + * + * @return Parsed/unparseable descriptors + * + * @since 1.9.0 + */ + public Iterable<Descriptor> parseDescriptors(byte[] rawDescriptorBytes, + File descriptorFile, String fileName); } diff --git a/src/main/java/org/torproject/descriptor/DescriptorReader.java b/src/main/java/org/torproject/descriptor/DescriptorReader.java index 85f8cdd..e15e5d9 100644 --- a/src/main/java/org/torproject/descriptor/DescriptorReader.java +++ b/src/main/java/org/torproject/descriptor/DescriptorReader.java @@ -23,18 +23,13 @@ import java.util.SortedMap; * DescriptorReader descriptorReader = * DescriptorSourceFactory.createDescriptorReader(); * // Read descriptors from local directory called in/. - * descriptorReader.addDirectory(new File("in")); - * Iterator<DescriptorFile> descriptorFiles = - * descriptorReader.readDescriptors(); - * while (descriptorFiles.hasNext()) { - * DescriptorFile descriptorFile = descriptorFiles.next(); - * for (Descriptor descriptor : descriptorFile.getDescriptors()) { - * if ((descriptor instanceof RelayNetworkStatusConsensus)) { - * // Only process network status consensuses, ignore the rest. - * RelayNetworkStatusConsensus consensus = - * (RelayNetworkStatusConsensus) descriptor; - * processConsensus(consensus); - * } + * for (Descriptor descriptor : + * descriptorReader.readDescriptors(new File("in")) { + * // Only process network status consensuses, ignore the rest. + * if ((descriptor instanceof RelayNetworkStatusConsensus)) { + * RelayNetworkStatusConsensus consensus = + * (RelayNetworkStatusConsensus) descriptor; + * processConsensus(consensus); * } * }}</pre> * @@ -46,6 +41,9 @@ public interface DescriptorReader { * Add a local directory to read descriptors from, which may contain * descriptor files or tarballs containing descriptor files. * + * @deprecated Replaced with a parameter in {@link #readDescriptors(File...)}, + * which ignores any directories added via this deprecated method. + * * @since 1.0.0 */ public void addDirectory(File directory); @@ -54,6 +52,9 @@ public interface DescriptorReader { * Add a tarball to read descriptors from, which may be uncompressed, * bz2-compressed, or xz-compressed. * + * @deprecated Replaced with a parameter in {@link #readDescriptors(File...)}, + * which ignores any tarballs added via this deprecated method. + * * @since 1.0.0 */ public void addTarball(File tarball); @@ -157,11 +158,23 @@ public interface DescriptorReader { * The default is 100, but if descriptor files contain hundreds or * even thousands of descriptors, that default may be too high. * + * @deprecated Replaced with {@link #setMaxDescriptorsInQueue(int)}. + * * @since 1.0.0 */ public void setMaxDescriptorFilesInQueue(int max); /** + * Don't keep more than this number of descriptors in the queue (default: + * 100). + * + * @param maxDescriptorsInQueue Maximum number of descriptors in the queue. + * + * @since 1.9.0 + */ + public void setMaxDescriptorsInQueue(int maxDescriptorsInQueue); + + /** * Read the previously configured descriptors and make them available * via the returned blocking iterator. * @@ -169,8 +182,27 @@ public interface DescriptorReader { * more shortly after, it blocks the caller. This method can only be * run once. * + * @deprecated Replaced with {@link #readDescriptors(File...)}. + * * @since 1.0.0 */ public Iterator<DescriptorFile> readDescriptors(); + + /** + * Read descriptors from the given descriptor file(s) and return the parsed + * descriptors. + * + * Whenever the reader runs out of descriptors and expects to provide + * more shortly after, it blocks the caller. This method can only be + * run once. + * + * @param descriptorFiles One or more directories, tarballs, or files + * containing descriptors. + * + * @return Parsed descriptors. + * + * @since 1.9.0 + */ + public Iterable<Descriptor> readDescriptors(File... descriptorFiles); } diff --git a/src/main/java/org/torproject/descriptor/UnparseableDescriptor.java b/src/main/java/org/torproject/descriptor/UnparseableDescriptor.java new file mode 100644 index 0000000..6282222 --- /dev/null +++ b/src/main/java/org/torproject/descriptor/UnparseableDescriptor.java @@ -0,0 +1,44 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.descriptor; + +import java.util.List; + +/** + * Contains a descriptor that could not be parsed. + * Only {@link UnparseableDescriptor#getRawDescriptorBytes} and + * {@link UnparseableDescriptor#getDescriptorFile} are supported. + * + * @since 1.9.0 + */ +public interface UnparseableDescriptor extends Descriptor { + + /** + * Return the first exception thrown while attempting to parse this + * descriptor. + * + * @return First exception thrown. + * + * @since 1.9.0 + */ + public DescriptorParseException getDescriptorParseException(); + + /** + * Will always throw an {@code UnsupportedOperationException}. + * + * @since 1.9.0 + */ + @Override + public List<String> getAnnotations(); + + /** + * Will always throw an {@code UnsupportedOperationException}. + * + * @since 1.9.0 + */ + @Override + public List<String> getUnrecognizedLines(); + +} + diff --git a/src/main/java/org/torproject/descriptor/impl/BlockingIteratorImpl.java b/src/main/java/org/torproject/descriptor/impl/BlockingIteratorImpl.java index 3509026..4921ac4 100644 --- a/src/main/java/org/torproject/descriptor/impl/BlockingIteratorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/BlockingIteratorImpl.java @@ -11,7 +11,7 @@ import java.util.Queue; /* Provide an iterator for a queue of objects and block when there are * currently no objects in the queue. Allow the producer to signal that * there won't be further objects and unblock any waiting consumers. */ -public class BlockingIteratorImpl<T> implements Iterator<T> { +public class BlockingIteratorImpl<T> implements Iterator<T>, Iterable<T> { /* Queue containing produced elemnts waiting for consumers. */ private Queue<T> queue = new LinkedList<>(); @@ -24,8 +24,10 @@ public class BlockingIteratorImpl<T> implements Iterator<T> { } /* Create instance with maximum queue size. */ - protected BlockingIteratorImpl(int maxQueueSize) { - this.maxQueueSize = maxQueueSize; + protected BlockingIteratorImpl(Integer maxQueueSize) { + if (null != maxQueueSize) { + this.maxQueueSize = maxQueueSize; + } } /* Add an object to the queue if there's still room. */ @@ -99,5 +101,10 @@ public class BlockingIteratorImpl<T> implements Iterator<T> { public void remove() { throw new UnsupportedOperationException(); } + + @Override + public Iterator<T> iterator() { + return this; + } } diff --git a/src/main/java/org/torproject/descriptor/impl/BridgeExtraInfoDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgeExtraInfoDescriptorImpl.java index 7d0af52..b2b3e99 100644 --- a/src/main/java/org/torproject/descriptor/impl/BridgeExtraInfoDescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/BridgeExtraInfoDescriptorImpl.java @@ -6,13 +6,16 @@ package org.torproject.descriptor.impl; import org.torproject.descriptor.BridgeExtraInfoDescriptor; import org.torproject.descriptor.DescriptorParseException; +import java.io.File; + public class BridgeExtraInfoDescriptorImpl extends ExtraInfoDescriptorImpl implements BridgeExtraInfoDescriptor { protected BridgeExtraInfoDescriptorImpl(byte[] descriptorBytes, - int[] offsetAndLimit, boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { - super(descriptorBytes, offsetAndLimit, failUnrecognizedDescriptorLines); + int[] offsetAndLimit, File descriptorFile, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { + super(descriptorBytes, offsetAndLimit, descriptorFile, + failUnrecognizedDescriptorLines); } } diff --git a/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java index 83a3a8a..a54c91c 100644 --- a/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java @@ -6,6 +6,7 @@ package org.torproject.descriptor.impl; import org.torproject.descriptor.BridgeNetworkStatus; import org.torproject.descriptor.DescriptorParseException; +import java.io.File; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -19,10 +20,10 @@ public class BridgeNetworkStatusImpl extends NetworkStatusImpl implements BridgeNetworkStatus { protected BridgeNetworkStatusImpl(byte[] rawDescriptorBytes, - int[] offsetAndLength, String fileName, + int[] offsetAndLength, File descriptorFile, String fileName, boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines, - false, false); + super(rawDescriptorBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines, false, false); this.setPublishedMillisFromFileName(fileName); } diff --git a/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java index ca07c29..b2015ff 100644 --- a/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java @@ -6,6 +6,7 @@ package org.torproject.descriptor.impl; import org.torproject.descriptor.BridgePoolAssignment; import org.torproject.descriptor.DescriptorParseException; +import java.io.File; import java.util.EnumSet; import java.util.Scanner; import java.util.SortedMap; @@ -15,10 +16,10 @@ public class BridgePoolAssignmentImpl extends DescriptorImpl implements BridgePoolAssignment { protected BridgePoolAssignmentImpl(byte[] rawDescriptorBytes, - int[] offsetAndlength, boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { - super(rawDescriptorBytes, offsetAndlength, failUnrecognizedDescriptorLines, - false); + int[] offsetAndlength, File descriptorFile, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { + super(rawDescriptorBytes, offsetAndlength, descriptorFile, + failUnrecognizedDescriptorLines, false); this.parseDescriptorBytes(); this.checkExactlyOnceKeys(EnumSet.of(Key.BRIDGE_POOL_ASSIGNMENT)); this.checkFirstKey(Key.BRIDGE_POOL_ASSIGNMENT); diff --git a/src/main/java/org/torproject/descriptor/impl/BridgeServerDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgeServerDescriptorImpl.java index ac45591..30da220 100644 --- a/src/main/java/org/torproject/descriptor/impl/BridgeServerDescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/BridgeServerDescriptorImpl.java @@ -6,13 +6,16 @@ package org.torproject.descriptor.impl; import org.torproject.descriptor.BridgeServerDescriptor; import org.torproject.descriptor.DescriptorParseException; +import java.io.File; + public class BridgeServerDescriptorImpl extends ServerDescriptorImpl implements BridgeServerDescriptor { protected BridgeServerDescriptorImpl(byte[] rawDescriptorBytes, - int[] offsetAndLength, boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { - super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines); + int[] offsetAndLength, File descriptorFile, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { + super(rawDescriptorBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines); } } diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java index 52ff6d4..d88b542 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java @@ -7,6 +7,7 @@ import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import java.io.ByteArrayInputStream; +import java.io.File; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -41,6 +42,17 @@ public abstract class DescriptorImpl implements Descriptor { protected int length; /** + * Optional reference to the descriptor file, if this descriptor was read from + * a descriptor file. + */ + private File descriptorFile; + + @Override + public File getDescriptorFile() { + return this.descriptorFile; + } + + /** * Returns a <emph>copy</emph> of the full raw descriptor bytes. * * If possible, subclasses should avoid retrieving raw descriptor bytes and @@ -178,8 +190,7 @@ public abstract class DescriptorImpl implements Descriptor { } protected DescriptorImpl(byte[] rawDescriptorBytes, int[] offsetAndLength, - boolean failUnrecognizedDescriptorLines, boolean blankLinesAllowed) - throws DescriptorParseException { + File descriptorFile, boolean failUnrecognizedDescriptorLines) { int offset = offsetAndLength[0]; int length = offsetAndLength[1]; if (offset < 0 || offset + length > rawDescriptorBytes.length @@ -191,8 +202,16 @@ public abstract class DescriptorImpl implements Descriptor { this.rawDescriptorBytes = rawDescriptorBytes; this.offset = offset; this.length = length; + this.descriptorFile = descriptorFile; this.failUnrecognizedDescriptorLines = failUnrecognizedDescriptorLines; + } + + protected DescriptorImpl(byte[] rawDescriptorBytes, int[] offsetAndLength, + File descriptorFile, boolean failUnrecognizedDescriptorLines, + boolean blankLinesAllowed) throws DescriptorParseException { + this(rawDescriptorBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines); this.cutOffAnnotations(); this.countKeys(rawDescriptorBytes, blankLinesAllowed); } diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java index e416d93..a0be85c 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java @@ -10,6 +10,7 @@ import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DescriptorParser; +import java.io.File; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; @@ -31,6 +32,29 @@ public class DescriptorParserImpl implements DescriptorParser { public List<Descriptor> parseDescriptors( byte[] rawDescriptorBytes, String fileName) throws DescriptorParseException { + return this.parseDescriptors(rawDescriptorBytes, null, fileName, false); + } + + @Override + public Iterable<Descriptor> parseDescriptors(byte[] rawDescriptorBytes, + File descriptorFile, String fileName) { + try { + return this.parseDescriptors(rawDescriptorBytes, descriptorFile, fileName, + true); + } catch (DescriptorParseException e) { + /* Looks like we attempted to parse the whole raw descriptor bytes at once + * below and ran into a parse issue. */ + List<Descriptor> parsedDescriptors = new ArrayList<>(); + parsedDescriptors.add(new UnparseableDescriptorImpl(rawDescriptorBytes, + new int[] { 0, rawDescriptorBytes.length }, descriptorFile, + failUnrecognizedDescriptorLines, e)); + return parsedDescriptors; + } + } + + private List<Descriptor> parseDescriptors( + byte[] rawDescriptorBytes, File descriptorFile, String fileName, + boolean includeUnparseableDescriptors) throws DescriptorParseException { byte[] first100Chars = new byte[Math.min(100, rawDescriptorBytes.length)]; System.arraycopy(rawDescriptorBytes, 0, first100Chars, 0, @@ -45,9 +69,9 @@ public class DescriptorParserImpl implements DescriptorParser { NL + Key.NETWORK_STATUS_VERSION.keyword + SP + "3")) && firstLines.contains( NL + Key.VOTE_STATUS.keyword + SP + "consensus" + NL))) { - return parseDescriptors(rawDescriptorBytes, Key.NETWORK_STATUS_VERSION, - RelayNetworkStatusConsensusImpl.class, - this.failUnrecognizedDescriptorLines); + return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + Key.NETWORK_STATUS_VERSION, RelayNetworkStatusConsensusImpl.class, + this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors); } else if (firstLines.startsWith("@type network-status-vote-3 1.") || ((firstLines.startsWith( Key.NETWORK_STATUS_VERSION.keyword + SP + "3" + NL) @@ -55,89 +79,95 @@ public class DescriptorParserImpl implements DescriptorParser { NL + Key.NETWORK_STATUS_VERSION.keyword + SP + "3" + NL)) && firstLines.contains( NL + Key.VOTE_STATUS.keyword + SP + "vote" + NL))) { - return parseDescriptors(rawDescriptorBytes, Key.NETWORK_STATUS_VERSION, - RelayNetworkStatusVoteImpl.class, - this.failUnrecognizedDescriptorLines); + return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + Key.NETWORK_STATUS_VERSION, RelayNetworkStatusVoteImpl.class, + this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors); } else if (firstLines.startsWith("@type bridge-network-status 1.") || firstLines.startsWith(Key.R.keyword + SP)) { List<Descriptor> parsedDescriptors = new ArrayList<>(); parsedDescriptors.add(new BridgeNetworkStatusImpl( rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length }, - fileName, this.failUnrecognizedDescriptorLines)); + descriptorFile, fileName, this.failUnrecognizedDescriptorLines)); return parsedDescriptors; } else if (firstLines.startsWith("@type bridge-server-descriptor 1.")) { - return parseDescriptors(rawDescriptorBytes, Key.ROUTER, - BridgeServerDescriptorImpl.class, - this.failUnrecognizedDescriptorLines); + return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + Key.ROUTER, BridgeServerDescriptorImpl.class, + this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors); } else if (firstLines.startsWith("@type server-descriptor 1.") || firstLines.startsWith(Key.ROUTER.keyword + SP) || firstLines.contains(NL + Key.ROUTER.keyword + SP)) { - return parseDescriptors(rawDescriptorBytes, Key.ROUTER, - RelayServerDescriptorImpl.class, - this.failUnrecognizedDescriptorLines); + return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + Key.ROUTER, RelayServerDescriptorImpl.class, + this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors); } else if (firstLines.startsWith("@type bridge-extra-info 1.")) { - return parseDescriptors(rawDescriptorBytes, Key.EXTRA_INFO, - BridgeExtraInfoDescriptorImpl.class, - this.failUnrecognizedDescriptorLines); + return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + Key.EXTRA_INFO, BridgeExtraInfoDescriptorImpl.class, + this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors); } else if (firstLines.startsWith("@type extra-info 1.") || firstLines.startsWith(Key.EXTRA_INFO.keyword + SP) || firstLines.contains(NL + Key.EXTRA_INFO.keyword + SP)) { - return parseDescriptors(rawDescriptorBytes, Key.EXTRA_INFO, - RelayExtraInfoDescriptorImpl.class, - this.failUnrecognizedDescriptorLines); + return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + Key.EXTRA_INFO, RelayExtraInfoDescriptorImpl.class, + this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors); } else if (firstLines.startsWith("@type microdescriptor 1.") || firstLines.startsWith(Key.ONION_KEY.keyword + NL) || firstLines.contains(NL + Key.ONION_KEY.keyword + NL)) { - return parseDescriptors(rawDescriptorBytes, Key.ONION_KEY, - MicrodescriptorImpl.class, this.failUnrecognizedDescriptorLines); + return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + Key.ONION_KEY, MicrodescriptorImpl.class, + this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors); } else if (firstLines.startsWith("@type bridge-pool-assignment 1.") || firstLines.startsWith(Key.BRIDGE_POOL_ASSIGNMENT.keyword + SP) || firstLines.contains(NL + Key.BRIDGE_POOL_ASSIGNMENT.keyword + SP)) { - return parseDescriptors(rawDescriptorBytes, Key.BRIDGE_POOL_ASSIGNMENT, - BridgePoolAssignmentImpl.class, this.failUnrecognizedDescriptorLines); + return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + Key.BRIDGE_POOL_ASSIGNMENT, BridgePoolAssignmentImpl.class, + this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors); } else if (firstLines.startsWith("@type dir-key-certificate-3 1.") || firstLines.startsWith(Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP) || firstLines.contains( NL + Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP)) { - return parseDescriptors(rawDescriptorBytes, + return this.parseDescriptors(rawDescriptorBytes, descriptorFile, Key.DIR_KEY_CERTIFICATE_VERSION, DirectoryKeyCertificateImpl.class, - this.failUnrecognizedDescriptorLines); + this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors); } else if (firstLines.startsWith("@type tordnsel 1.") || firstLines.startsWith("ExitNode" + SP) || firstLines.contains(NL + "ExitNode" + SP)) { List<Descriptor> parsedDescriptors = new ArrayList<>(); - parsedDescriptors.add(new ExitListImpl(rawDescriptorBytes, fileName, - this.failUnrecognizedDescriptorLines)); + parsedDescriptors.add(new ExitListImpl(rawDescriptorBytes, descriptorFile, + fileName, this.failUnrecognizedDescriptorLines)); return parsedDescriptors; } else if (firstLines.startsWith("@type network-status-2 1.") || firstLines.startsWith( Key.NETWORK_STATUS_VERSION.keyword + SP + "2" + NL) || firstLines.contains( NL + Key.NETWORK_STATUS_VERSION.keyword + SP + "2" + NL)) { - return parseDescriptors(rawDescriptorBytes, Key.NETWORK_STATUS_VERSION, - RelayNetworkStatusImpl.class, this.failUnrecognizedDescriptorLines); + return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + Key.NETWORK_STATUS_VERSION, RelayNetworkStatusImpl.class, + this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors); } else if (firstLines.startsWith("@type directory 1.") || firstLines.startsWith(Key.SIGNED_DIRECTORY.keyword + NL) || firstLines.contains(NL + Key.SIGNED_DIRECTORY.keyword + NL)) { - return parseDescriptors(rawDescriptorBytes, Key.SIGNED_DIRECTORY, - RelayDirectoryImpl.class, this.failUnrecognizedDescriptorLines); + return this.parseDescriptors(rawDescriptorBytes, descriptorFile, + Key.SIGNED_DIRECTORY, RelayDirectoryImpl.class, + this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors); } else if (firstLines.startsWith("@type torperf 1.")) { - return TorperfResultImpl.parseTorperfResults( - rawDescriptorBytes, this.failUnrecognizedDescriptorLines); + return TorperfResultImpl.parseTorperfResults(rawDescriptorBytes, + descriptorFile, this.failUnrecognizedDescriptorLines); } else { throw new DescriptorParseException("Could not detect descriptor " + "type in descriptor starting with '" + firstLines + "'."); } } - private static List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes, - Key key, Class<? extends DescriptorImpl> descriptorClass, - boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { + private List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes, + File descriptorFile, Key key, + Class<? extends DescriptorImpl> descriptorClass, + boolean failUnrecognizedDescriptorLines, + boolean includeUnparseableDescriptors) throws DescriptorParseException { List<Descriptor> parsedDescriptors = new ArrayList<>(); Constructor<? extends DescriptorImpl> constructor; try { constructor = descriptorClass.getDeclaredConstructor(byte[].class, - int[].class, boolean.class); + int[].class, File.class, boolean.class); } catch (NoSuchMethodException e) { throw new RuntimeException(e); } @@ -181,20 +211,31 @@ public class DescriptorParserImpl implements DescriptorParser { endDescriptor += 1; int[] offsetAndLength = new int[] { startAnnotations, endDescriptor - startAnnotations }; - parsedDescriptors.add(parseDescriptor(rawDescriptorBytes, - offsetAndLength, constructor, failUnrecognizedDescriptorLines)); + try { + parsedDescriptors.add(this.parseDescriptor(rawDescriptorBytes, + offsetAndLength, descriptorFile, constructor, + failUnrecognizedDescriptorLines)); + } catch (DescriptorParseException e) { + if (includeUnparseableDescriptors) { + parsedDescriptors.add(new UnparseableDescriptorImpl( + rawDescriptorBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines, e)); + } else { + throw e; + } + } startAnnotations = endDescriptor; } return parsedDescriptors; } - static Descriptor parseDescriptor(byte[] rawDescriptorBytes, - int[] offsetAndLength, Constructor<? extends DescriptorImpl> constructor, - boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { + Descriptor parseDescriptor(byte[] rawDescriptorBytes, + int[] offsetAndLength, File descriptorFile, + Constructor<? extends DescriptorImpl> constructor, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { try { return constructor.newInstance(rawDescriptorBytes, - offsetAndLength, failUnrecognizedDescriptorLines); + offsetAndLength, descriptorFile, failUnrecognizedDescriptorLines); } catch (InvocationTargetException e) { if (null != e.getCause() && e.getCause() instanceof DescriptorParseException) { @@ -206,5 +247,4 @@ public class DescriptorParserImpl implements DescriptorParser { throw new RuntimeException(e); } } - } diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java index 12d73ad..54e26fc 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java @@ -135,6 +135,17 @@ public class DescriptorReaderImpl implements DescriptorReader { this.maxDescriptorFilesInQueue = max; } + private int maxDescriptorsInQueue = 100; + + @Override + public void setMaxDescriptorsInQueue(int maxDescriptorsInQueue) { + if (this.hasStartedReading) { + throw new IllegalStateException("Reconfiguration is not permitted " + + "after starting to read."); + } + this.maxDescriptorsInQueue = maxDescriptorsInQueue; + } + private DescriptorReaderRunnable reader; @Override @@ -160,6 +171,25 @@ public class DescriptorReaderImpl implements DescriptorReader { } @Override + public Iterable<Descriptor> readDescriptors(File... descriptorFiles) { + if (this.hasStartedReading) { + throw new IllegalStateException("Initiating reading is only " + + "permitted once."); + } + this.hasStartedReading = true; + BlockingIteratorImpl<Descriptor> descriptorQueue = + new BlockingIteratorImpl<>(this.maxDescriptorsInQueue); + this.reader = new DescriptorReaderRunnable( + descriptorFiles, descriptorQueue, this.autoSaveHistoryFile, + this.manualSaveHistoryFile, this.excludedFiles, + this.failUnrecognizedDescriptorLines); + Thread readerThread = new Thread(this.reader); + readerThread.setDaemon(true); + readerThread.start(); + return descriptorQueue; + } + + @Override public void saveHistoryFile(File historyFile) { if (!this.reader.hasFinishedReading) { throw new IllegalStateException("Saving history is only permitted after " @@ -170,11 +200,15 @@ public class DescriptorReaderImpl implements DescriptorReader { private static class DescriptorReaderRunnable implements Runnable { + private File[] descriptorFiles; + private List<File> directories; private List<File> tarballs; - private BlockingIteratorImpl<DescriptorFile> descriptorQueue; + private BlockingIteratorImpl<Descriptor> descriptorQueue; + + private BlockingIteratorImpl<DescriptorFile> descriptorFileQueue; private File autoSaveHistoryFile; @@ -186,24 +220,45 @@ public class DescriptorReaderImpl implements DescriptorReader { private SortedMap<String, Long> parsedFilesAfter = new TreeMap<>(); + private boolean failUnrecognizedDescriptorLines; + private DescriptorParser descriptorParser; private boolean hasFinishedReading = false; private DescriptorReaderRunnable(List<File> directories, List<File> tarballs, - BlockingIteratorImpl<DescriptorFile> descriptorQueue, + BlockingIteratorImpl<DescriptorFile> descriptorFileQueue, File autoSaveHistoryFile, File manualSaveHistoryFile, SortedMap<String, Long> excludedFiles, boolean failUnrecognizedDescriptorLines) { this.directories = directories; this.tarballs = tarballs; + this.descriptorFileQueue = descriptorFileQueue; + this.autoSaveHistoryFile = autoSaveHistoryFile; + this.manualSaveHistoryFile = manualSaveHistoryFile; + if (excludedFiles != null) { + this.excludedFilesBefore = excludedFiles; + } + this.failUnrecognizedDescriptorLines = failUnrecognizedDescriptorLines; + this.descriptorParser = new DescriptorParserImpl(); + this.descriptorParser.setFailUnrecognizedDescriptorLines( + failUnrecognizedDescriptorLines); + } + + private DescriptorReaderRunnable(File[] descriptorFiles, + BlockingIteratorImpl<Descriptor> descriptorQueue, + File autoSaveHistoryFile, File manualSaveHistoryFile, + SortedMap<String, Long> excludedFiles, + boolean failUnrecognizedDescriptorLines) { + this.descriptorFiles = descriptorFiles; this.descriptorQueue = descriptorQueue; this.autoSaveHistoryFile = autoSaveHistoryFile; this.manualSaveHistoryFile = manualSaveHistoryFile; if (excludedFiles != null) { this.excludedFilesBefore = excludedFiles; } + this.failUnrecognizedDescriptorLines = failUnrecognizedDescriptorLines; this.descriptorParser = new DescriptorParserImpl(); this.descriptorParser.setFailUnrecognizedDescriptorLines( failUnrecognizedDescriptorLines); @@ -213,6 +268,7 @@ public class DescriptorReaderImpl implements DescriptorReader { try { this.readOldHistory(this.autoSaveHistoryFile); this.readOldHistory(this.manualSaveHistoryFile); + this.readDescriptorFiles(); this.readDescriptors(); this.readTarballs(); this.hasFinishedReading = true; @@ -220,7 +276,12 @@ public class DescriptorReaderImpl implements DescriptorReader { log.error("Bug: uncaught exception or error while " + "reading descriptors: " + t.getMessage(), t); } finally { - this.descriptorQueue.setOutOfDescriptors(); + if (null != this.descriptorFileQueue) { + this.descriptorFileQueue.setOutOfDescriptors(); + } + if (null != this.descriptorQueue) { + this.descriptorQueue.setOutOfDescriptors(); + } } if (this.hasFinishedReading) { this.writeNewHistory(this.autoSaveHistoryFile); @@ -275,7 +336,97 @@ public class DescriptorReaderImpl implements DescriptorReader { } } + private void readDescriptorFiles() { + if (null == this.descriptorFiles) { + return; + } + Stack<File> files = new Stack<>(); + for (File descriptorFile : this.descriptorFiles) { + if (!descriptorFile.exists()) { + continue; + } + files.add(descriptorFile); + while (!files.isEmpty()) { + File file = files.pop(); + try { + String absolutePath = file.getAbsolutePath(); + long lastModifiedMillis = file.lastModified(); + if (this.excludedFilesBefore.containsKey(absolutePath) + && this.excludedFilesBefore.get(absolutePath) + == lastModifiedMillis) { + this.excludedFilesAfter.put(absolutePath, lastModifiedMillis); + continue; + } + if (file.isDirectory()) { + files.addAll(Arrays.asList(file.listFiles())); + continue; + } else if (file.getName().endsWith(".tar") + || file.getName().endsWith(".tar.bz2") + || file.getName().endsWith(".tar.xz")) { + this.readTarball(file); + } else { + this.readDescriptorFile(file); + } + this.parsedFilesAfter.put(absolutePath, lastModifiedMillis); + } catch (IOException e) { + log.warn("Unable to read descriptor file {}.", file, e); + } + } + } + } + + private void readTarball(File file) throws IOException { + FileInputStream in = new FileInputStream(file); + if (in.available() <= 0) { + return; + } + TarArchiveInputStream tais; + if (file.getName().endsWith(".tar.bz2")) { + tais = new TarArchiveInputStream(new BZip2CompressorInputStream(in)); + } else if (file.getName().endsWith(".tar.xz")) { + tais = new TarArchiveInputStream(new XZCompressorInputStream(in)); + } else if (file.getName().endsWith(".tar")) { + tais = new TarArchiveInputStream(in); + } else { + return; + } + BufferedInputStream bis = new BufferedInputStream(tais); + TarArchiveEntry tae; + while ((tae = tais.getNextTarEntry()) != null) { + if (tae.isDirectory()) { + continue; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + int len; + byte[] data = new byte[1024]; + while ((len = bis.read(data, 0, 1024)) >= 0) { + baos.write(data, 0, len); + } + byte[] rawDescriptorBytes = baos.toByteArray(); + if (rawDescriptorBytes.length < 1) { + continue; + } + String fileName = tae.getName().substring( + tae.getName().lastIndexOf("/") + 1); + for (Descriptor descriptor : this.descriptorParser.parseDescriptors( + rawDescriptorBytes, file, fileName)) { + this.descriptorQueue.add(descriptor); + } + } + } + + private void readDescriptorFile(File file) throws IOException { + byte[] rawDescriptorBytes = Files.readAllBytes(file.toPath()); + for (Descriptor descriptor : this.descriptorParser.parseDescriptors( + rawDescriptorBytes, file, file.getName())) { + this.descriptorQueue.add(descriptor); + } + } + private void readDescriptors() { + if (null == this.directories) { + return; + } for (File directory : this.directories) { if (!directory.exists() || !directory.isDirectory()) { continue; @@ -315,13 +466,16 @@ public class DescriptorReaderImpl implements DescriptorReader { descriptorFile.setException(e); abortReading = true; } - this.descriptorQueue.add(descriptorFile); + this.descriptorFileQueue.add(descriptorFile); } } } } private void readTarballs() { + if (null == this.tarballs) { + return; + } List<File> files = new ArrayList<>(this.tarballs); boolean abortReading = false; while (!abortReading && !files.isEmpty()) { @@ -385,7 +539,7 @@ public class DescriptorReaderImpl implements DescriptorReader { } catch (DescriptorParseException e) { descriptorFile.setException(e); } - this.descriptorQueue.add(descriptorFile); + this.descriptorFileQueue.add(descriptorFile); } } } catch (IOException e) { diff --git a/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java b/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java index 9a06a93..6beb9b5 100644 --- a/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java @@ -6,6 +6,7 @@ package org.torproject.descriptor.impl; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DirectoryKeyCertificate; +import java.io.File; import java.util.ArrayList; import java.util.EnumSet; import java.util.Scanner; @@ -15,10 +16,10 @@ public class DirectoryKeyCertificateImpl extends DescriptorImpl implements DirectoryKeyCertificate { protected DirectoryKeyCertificateImpl(byte[] rawDescriptorBytes, - int[] offsetAndLength, boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { - super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines, - false); + int[] offsetAndLength, File descriptorFile, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { + super(rawDescriptorBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines, false); this.parseDescriptorBytes(); this.calculateDigestSha1Hex(Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP, NL + Key.DIR_KEY_CERTIFICATION.keyword + NL); diff --git a/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java b/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java index a87a55f..d16856a 100644 --- a/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java @@ -7,6 +7,7 @@ import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.ExitList; import org.torproject.descriptor.ExitListEntry; +import java.io.File; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -18,11 +19,11 @@ import java.util.TimeZone; public class ExitListImpl extends DescriptorImpl implements ExitList { - protected ExitListImpl(byte[] rawDescriptorBytes, String fileName, - boolean failUnrecognizedDescriptorLines) + protected ExitListImpl(byte[] rawDescriptorBytes, File descriptorfile, + String fileName, boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { super(rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length }, - failUnrecognizedDescriptorLines, false); + descriptorfile, failUnrecognizedDescriptorLines, false); this.splitAndParseExitListEntries(); this.setPublishedMillisFromFileName(fileName); } diff --git a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java index ca7d376..32a4dcc 100644 --- a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java @@ -7,6 +7,7 @@ import org.torproject.descriptor.BandwidthHistory; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.ExtraInfoDescriptor; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -34,10 +35,10 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl Key.PADDING_COUNTS); protected ExtraInfoDescriptorImpl(byte[] descriptorBytes, - int[] offsetAndLimit, boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { - super(descriptorBytes, offsetAndLimit, failUnrecognizedDescriptorLines, - false); + int[] offsetAndLimit, File descriptorFile, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { + super(descriptorBytes, offsetAndLimit, descriptorFile, + failUnrecognizedDescriptorLines, false); this.parseDescriptorBytes(); this.checkExactlyOnceKeys(exactlyOnceKeys); Set<Key> dirreqStatsKeys = EnumSet.of( diff --git a/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java index c436120..d9c284e 100644 --- a/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java @@ -6,6 +6,7 @@ package org.torproject.descriptor.impl; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.Microdescriptor; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -18,10 +19,10 @@ public class MicrodescriptorImpl extends DescriptorImpl implements Microdescriptor { protected MicrodescriptorImpl(byte[] descriptorBytes, int[] offsetAndLength, - boolean failUnrecognizedDescriptorLines) + File descriptorFile, boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(descriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines, - false); + super(descriptorBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines, false); this.parseDescriptorBytes(); this.calculateDigestSha256Base64(Key.ONION_KEY.keyword + NL); this.checkExactlyOnceKeys(EnumSet.of(Key.ONION_KEY)); diff --git a/src/main/java/org/torproject/descriptor/impl/NetworkStatusImpl.java b/src/main/java/org/torproject/descriptor/impl/NetworkStatusImpl.java index 6765b6f..5695353 100644 --- a/src/main/java/org/torproject/descriptor/impl/NetworkStatusImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/NetworkStatusImpl.java @@ -8,6 +8,7 @@ import org.torproject.descriptor.DirSourceEntry; import org.torproject.descriptor.DirectorySignature; import org.torproject.descriptor.NetworkStatusEntry; +import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.SortedMap; @@ -19,11 +20,11 @@ import java.util.TreeMap; public abstract class NetworkStatusImpl extends DescriptorImpl { protected NetworkStatusImpl(byte[] rawDescriptorBytes, int[] offsetAndLength, - boolean failUnrecognizedDescriptorLines, + File descriptorFile, boolean failUnrecognizedDescriptorLines, boolean containsDirSourceEntries, boolean blankLinesAllowed) throws DescriptorParseException { - super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines, - blankLinesAllowed); + super(rawDescriptorBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines, blankLinesAllowed); this.splitAndParseParts(containsDirSourceEntries); } diff --git a/src/main/java/org/torproject/descriptor/impl/RelayDirectoryImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayDirectoryImpl.java index d5108ca..79ab862 100644 --- a/src/main/java/org/torproject/descriptor/impl/RelayDirectoryImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/RelayDirectoryImpl.java @@ -8,6 +8,7 @@ import org.torproject.descriptor.RelayDirectory; import org.torproject.descriptor.RouterStatusEntry; import org.torproject.descriptor.ServerDescriptor; +import java.io.File; import java.util.ArrayList; import java.util.EnumSet; import java.util.List; @@ -18,10 +19,10 @@ public class RelayDirectoryImpl extends DescriptorImpl implements RelayDirectory { protected RelayDirectoryImpl(byte[] directoryBytes, int[] offsetAndLength, - boolean failUnrecognizedDescriptorLines) + File descriptorFile, boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(directoryBytes, offsetAndLength, failUnrecognizedDescriptorLines, - true); + super(directoryBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines, true); this.splitAndParseParts(); this.calculateDigestSha1Hex(Key.SIGNED_DIRECTORY.keyword + NL, NL + Key.DIRECTORY_SIGNATURE.keyword + SP); @@ -194,7 +195,8 @@ public class RelayDirectoryImpl extends DescriptorImpl try { ServerDescriptorImpl serverDescriptor = new RelayServerDescriptorImpl(this.rawDescriptorBytes, - new int[] { offset, length }, this.failUnrecognizedDescriptorLines); + new int[] { offset, length }, this.getDescriptorFile(), + this.failUnrecognizedDescriptorLines); this.serverDescriptors.add(serverDescriptor); } catch (DescriptorParseException e) { this.serverDescriptorParseExceptions.add(e); diff --git a/src/main/java/org/torproject/descriptor/impl/RelayExtraInfoDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayExtraInfoDescriptorImpl.java index 92cff65..9644f90 100644 --- a/src/main/java/org/torproject/descriptor/impl/RelayExtraInfoDescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/RelayExtraInfoDescriptorImpl.java @@ -6,13 +6,16 @@ package org.torproject.descriptor.impl; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.RelayExtraInfoDescriptor; +import java.io.File; + public class RelayExtraInfoDescriptorImpl extends ExtraInfoDescriptorImpl implements RelayExtraInfoDescriptor { protected RelayExtraInfoDescriptorImpl(byte[] descriptorBytes, - int[] offsetAndLimit, boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { - super(descriptorBytes, offsetAndLimit, failUnrecognizedDescriptorLines); + int[] offsetAndLimit, File descriptorFile, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { + super(descriptorBytes, offsetAndLimit, descriptorFile, + failUnrecognizedDescriptorLines); this.calculateDigestSha1Hex(Key.EXTRA_INFO.keyword + SP, NL + Key.ROUTER_SIGNATURE.keyword + NL); this.calculateDigestSha256Base64(Key.EXTRA_INFO.keyword + SP, diff --git a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java index 12a9dc3..0e853bb 100644 --- a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java @@ -6,6 +6,7 @@ package org.torproject.descriptor.impl; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.RelayNetworkStatusConsensus; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -22,10 +23,10 @@ public class RelayNetworkStatusConsensusImpl extends NetworkStatusImpl implements RelayNetworkStatusConsensus { protected RelayNetworkStatusConsensusImpl(byte[] consensusBytes, - int[] offsetAndLimit, boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { - super(consensusBytes, offsetAndLimit, failUnrecognizedDescriptorLines, true, - false); + int[] offsetAndLimit, File descriptorFile, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { + super(consensusBytes, offsetAndLimit, descriptorFile, + failUnrecognizedDescriptorLines, true, false); Set<Key> exactlyOnceKeys = EnumSet.of( Key.VOTE_STATUS, Key.CONSENSUS_METHOD, Key.VALID_AFTER, Key.FRESH_UNTIL, Key.VALID_UNTIL, Key.VOTING_DELAY, Key.KNOWN_FLAGS); diff --git a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java index edc483a..8ebea2d 100644 --- a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java @@ -6,6 +6,7 @@ package org.torproject.descriptor.impl; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.RelayNetworkStatus; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -19,10 +20,10 @@ public class RelayNetworkStatusImpl extends NetworkStatusImpl implements RelayNetworkStatus { protected RelayNetworkStatusImpl(byte[] statusBytes, int[] offsetAndLength, - boolean failUnrecognizedDescriptorLines) + File descriptorFile, boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(statusBytes, offsetAndLength, failUnrecognizedDescriptorLines, false, - true); + super(statusBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines, false, true); Set<Key> exactlyOnceKeys = EnumSet.of( Key.NETWORK_STATUS_VERSION, Key.DIR_SOURCE, Key.FINGERPRINT, Key.CONTACT, Key.DIR_SIGNING_KEY, Key.PUBLISHED); diff --git a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java index cad5c15..93d6f12 100644 --- a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java @@ -7,6 +7,7 @@ import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.DirectorySignature; import org.torproject.descriptor.RelayNetworkStatusVote; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -24,10 +25,10 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl implements RelayNetworkStatusVote { protected RelayNetworkStatusVoteImpl(byte[] voteBytes, int[] offsetAndLength, - boolean failUnrecognizedDescriptorLines) + File descriptorFile, boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(voteBytes, offsetAndLength, failUnrecognizedDescriptorLines, false, - false); + super(voteBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines, false, false); Set<Key> exactlyOnceKeys = EnumSet.of( Key.VOTE_STATUS, Key.PUBLISHED, Key.VALID_AFTER, Key.FRESH_UNTIL, Key.VALID_UNTIL, Key.VOTING_DELAY, Key.KNOWN_FLAGS, Key.DIR_SOURCE, diff --git a/src/main/java/org/torproject/descriptor/impl/RelayServerDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayServerDescriptorImpl.java index 4ccb35a..23e9152 100644 --- a/src/main/java/org/torproject/descriptor/impl/RelayServerDescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/RelayServerDescriptorImpl.java @@ -6,13 +6,16 @@ package org.torproject.descriptor.impl; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.RelayServerDescriptor; +import java.io.File; + public class RelayServerDescriptorImpl extends ServerDescriptorImpl implements RelayServerDescriptor { protected RelayServerDescriptorImpl(byte[] descriptorBytes, - int[] offsetAndLength, boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { - super(descriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines); + int[] offsetAndLength, File descriptorFile, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { + super(descriptorBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines); this.calculateDigestSha1Hex(Key.ROUTER.keyword + SP, NL + Key.ROUTER_SIGNATURE.keyword + NL); this.calculateDigestSha256Base64(Key.ROUTER.keyword + SP, diff --git a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java index d438702..f434d84 100644 --- a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java @@ -7,6 +7,7 @@ import org.torproject.descriptor.BandwidthHistory; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.ServerDescriptor; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; @@ -35,10 +36,10 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl Key.ROUTER, Key.BANDWIDTH, Key.PUBLISHED); protected ServerDescriptorImpl(byte[] descriptorBytes, int[] offsetAndLength, - boolean failUnrecognizedDescriptorLines) + File descriptorFile, boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(descriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines, - false); + super(descriptorBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines, false); this.parseDescriptorBytes(); this.checkExactlyOnceKeys(exactlyOnce); this.checkAtMostOnceKeys(atMostOnce); diff --git a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java index fb00a37..ea9eb4b 100644 --- a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java @@ -7,6 +7,7 @@ import org.torproject.descriptor.Descriptor; import org.torproject.descriptor.DescriptorParseException; import org.torproject.descriptor.TorperfResult; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -20,8 +21,8 @@ public class TorperfResultImpl extends DescriptorImpl implements TorperfResult { protected static List<Descriptor> parseTorperfResults( - byte[] rawDescriptorBytes, boolean failUnrecognizedDescriptorLines) - throws DescriptorParseException { + byte[] rawDescriptorBytes, File descriptorFile, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { if (rawDescriptorBytes.length == 0) { throw new DescriptorParseException("Descriptor is empty."); } @@ -47,7 +48,7 @@ public class TorperfResultImpl extends DescriptorImpl } else { /* XXX21932 */ parsedDescriptors.add(new TorperfResultImpl( - (typeAnnotation + line).getBytes(), + (typeAnnotation + line).getBytes(), descriptorFile, failUnrecognizedDescriptorLines)); typeAnnotation = ""; } @@ -55,11 +56,11 @@ public class TorperfResultImpl extends DescriptorImpl return parsedDescriptors; } - protected TorperfResultImpl(byte[] rawDescriptorBytes, + protected TorperfResultImpl(byte[] rawDescriptorBytes, File descriptorFile, boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { super(rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length }, - failUnrecognizedDescriptorLines, false); + descriptorFile, failUnrecognizedDescriptorLines, false); this.parseTorperfResultLine(); } diff --git a/src/main/java/org/torproject/descriptor/impl/UnparseableDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/UnparseableDescriptorImpl.java new file mode 100644 index 0000000..4a0e772 --- /dev/null +++ b/src/main/java/org/torproject/descriptor/impl/UnparseableDescriptorImpl.java @@ -0,0 +1,40 @@ +/* Copyright 2017 The Tor Project + * See LICENSE for licensing information */ + +package org.torproject.descriptor.impl; + +import org.torproject.descriptor.DescriptorParseException; +import org.torproject.descriptor.UnparseableDescriptor; + +import java.io.File; +import java.util.List; + +public class UnparseableDescriptorImpl extends DescriptorImpl + implements UnparseableDescriptor { + + protected UnparseableDescriptorImpl(byte[] rawDescriptorBytes, + int[] offsetAndLength, File descriptorFile, + boolean failUnrecognizedDescriptorLines, + DescriptorParseException descriptorParseException) { + super(rawDescriptorBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines); + this.descriptorParseException = descriptorParseException; + } + + private DescriptorParseException descriptorParseException; + + public DescriptorParseException getDescriptorParseException() { + return this.descriptorParseException; + } + + @Override + public List<String> getAnnotations() { + throw new UnsupportedOperationException("This operation is not supported."); + } + + @Override + public List<String> getUnrecognizedLines() { + throw new UnsupportedOperationException("This operation is not supported."); + } +} + diff --git a/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java b/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java index 3739e5d..c44de6d 100644 --- a/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java +++ b/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java @@ -100,7 +100,7 @@ public class BridgeNetworkStatusTest { throws DescriptorParseException { byte[] statusBytes = this.buildStatusBytes(); return new BridgeNetworkStatusImpl(statusBytes, - new int[] { 0, statusBytes.length }, this.fileName, + new int[] { 0, statusBytes.length }, null, this.fileName, failUnrecognizedDescriptorLines); } diff --git a/src/test/java/org/torproject/descriptor/impl/ConsensusBuilder.java b/src/test/java/org/torproject/descriptor/impl/ConsensusBuilder.java index e0d0c99..71bcd94 100644 --- a/src/test/java/org/torproject/descriptor/impl/ConsensusBuilder.java +++ b/src/test/java/org/torproject/descriptor/impl/ConsensusBuilder.java @@ -360,7 +360,7 @@ public class ConsensusBuilder { throws DescriptorParseException { byte[] consensusBytes = this.buildConsensusBytes(); return new RelayNetworkStatusConsensusImpl(consensusBytes, - new int[] { 0, consensusBytes.length }, + new int[] { 0, consensusBytes.length }, null, failUnrecognizedDescriptorLines); } diff --git a/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java index f1a8935..543254d 100644 --- a/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java @@ -131,7 +131,6 @@ public class DescriptorImplTest { digest = des.getDigestSha256Base64(); assertEquals("1bEECw9nT5KRzPG8dAzEFJgSI4OBQfyWn+wjREb8oa8", digest); } - } diff --git a/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java b/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java index 5a328ec..558a395 100644 --- a/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java @@ -11,6 +11,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import java.io.File; import java.lang.reflect.Constructor; public class DescriptorParserImplTest { @@ -46,15 +47,17 @@ public class DescriptorParserImplTest { Constructor<? extends DescriptorImpl> constructor; try { constructor = TestServerDescriptor.class - .getDeclaredConstructor(byte[].class, int[].class, boolean.class); + .getDeclaredConstructor(byte[].class, int[].class, File.class, + boolean.class); } catch (NoSuchMethodException e) { throw new RuntimeException(e); } this.thrown.expect(DescriptorParseException.class); this.thrown.expectMessage("'176x.158.53.63' in line 'router UbuntuCore169 " + "176x.158.53.63 44583 0 0' is not a valid IPv4 address."); - DescriptorParserImpl.parseDescriptor(DEFECT.getBytes(), - new int[]{0, DEFECT.getBytes().length}, constructor, false); + DescriptorParserImpl dpi = new DescriptorParserImpl(); + dpi.parseDescriptor(DEFECT.getBytes(), + new int[]{0, DEFECT.getBytes().length}, null, constructor, false); } private static final String DEFECT = diff --git a/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java b/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java index 74417d9..5f31341 100644 --- a/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java @@ -24,7 +24,7 @@ public class ExitListImplTest { @Test public void testAnnotatedInput() throws Exception { ExitListImpl result = new ExitListImpl((tordnselAnnotation + input) - .getBytes("US-ASCII"), fileName, false); + .getBytes("US-ASCII"), null, fileName, false); assertEquals("Expected one annotation.", 1, result.getAnnotations().size()); assertEquals(tordnselAnnotation.substring(0, 18), @@ -42,7 +42,7 @@ public class ExitListImplTest { public void testMultipleOldExitAddresses() throws Exception { ExitListImpl result = new ExitListImpl( (tordnselAnnotation + multiExitAddressInput) - .getBytes("US-ASCII"), fileName, false); + .getBytes("US-ASCII"), null, fileName, false); assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(), result.getUnrecognizedLines().isEmpty()); assertEquals("Found: " + result.getExitListEntries(), @@ -68,7 +68,7 @@ public class ExitListImplTest { public void testMultipleExitAddresses() throws Exception { ExitListImpl result = new ExitListImpl( (tordnselAnnotation + multiExitAddressInput) - .getBytes("US-ASCII"), fileName, false); + .getBytes("US-ASCII"), null, fileName, false); assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(), result.getUnrecognizedLines().isEmpty()); Map<String, Long> map = result.getEntries() @@ -84,7 +84,7 @@ public class ExitListImplTest { this.thrown.expect(DescriptorParseException.class); this.thrown.expectMessage("Missing 'ExitAddress' line in exit list entry."); new ExitListImpl((tordnselAnnotation + insufficientInput[0]) - .getBytes("US-ASCII"), fileName, false); + .getBytes("US-ASCII"), null, fileName, false); } @Test @@ -92,7 +92,7 @@ public class ExitListImplTest { this.thrown.expect(DescriptorParseException.class); this.thrown.expectMessage("Missing 'Published' line in exit list entry."); new ExitListImpl((tordnselAnnotation + insufficientInput[1]) - .getBytes("US-ASCII"), fileName, false); + .getBytes("US-ASCII"), null, fileName, false); } private static final String tordnselAnnotation = "@type tordnsel 1.0\n"; diff --git a/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java index 7b62393..82dd365 100644 --- a/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java @@ -336,7 +336,7 @@ public class ExtraInfoDescriptorImplTest { throws DescriptorParseException { byte[] descriptorBytes = this.buildDescriptorBytes(); return new RelayExtraInfoDescriptorImpl(descriptorBytes, - new int[] { 0, descriptorBytes.length}, + new int[] { 0, descriptorBytes.length}, null, failUnrecognizedDescriptorLines); } } @@ -2332,7 +2332,7 @@ public class ExtraInfoDescriptorImplTest { + "").getBytes(); RelayExtraInfoDescriptor descriptor = new RelayExtraInfoDescriptorImpl(descriptorBytes, - new int[] { 0, descriptorBytes.length }, true); + new int[] { 0, descriptorBytes.length }, null, true); assertEquals("Pt1BtzfRwhYqGCDo8jjchS8nJP3ovrDyHGn+dqPbMgw", descriptor.getDigestSha256Base64()); } @@ -2357,7 +2357,7 @@ public class ExtraInfoDescriptorImplTest { .getBytes(); BridgeExtraInfoDescriptor descriptor = new BridgeExtraInfoDescriptorImpl(descriptorBytes, - new int[] { 0, descriptorBytes.length }, true); + new int[] { 0, descriptorBytes.length }, null, true); assertEquals("TvrqpjI7OmCtwGwair/NHUxg5ROVVQYz6/EDyXsDHR4", descriptor.getDigestSha256Base64()); } @@ -2379,7 +2379,7 @@ public class ExtraInfoDescriptorImplTest { .getBytes(); BridgeExtraInfoDescriptor descriptor = new BridgeExtraInfoDescriptorImpl(descriptorBytes, - new int[] { 0, descriptorBytes.length }, true); + new int[] { 0, descriptorBytes.length }, null, true); assertNull(descriptor.getDigestSha1Hex()); assertNull(descriptor.getDigestSha256Base64()); } diff --git a/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java index b1d6474..f5c9131 100644 --- a/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java @@ -66,7 +66,7 @@ public class MicrodescriptorImplTest { throws DescriptorParseException { byte[] descriptorBytes = this.buildDescriptorBytes(); return new MicrodescriptorImpl(descriptorBytes, - new int[] { 0, descriptorBytes.length }, + new int[] { 0, descriptorBytes.length }, null, failUnrecognizedDescriptorLines); } } diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java index 90128c2..4793e76 100644 --- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java @@ -1599,7 +1599,7 @@ public class RelayNetworkStatusConsensusImplTest { byte[] consensusBytes = cb.buildConsensusBytes(); consensusBytes[20] = (byte) 200; new RelayNetworkStatusConsensusImpl(consensusBytes, - new int[] { 0, consensusBytes.length }, true); + new int[] { 0, consensusBytes.length }, null, true); } @Test @@ -1613,7 +1613,7 @@ public class RelayNetworkStatusConsensusImplTest { byte[] consensusBytes = cb.buildConsensusBytes(); consensusBytes[0] = (byte) 200; new RelayNetworkStatusConsensusImpl(consensusBytes, - new int[] { 0, consensusBytes.length }, true); + new int[] { 0, consensusBytes.length }, null, true); } @Test diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java index e69a26b..760d04c 100644 --- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java @@ -49,14 +49,14 @@ public class RelayNetworkStatusImplTest { byte[] statusBytes = (validAnnotation + invalidHeader + validFooter) .getBytes(); new RelayNetworkStatusImpl(statusBytes, new int[] { 0, statusBytes.length }, - true); + null, true); } @Test public void testValidHeader() throws DescriptorParseException { byte[] statusBytes = validStatus.getBytes(); RelayNetworkStatusImpl rnsi = new RelayNetworkStatusImpl(statusBytes, - new int[] { 0, statusBytes.length }, true); + new int[] { 0, statusBytes.length }, null, true); assertEquals(rnsi.getContactLine(), "1024R/8D56913D Alex de Joode <adejoode(a)sabotage.org>"); } diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java index 8a85145..9727f50 100644 --- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java @@ -557,7 +557,8 @@ public class RelayNetworkStatusVoteImplTest { throws DescriptorParseException { byte[] voteBytes = this.buildVoteBytes(); return new RelayNetworkStatusVoteImpl(voteBytes, - new int[] { 0, voteBytes.length }, failUnrecognizedDescriptorLines); + new int[] { 0, voteBytes.length }, null, + failUnrecognizedDescriptorLines); } private void appendHeader(StringBuilder sb) { diff --git a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java index 9533de3..d0c2561 100644 --- a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java @@ -456,7 +456,7 @@ public class ServerDescriptorImplTest { throws DescriptorParseException { byte[] descriptorBytes = this.buildDescriptorBytes(); return new RelayServerDescriptorImpl(descriptorBytes, - new int[] { 0, descriptorBytes.length }, + new int[] { 0, descriptorBytes.length }, null, failUnrecognizedDescriptorLines); } } @@ -1963,7 +1963,7 @@ public class ServerDescriptorImplTest { + "\nrouter-digest " + digestSha1Hex; byte[] descriptorBytes = db.buildDescriptorBytes(); BridgeServerDescriptor descriptor = new BridgeServerDescriptorImpl( - descriptorBytes, new int[] { 0, descriptorBytes.length }, true); + descriptorBytes, new int[] { 0, descriptorBytes.length }, null, true); assertEquals(digestSha1Hex, descriptor.getDigestSha1Hex()); assertEquals(digestSha256Base64, descriptor.getDigestSha256Base64()); } @@ -1974,7 +1974,7 @@ public class ServerDescriptorImplTest { DescriptorBuilder db = new DescriptorBuilder(); byte[] descriptorBytes = db.buildDescriptorBytes(); BridgeServerDescriptor descriptor = new BridgeServerDescriptorImpl( - descriptorBytes, new int[] { 0, descriptorBytes.length }, true); + descriptorBytes, new int[] { 0, descriptorBytes.length }, null, true); assertNull(descriptor.getDigestSha1Hex()); assertNull(descriptor.getDigestSha256Base64()); } diff --git a/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java b/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java index 84aedee..e3a0c7c 100644 --- a/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java +++ b/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java @@ -3,12 +3,14 @@ package org.torproject.descriptor.impl; +import org.torproject.descriptor.DescriptorParseException; + public class TestDescriptor extends DescriptorImpl { protected TestDescriptor(byte[] rawDescriptorBytes, int[] offsetAndLength, boolean failUnrecognizedDescriptorLines, boolean blankLinesAllowed) throws DescriptorParseException { - super(rawDescriptorBytes, offsetAndLength, + super(rawDescriptorBytes, offsetAndLength, null, failUnrecognizedDescriptorLines, blankLinesAllowed); } } diff --git a/src/test/java/org/torproject/descriptor/impl/TestServerDescriptor.java b/src/test/java/org/torproject/descriptor/impl/TestServerDescriptor.java index 194df82..0afd7b4 100644 --- a/src/test/java/org/torproject/descriptor/impl/TestServerDescriptor.java +++ b/src/test/java/org/torproject/descriptor/impl/TestServerDescriptor.java @@ -3,12 +3,16 @@ package org.torproject.descriptor.impl; +import java.io.File; + public class TestServerDescriptor extends ServerDescriptorImpl { protected TestServerDescriptor(byte[] rawDescriptorBytes, - int[] offsetAndLength, boolean failUnrecognizedDescriptorLines) + int[] offsetAndLength, File descriptorFile, + boolean failUnrecognizedDescriptorLines) throws DescriptorParseException { - super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines); + super(rawDescriptorBytes, offsetAndLength, descriptorFile, + failUnrecognizedDescriptorLines); } } diff --git a/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java b/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java index 122e301..7df4269 100644 --- a/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java +++ b/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java @@ -20,7 +20,7 @@ public class TorperfResultImplTest { public void testAnnotatedInput() throws Exception { TorperfResultImpl result = (TorperfResultImpl) (TorperfResultImpl.parseTorperfResults((torperfAnnotation + input) - .getBytes("US-ASCII"), false).get(0)); + .getBytes("US-ASCII"), null, false).get(0)); assertEquals("Expected one annotation.", 1, result.getAnnotations().size()); assertEquals(torperfAnnotation.substring(0, 17), @@ -37,7 +37,7 @@ public class TorperfResultImplTest { byte[] asciiBytes = (torperfAnnotation + input + input + input).getBytes("US-ASCII"); List<Descriptor> result = TorperfResultImpl.parseTorperfResults( - asciiBytes, false); + asciiBytes, null, false); assertEquals("Expected one annotation.", 1, ((TorperfResultImpl)(result.get(0))).getAnnotations().size()); assertEquals(3, result.size()); @@ -53,7 +53,7 @@ public class TorperfResultImplTest { + torperfAnnotation + input + torperfAnnotation + input).getBytes("US-ASCII"); List<Descriptor> result = TorperfResultImpl.parseTorperfResults( - asciiBytes, false); + asciiBytes, null, false); assertEquals("Expected one annotation.", 1, ((TorperfResultImpl)(result.get(0))).getAnnotations().size()); assertEquals(3, result.size()); @@ -90,7 +90,7 @@ public class TorperfResultImplTest { @Test public void testDatapercNonNumeric() throws Exception { List<Descriptor> result = TorperfResultImpl.parseTorperfResults( - ("DATAPERMILLE=2.0 " + input).getBytes(), false); + ("DATAPERMILLE=2.0 " + input).getBytes(), null, false); assertEquals(1, result.size()); TorperfResultImpl torperfResult = (TorperfResultImpl) result.get(0); assertEquals(1, torperfResult.getUnrecognizedKeys().size()); @@ -125,7 +125,7 @@ public class TorperfResultImplTest { public void testEndpointsHostnamesSourceAddress() throws DescriptorParseException { List<Descriptor> result = TorperfResultImpl.parseTorperfResults( - input2.getBytes(), true); + input2.getBytes(), null, true); assertEquals(1, result.size()); TorperfResultImpl torperfResult = (TorperfResultImpl) result.get(0); assertNull(torperfResult.getUnrecognizedKeys());

1 0

[metrics-lib/release] Bump version to 1.8.2-dev.
by karsten＠torproject.org 21 Jun '17

21 Jun '17

commit 59dc55fbe1b3c8afa7174e8b6b5f868d92e5c921 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Fri Jun 16 17:03:41 2017 +0200 Bump version to 1.8.2-dev. --- build.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.xml b/build.xml index d755932..d5aaa8f 100644 --- a/build.xml +++ b/build.xml @@ -6,7 +6,7 @@ <project default="usage" name="descriptor" basedir="."> - <property name="release.version" value="1.8.2" /> + <property name="release.version" value="1.8.2-dev" /> <property name="javadoc-title" value="DescripTor API Documentation"/> <property name="javadoc-excludes" value="**/impl/** **/index/**" /> <property name="implementation-title" value="DescripTor" />

1 0

[metrics-lib/release] Add new method to retrieve raw descriptor length.
by karsten＠torproject.org 21 Jun '17

21 Jun '17

commit a113feb2c4f195dbc607e3da8c5fc2be19a6d475 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Thu Jun 15 14:25:29 2017 +0200 Add new method to retrieve raw descriptor length. When we switched from storing copies of byte[] to storing byte[] references, offset, and length, getRawDescriptor() suddenly became more expensive than before. (Before that, copying bytes in the first place was always expensive.) If an application only calls Descriptor#getRawDescriptorBytes() to learn the array length, there's now a more efficient way to provide that information. --- CHANGELOG.md | 3 +++ src/main/java/org/torproject/descriptor/Descriptor.java | 15 +++++++++++++++ .../org/torproject/descriptor/impl/DescriptorImpl.java | 5 +++++ 3 files changed, 23 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44f1130..95cd851 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ * Medium changes - Let DescriptorParser return an Iterable instead of a List, which prepares parsing large descriptor files descriptor by descriptor. + - Add new method to retrieve the raw descriptor length, rather than + forcing applications to request (a copy of) raw descriptor bytes + only to determine the raw descriptor length. * Minor changes - Fix a bug where NetworkStatusEntry's getMicrodescriptorDigests() diff --git a/src/main/java/org/torproject/descriptor/Descriptor.java b/src/main/java/org/torproject/descriptor/Descriptor.java index fb511bf..ca2a4cf 100644 --- a/src/main/java/org/torproject/descriptor/Descriptor.java +++ b/src/main/java/org/torproject/descriptor/Descriptor.java @@ -17,11 +17,26 @@ public interface Descriptor { /** * Return the raw descriptor bytes. * + * This method creates a new copy of raw descriptor bytes from a + * potentially larger byte array containing multiple descriptors. + * Applications that only want to learn about raw descriptor length in bytes + * should instead use {@link #getRawDescriptorLength()}. + * * @since 1.0.0 */ public byte[] getRawDescriptorBytes(); /** + * Return the raw descriptor length in bytes. + * + * Returns the exact same result as {@code getRawDescriptorBytes().length}, + * but much more efficiently. + * + * @since 1.9.0 + */ + public int getRawDescriptorLength(); + + /** * Return the (possibly empty) list of annotations in the format * {@code "@key( value)*"}. * diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java index d88b542..e0f94f0 100644 --- a/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java @@ -41,6 +41,11 @@ public abstract class DescriptorImpl implements Descriptor { */ protected int length; + @Override + public int getRawDescriptorLength() { + return this.length; + } + /** * Optional reference to the descriptor file, if this descriptor was read from * a descriptor file.

1 0

[metrics-lib/release] Change encoding of microdescriptor digests in network status entries.
by karsten＠torproject.org 21 Jun '17

21 Jun '17

commit de9600a5dc937c4c453ed261266f38f21978f61e Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Sat Jun 17 21:20:01 2017 +0200 Change encoding of microdescriptor digests in network status entries. Fixes #22640. --- CHANGELOG.md | 8 ++++++++ .../java/org/torproject/descriptor/NetworkStatusEntry.java | 12 ++++++------ .../torproject/descriptor/impl/NetworkStatusEntryImpl.java | 14 +++++++------- .../java/org/torproject/descriptor/impl/ParseHelper.java | 14 ++++++++++++-- 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 409d0e8..e79811a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# Changes in version 1.9.0 - 2017-06-?? + + * Minor changes + - Fix a bug where NetworkStatusEntry's getMicrodescriptorDigests() + and getMicrodescriptorDigestsSha256Base64() return hex strings + rather than base64 strings. + + # Changes in version 1.8.2 - 2017-06-16 * Medium changes diff --git a/src/main/java/org/torproject/descriptor/NetworkStatusEntry.java b/src/main/java/org/torproject/descriptor/NetworkStatusEntry.java index c59eab9..9c5dae5 100644 --- a/src/main/java/org/torproject/descriptor/NetworkStatusEntry.java +++ b/src/main/java/org/torproject/descriptor/NetworkStatusEntry.java @@ -89,9 +89,9 @@ public interface NetworkStatusEntry { public int getDirPort(); /** - * Return the (possibly empty) set of microdescriptor digests if the - * containing network status is a vote or microdesc consensus, or null - * otherwise. + * Return the (possibly empty) set of microdescriptor digests, encoded as 43 + * base64 characters without padding characters, if the containing network + * status is a vote or microdesc consensus, or null otherwise. * * @deprecated Renamed to {@link #getMicrodescriptorDigestsSha256Base64()}. * @@ -100,9 +100,9 @@ public interface NetworkStatusEntry { public Set<String> getMicrodescriptorDigests(); /** - * Return the (possibly empty) set of microdescriptor digests if the - * containing network status is a vote or microdesc consensus, or null - * otherwise. + * Return the (possibly empty) set of microdescriptor digests, encoded as 43 + * base64 characters without padding characters, if the containing network + * status is a vote or microdesc consensus, or null otherwise. * * @since 1.7.0 */ diff --git a/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java b/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java index 987b530..337f0b2 100644 --- a/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java @@ -141,11 +141,11 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry { + "fewer space-separated elements than expected."); } this.nickname = ParseHelper.parseNickname(line, parts[1]); - this.fingerprint = ParseHelper.parseTwentyByteBase64String(line, + this.fingerprint = ParseHelper.convertTwentyByteBase64StringToHex(line, parts[2]); int descriptorOffset = 0; if (!this.microdescConsensus) { - this.descriptor = ParseHelper.parseTwentyByteBase64String(line, + this.descriptor = ParseHelper.convertTwentyByteBase64StringToHex(line, parts[3]); descriptorOffset = 1; } @@ -266,13 +266,13 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry { this.microdescriptorDigests = new HashSet<>(); } if (parts.length == 2) { - this.microdescriptorDigests.add( - ParseHelper.parseThirtyTwoByteBase64String(line, parts[1])); + ParseHelper.parseThirtyTwoByteBase64String(line, parts[1]); + this.microdescriptorDigests.add(parts[1]); } else if (parts.length == 3 && parts[2].length() > 7) { /* 7 == "sha256=".length() */ - this.microdescriptorDigests.add( - ParseHelper.parseThirtyTwoByteBase64String(line, - parts[2].substring(7))); + ParseHelper.parseThirtyTwoByteBase64String(line, + parts[2].substring(7)); + this.microdescriptorDigests.add(parts[2].substring(7)); } } diff --git a/src/main/java/org/torproject/descriptor/impl/ParseHelper.java b/src/main/java/org/torproject/descriptor/impl/ParseHelper.java index 31454e5..dec33f5 100644 --- a/src/main/java/org/torproject/descriptor/impl/ParseHelper.java +++ b/src/main/java/org/torproject/descriptor/impl/ParseHelper.java @@ -281,7 +281,12 @@ public class ParseHelper { private static Pattern twentyByteBase64Pattern = Pattern.compile("^[0-9a-zA-Z+/]{27}$"); - protected static String parseTwentyByteBase64String(String line, + protected static void parseTwentyByteBase64String(String line, + String base64String) throws DescriptorParseException { + convertTwentyByteBase64StringToHex(line, base64String); + } + + protected static String convertTwentyByteBase64StringToHex(String line, String base64String) throws DescriptorParseException { if (!twentyByteBase64Pattern.matcher(base64String).matches()) { throw new DescriptorParseException("'" + base64String @@ -296,7 +301,12 @@ public class ParseHelper { private static Pattern thirtyTwoByteBase64Pattern = Pattern.compile("^[0-9a-zA-Z+/]{43}$"); - protected static String parseThirtyTwoByteBase64String(String line, + protected static void parseThirtyTwoByteBase64String(String line, + String base64String) throws DescriptorParseException { + convertThirtyTwoByteBase64StringToHex(line, base64String); + } + + private static String convertThirtyTwoByteBase64StringToHex(String line, String base64String) throws DescriptorParseException { if (!thirtyTwoByteBase64Pattern.matcher(base64String).matches()) { throw new DescriptorParseException("'" + base64String

1 0

[metrics-lib/release] Rename two parse* methods to verify*.
by karsten＠torproject.org 21 Jun '17

21 Jun '17

commit 0236dc77715c272e7201e6111b98fbe4af747a4b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Jun 19 11:42:18 2017 +0200 Rename two parse* methods to verify*. --- .../torproject/descriptor/impl/ExtraInfoDescriptorImpl.java | 2 +- .../org/torproject/descriptor/impl/MicrodescriptorImpl.java | 4 ++-- .../torproject/descriptor/impl/NetworkStatusEntryImpl.java | 6 +++--- .../java/org/torproject/descriptor/impl/ParseHelper.java | 13 +++++++++++-- .../torproject/descriptor/impl/ServerDescriptorImpl.java | 4 ++-- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java index 0d67af9..ca7d376 100644 --- a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java @@ -828,7 +828,7 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl if (partsNoOpt.length != 2) { throw new DescriptorParseException("Illegal line '" + line + "'."); } - ParseHelper.parseThirtyTwoByteBase64String(line, partsNoOpt[1]); + ParseHelper.verifyThirtyTwoByteBase64String(line, partsNoOpt[1]); this.setDigestSha256Base64(partsNoOpt[1]); } diff --git a/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java index 65c20d4..c436120 100644 --- a/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java @@ -198,11 +198,11 @@ public class MicrodescriptorImpl extends DescriptorImpl } else { switch (parts[1]) { case "ed25519": - ParseHelper.parseThirtyTwoByteBase64String(line, parts[2]); + ParseHelper.verifyThirtyTwoByteBase64String(line, parts[2]); this.ed25519Identity = parts[2]; break; case "rsa1024": - ParseHelper.parseTwentyByteBase64String(line, parts[2]); + ParseHelper.verifyTwentyByteBase64String(line, parts[2]); this.rsa1024Identity = parts[2]; break; default: diff --git a/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java b/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java index 337f0b2..64c688e 100644 --- a/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java @@ -266,11 +266,11 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry { this.microdescriptorDigests = new HashSet<>(); } if (parts.length == 2) { - ParseHelper.parseThirtyTwoByteBase64String(line, parts[1]); + ParseHelper.verifyThirtyTwoByteBase64String(line, parts[1]); this.microdescriptorDigests.add(parts[1]); } else if (parts.length == 3 && parts[2].length() > 7) { /* 7 == "sha256=".length() */ - ParseHelper.parseThirtyTwoByteBase64String(line, + ParseHelper.verifyThirtyTwoByteBase64String(line, parts[2].substring(7)); this.microdescriptorDigests.add(parts[2].substring(7)); } @@ -283,7 +283,7 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry { } else if ("none".equals(parts[2])) { this.masterKeyEd25519 = "none"; } else { - ParseHelper.parseThirtyTwoByteBase64String(line, parts[2]); + ParseHelper.verifyThirtyTwoByteBase64String(line, parts[2]); this.masterKeyEd25519 = parts[2]; } } diff --git a/src/main/java/org/torproject/descriptor/impl/ParseHelper.java b/src/main/java/org/torproject/descriptor/impl/ParseHelper.java index dec33f5..1c7aa93 100644 --- a/src/main/java/org/torproject/descriptor/impl/ParseHelper.java +++ b/src/main/java/org/torproject/descriptor/impl/ParseHelper.java @@ -21,6 +21,15 @@ import java.util.regex.Pattern; import javax.xml.bind.DatatypeConverter; +/** + * Parse helper for descriptor contents. + * + * Naming convention: methods starting with {@code parse} return valid and + * sometimes changed data items (e.g. string to port int), methods starting with + * {@code verify} only check data items without returning them, and methods + * starting with {@code convert} return checked and explicitly changed data + * items (e.g. base64 to hex encoding). + */ public class ParseHelper { private static Pattern keywordPattern = Pattern.compile("^[A-Za-z0-9-]+$"); @@ -281,7 +290,7 @@ public class ParseHelper { private static Pattern twentyByteBase64Pattern = Pattern.compile("^[0-9a-zA-Z+/]{27}$"); - protected static void parseTwentyByteBase64String(String line, + protected static void verifyTwentyByteBase64String(String line, String base64String) throws DescriptorParseException { convertTwentyByteBase64StringToHex(line, base64String); } @@ -301,7 +310,7 @@ public class ParseHelper { private static Pattern thirtyTwoByteBase64Pattern = Pattern.compile("^[0-9a-zA-Z+/]{43}$"); - protected static void parseThirtyTwoByteBase64String(String line, + protected static void verifyThirtyTwoByteBase64String(String line, String base64String) throws DescriptorParseException { convertThirtyTwoByteBase64StringToHex(line, base64String); } diff --git a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java index 428b2f0..d438702 100644 --- a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java +++ b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java @@ -467,7 +467,7 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl this.extraInfoDigest = ParseHelper.parseTwentyByteHexString(line, partsNoOpt[1]); if (partsNoOpt.length >= 3) { - ParseHelper.parseThirtyTwoByteBase64String(line, partsNoOpt[2]); + ParseHelper.verifyThirtyTwoByteBase64String(line, partsNoOpt[2]); this.extraInfoDigestSha256 = partsNoOpt[2]; } } @@ -671,7 +671,7 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl if (partsNoOpt.length != 2) { throw new DescriptorParseException("Illegal line '" + line + "'."); } - ParseHelper.parseThirtyTwoByteBase64String(line, partsNoOpt[1]); + ParseHelper.verifyThirtyTwoByteBase64String(line, partsNoOpt[1]); this.setDigestSha256Base64(partsNoOpt[1]); }

1 0