tor-commits
Threads by month
- ----- 2025 -----
- July
- June
- May
- April
- March
- February
- January
- ----- 2024 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2023 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2022 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2021 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2020 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2019 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2018 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2017 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2016 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2015 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2014 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2013 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2012 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
- January
- ----- 2011 -----
- December
- November
- October
- September
- August
- July
- June
- May
- April
- March
- February
June 2017
- 13 participants
- 1918 discussions

21 Jun '17
commit 78eaa445c9344b878e55c3e114f73358faa8d008
Author: Damian Johnson <atagar(a)torproject.org>
Date: Wed Jun 21 08:51:24 2017 -0700
Python3 support for server descriptor signing
While writing it I consciously didn't invest effort into making it compatible.
Now that we've got it working fixing that. Most interesting issue was...
======================================================================
ERROR: test_descriptor_signing
----------------------------------------------------------------------
Traceback (most recent call last):
File "/home/atagar/Desktop/stem/test/require.py", line 58, in wrapped
return func(self, *args, **kwargs)
File "/home/atagar/Desktop/stem/test/unit/descriptor/server_descriptor.py", line 260, in test_descriptor_signing
RelayDescriptor.create(sign = True)
File "/home/atagar/Desktop/stem/stem/descriptor/server_descriptor.py", line 873, in create
return cls(cls.content(attr, exclude, sign, private_signing_key), validate = validate, skip_crypto_validation = not sign)
File "/home/atagar/Desktop/stem/stem/descriptor/server_descriptor.py", line 865, in content
signature = base64.b64encode(private_signing_key.sign(content, padding.PKCS1v15(), hashes.SHA1()))
File "/usr/local/lib/python3.2/dist-packages/cryptography/hazmat/backends/openssl/rsa.py", line 463, in sign
algorithm, self, data
File "/usr/local/lib/python3.2/dist-packages/cryptography/hazmat/backends/openssl/rsa.py", line 199, in _rsa_sig_sign
backend.openssl_assert(res > 0)
TypeError: unorderable types: NoneType() > int()
----------------------------------------------------------------------
This was due to one of our no_op mocks.
---
stem/descriptor/__init__.py | 2 +-
stem/descriptor/server_descriptor.py | 8 ++++----
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/stem/descriptor/__init__.py b/stem/descriptor/__init__.py
index 96cf0b4..bf336fb 100644
--- a/stem/descriptor/__init__.py
+++ b/stem/descriptor/__init__.py
@@ -391,7 +391,7 @@ def _descriptor_content(attr = None, exclude = (), sign = False, header_template
if keyword in exclude:
continue
- value = attr.pop(keyword, value)
+ value = stem.util.str_tools._to_unicode(attr.pop(keyword, value))
if value is None:
continue
diff --git a/stem/descriptor/server_descriptor.py b/stem/descriptor/server_descriptor.py
index 4e55ee2..2c7258a 100644
--- a/stem/descriptor/server_descriptor.py
+++ b/stem/descriptor/server_descriptor.py
@@ -847,7 +847,7 @@ class RelayDescriptor(ServerDescriptor):
# ewww.
def no_op(*args, **kwargs):
- pass
+ return 1
private_signing_key._backend._lib.EVP_PKEY_CTX_set_signature_md = no_op
private_signing_key._backend.openssl_assert = no_op
@@ -855,15 +855,15 @@ class RelayDescriptor(ServerDescriptor):
# create descriptor content without the router-signature, then
# appending the content signature
- attr['signing-key'] = '\n' + private_signing_key.public_key().public_bytes(
+ attr['signing-key'] = b'\n' + private_signing_key.public_key().public_bytes(
encoding = serialization.Encoding.PEM,
format = serialization.PublicFormat.PKCS1,
).strip()
- content = _descriptor_content(attr, exclude, sign, RELAY_SERVER_HEADER) + '\nrouter-signature\n'
+ content = _descriptor_content(attr, exclude, sign, RELAY_SERVER_HEADER) + b'\nrouter-signature\n'
signature = base64.b64encode(private_signing_key.sign(content, padding.PKCS1v15(), hashes.SHA1()))
- return content + '\n'.join(['-----BEGIN SIGNATURE-----'] + stem.util.str_tools._split_by_length(signature, 64) + ['-----END SIGNATURE-----\n'])
+ return content + b'\n'.join([b'-----BEGIN SIGNATURE-----'] + stem.util.str_tools._split_by_length(signature, 64) + [b'-----END SIGNATURE-----\n'])
else:
return _descriptor_content(attr, exclude, sign, RELAY_SERVER_HEADER, RELAY_SERVER_FOOTER)
1
0
commit f0a5416f95e06ebfd27eb95436eb85cd66494b9a
Merge: 884c0ff 0a96d11
Author: Nick Mathewson <nickm(a)torproject.org>
Date: Wed Jun 21 12:19:55 2017 -0400
Merge branch 'Ryman-bug5847-squashed'
changes/bug5847 | 5 +++++
src/or/control.c | 36 ++++++++++++++++++++++++++++--------
2 files changed, 33 insertions(+), 8 deletions(-)
1
0

[tor/master] Better error message for GETINFO desc/(id|name) whenever microdescriptors are in use. Fixes #5847.
by nickm@torproject.org 21 Jun '17
by nickm@torproject.org 21 Jun '17
21 Jun '17
commit 0a96d11539e27f8ef0577cc8ee4e503f046e637b
Author: Kevin Butler <haqkrs(a)gmail.com>
Date: Wed Sep 4 02:10:18 2013 +0100
Better error message for GETINFO desc/(id|name) whenever microdescriptors are in use. Fixes #5847.
---
changes/bug5847 | 5 +++++
src/or/control.c | 36 ++++++++++++++++++++++++++++--------
2 files changed, 33 insertions(+), 8 deletions(-)
diff --git a/changes/bug5847 b/changes/bug5847
new file mode 100644
index 0000000..782fc7b
--- /dev/null
+++ b/changes/bug5847
@@ -0,0 +1,5 @@
+ o Minor features (control port):
+ - Provide better error message for GETINFO desc/(id|name) when
+ microdescriptors are in use and router descriptors are not fetched.
+ Closes ticket 5847. Patch by Kevin Butler.
+
diff --git a/src/or/control.c b/src/or/control.c
index 9454a7a..b971725 100644
--- a/src/or/control.c
+++ b/src/or/control.c
@@ -60,6 +60,7 @@
#include "hibernate.h"
#include "hs_common.h"
#include "main.h"
+#include "microdesc.h"
#include "networkstatus.h"
#include "nodelist.h"
#include "policies.h"
@@ -1892,6 +1893,12 @@ getinfo_helper_dir(control_connection_t *control_conn,
const char *body = signed_descriptor_get_body(&ri->cache_info);
if (body)
*answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+ } else if (! we_fetch_router_descriptors(get_options())) {
+ /* Descriptors won't be available, provide proper error */
+ *errmsg = "We fetch microdescriptors, not router "
+ "descriptors. You'll need to use md/id/* "
+ "instead of desc/id/*.";
+ return 0;
}
} else if (!strcmpstart(question, "desc/name/")) {
const routerinfo_t *ri = NULL;
@@ -1905,6 +1912,12 @@ getinfo_helper_dir(control_connection_t *control_conn,
const char *body = signed_descriptor_get_body(&ri->cache_info);
if (body)
*answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+ } else if (! we_fetch_router_descriptors(get_options())) {
+ /* Descriptors won't be available, provide proper error */
+ *errmsg = "We fetch microdescriptors, not router "
+ "descriptors. You'll need to use md/name/* "
+ "instead of desc/name/*.";
+ return 0;
}
} else if (!strcmp(question, "desc/all-recent")) {
routerlist_t *routerlist = router_get_routerlist();
@@ -2907,7 +2920,8 @@ getinfo_helper_sr(control_connection_t *control_conn,
* *<b>a</b>. If an internal error occurs, return -1 and optionally set
* *<b>error_out</b> to point to an error message to be delivered to the
* controller. On success, _or if the key is not recognized_, return 0. Do not
- * set <b>a</b> if the key is not recognized.
+ * set <b>a</b> if the key is not recognized but you may set <b>error_out</b>
+ * to improve the error message.
*/
typedef int (*getinfo_helper_t)(control_connection_t *,
const char *q, char **a,
@@ -3162,7 +3176,7 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
smartlist_t *questions = smartlist_new();
smartlist_t *answers = smartlist_new();
smartlist_t *unrecognized = smartlist_new();
- char *msg = NULL, *ans = NULL;
+ char *ans = NULL;
int i;
(void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
@@ -3177,20 +3191,26 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
goto done;
}
if (!ans) {
- smartlist_add(unrecognized, (char*)q);
+ if (errmsg) /* use provided error message */
+ smartlist_add_strdup(unrecognized, errmsg);
+ else /* use default error message */
+ smartlist_add_asprintf(unrecognized, "Unrecognized key \"%s\"", q);
} else {
smartlist_add_strdup(answers, q);
smartlist_add(answers, ans);
}
} SMARTLIST_FOREACH_END(q);
+
if (smartlist_len(unrecognized)) {
+ /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */
for (i=0; i < smartlist_len(unrecognized)-1; ++i)
connection_printf_to_buf(conn,
- "552-Unrecognized key \"%s\"\r\n",
- (char*)smartlist_get(unrecognized, i));
+ "552-%s\r\n",
+ (char *)smartlist_get(unrecognized, i));
+
connection_printf_to_buf(conn,
- "552 Unrecognized key \"%s\"\r\n",
- (char*)smartlist_get(unrecognized, i));
+ "552 %s\r\n",
+ (char *)smartlist_get(unrecognized, i));
goto done;
}
@@ -3217,8 +3237,8 @@ handle_control_getinfo(control_connection_t *conn, uint32_t len,
smartlist_free(answers);
SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp));
smartlist_free(questions);
+ SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp));
smartlist_free(unrecognized);
- tor_free(msg);
return 0;
}
1
0
commit cad14ead4c72744698ef013404f94dfe3842df0b
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Wed Jun 21 14:21:19 2017 +0200
Bump version to 1.9.0-dev.
---
build.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/build.xml b/build.xml
index 18ba365..0de365c 100644
--- a/build.xml
+++ b/build.xml
@@ -6,7 +6,7 @@
<project default="usage" name="descriptor" basedir=".">
- <property name="release.version" value="1.9.0" />
+ <property name="release.version" value="1.9.0-dev" />
<property name="javadoc-title" value="DescripTor API Documentation"/>
<property name="javadoc-excludes" value="**/impl/** **/index/**" />
<property name="implementation-title" value="DescripTor" />
1
0
commit 3c98457777aa4d96b2e5da6cd640c2704dd42e9f
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Tue Jun 20 22:36:58 2017 +0200
Prepare for 1.9.0 release.
---
CHANGELOG.md | 2 +-
build.xml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 95cd851..98728b1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,4 @@
-# Changes in version 1.9.0 - 2017-06-??
+# Changes in version 1.9.0 - 2017-06-21
* Major changes
- Simplify DescriptorReader by returning Descriptor instances
diff --git a/build.xml b/build.xml
index d5aaa8f..18ba365 100644
--- a/build.xml
+++ b/build.xml
@@ -6,7 +6,7 @@
<project default="usage" name="descriptor" basedir=".">
- <property name="release.version" value="1.8.2-dev" />
+ <property name="release.version" value="1.9.0" />
<property name="javadoc-title" value="DescripTor API Documentation"/>
<property name="javadoc-excludes" value="**/impl/** **/index/**" />
<property name="implementation-title" value="DescripTor" />
1
0
commit 1062da637f6f55313f52191314f7fb939355af23
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Wed Jun 7 17:52:02 2017 +0200
Make DescriptorFile obsolete.
This change simplifies the DescriptorReader interface and allows for
shorter and more concise application code. The result is that
DescriptorReader returns Descriptor instances rather than
DescriptorFile instances containing Descriptors and accepts a maximum
queue size in Descriptors, DescriptorFile is deprecated, and
Descriptor contains a File reference to the descriptor file.
Implements #22141.
Related to this change, this commit introduces a new
UnparseableDescriptor to be returned by DescriptorParser and
DescriptorReader if a descriptor cannot be parsed, as opposed to
throwing a DescriptorParseException or skipping the entire descriptor
file (fixes #22139), respectively.
Also related to this change, DescriptorParser now returns an Iterable
instead of a List, which prepares parsing large descriptor files
descriptor by descriptor (will be tackled in #20395).
---
CHANGELOG.md | 13 ++
.../java/org/torproject/descriptor/Descriptor.java | 19 +++
.../org/torproject/descriptor/DescriptorFile.java | 7 +
.../torproject/descriptor/DescriptorParser.java | 19 +++
.../torproject/descriptor/DescriptorReader.java | 56 +++++--
.../descriptor/UnparseableDescriptor.java | 44 ++++++
.../descriptor/impl/BlockingIteratorImpl.java | 13 +-
.../impl/BridgeExtraInfoDescriptorImpl.java | 9 +-
.../descriptor/impl/BridgeNetworkStatusImpl.java | 7 +-
.../descriptor/impl/BridgePoolAssignmentImpl.java | 9 +-
.../impl/BridgeServerDescriptorImpl.java | 9 +-
.../torproject/descriptor/impl/DescriptorImpl.java | 23 ++-
.../descriptor/impl/DescriptorParserImpl.java | 130 ++++++++++------
.../descriptor/impl/DescriptorReaderImpl.java | 164 ++++++++++++++++++++-
.../impl/DirectoryKeyCertificateImpl.java | 9 +-
.../torproject/descriptor/impl/ExitListImpl.java | 7 +-
.../descriptor/impl/ExtraInfoDescriptorImpl.java | 9 +-
.../descriptor/impl/MicrodescriptorImpl.java | 7 +-
.../descriptor/impl/NetworkStatusImpl.java | 7 +-
.../descriptor/impl/RelayDirectoryImpl.java | 10 +-
.../impl/RelayExtraInfoDescriptorImpl.java | 9 +-
.../impl/RelayNetworkStatusConsensusImpl.java | 9 +-
.../descriptor/impl/RelayNetworkStatusImpl.java | 7 +-
.../impl/RelayNetworkStatusVoteImpl.java | 7 +-
.../descriptor/impl/RelayServerDescriptorImpl.java | 9 +-
.../descriptor/impl/ServerDescriptorImpl.java | 7 +-
.../descriptor/impl/TorperfResultImpl.java | 11 +-
.../descriptor/impl/UnparseableDescriptorImpl.java | 40 +++++
.../descriptor/impl/BridgeNetworkStatusTest.java | 2 +-
.../descriptor/impl/ConsensusBuilder.java | 2 +-
.../descriptor/impl/DescriptorImplTest.java | 1 -
.../descriptor/impl/DescriptorParserImplTest.java | 9 +-
.../descriptor/impl/ExitListImplTest.java | 10 +-
.../impl/ExtraInfoDescriptorImplTest.java | 8 +-
.../descriptor/impl/MicrodescriptorImplTest.java | 2 +-
.../impl/RelayNetworkStatusConsensusImplTest.java | 4 +-
.../impl/RelayNetworkStatusImplTest.java | 4 +-
.../impl/RelayNetworkStatusVoteImplTest.java | 3 +-
.../descriptor/impl/ServerDescriptorImplTest.java | 6 +-
.../torproject/descriptor/impl/TestDescriptor.java | 4 +-
.../descriptor/impl/TestServerDescriptor.java | 8 +-
.../descriptor/impl/TorperfResultImplTest.java | 10 +-
42 files changed, 586 insertions(+), 157 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e79811a..44f1130 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,18 @@
# Changes in version 1.9.0 - 2017-06-??
+ * Major changes
+ - Simplify DescriptorReader by returning Descriptor instances
+ rather than DescriptorFile instances containing Descriptors,
+ deprecate DescriptorFile, and add a File reference to Descriptor.
+ - Introduce a new UnparseableDescriptor to be returned by
+ DescriptorParser and DescriptorReader if a descriptor cannot be
+ parsed, as opposed to throwing a DescriptorParseException or
+ skipping the entire descriptor file, respectively.
+
+ * Medium changes
+ - Let DescriptorParser return an Iterable instead of a List, which
+ prepares parsing large descriptor files descriptor by descriptor.
+
* Minor changes
- Fix a bug where NetworkStatusEntry's getMicrodescriptorDigests()
and getMicrodescriptorDigestsSha256Base64() return hex strings
diff --git a/src/main/java/org/torproject/descriptor/Descriptor.java b/src/main/java/org/torproject/descriptor/Descriptor.java
index be6e02f..fb511bf 100644
--- a/src/main/java/org/torproject/descriptor/Descriptor.java
+++ b/src/main/java/org/torproject/descriptor/Descriptor.java
@@ -3,6 +3,7 @@
package org.torproject.descriptor;
+import java.io.File;
import java.util.List;
/**
@@ -24,6 +25,10 @@ public interface Descriptor {
* Return the (possibly empty) list of annotations in the format
* {@code "@key( value)*"}.
*
+ * <p>Some implementations might not support this operation and will throw an
+ * {@code UnsupportedOperationException}, e.g.,
+ * {@link UnparseableDescriptor}.</p>
+ *
* @since 1.0.0
*/
public List<String> getAnnotations();
@@ -32,8 +37,22 @@ public interface Descriptor {
* Return any unrecognized lines when parsing this descriptor, or an
* empty list if there were no unrecognized lines.
*
+ * <p>Some implementations might not support this operation and will throw an
+ * {@code UnsupportedOperationException}, * e.g.,
+ * {@link UnparseableDescriptor}.</p>
+ *
* @since 1.0.0
*/
public List<String> getUnrecognizedLines();
+
+ /**
+ * Return the file, tarball or plain file, that contained this descriptor, or
+ * {@code null} if this descriptor was not read from a file.
+ *
+ * @return Descriptor file that contained this descriptor.
+ *
+ * @since 1.9.0
+ */
+ public File getDescriptorFile();
}
diff --git a/src/main/java/org/torproject/descriptor/DescriptorFile.java b/src/main/java/org/torproject/descriptor/DescriptorFile.java
index b501186..e8c0d21 100644
--- a/src/main/java/org/torproject/descriptor/DescriptorFile.java
+++ b/src/main/java/org/torproject/descriptor/DescriptorFile.java
@@ -15,6 +15,13 @@ import java.util.List;
* This container also stores potentially useful meta-data about the
* descriptor file.</p>
*
+ * @deprecated Irrelevant after introducing
+ * {@link DescriptorReader#readDescriptors(File...)}, which returns
+ * {@link Descriptor}s rather than {@link DescriptorFile}s, and including
+ * the descriptor file reference in {@link Descriptor#getDescriptorFile()}
+ * and the first thrown {@link DescriptorParseException} in
+ * {@link UnparseableDescriptor#getDescriptorParseException()}.
+ *
* @since 1.0.0
*/
public interface DescriptorFile {
diff --git a/src/main/java/org/torproject/descriptor/DescriptorParser.java b/src/main/java/org/torproject/descriptor/DescriptorParser.java
index 537a259..61f69f4 100644
--- a/src/main/java/org/torproject/descriptor/DescriptorParser.java
+++ b/src/main/java/org/torproject/descriptor/DescriptorParser.java
@@ -3,6 +3,7 @@
package org.torproject.descriptor;
+import java.io.File;
import java.util.List;
/**
@@ -48,4 +49,22 @@ public interface DescriptorParser {
*/
public List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes,
String fileName) throws DescriptorParseException;
+
+ /**
+ * Parse descriptors in the given byte array and return the parsed/unparseable
+ * descriptors.
+ *
+ * @param rawDescriptorBytes Raw descriptor bytes containing one or more
+ * descriptors
+ * @param descriptorFile Optional descriptor file reference included in
+ * parsed/unparseable descriptors
+ * @param fileName Descriptor file name used for parsing the descriptor
+ * publication time of some descriptor types
+ *
+ * @return Parsed/unparseable descriptors
+ *
+ * @since 1.9.0
+ */
+ public Iterable<Descriptor> parseDescriptors(byte[] rawDescriptorBytes,
+ File descriptorFile, String fileName);
}
diff --git a/src/main/java/org/torproject/descriptor/DescriptorReader.java b/src/main/java/org/torproject/descriptor/DescriptorReader.java
index 85f8cdd..e15e5d9 100644
--- a/src/main/java/org/torproject/descriptor/DescriptorReader.java
+++ b/src/main/java/org/torproject/descriptor/DescriptorReader.java
@@ -23,18 +23,13 @@ import java.util.SortedMap;
* DescriptorReader descriptorReader =
* DescriptorSourceFactory.createDescriptorReader();
* // Read descriptors from local directory called in/.
- * descriptorReader.addDirectory(new File("in"));
- * Iterator<DescriptorFile> descriptorFiles =
- * descriptorReader.readDescriptors();
- * while (descriptorFiles.hasNext()) {
- * DescriptorFile descriptorFile = descriptorFiles.next();
- * for (Descriptor descriptor : descriptorFile.getDescriptors()) {
- * if ((descriptor instanceof RelayNetworkStatusConsensus)) {
- * // Only process network status consensuses, ignore the rest.
- * RelayNetworkStatusConsensus consensus =
- * (RelayNetworkStatusConsensus) descriptor;
- * processConsensus(consensus);
- * }
+ * for (Descriptor descriptor :
+ * descriptorReader.readDescriptors(new File("in")) {
+ * // Only process network status consensuses, ignore the rest.
+ * if ((descriptor instanceof RelayNetworkStatusConsensus)) {
+ * RelayNetworkStatusConsensus consensus =
+ * (RelayNetworkStatusConsensus) descriptor;
+ * processConsensus(consensus);
* }
* }}</pre>
*
@@ -46,6 +41,9 @@ public interface DescriptorReader {
* Add a local directory to read descriptors from, which may contain
* descriptor files or tarballs containing descriptor files.
*
+ * @deprecated Replaced with a parameter in {@link #readDescriptors(File...)},
+ * which ignores any directories added via this deprecated method.
+ *
* @since 1.0.0
*/
public void addDirectory(File directory);
@@ -54,6 +52,9 @@ public interface DescriptorReader {
* Add a tarball to read descriptors from, which may be uncompressed,
* bz2-compressed, or xz-compressed.
*
+ * @deprecated Replaced with a parameter in {@link #readDescriptors(File...)},
+ * which ignores any tarballs added via this deprecated method.
+ *
* @since 1.0.0
*/
public void addTarball(File tarball);
@@ -157,11 +158,23 @@ public interface DescriptorReader {
* <p>The default is 100, but if descriptor files contain hundreds or
* even thousands of descriptors, that default may be too high.</p>
*
+ * @deprecated Replaced with {@link #setMaxDescriptorsInQueue(int)}.
+ *
* @since 1.0.0
*/
public void setMaxDescriptorFilesInQueue(int max);
/**
+ * Don't keep more than this number of descriptors in the queue (default:
+ * 100).
+ *
+ * @param maxDescriptorsInQueue Maximum number of descriptors in the queue.
+ *
+ * @since 1.9.0
+ */
+ public void setMaxDescriptorsInQueue(int maxDescriptorsInQueue);
+
+ /**
* Read the previously configured descriptors and make them available
* via the returned blocking iterator.
*
@@ -169,8 +182,27 @@ public interface DescriptorReader {
* more shortly after, it blocks the caller. This method can only be
* run once.</p>
*
+ * @deprecated Replaced with {@link #readDescriptors(File...)}.
+ *
* @since 1.0.0
*/
public Iterator<DescriptorFile> readDescriptors();
+
+ /**
+ * Read descriptors from the given descriptor file(s) and return the parsed
+ * descriptors.
+ *
+ * <p>Whenever the reader runs out of descriptors and expects to provide
+ * more shortly after, it blocks the caller. This method can only be
+ * run once.</p>
+ *
+ * @param descriptorFiles One or more directories, tarballs, or files
+ * containing descriptors.
+ *
+ * @return Parsed descriptors.
+ *
+ * @since 1.9.0
+ */
+ public Iterable<Descriptor> readDescriptors(File... descriptorFiles);
}
diff --git a/src/main/java/org/torproject/descriptor/UnparseableDescriptor.java b/src/main/java/org/torproject/descriptor/UnparseableDescriptor.java
new file mode 100644
index 0000000..6282222
--- /dev/null
+++ b/src/main/java/org/torproject/descriptor/UnparseableDescriptor.java
@@ -0,0 +1,44 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor;
+
+import java.util.List;
+
+/**
+ * Contains a descriptor that could not be parsed.
+ * Only {@link UnparseableDescriptor#getRawDescriptorBytes} and
+ * {@link UnparseableDescriptor#getDescriptorFile} are supported.
+ *
+ * @since 1.9.0
+ */
+public interface UnparseableDescriptor extends Descriptor {
+
+ /**
+ * Return the first exception thrown while attempting to parse this
+ * descriptor.
+ *
+ * @return First exception thrown.
+ *
+ * @since 1.9.0
+ */
+ public DescriptorParseException getDescriptorParseException();
+
+ /**
+ * Will always throw an {@code UnsupportedOperationException}.
+ *
+ * @since 1.9.0
+ */
+ @Override
+ public List<String> getAnnotations();
+
+ /**
+ * Will always throw an {@code UnsupportedOperationException}.
+ *
+ * @since 1.9.0
+ */
+ @Override
+ public List<String> getUnrecognizedLines();
+
+}
+
diff --git a/src/main/java/org/torproject/descriptor/impl/BlockingIteratorImpl.java b/src/main/java/org/torproject/descriptor/impl/BlockingIteratorImpl.java
index 3509026..4921ac4 100644
--- a/src/main/java/org/torproject/descriptor/impl/BlockingIteratorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/BlockingIteratorImpl.java
@@ -11,7 +11,7 @@ import java.util.Queue;
/* Provide an iterator for a queue of objects and block when there are
* currently no objects in the queue. Allow the producer to signal that
* there won't be further objects and unblock any waiting consumers. */
-public class BlockingIteratorImpl<T> implements Iterator<T> {
+public class BlockingIteratorImpl<T> implements Iterator<T>, Iterable<T> {
/* Queue containing produced elemnts waiting for consumers. */
private Queue<T> queue = new LinkedList<>();
@@ -24,8 +24,10 @@ public class BlockingIteratorImpl<T> implements Iterator<T> {
}
/* Create instance with maximum queue size. */
- protected BlockingIteratorImpl(int maxQueueSize) {
- this.maxQueueSize = maxQueueSize;
+ protected BlockingIteratorImpl(Integer maxQueueSize) {
+ if (null != maxQueueSize) {
+ this.maxQueueSize = maxQueueSize;
+ }
}
/* Add an object to the queue if there's still room. */
@@ -99,5 +101,10 @@ public class BlockingIteratorImpl<T> implements Iterator<T> {
public void remove() {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public Iterator<T> iterator() {
+ return this;
+ }
}
diff --git a/src/main/java/org/torproject/descriptor/impl/BridgeExtraInfoDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgeExtraInfoDescriptorImpl.java
index 7d0af52..b2b3e99 100644
--- a/src/main/java/org/torproject/descriptor/impl/BridgeExtraInfoDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/BridgeExtraInfoDescriptorImpl.java
@@ -6,13 +6,16 @@ package org.torproject.descriptor.impl;
import org.torproject.descriptor.BridgeExtraInfoDescriptor;
import org.torproject.descriptor.DescriptorParseException;
+import java.io.File;
+
public class BridgeExtraInfoDescriptorImpl
extends ExtraInfoDescriptorImpl implements BridgeExtraInfoDescriptor {
protected BridgeExtraInfoDescriptorImpl(byte[] descriptorBytes,
- int[] offsetAndLimit, boolean failUnrecognizedDescriptorLines)
- throws DescriptorParseException {
- super(descriptorBytes, offsetAndLimit, failUnrecognizedDescriptorLines);
+ int[] offsetAndLimit, File descriptorFile,
+ boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+ super(descriptorBytes, offsetAndLimit, descriptorFile,
+ failUnrecognizedDescriptorLines);
}
}
diff --git a/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java
index 83a3a8a..a54c91c 100644
--- a/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/BridgeNetworkStatusImpl.java
@@ -6,6 +6,7 @@ package org.torproject.descriptor.impl;
import org.torproject.descriptor.BridgeNetworkStatus;
import org.torproject.descriptor.DescriptorParseException;
+import java.io.File;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -19,10 +20,10 @@ public class BridgeNetworkStatusImpl extends NetworkStatusImpl
implements BridgeNetworkStatus {
protected BridgeNetworkStatusImpl(byte[] rawDescriptorBytes,
- int[] offsetAndLength, String fileName,
+ int[] offsetAndLength, File descriptorFile, String fileName,
boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
- super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines,
- false, false);
+ super(rawDescriptorBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines, false, false);
this.setPublishedMillisFromFileName(fileName);
}
diff --git a/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java
index ca07c29..b2015ff 100644
--- a/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/BridgePoolAssignmentImpl.java
@@ -6,6 +6,7 @@ package org.torproject.descriptor.impl;
import org.torproject.descriptor.BridgePoolAssignment;
import org.torproject.descriptor.DescriptorParseException;
+import java.io.File;
import java.util.EnumSet;
import java.util.Scanner;
import java.util.SortedMap;
@@ -15,10 +16,10 @@ public class BridgePoolAssignmentImpl extends DescriptorImpl
implements BridgePoolAssignment {
protected BridgePoolAssignmentImpl(byte[] rawDescriptorBytes,
- int[] offsetAndlength, boolean failUnrecognizedDescriptorLines)
- throws DescriptorParseException {
- super(rawDescriptorBytes, offsetAndlength, failUnrecognizedDescriptorLines,
- false);
+ int[] offsetAndlength, File descriptorFile,
+ boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+ super(rawDescriptorBytes, offsetAndlength, descriptorFile,
+ failUnrecognizedDescriptorLines, false);
this.parseDescriptorBytes();
this.checkExactlyOnceKeys(EnumSet.of(Key.BRIDGE_POOL_ASSIGNMENT));
this.checkFirstKey(Key.BRIDGE_POOL_ASSIGNMENT);
diff --git a/src/main/java/org/torproject/descriptor/impl/BridgeServerDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/BridgeServerDescriptorImpl.java
index ac45591..30da220 100644
--- a/src/main/java/org/torproject/descriptor/impl/BridgeServerDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/BridgeServerDescriptorImpl.java
@@ -6,13 +6,16 @@ package org.torproject.descriptor.impl;
import org.torproject.descriptor.BridgeServerDescriptor;
import org.torproject.descriptor.DescriptorParseException;
+import java.io.File;
+
public class BridgeServerDescriptorImpl extends ServerDescriptorImpl
implements BridgeServerDescriptor {
protected BridgeServerDescriptorImpl(byte[] rawDescriptorBytes,
- int[] offsetAndLength, boolean failUnrecognizedDescriptorLines)
- throws DescriptorParseException {
- super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines);
+ int[] offsetAndLength, File descriptorFile,
+ boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+ super(rawDescriptorBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines);
}
}
diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java
index 52ff6d4..d88b542 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorParseException;
import java.io.ByteArrayInputStream;
+import java.io.File;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
@@ -41,6 +42,17 @@ public abstract class DescriptorImpl implements Descriptor {
protected int length;
/**
+ * Optional reference to the descriptor file, if this descriptor was read from
+ * a descriptor file.
+ */
+ private File descriptorFile;
+
+ @Override
+ public File getDescriptorFile() {
+ return this.descriptorFile;
+ }
+
+ /**
* Returns a <emph>copy</emph> of the full raw descriptor bytes.
*
* <p>If possible, subclasses should avoid retrieving raw descriptor bytes and
@@ -178,8 +190,7 @@ public abstract class DescriptorImpl implements Descriptor {
}
protected DescriptorImpl(byte[] rawDescriptorBytes, int[] offsetAndLength,
- boolean failUnrecognizedDescriptorLines, boolean blankLinesAllowed)
- throws DescriptorParseException {
+ File descriptorFile, boolean failUnrecognizedDescriptorLines) {
int offset = offsetAndLength[0];
int length = offsetAndLength[1];
if (offset < 0 || offset + length > rawDescriptorBytes.length
@@ -191,8 +202,16 @@ public abstract class DescriptorImpl implements Descriptor {
this.rawDescriptorBytes = rawDescriptorBytes;
this.offset = offset;
this.length = length;
+ this.descriptorFile = descriptorFile;
this.failUnrecognizedDescriptorLines =
failUnrecognizedDescriptorLines;
+ }
+
+ protected DescriptorImpl(byte[] rawDescriptorBytes, int[] offsetAndLength,
+ File descriptorFile, boolean failUnrecognizedDescriptorLines,
+ boolean blankLinesAllowed) throws DescriptorParseException {
+ this(rawDescriptorBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines);
this.cutOffAnnotations();
this.countKeys(rawDescriptorBytes, blankLinesAllowed);
}
diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
index e416d93..a0be85c 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorParserImpl.java
@@ -10,6 +10,7 @@ import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.DescriptorParser;
+import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.charset.StandardCharsets;
@@ -31,6 +32,29 @@ public class DescriptorParserImpl implements DescriptorParser {
public List<Descriptor> parseDescriptors(
byte[] rawDescriptorBytes, String fileName)
throws DescriptorParseException {
+ return this.parseDescriptors(rawDescriptorBytes, null, fileName, false);
+ }
+
+ @Override
+ public Iterable<Descriptor> parseDescriptors(byte[] rawDescriptorBytes,
+ File descriptorFile, String fileName) {
+ try {
+ return this.parseDescriptors(rawDescriptorBytes, descriptorFile, fileName,
+ true);
+ } catch (DescriptorParseException e) {
+ /* Looks like we attempted to parse the whole raw descriptor bytes at once
+ * below and ran into a parse issue. */
+ List<Descriptor> parsedDescriptors = new ArrayList<>();
+ parsedDescriptors.add(new UnparseableDescriptorImpl(rawDescriptorBytes,
+ new int[] { 0, rawDescriptorBytes.length }, descriptorFile,
+ failUnrecognizedDescriptorLines, e));
+ return parsedDescriptors;
+ }
+ }
+
+ private List<Descriptor> parseDescriptors(
+ byte[] rawDescriptorBytes, File descriptorFile, String fileName,
+ boolean includeUnparseableDescriptors) throws DescriptorParseException {
byte[] first100Chars = new byte[Math.min(100,
rawDescriptorBytes.length)];
System.arraycopy(rawDescriptorBytes, 0, first100Chars, 0,
@@ -45,9 +69,9 @@ public class DescriptorParserImpl implements DescriptorParser {
NL + Key.NETWORK_STATUS_VERSION.keyword + SP + "3"))
&& firstLines.contains(
NL + Key.VOTE_STATUS.keyword + SP + "consensus" + NL))) {
- return parseDescriptors(rawDescriptorBytes, Key.NETWORK_STATUS_VERSION,
- RelayNetworkStatusConsensusImpl.class,
- this.failUnrecognizedDescriptorLines);
+ return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+ Key.NETWORK_STATUS_VERSION, RelayNetworkStatusConsensusImpl.class,
+ this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
} else if (firstLines.startsWith("@type network-status-vote-3 1.")
|| ((firstLines.startsWith(
Key.NETWORK_STATUS_VERSION.keyword + SP + "3" + NL)
@@ -55,89 +79,95 @@ public class DescriptorParserImpl implements DescriptorParser {
NL + Key.NETWORK_STATUS_VERSION.keyword + SP + "3" + NL))
&& firstLines.contains(
NL + Key.VOTE_STATUS.keyword + SP + "vote" + NL))) {
- return parseDescriptors(rawDescriptorBytes, Key.NETWORK_STATUS_VERSION,
- RelayNetworkStatusVoteImpl.class,
- this.failUnrecognizedDescriptorLines);
+ return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+ Key.NETWORK_STATUS_VERSION, RelayNetworkStatusVoteImpl.class,
+ this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
} else if (firstLines.startsWith("@type bridge-network-status 1.")
|| firstLines.startsWith(Key.R.keyword + SP)) {
List<Descriptor> parsedDescriptors = new ArrayList<>();
parsedDescriptors.add(new BridgeNetworkStatusImpl(
rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length },
- fileName, this.failUnrecognizedDescriptorLines));
+ descriptorFile, fileName, this.failUnrecognizedDescriptorLines));
return parsedDescriptors;
} else if (firstLines.startsWith("@type bridge-server-descriptor 1.")) {
- return parseDescriptors(rawDescriptorBytes, Key.ROUTER,
- BridgeServerDescriptorImpl.class,
- this.failUnrecognizedDescriptorLines);
+ return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+ Key.ROUTER, BridgeServerDescriptorImpl.class,
+ this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
} else if (firstLines.startsWith("@type server-descriptor 1.")
|| firstLines.startsWith(Key.ROUTER.keyword + SP)
|| firstLines.contains(NL + Key.ROUTER.keyword + SP)) {
- return parseDescriptors(rawDescriptorBytes, Key.ROUTER,
- RelayServerDescriptorImpl.class,
- this.failUnrecognizedDescriptorLines);
+ return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+ Key.ROUTER, RelayServerDescriptorImpl.class,
+ this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
} else if (firstLines.startsWith("@type bridge-extra-info 1.")) {
- return parseDescriptors(rawDescriptorBytes, Key.EXTRA_INFO,
- BridgeExtraInfoDescriptorImpl.class,
- this.failUnrecognizedDescriptorLines);
+ return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+ Key.EXTRA_INFO, BridgeExtraInfoDescriptorImpl.class,
+ this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
} else if (firstLines.startsWith("@type extra-info 1.")
|| firstLines.startsWith(Key.EXTRA_INFO.keyword + SP)
|| firstLines.contains(NL + Key.EXTRA_INFO.keyword + SP)) {
- return parseDescriptors(rawDescriptorBytes, Key.EXTRA_INFO,
- RelayExtraInfoDescriptorImpl.class,
- this.failUnrecognizedDescriptorLines);
+ return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+ Key.EXTRA_INFO, RelayExtraInfoDescriptorImpl.class,
+ this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
} else if (firstLines.startsWith("@type microdescriptor 1.")
|| firstLines.startsWith(Key.ONION_KEY.keyword + NL)
|| firstLines.contains(NL + Key.ONION_KEY.keyword + NL)) {
- return parseDescriptors(rawDescriptorBytes, Key.ONION_KEY,
- MicrodescriptorImpl.class, this.failUnrecognizedDescriptorLines);
+ return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+ Key.ONION_KEY, MicrodescriptorImpl.class,
+ this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
} else if (firstLines.startsWith("@type bridge-pool-assignment 1.")
|| firstLines.startsWith(Key.BRIDGE_POOL_ASSIGNMENT.keyword + SP)
|| firstLines.contains(NL + Key.BRIDGE_POOL_ASSIGNMENT.keyword + SP)) {
- return parseDescriptors(rawDescriptorBytes, Key.BRIDGE_POOL_ASSIGNMENT,
- BridgePoolAssignmentImpl.class, this.failUnrecognizedDescriptorLines);
+ return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+ Key.BRIDGE_POOL_ASSIGNMENT, BridgePoolAssignmentImpl.class,
+ this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
} else if (firstLines.startsWith("@type dir-key-certificate-3 1.")
|| firstLines.startsWith(Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP)
|| firstLines.contains(
NL + Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP)) {
- return parseDescriptors(rawDescriptorBytes,
+ return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
Key.DIR_KEY_CERTIFICATE_VERSION, DirectoryKeyCertificateImpl.class,
- this.failUnrecognizedDescriptorLines);
+ this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
} else if (firstLines.startsWith("@type tordnsel 1.")
|| firstLines.startsWith("ExitNode" + SP)
|| firstLines.contains(NL + "ExitNode" + SP)) {
List<Descriptor> parsedDescriptors = new ArrayList<>();
- parsedDescriptors.add(new ExitListImpl(rawDescriptorBytes, fileName,
- this.failUnrecognizedDescriptorLines));
+ parsedDescriptors.add(new ExitListImpl(rawDescriptorBytes, descriptorFile,
+ fileName, this.failUnrecognizedDescriptorLines));
return parsedDescriptors;
} else if (firstLines.startsWith("@type network-status-2 1.")
|| firstLines.startsWith(
Key.NETWORK_STATUS_VERSION.keyword + SP + "2" + NL)
|| firstLines.contains(
NL + Key.NETWORK_STATUS_VERSION.keyword + SP + "2" + NL)) {
- return parseDescriptors(rawDescriptorBytes, Key.NETWORK_STATUS_VERSION,
- RelayNetworkStatusImpl.class, this.failUnrecognizedDescriptorLines);
+ return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+ Key.NETWORK_STATUS_VERSION, RelayNetworkStatusImpl.class,
+ this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
} else if (firstLines.startsWith("@type directory 1.")
|| firstLines.startsWith(Key.SIGNED_DIRECTORY.keyword + NL)
|| firstLines.contains(NL + Key.SIGNED_DIRECTORY.keyword + NL)) {
- return parseDescriptors(rawDescriptorBytes, Key.SIGNED_DIRECTORY,
- RelayDirectoryImpl.class, this.failUnrecognizedDescriptorLines);
+ return this.parseDescriptors(rawDescriptorBytes, descriptorFile,
+ Key.SIGNED_DIRECTORY, RelayDirectoryImpl.class,
+ this.failUnrecognizedDescriptorLines, includeUnparseableDescriptors);
} else if (firstLines.startsWith("@type torperf 1.")) {
- return TorperfResultImpl.parseTorperfResults(
- rawDescriptorBytes, this.failUnrecognizedDescriptorLines);
+ return TorperfResultImpl.parseTorperfResults(rawDescriptorBytes,
+ descriptorFile, this.failUnrecognizedDescriptorLines);
} else {
throw new DescriptorParseException("Could not detect descriptor "
+ "type in descriptor starting with '" + firstLines + "'.");
}
}
- private static List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes,
- Key key, Class<? extends DescriptorImpl> descriptorClass,
- boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+ private List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes,
+ File descriptorFile, Key key,
+ Class<? extends DescriptorImpl> descriptorClass,
+ boolean failUnrecognizedDescriptorLines,
+ boolean includeUnparseableDescriptors) throws DescriptorParseException {
List<Descriptor> parsedDescriptors = new ArrayList<>();
Constructor<? extends DescriptorImpl> constructor;
try {
constructor = descriptorClass.getDeclaredConstructor(byte[].class,
- int[].class, boolean.class);
+ int[].class, File.class, boolean.class);
} catch (NoSuchMethodException e) {
throw new RuntimeException(e);
}
@@ -181,20 +211,31 @@ public class DescriptorParserImpl implements DescriptorParser {
endDescriptor += 1;
int[] offsetAndLength = new int[] { startAnnotations,
endDescriptor - startAnnotations };
- parsedDescriptors.add(parseDescriptor(rawDescriptorBytes,
- offsetAndLength, constructor, failUnrecognizedDescriptorLines));
+ try {
+ parsedDescriptors.add(this.parseDescriptor(rawDescriptorBytes,
+ offsetAndLength, descriptorFile, constructor,
+ failUnrecognizedDescriptorLines));
+ } catch (DescriptorParseException e) {
+ if (includeUnparseableDescriptors) {
+ parsedDescriptors.add(new UnparseableDescriptorImpl(
+ rawDescriptorBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines, e));
+ } else {
+ throw e;
+ }
+ }
startAnnotations = endDescriptor;
}
return parsedDescriptors;
}
- static Descriptor parseDescriptor(byte[] rawDescriptorBytes,
- int[] offsetAndLength, Constructor<? extends DescriptorImpl> constructor,
- boolean failUnrecognizedDescriptorLines)
- throws DescriptorParseException {
+ Descriptor parseDescriptor(byte[] rawDescriptorBytes,
+ int[] offsetAndLength, File descriptorFile,
+ Constructor<? extends DescriptorImpl> constructor,
+ boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
try {
return constructor.newInstance(rawDescriptorBytes,
- offsetAndLength, failUnrecognizedDescriptorLines);
+ offsetAndLength, descriptorFile, failUnrecognizedDescriptorLines);
} catch (InvocationTargetException e) {
if (null != e.getCause()
&& e.getCause() instanceof DescriptorParseException) {
@@ -206,5 +247,4 @@ public class DescriptorParserImpl implements DescriptorParser {
throw new RuntimeException(e);
}
}
-
}
diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
index 12d73ad..54e26fc 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorReaderImpl.java
@@ -135,6 +135,17 @@ public class DescriptorReaderImpl implements DescriptorReader {
this.maxDescriptorFilesInQueue = max;
}
+ private int maxDescriptorsInQueue = 100;
+
+ @Override
+ public void setMaxDescriptorsInQueue(int maxDescriptorsInQueue) {
+ if (this.hasStartedReading) {
+ throw new IllegalStateException("Reconfiguration is not permitted "
+ + "after starting to read.");
+ }
+ this.maxDescriptorsInQueue = maxDescriptorsInQueue;
+ }
+
private DescriptorReaderRunnable reader;
@Override
@@ -160,6 +171,25 @@ public class DescriptorReaderImpl implements DescriptorReader {
}
@Override
+ public Iterable<Descriptor> readDescriptors(File... descriptorFiles) {
+ if (this.hasStartedReading) {
+ throw new IllegalStateException("Initiating reading is only "
+ + "permitted once.");
+ }
+ this.hasStartedReading = true;
+ BlockingIteratorImpl<Descriptor> descriptorQueue =
+ new BlockingIteratorImpl<>(this.maxDescriptorsInQueue);
+ this.reader = new DescriptorReaderRunnable(
+ descriptorFiles, descriptorQueue, this.autoSaveHistoryFile,
+ this.manualSaveHistoryFile, this.excludedFiles,
+ this.failUnrecognizedDescriptorLines);
+ Thread readerThread = new Thread(this.reader);
+ readerThread.setDaemon(true);
+ readerThread.start();
+ return descriptorQueue;
+ }
+
+ @Override
public void saveHistoryFile(File historyFile) {
if (!this.reader.hasFinishedReading) {
throw new IllegalStateException("Saving history is only permitted after "
@@ -170,11 +200,15 @@ public class DescriptorReaderImpl implements DescriptorReader {
private static class DescriptorReaderRunnable implements Runnable {
+ private File[] descriptorFiles;
+
private List<File> directories;
private List<File> tarballs;
- private BlockingIteratorImpl<DescriptorFile> descriptorQueue;
+ private BlockingIteratorImpl<Descriptor> descriptorQueue;
+
+ private BlockingIteratorImpl<DescriptorFile> descriptorFileQueue;
private File autoSaveHistoryFile;
@@ -186,24 +220,45 @@ public class DescriptorReaderImpl implements DescriptorReader {
private SortedMap<String, Long> parsedFilesAfter = new TreeMap<>();
+ private boolean failUnrecognizedDescriptorLines;
+
private DescriptorParser descriptorParser;
private boolean hasFinishedReading = false;
private DescriptorReaderRunnable(List<File> directories,
List<File> tarballs,
- BlockingIteratorImpl<DescriptorFile> descriptorQueue,
+ BlockingIteratorImpl<DescriptorFile> descriptorFileQueue,
File autoSaveHistoryFile, File manualSaveHistoryFile,
SortedMap<String, Long> excludedFiles,
boolean failUnrecognizedDescriptorLines) {
this.directories = directories;
this.tarballs = tarballs;
+ this.descriptorFileQueue = descriptorFileQueue;
+ this.autoSaveHistoryFile = autoSaveHistoryFile;
+ this.manualSaveHistoryFile = manualSaveHistoryFile;
+ if (excludedFiles != null) {
+ this.excludedFilesBefore = excludedFiles;
+ }
+ this.failUnrecognizedDescriptorLines = failUnrecognizedDescriptorLines;
+ this.descriptorParser = new DescriptorParserImpl();
+ this.descriptorParser.setFailUnrecognizedDescriptorLines(
+ failUnrecognizedDescriptorLines);
+ }
+
+ private DescriptorReaderRunnable(File[] descriptorFiles,
+ BlockingIteratorImpl<Descriptor> descriptorQueue,
+ File autoSaveHistoryFile, File manualSaveHistoryFile,
+ SortedMap<String, Long> excludedFiles,
+ boolean failUnrecognizedDescriptorLines) {
+ this.descriptorFiles = descriptorFiles;
this.descriptorQueue = descriptorQueue;
this.autoSaveHistoryFile = autoSaveHistoryFile;
this.manualSaveHistoryFile = manualSaveHistoryFile;
if (excludedFiles != null) {
this.excludedFilesBefore = excludedFiles;
}
+ this.failUnrecognizedDescriptorLines = failUnrecognizedDescriptorLines;
this.descriptorParser = new DescriptorParserImpl();
this.descriptorParser.setFailUnrecognizedDescriptorLines(
failUnrecognizedDescriptorLines);
@@ -213,6 +268,7 @@ public class DescriptorReaderImpl implements DescriptorReader {
try {
this.readOldHistory(this.autoSaveHistoryFile);
this.readOldHistory(this.manualSaveHistoryFile);
+ this.readDescriptorFiles();
this.readDescriptors();
this.readTarballs();
this.hasFinishedReading = true;
@@ -220,7 +276,12 @@ public class DescriptorReaderImpl implements DescriptorReader {
log.error("Bug: uncaught exception or error while "
+ "reading descriptors: " + t.getMessage(), t);
} finally {
- this.descriptorQueue.setOutOfDescriptors();
+ if (null != this.descriptorFileQueue) {
+ this.descriptorFileQueue.setOutOfDescriptors();
+ }
+ if (null != this.descriptorQueue) {
+ this.descriptorQueue.setOutOfDescriptors();
+ }
}
if (this.hasFinishedReading) {
this.writeNewHistory(this.autoSaveHistoryFile);
@@ -275,7 +336,97 @@ public class DescriptorReaderImpl implements DescriptorReader {
}
}
+ private void readDescriptorFiles() {
+ if (null == this.descriptorFiles) {
+ return;
+ }
+ Stack<File> files = new Stack<>();
+ for (File descriptorFile : this.descriptorFiles) {
+ if (!descriptorFile.exists()) {
+ continue;
+ }
+ files.add(descriptorFile);
+ while (!files.isEmpty()) {
+ File file = files.pop();
+ try {
+ String absolutePath = file.getAbsolutePath();
+ long lastModifiedMillis = file.lastModified();
+ if (this.excludedFilesBefore.containsKey(absolutePath)
+ && this.excludedFilesBefore.get(absolutePath)
+ == lastModifiedMillis) {
+ this.excludedFilesAfter.put(absolutePath, lastModifiedMillis);
+ continue;
+ }
+ if (file.isDirectory()) {
+ files.addAll(Arrays.asList(file.listFiles()));
+ continue;
+ } else if (file.getName().endsWith(".tar")
+ || file.getName().endsWith(".tar.bz2")
+ || file.getName().endsWith(".tar.xz")) {
+ this.readTarball(file);
+ } else {
+ this.readDescriptorFile(file);
+ }
+ this.parsedFilesAfter.put(absolutePath, lastModifiedMillis);
+ } catch (IOException e) {
+ log.warn("Unable to read descriptor file {}.", file, e);
+ }
+ }
+ }
+ }
+
+ private void readTarball(File file) throws IOException {
+ FileInputStream in = new FileInputStream(file);
+ if (in.available() <= 0) {
+ return;
+ }
+ TarArchiveInputStream tais;
+ if (file.getName().endsWith(".tar.bz2")) {
+ tais = new TarArchiveInputStream(new BZip2CompressorInputStream(in));
+ } else if (file.getName().endsWith(".tar.xz")) {
+ tais = new TarArchiveInputStream(new XZCompressorInputStream(in));
+ } else if (file.getName().endsWith(".tar")) {
+ tais = new TarArchiveInputStream(in);
+ } else {
+ return;
+ }
+ BufferedInputStream bis = new BufferedInputStream(tais);
+ TarArchiveEntry tae;
+ while ((tae = tais.getNextTarEntry()) != null) {
+ if (tae.isDirectory()) {
+ continue;
+ }
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ byte[] rawDescriptorBytes = baos.toByteArray();
+ if (rawDescriptorBytes.length < 1) {
+ continue;
+ }
+ String fileName = tae.getName().substring(
+ tae.getName().lastIndexOf("/") + 1);
+ for (Descriptor descriptor : this.descriptorParser.parseDescriptors(
+ rawDescriptorBytes, file, fileName)) {
+ this.descriptorQueue.add(descriptor);
+ }
+ }
+ }
+
+ private void readDescriptorFile(File file) throws IOException {
+ byte[] rawDescriptorBytes = Files.readAllBytes(file.toPath());
+ for (Descriptor descriptor : this.descriptorParser.parseDescriptors(
+ rawDescriptorBytes, file, file.getName())) {
+ this.descriptorQueue.add(descriptor);
+ }
+ }
+
private void readDescriptors() {
+ if (null == this.directories) {
+ return;
+ }
for (File directory : this.directories) {
if (!directory.exists() || !directory.isDirectory()) {
continue;
@@ -315,13 +466,16 @@ public class DescriptorReaderImpl implements DescriptorReader {
descriptorFile.setException(e);
abortReading = true;
}
- this.descriptorQueue.add(descriptorFile);
+ this.descriptorFileQueue.add(descriptorFile);
}
}
}
}
private void readTarballs() {
+ if (null == this.tarballs) {
+ return;
+ }
List<File> files = new ArrayList<>(this.tarballs);
boolean abortReading = false;
while (!abortReading && !files.isEmpty()) {
@@ -385,7 +539,7 @@ public class DescriptorReaderImpl implements DescriptorReader {
} catch (DescriptorParseException e) {
descriptorFile.setException(e);
}
- this.descriptorQueue.add(descriptorFile);
+ this.descriptorFileQueue.add(descriptorFile);
}
}
} catch (IOException e) {
diff --git a/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java b/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java
index 9a06a93..6beb9b5 100644
--- a/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DirectoryKeyCertificateImpl.java
@@ -6,6 +6,7 @@ package org.torproject.descriptor.impl;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.DirectoryKeyCertificate;
+import java.io.File;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.Scanner;
@@ -15,10 +16,10 @@ public class DirectoryKeyCertificateImpl extends DescriptorImpl
implements DirectoryKeyCertificate {
protected DirectoryKeyCertificateImpl(byte[] rawDescriptorBytes,
- int[] offsetAndLength, boolean failUnrecognizedDescriptorLines)
- throws DescriptorParseException {
- super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines,
- false);
+ int[] offsetAndLength, File descriptorFile,
+ boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+ super(rawDescriptorBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines, false);
this.parseDescriptorBytes();
this.calculateDigestSha1Hex(Key.DIR_KEY_CERTIFICATE_VERSION.keyword + SP,
NL + Key.DIR_KEY_CERTIFICATION.keyword + NL);
diff --git a/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java b/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java
index a87a55f..d16856a 100644
--- a/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/ExitListImpl.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.ExitList;
import org.torproject.descriptor.ExitListEntry;
+import java.io.File;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
@@ -18,11 +19,11 @@ import java.util.TimeZone;
public class ExitListImpl extends DescriptorImpl implements ExitList {
- protected ExitListImpl(byte[] rawDescriptorBytes, String fileName,
- boolean failUnrecognizedDescriptorLines)
+ protected ExitListImpl(byte[] rawDescriptorBytes, File descriptorfile,
+ String fileName, boolean failUnrecognizedDescriptorLines)
throws DescriptorParseException {
super(rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length },
- failUnrecognizedDescriptorLines, false);
+ descriptorfile, failUnrecognizedDescriptorLines, false);
this.splitAndParseExitListEntries();
this.setPublishedMillisFromFileName(fileName);
}
diff --git a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
index ca7d376..32a4dcc 100644
--- a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.BandwidthHistory;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.ExtraInfoDescriptor;
+import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -34,10 +35,10 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl
Key.PADDING_COUNTS);
protected ExtraInfoDescriptorImpl(byte[] descriptorBytes,
- int[] offsetAndLimit, boolean failUnrecognizedDescriptorLines)
- throws DescriptorParseException {
- super(descriptorBytes, offsetAndLimit, failUnrecognizedDescriptorLines,
- false);
+ int[] offsetAndLimit, File descriptorFile,
+ boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+ super(descriptorBytes, offsetAndLimit, descriptorFile,
+ failUnrecognizedDescriptorLines, false);
this.parseDescriptorBytes();
this.checkExactlyOnceKeys(exactlyOnceKeys);
Set<Key> dirreqStatsKeys = EnumSet.of(
diff --git a/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java
index c436120..d9c284e 100644
--- a/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java
@@ -6,6 +6,7 @@ package org.torproject.descriptor.impl;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.Microdescriptor;
+import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
@@ -18,10 +19,10 @@ public class MicrodescriptorImpl extends DescriptorImpl
implements Microdescriptor {
protected MicrodescriptorImpl(byte[] descriptorBytes, int[] offsetAndLength,
- boolean failUnrecognizedDescriptorLines)
+ File descriptorFile, boolean failUnrecognizedDescriptorLines)
throws DescriptorParseException {
- super(descriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines,
- false);
+ super(descriptorBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines, false);
this.parseDescriptorBytes();
this.calculateDigestSha256Base64(Key.ONION_KEY.keyword + NL);
this.checkExactlyOnceKeys(EnumSet.of(Key.ONION_KEY));
diff --git a/src/main/java/org/torproject/descriptor/impl/NetworkStatusImpl.java b/src/main/java/org/torproject/descriptor/impl/NetworkStatusImpl.java
index 6765b6f..5695353 100644
--- a/src/main/java/org/torproject/descriptor/impl/NetworkStatusImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/NetworkStatusImpl.java
@@ -8,6 +8,7 @@ import org.torproject.descriptor.DirSourceEntry;
import org.torproject.descriptor.DirectorySignature;
import org.torproject.descriptor.NetworkStatusEntry;
+import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.SortedMap;
@@ -19,11 +20,11 @@ import java.util.TreeMap;
public abstract class NetworkStatusImpl extends DescriptorImpl {
protected NetworkStatusImpl(byte[] rawDescriptorBytes, int[] offsetAndLength,
- boolean failUnrecognizedDescriptorLines,
+ File descriptorFile, boolean failUnrecognizedDescriptorLines,
boolean containsDirSourceEntries, boolean blankLinesAllowed)
throws DescriptorParseException {
- super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines,
- blankLinesAllowed);
+ super(rawDescriptorBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines, blankLinesAllowed);
this.splitAndParseParts(containsDirSourceEntries);
}
diff --git a/src/main/java/org/torproject/descriptor/impl/RelayDirectoryImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayDirectoryImpl.java
index d5108ca..79ab862 100644
--- a/src/main/java/org/torproject/descriptor/impl/RelayDirectoryImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/RelayDirectoryImpl.java
@@ -8,6 +8,7 @@ import org.torproject.descriptor.RelayDirectory;
import org.torproject.descriptor.RouterStatusEntry;
import org.torproject.descriptor.ServerDescriptor;
+import java.io.File;
import java.util.ArrayList;
import java.util.EnumSet;
import java.util.List;
@@ -18,10 +19,10 @@ public class RelayDirectoryImpl extends DescriptorImpl
implements RelayDirectory {
protected RelayDirectoryImpl(byte[] directoryBytes, int[] offsetAndLength,
- boolean failUnrecognizedDescriptorLines)
+ File descriptorFile, boolean failUnrecognizedDescriptorLines)
throws DescriptorParseException {
- super(directoryBytes, offsetAndLength, failUnrecognizedDescriptorLines,
- true);
+ super(directoryBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines, true);
this.splitAndParseParts();
this.calculateDigestSha1Hex(Key.SIGNED_DIRECTORY.keyword + NL,
NL + Key.DIRECTORY_SIGNATURE.keyword + SP);
@@ -194,7 +195,8 @@ public class RelayDirectoryImpl extends DescriptorImpl
try {
ServerDescriptorImpl serverDescriptor =
new RelayServerDescriptorImpl(this.rawDescriptorBytes,
- new int[] { offset, length }, this.failUnrecognizedDescriptorLines);
+ new int[] { offset, length }, this.getDescriptorFile(),
+ this.failUnrecognizedDescriptorLines);
this.serverDescriptors.add(serverDescriptor);
} catch (DescriptorParseException e) {
this.serverDescriptorParseExceptions.add(e);
diff --git a/src/main/java/org/torproject/descriptor/impl/RelayExtraInfoDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayExtraInfoDescriptorImpl.java
index 92cff65..9644f90 100644
--- a/src/main/java/org/torproject/descriptor/impl/RelayExtraInfoDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/RelayExtraInfoDescriptorImpl.java
@@ -6,13 +6,16 @@ package org.torproject.descriptor.impl;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.RelayExtraInfoDescriptor;
+import java.io.File;
+
public class RelayExtraInfoDescriptorImpl
extends ExtraInfoDescriptorImpl implements RelayExtraInfoDescriptor {
protected RelayExtraInfoDescriptorImpl(byte[] descriptorBytes,
- int[] offsetAndLimit, boolean failUnrecognizedDescriptorLines)
- throws DescriptorParseException {
- super(descriptorBytes, offsetAndLimit, failUnrecognizedDescriptorLines);
+ int[] offsetAndLimit, File descriptorFile,
+ boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+ super(descriptorBytes, offsetAndLimit, descriptorFile,
+ failUnrecognizedDescriptorLines);
this.calculateDigestSha1Hex(Key.EXTRA_INFO.keyword + SP,
NL + Key.ROUTER_SIGNATURE.keyword + NL);
this.calculateDigestSha256Base64(Key.EXTRA_INFO.keyword + SP,
diff --git a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java
index 12a9dc3..0e853bb 100644
--- a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImpl.java
@@ -6,6 +6,7 @@ package org.torproject.descriptor.impl;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
@@ -22,10 +23,10 @@ public class RelayNetworkStatusConsensusImpl extends NetworkStatusImpl
implements RelayNetworkStatusConsensus {
protected RelayNetworkStatusConsensusImpl(byte[] consensusBytes,
- int[] offsetAndLimit, boolean failUnrecognizedDescriptorLines)
- throws DescriptorParseException {
- super(consensusBytes, offsetAndLimit, failUnrecognizedDescriptorLines, true,
- false);
+ int[] offsetAndLimit, File descriptorFile,
+ boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+ super(consensusBytes, offsetAndLimit, descriptorFile,
+ failUnrecognizedDescriptorLines, true, false);
Set<Key> exactlyOnceKeys = EnumSet.of(
Key.VOTE_STATUS, Key.CONSENSUS_METHOD, Key.VALID_AFTER, Key.FRESH_UNTIL,
Key.VALID_UNTIL, Key.VOTING_DELAY, Key.KNOWN_FLAGS);
diff --git a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java
index edc483a..8ebea2d 100644
--- a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusImpl.java
@@ -6,6 +6,7 @@ package org.torproject.descriptor.impl;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.RelayNetworkStatus;
+import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
@@ -19,10 +20,10 @@ public class RelayNetworkStatusImpl extends NetworkStatusImpl
implements RelayNetworkStatus {
protected RelayNetworkStatusImpl(byte[] statusBytes, int[] offsetAndLength,
- boolean failUnrecognizedDescriptorLines)
+ File descriptorFile, boolean failUnrecognizedDescriptorLines)
throws DescriptorParseException {
- super(statusBytes, offsetAndLength, failUnrecognizedDescriptorLines, false,
- true);
+ super(statusBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines, false, true);
Set<Key> exactlyOnceKeys = EnumSet.of(
Key.NETWORK_STATUS_VERSION, Key.DIR_SOURCE, Key.FINGERPRINT,
Key.CONTACT, Key.DIR_SIGNING_KEY, Key.PUBLISHED);
diff --git a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java
index cad5c15..93d6f12 100644
--- a/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImpl.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.DirectorySignature;
import org.torproject.descriptor.RelayNetworkStatusVote;
+import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
@@ -24,10 +25,10 @@ public class RelayNetworkStatusVoteImpl extends NetworkStatusImpl
implements RelayNetworkStatusVote {
protected RelayNetworkStatusVoteImpl(byte[] voteBytes, int[] offsetAndLength,
- boolean failUnrecognizedDescriptorLines)
+ File descriptorFile, boolean failUnrecognizedDescriptorLines)
throws DescriptorParseException {
- super(voteBytes, offsetAndLength, failUnrecognizedDescriptorLines, false,
- false);
+ super(voteBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines, false, false);
Set<Key> exactlyOnceKeys = EnumSet.of(
Key.VOTE_STATUS, Key.PUBLISHED, Key.VALID_AFTER, Key.FRESH_UNTIL,
Key.VALID_UNTIL, Key.VOTING_DELAY, Key.KNOWN_FLAGS, Key.DIR_SOURCE,
diff --git a/src/main/java/org/torproject/descriptor/impl/RelayServerDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/RelayServerDescriptorImpl.java
index 4ccb35a..23e9152 100644
--- a/src/main/java/org/torproject/descriptor/impl/RelayServerDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/RelayServerDescriptorImpl.java
@@ -6,13 +6,16 @@ package org.torproject.descriptor.impl;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.RelayServerDescriptor;
+import java.io.File;
+
public class RelayServerDescriptorImpl extends ServerDescriptorImpl
implements RelayServerDescriptor {
protected RelayServerDescriptorImpl(byte[] descriptorBytes,
- int[] offsetAndLength, boolean failUnrecognizedDescriptorLines)
- throws DescriptorParseException {
- super(descriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines);
+ int[] offsetAndLength, File descriptorFile,
+ boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
+ super(descriptorBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines);
this.calculateDigestSha1Hex(Key.ROUTER.keyword + SP,
NL + Key.ROUTER_SIGNATURE.keyword + NL);
this.calculateDigestSha256Base64(Key.ROUTER.keyword + SP,
diff --git a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java
index d438702..f434d84 100644
--- a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.BandwidthHistory;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.ServerDescriptor;
+import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
@@ -35,10 +36,10 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl
Key.ROUTER, Key.BANDWIDTH, Key.PUBLISHED);
protected ServerDescriptorImpl(byte[] descriptorBytes, int[] offsetAndLength,
- boolean failUnrecognizedDescriptorLines)
+ File descriptorFile, boolean failUnrecognizedDescriptorLines)
throws DescriptorParseException {
- super(descriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines,
- false);
+ super(descriptorBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines, false);
this.parseDescriptorBytes();
this.checkExactlyOnceKeys(exactlyOnce);
this.checkAtMostOnceKeys(atMostOnce);
diff --git a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
index fb00a37..ea9eb4b 100644
--- a/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/TorperfResultImpl.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorParseException;
import org.torproject.descriptor.TorperfResult;
+import java.io.File;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
@@ -20,8 +21,8 @@ public class TorperfResultImpl extends DescriptorImpl
implements TorperfResult {
protected static List<Descriptor> parseTorperfResults(
- byte[] rawDescriptorBytes, boolean failUnrecognizedDescriptorLines)
- throws DescriptorParseException {
+ byte[] rawDescriptorBytes, File descriptorFile,
+ boolean failUnrecognizedDescriptorLines) throws DescriptorParseException {
if (rawDescriptorBytes.length == 0) {
throw new DescriptorParseException("Descriptor is empty.");
}
@@ -47,7 +48,7 @@ public class TorperfResultImpl extends DescriptorImpl
} else {
/* XXX21932 */
parsedDescriptors.add(new TorperfResultImpl(
- (typeAnnotation + line).getBytes(),
+ (typeAnnotation + line).getBytes(), descriptorFile,
failUnrecognizedDescriptorLines));
typeAnnotation = "";
}
@@ -55,11 +56,11 @@ public class TorperfResultImpl extends DescriptorImpl
return parsedDescriptors;
}
- protected TorperfResultImpl(byte[] rawDescriptorBytes,
+ protected TorperfResultImpl(byte[] rawDescriptorBytes, File descriptorFile,
boolean failUnrecognizedDescriptorLines)
throws DescriptorParseException {
super(rawDescriptorBytes, new int[] { 0, rawDescriptorBytes.length },
- failUnrecognizedDescriptorLines, false);
+ descriptorFile, failUnrecognizedDescriptorLines, false);
this.parseTorperfResultLine();
}
diff --git a/src/main/java/org/torproject/descriptor/impl/UnparseableDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/UnparseableDescriptorImpl.java
new file mode 100644
index 0000000..4a0e772
--- /dev/null
+++ b/src/main/java/org/torproject/descriptor/impl/UnparseableDescriptorImpl.java
@@ -0,0 +1,40 @@
+/* Copyright 2017 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.descriptor.impl;
+
+import org.torproject.descriptor.DescriptorParseException;
+import org.torproject.descriptor.UnparseableDescriptor;
+
+import java.io.File;
+import java.util.List;
+
+public class UnparseableDescriptorImpl extends DescriptorImpl
+ implements UnparseableDescriptor {
+
+ protected UnparseableDescriptorImpl(byte[] rawDescriptorBytes,
+ int[] offsetAndLength, File descriptorFile,
+ boolean failUnrecognizedDescriptorLines,
+ DescriptorParseException descriptorParseException) {
+ super(rawDescriptorBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines);
+ this.descriptorParseException = descriptorParseException;
+ }
+
+ private DescriptorParseException descriptorParseException;
+
+ public DescriptorParseException getDescriptorParseException() {
+ return this.descriptorParseException;
+ }
+
+ @Override
+ public List<String> getAnnotations() {
+ throw new UnsupportedOperationException("This operation is not supported.");
+ }
+
+ @Override
+ public List<String> getUnrecognizedLines() {
+ throw new UnsupportedOperationException("This operation is not supported.");
+ }
+}
+
diff --git a/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java b/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java
index 3739e5d..c44de6d 100644
--- a/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/BridgeNetworkStatusTest.java
@@ -100,7 +100,7 @@ public class BridgeNetworkStatusTest {
throws DescriptorParseException {
byte[] statusBytes = this.buildStatusBytes();
return new BridgeNetworkStatusImpl(statusBytes,
- new int[] { 0, statusBytes.length }, this.fileName,
+ new int[] { 0, statusBytes.length }, null, this.fileName,
failUnrecognizedDescriptorLines);
}
diff --git a/src/test/java/org/torproject/descriptor/impl/ConsensusBuilder.java b/src/test/java/org/torproject/descriptor/impl/ConsensusBuilder.java
index e0d0c99..71bcd94 100644
--- a/src/test/java/org/torproject/descriptor/impl/ConsensusBuilder.java
+++ b/src/test/java/org/torproject/descriptor/impl/ConsensusBuilder.java
@@ -360,7 +360,7 @@ public class ConsensusBuilder {
throws DescriptorParseException {
byte[] consensusBytes = this.buildConsensusBytes();
return new RelayNetworkStatusConsensusImpl(consensusBytes,
- new int[] { 0, consensusBytes.length },
+ new int[] { 0, consensusBytes.length }, null,
failUnrecognizedDescriptorLines);
}
diff --git a/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java
index f1a8935..543254d 100644
--- a/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/DescriptorImplTest.java
@@ -131,7 +131,6 @@ public class DescriptorImplTest {
digest = des.getDigestSha256Base64();
assertEquals("1bEECw9nT5KRzPG8dAzEFJgSI4OBQfyWn+wjREb8oa8", digest);
}
-
}
diff --git a/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java b/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java
index 5a328ec..558a395 100644
--- a/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/DescriptorParserImplTest.java
@@ -11,6 +11,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
+import java.io.File;
import java.lang.reflect.Constructor;
public class DescriptorParserImplTest {
@@ -46,15 +47,17 @@ public class DescriptorParserImplTest {
Constructor<? extends DescriptorImpl> constructor;
try {
constructor = TestServerDescriptor.class
- .getDeclaredConstructor(byte[].class, int[].class, boolean.class);
+ .getDeclaredConstructor(byte[].class, int[].class, File.class,
+ boolean.class);
} catch (NoSuchMethodException e) {
throw new RuntimeException(e);
}
this.thrown.expect(DescriptorParseException.class);
this.thrown.expectMessage("'176x.158.53.63' in line 'router UbuntuCore169 "
+ "176x.158.53.63 44583 0 0' is not a valid IPv4 address.");
- DescriptorParserImpl.parseDescriptor(DEFECT.getBytes(),
- new int[]{0, DEFECT.getBytes().length}, constructor, false);
+ DescriptorParserImpl dpi = new DescriptorParserImpl();
+ dpi.parseDescriptor(DEFECT.getBytes(),
+ new int[]{0, DEFECT.getBytes().length}, null, constructor, false);
}
private static final String DEFECT =
diff --git a/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java b/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java
index 74417d9..5f31341 100644
--- a/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/ExitListImplTest.java
@@ -24,7 +24,7 @@ public class ExitListImplTest {
@Test
public void testAnnotatedInput() throws Exception {
ExitListImpl result = new ExitListImpl((tordnselAnnotation + input)
- .getBytes("US-ASCII"), fileName, false);
+ .getBytes("US-ASCII"), null, fileName, false);
assertEquals("Expected one annotation.", 1,
result.getAnnotations().size());
assertEquals(tordnselAnnotation.substring(0, 18),
@@ -42,7 +42,7 @@ public class ExitListImplTest {
public void testMultipleOldExitAddresses() throws Exception {
ExitListImpl result = new ExitListImpl(
(tordnselAnnotation + multiExitAddressInput)
- .getBytes("US-ASCII"), fileName, false);
+ .getBytes("US-ASCII"), null, fileName, false);
assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(),
result.getUnrecognizedLines().isEmpty());
assertEquals("Found: " + result.getExitListEntries(),
@@ -68,7 +68,7 @@ public class ExitListImplTest {
public void testMultipleExitAddresses() throws Exception {
ExitListImpl result = new ExitListImpl(
(tordnselAnnotation + multiExitAddressInput)
- .getBytes("US-ASCII"), fileName, false);
+ .getBytes("US-ASCII"), null, fileName, false);
assertTrue("Unrecognized lines: " + result.getUnrecognizedLines(),
result.getUnrecognizedLines().isEmpty());
Map<String, Long> map = result.getEntries()
@@ -84,7 +84,7 @@ public class ExitListImplTest {
this.thrown.expect(DescriptorParseException.class);
this.thrown.expectMessage("Missing 'ExitAddress' line in exit list entry.");
new ExitListImpl((tordnselAnnotation + insufficientInput[0])
- .getBytes("US-ASCII"), fileName, false);
+ .getBytes("US-ASCII"), null, fileName, false);
}
@Test
@@ -92,7 +92,7 @@ public class ExitListImplTest {
this.thrown.expect(DescriptorParseException.class);
this.thrown.expectMessage("Missing 'Published' line in exit list entry.");
new ExitListImpl((tordnselAnnotation + insufficientInput[1])
- .getBytes("US-ASCII"), fileName, false);
+ .getBytes("US-ASCII"), null, fileName, false);
}
private static final String tordnselAnnotation = "@type tordnsel 1.0\n";
diff --git a/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java
index 7b62393..82dd365 100644
--- a/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImplTest.java
@@ -336,7 +336,7 @@ public class ExtraInfoDescriptorImplTest {
throws DescriptorParseException {
byte[] descriptorBytes = this.buildDescriptorBytes();
return new RelayExtraInfoDescriptorImpl(descriptorBytes,
- new int[] { 0, descriptorBytes.length},
+ new int[] { 0, descriptorBytes.length}, null,
failUnrecognizedDescriptorLines);
}
}
@@ -2332,7 +2332,7 @@ public class ExtraInfoDescriptorImplTest {
+ "").getBytes();
RelayExtraInfoDescriptor descriptor =
new RelayExtraInfoDescriptorImpl(descriptorBytes,
- new int[] { 0, descriptorBytes.length }, true);
+ new int[] { 0, descriptorBytes.length }, null, true);
assertEquals("Pt1BtzfRwhYqGCDo8jjchS8nJP3ovrDyHGn+dqPbMgw",
descriptor.getDigestSha256Base64());
}
@@ -2357,7 +2357,7 @@ public class ExtraInfoDescriptorImplTest {
.getBytes();
BridgeExtraInfoDescriptor descriptor =
new BridgeExtraInfoDescriptorImpl(descriptorBytes,
- new int[] { 0, descriptorBytes.length }, true);
+ new int[] { 0, descriptorBytes.length }, null, true);
assertEquals("TvrqpjI7OmCtwGwair/NHUxg5ROVVQYz6/EDyXsDHR4",
descriptor.getDigestSha256Base64());
}
@@ -2379,7 +2379,7 @@ public class ExtraInfoDescriptorImplTest {
.getBytes();
BridgeExtraInfoDescriptor descriptor =
new BridgeExtraInfoDescriptorImpl(descriptorBytes,
- new int[] { 0, descriptorBytes.length }, true);
+ new int[] { 0, descriptorBytes.length }, null, true);
assertNull(descriptor.getDigestSha1Hex());
assertNull(descriptor.getDigestSha256Base64());
}
diff --git a/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java
index b1d6474..f5c9131 100644
--- a/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/MicrodescriptorImplTest.java
@@ -66,7 +66,7 @@ public class MicrodescriptorImplTest {
throws DescriptorParseException {
byte[] descriptorBytes = this.buildDescriptorBytes();
return new MicrodescriptorImpl(descriptorBytes,
- new int[] { 0, descriptorBytes.length },
+ new int[] { 0, descriptorBytes.length }, null,
failUnrecognizedDescriptorLines);
}
}
diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java
index 90128c2..4793e76 100644
--- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusConsensusImplTest.java
@@ -1599,7 +1599,7 @@ public class RelayNetworkStatusConsensusImplTest {
byte[] consensusBytes = cb.buildConsensusBytes();
consensusBytes[20] = (byte) 200;
new RelayNetworkStatusConsensusImpl(consensusBytes,
- new int[] { 0, consensusBytes.length }, true);
+ new int[] { 0, consensusBytes.length }, null, true);
}
@Test
@@ -1613,7 +1613,7 @@ public class RelayNetworkStatusConsensusImplTest {
byte[] consensusBytes = cb.buildConsensusBytes();
consensusBytes[0] = (byte) 200;
new RelayNetworkStatusConsensusImpl(consensusBytes,
- new int[] { 0, consensusBytes.length }, true);
+ new int[] { 0, consensusBytes.length }, null, true);
}
@Test
diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java
index e69a26b..760d04c 100644
--- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusImplTest.java
@@ -49,14 +49,14 @@ public class RelayNetworkStatusImplTest {
byte[] statusBytes = (validAnnotation + invalidHeader + validFooter)
.getBytes();
new RelayNetworkStatusImpl(statusBytes, new int[] { 0, statusBytes.length },
- true);
+ null, true);
}
@Test
public void testValidHeader() throws DescriptorParseException {
byte[] statusBytes = validStatus.getBytes();
RelayNetworkStatusImpl rnsi = new RelayNetworkStatusImpl(statusBytes,
- new int[] { 0, statusBytes.length }, true);
+ new int[] { 0, statusBytes.length }, null, true);
assertEquals(rnsi.getContactLine(),
"1024R/8D56913D Alex de Joode <adejoode(a)sabotage.org>");
}
diff --git a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java
index 8a85145..9727f50 100644
--- a/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/RelayNetworkStatusVoteImplTest.java
@@ -557,7 +557,8 @@ public class RelayNetworkStatusVoteImplTest {
throws DescriptorParseException {
byte[] voteBytes = this.buildVoteBytes();
return new RelayNetworkStatusVoteImpl(voteBytes,
- new int[] { 0, voteBytes.length }, failUnrecognizedDescriptorLines);
+ new int[] { 0, voteBytes.length }, null,
+ failUnrecognizedDescriptorLines);
}
private void appendHeader(StringBuilder sb) {
diff --git a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java
index 9533de3..d0c2561 100644
--- a/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/ServerDescriptorImplTest.java
@@ -456,7 +456,7 @@ public class ServerDescriptorImplTest {
throws DescriptorParseException {
byte[] descriptorBytes = this.buildDescriptorBytes();
return new RelayServerDescriptorImpl(descriptorBytes,
- new int[] { 0, descriptorBytes.length },
+ new int[] { 0, descriptorBytes.length }, null,
failUnrecognizedDescriptorLines);
}
}
@@ -1963,7 +1963,7 @@ public class ServerDescriptorImplTest {
+ "\nrouter-digest " + digestSha1Hex;
byte[] descriptorBytes = db.buildDescriptorBytes();
BridgeServerDescriptor descriptor = new BridgeServerDescriptorImpl(
- descriptorBytes, new int[] { 0, descriptorBytes.length }, true);
+ descriptorBytes, new int[] { 0, descriptorBytes.length }, null, true);
assertEquals(digestSha1Hex, descriptor.getDigestSha1Hex());
assertEquals(digestSha256Base64, descriptor.getDigestSha256Base64());
}
@@ -1974,7 +1974,7 @@ public class ServerDescriptorImplTest {
DescriptorBuilder db = new DescriptorBuilder();
byte[] descriptorBytes = db.buildDescriptorBytes();
BridgeServerDescriptor descriptor = new BridgeServerDescriptorImpl(
- descriptorBytes, new int[] { 0, descriptorBytes.length }, true);
+ descriptorBytes, new int[] { 0, descriptorBytes.length }, null, true);
assertNull(descriptor.getDigestSha1Hex());
assertNull(descriptor.getDigestSha256Base64());
}
diff --git a/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java b/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java
index 84aedee..e3a0c7c 100644
--- a/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java
+++ b/src/test/java/org/torproject/descriptor/impl/TestDescriptor.java
@@ -3,12 +3,14 @@
package org.torproject.descriptor.impl;
+import org.torproject.descriptor.DescriptorParseException;
+
public class TestDescriptor extends DescriptorImpl {
protected TestDescriptor(byte[] rawDescriptorBytes, int[] offsetAndLength,
boolean failUnrecognizedDescriptorLines, boolean blankLinesAllowed)
throws DescriptorParseException {
- super(rawDescriptorBytes, offsetAndLength,
+ super(rawDescriptorBytes, offsetAndLength, null,
failUnrecognizedDescriptorLines, blankLinesAllowed);
}
}
diff --git a/src/test/java/org/torproject/descriptor/impl/TestServerDescriptor.java b/src/test/java/org/torproject/descriptor/impl/TestServerDescriptor.java
index 194df82..0afd7b4 100644
--- a/src/test/java/org/torproject/descriptor/impl/TestServerDescriptor.java
+++ b/src/test/java/org/torproject/descriptor/impl/TestServerDescriptor.java
@@ -3,12 +3,16 @@
package org.torproject.descriptor.impl;
+import java.io.File;
+
public class TestServerDescriptor extends ServerDescriptorImpl {
protected TestServerDescriptor(byte[] rawDescriptorBytes,
- int[] offsetAndLength, boolean failUnrecognizedDescriptorLines)
+ int[] offsetAndLength, File descriptorFile,
+ boolean failUnrecognizedDescriptorLines)
throws DescriptorParseException {
- super(rawDescriptorBytes, offsetAndLength, failUnrecognizedDescriptorLines);
+ super(rawDescriptorBytes, offsetAndLength, descriptorFile,
+ failUnrecognizedDescriptorLines);
}
}
diff --git a/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java b/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java
index 122e301..7df4269 100644
--- a/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java
+++ b/src/test/java/org/torproject/descriptor/impl/TorperfResultImplTest.java
@@ -20,7 +20,7 @@ public class TorperfResultImplTest {
public void testAnnotatedInput() throws Exception {
TorperfResultImpl result = (TorperfResultImpl)
(TorperfResultImpl.parseTorperfResults((torperfAnnotation + input)
- .getBytes("US-ASCII"), false).get(0));
+ .getBytes("US-ASCII"), null, false).get(0));
assertEquals("Expected one annotation.", 1,
result.getAnnotations().size());
assertEquals(torperfAnnotation.substring(0, 17),
@@ -37,7 +37,7 @@ public class TorperfResultImplTest {
byte[] asciiBytes = (torperfAnnotation
+ input + input + input).getBytes("US-ASCII");
List<Descriptor> result = TorperfResultImpl.parseTorperfResults(
- asciiBytes, false);
+ asciiBytes, null, false);
assertEquals("Expected one annotation.", 1,
((TorperfResultImpl)(result.get(0))).getAnnotations().size());
assertEquals(3, result.size());
@@ -53,7 +53,7 @@ public class TorperfResultImplTest {
+ torperfAnnotation + input
+ torperfAnnotation + input).getBytes("US-ASCII");
List<Descriptor> result = TorperfResultImpl.parseTorperfResults(
- asciiBytes, false);
+ asciiBytes, null, false);
assertEquals("Expected one annotation.", 1,
((TorperfResultImpl)(result.get(0))).getAnnotations().size());
assertEquals(3, result.size());
@@ -90,7 +90,7 @@ public class TorperfResultImplTest {
@Test
public void testDatapercNonNumeric() throws Exception {
List<Descriptor> result = TorperfResultImpl.parseTorperfResults(
- ("DATAPERMILLE=2.0 " + input).getBytes(), false);
+ ("DATAPERMILLE=2.0 " + input).getBytes(), null, false);
assertEquals(1, result.size());
TorperfResultImpl torperfResult = (TorperfResultImpl) result.get(0);
assertEquals(1, torperfResult.getUnrecognizedKeys().size());
@@ -125,7 +125,7 @@ public class TorperfResultImplTest {
public void testEndpointsHostnamesSourceAddress()
throws DescriptorParseException {
List<Descriptor> result = TorperfResultImpl.parseTorperfResults(
- input2.getBytes(), true);
+ input2.getBytes(), null, true);
assertEquals(1, result.size());
TorperfResultImpl torperfResult = (TorperfResultImpl) result.get(0);
assertNull(torperfResult.getUnrecognizedKeys());
1
0
commit 59dc55fbe1b3c8afa7174e8b6b5f868d92e5c921
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Fri Jun 16 17:03:41 2017 +0200
Bump version to 1.8.2-dev.
---
build.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/build.xml b/build.xml
index d755932..d5aaa8f 100644
--- a/build.xml
+++ b/build.xml
@@ -6,7 +6,7 @@
<project default="usage" name="descriptor" basedir=".">
- <property name="release.version" value="1.8.2" />
+ <property name="release.version" value="1.8.2-dev" />
<property name="javadoc-title" value="DescripTor API Documentation"/>
<property name="javadoc-excludes" value="**/impl/** **/index/**" />
<property name="implementation-title" value="DescripTor" />
1
0

[metrics-lib/release] Add new method to retrieve raw descriptor length.
by karsten@torproject.org 21 Jun '17
by karsten@torproject.org 21 Jun '17
21 Jun '17
commit a113feb2c4f195dbc607e3da8c5fc2be19a6d475
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Thu Jun 15 14:25:29 2017 +0200
Add new method to retrieve raw descriptor length.
When we switched from storing copies of byte[] to storing byte[]
references, offset, and length, getRawDescriptor() suddenly became
more expensive than before. (Before that, copying bytes in the first
place was always expensive.) If an application only calls
Descriptor#getRawDescriptorBytes() to learn the array length, there's
now a more efficient way to provide that information.
---
CHANGELOG.md | 3 +++
src/main/java/org/torproject/descriptor/Descriptor.java | 15 +++++++++++++++
.../org/torproject/descriptor/impl/DescriptorImpl.java | 5 +++++
3 files changed, 23 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 44f1130..95cd851 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,9 @@
* Medium changes
- Let DescriptorParser return an Iterable instead of a List, which
prepares parsing large descriptor files descriptor by descriptor.
+ - Add new method to retrieve the raw descriptor length, rather than
+ forcing applications to request (a copy of) raw descriptor bytes
+ only to determine the raw descriptor length.
* Minor changes
- Fix a bug where NetworkStatusEntry's getMicrodescriptorDigests()
diff --git a/src/main/java/org/torproject/descriptor/Descriptor.java b/src/main/java/org/torproject/descriptor/Descriptor.java
index fb511bf..ca2a4cf 100644
--- a/src/main/java/org/torproject/descriptor/Descriptor.java
+++ b/src/main/java/org/torproject/descriptor/Descriptor.java
@@ -17,11 +17,26 @@ public interface Descriptor {
/**
* Return the raw descriptor bytes.
*
+ * <p>This method creates a new copy of raw descriptor bytes from a
+ * potentially larger byte array containing multiple descriptors.
+ * Applications that only want to learn about raw descriptor length in bytes
+ * should instead use {@link #getRawDescriptorLength()}.</p>
+ *
* @since 1.0.0
*/
public byte[] getRawDescriptorBytes();
/**
+ * Return the raw descriptor length in bytes.
+ *
+ * <p>Returns the exact same result as {@code getRawDescriptorBytes().length},
+ * but much more efficiently.</p>
+ *
+ * @since 1.9.0
+ */
+ public int getRawDescriptorLength();
+
+ /**
* Return the (possibly empty) list of annotations in the format
* {@code "@key( value)*"}.
*
diff --git a/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java
index d88b542..e0f94f0 100644
--- a/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/DescriptorImpl.java
@@ -41,6 +41,11 @@ public abstract class DescriptorImpl implements Descriptor {
*/
protected int length;
+ @Override
+ public int getRawDescriptorLength() {
+ return this.length;
+ }
+
/**
* Optional reference to the descriptor file, if this descriptor was read from
* a descriptor file.
1
0

[metrics-lib/release] Change encoding of microdescriptor digests in network status entries.
by karsten@torproject.org 21 Jun '17
by karsten@torproject.org 21 Jun '17
21 Jun '17
commit de9600a5dc937c4c453ed261266f38f21978f61e
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Sat Jun 17 21:20:01 2017 +0200
Change encoding of microdescriptor digests in network status entries.
Fixes #22640.
---
CHANGELOG.md | 8 ++++++++
.../java/org/torproject/descriptor/NetworkStatusEntry.java | 12 ++++++------
.../torproject/descriptor/impl/NetworkStatusEntryImpl.java | 14 +++++++-------
.../java/org/torproject/descriptor/impl/ParseHelper.java | 14 ++++++++++++--
4 files changed, 33 insertions(+), 15 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 409d0e8..e79811a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,11 @@
+# Changes in version 1.9.0 - 2017-06-??
+
+ * Minor changes
+ - Fix a bug where NetworkStatusEntry's getMicrodescriptorDigests()
+ and getMicrodescriptorDigestsSha256Base64() return hex strings
+ rather than base64 strings.
+
+
# Changes in version 1.8.2 - 2017-06-16
* Medium changes
diff --git a/src/main/java/org/torproject/descriptor/NetworkStatusEntry.java b/src/main/java/org/torproject/descriptor/NetworkStatusEntry.java
index c59eab9..9c5dae5 100644
--- a/src/main/java/org/torproject/descriptor/NetworkStatusEntry.java
+++ b/src/main/java/org/torproject/descriptor/NetworkStatusEntry.java
@@ -89,9 +89,9 @@ public interface NetworkStatusEntry {
public int getDirPort();
/**
- * Return the (possibly empty) set of microdescriptor digests if the
- * containing network status is a vote or microdesc consensus, or null
- * otherwise.
+ * Return the (possibly empty) set of microdescriptor digests, encoded as 43
+ * base64 characters without padding characters, if the containing network
+ * status is a vote or microdesc consensus, or null otherwise.
*
* @deprecated Renamed to {@link #getMicrodescriptorDigestsSha256Base64()}.
*
@@ -100,9 +100,9 @@ public interface NetworkStatusEntry {
public Set<String> getMicrodescriptorDigests();
/**
- * Return the (possibly empty) set of microdescriptor digests if the
- * containing network status is a vote or microdesc consensus, or null
- * otherwise.
+ * Return the (possibly empty) set of microdescriptor digests, encoded as 43
+ * base64 characters without padding characters, if the containing network
+ * status is a vote or microdesc consensus, or null otherwise.
*
* @since 1.7.0
*/
diff --git a/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java b/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java
index 987b530..337f0b2 100644
--- a/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java
@@ -141,11 +141,11 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry {
+ "fewer space-separated elements than expected.");
}
this.nickname = ParseHelper.parseNickname(line, parts[1]);
- this.fingerprint = ParseHelper.parseTwentyByteBase64String(line,
+ this.fingerprint = ParseHelper.convertTwentyByteBase64StringToHex(line,
parts[2]);
int descriptorOffset = 0;
if (!this.microdescConsensus) {
- this.descriptor = ParseHelper.parseTwentyByteBase64String(line,
+ this.descriptor = ParseHelper.convertTwentyByteBase64StringToHex(line,
parts[3]);
descriptorOffset = 1;
}
@@ -266,13 +266,13 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry {
this.microdescriptorDigests = new HashSet<>();
}
if (parts.length == 2) {
- this.microdescriptorDigests.add(
- ParseHelper.parseThirtyTwoByteBase64String(line, parts[1]));
+ ParseHelper.parseThirtyTwoByteBase64String(line, parts[1]);
+ this.microdescriptorDigests.add(parts[1]);
} else if (parts.length == 3 && parts[2].length() > 7) {
/* 7 == "sha256=".length() */
- this.microdescriptorDigests.add(
- ParseHelper.parseThirtyTwoByteBase64String(line,
- parts[2].substring(7)));
+ ParseHelper.parseThirtyTwoByteBase64String(line,
+ parts[2].substring(7));
+ this.microdescriptorDigests.add(parts[2].substring(7));
}
}
diff --git a/src/main/java/org/torproject/descriptor/impl/ParseHelper.java b/src/main/java/org/torproject/descriptor/impl/ParseHelper.java
index 31454e5..dec33f5 100644
--- a/src/main/java/org/torproject/descriptor/impl/ParseHelper.java
+++ b/src/main/java/org/torproject/descriptor/impl/ParseHelper.java
@@ -281,7 +281,12 @@ public class ParseHelper {
private static Pattern twentyByteBase64Pattern =
Pattern.compile("^[0-9a-zA-Z+/]{27}$");
- protected static String parseTwentyByteBase64String(String line,
+ protected static void parseTwentyByteBase64String(String line,
+ String base64String) throws DescriptorParseException {
+ convertTwentyByteBase64StringToHex(line, base64String);
+ }
+
+ protected static String convertTwentyByteBase64StringToHex(String line,
String base64String) throws DescriptorParseException {
if (!twentyByteBase64Pattern.matcher(base64String).matches()) {
throw new DescriptorParseException("'" + base64String
@@ -296,7 +301,12 @@ public class ParseHelper {
private static Pattern thirtyTwoByteBase64Pattern =
Pattern.compile("^[0-9a-zA-Z+/]{43}$");
- protected static String parseThirtyTwoByteBase64String(String line,
+ protected static void parseThirtyTwoByteBase64String(String line,
+ String base64String) throws DescriptorParseException {
+ convertThirtyTwoByteBase64StringToHex(line, base64String);
+ }
+
+ private static String convertThirtyTwoByteBase64StringToHex(String line,
String base64String) throws DescriptorParseException {
if (!thirtyTwoByteBase64Pattern.matcher(base64String).matches()) {
throw new DescriptorParseException("'" + base64String
1
0

21 Jun '17
commit 0236dc77715c272e7201e6111b98fbe4af747a4b
Author: Karsten Loesing <karsten.loesing(a)gmx.net>
Date: Mon Jun 19 11:42:18 2017 +0200
Rename two parse* methods to verify*.
---
.../torproject/descriptor/impl/ExtraInfoDescriptorImpl.java | 2 +-
.../org/torproject/descriptor/impl/MicrodescriptorImpl.java | 4 ++--
.../torproject/descriptor/impl/NetworkStatusEntryImpl.java | 6 +++---
.../java/org/torproject/descriptor/impl/ParseHelper.java | 13 +++++++++++--
.../torproject/descriptor/impl/ServerDescriptorImpl.java | 4 ++--
5 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
index 0d67af9..ca7d376 100644
--- a/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/ExtraInfoDescriptorImpl.java
@@ -828,7 +828,7 @@ public abstract class ExtraInfoDescriptorImpl extends DescriptorImpl
if (partsNoOpt.length != 2) {
throw new DescriptorParseException("Illegal line '" + line + "'.");
}
- ParseHelper.parseThirtyTwoByteBase64String(line, partsNoOpt[1]);
+ ParseHelper.verifyThirtyTwoByteBase64String(line, partsNoOpt[1]);
this.setDigestSha256Base64(partsNoOpt[1]);
}
diff --git a/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java
index 65c20d4..c436120 100644
--- a/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/MicrodescriptorImpl.java
@@ -198,11 +198,11 @@ public class MicrodescriptorImpl extends DescriptorImpl
} else {
switch (parts[1]) {
case "ed25519":
- ParseHelper.parseThirtyTwoByteBase64String(line, parts[2]);
+ ParseHelper.verifyThirtyTwoByteBase64String(line, parts[2]);
this.ed25519Identity = parts[2];
break;
case "rsa1024":
- ParseHelper.parseTwentyByteBase64String(line, parts[2]);
+ ParseHelper.verifyTwentyByteBase64String(line, parts[2]);
this.rsa1024Identity = parts[2];
break;
default:
diff --git a/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java b/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java
index 337f0b2..64c688e 100644
--- a/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/NetworkStatusEntryImpl.java
@@ -266,11 +266,11 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry {
this.microdescriptorDigests = new HashSet<>();
}
if (parts.length == 2) {
- ParseHelper.parseThirtyTwoByteBase64String(line, parts[1]);
+ ParseHelper.verifyThirtyTwoByteBase64String(line, parts[1]);
this.microdescriptorDigests.add(parts[1]);
} else if (parts.length == 3 && parts[2].length() > 7) {
/* 7 == "sha256=".length() */
- ParseHelper.parseThirtyTwoByteBase64String(line,
+ ParseHelper.verifyThirtyTwoByteBase64String(line,
parts[2].substring(7));
this.microdescriptorDigests.add(parts[2].substring(7));
}
@@ -283,7 +283,7 @@ public class NetworkStatusEntryImpl implements NetworkStatusEntry {
} else if ("none".equals(parts[2])) {
this.masterKeyEd25519 = "none";
} else {
- ParseHelper.parseThirtyTwoByteBase64String(line, parts[2]);
+ ParseHelper.verifyThirtyTwoByteBase64String(line, parts[2]);
this.masterKeyEd25519 = parts[2];
}
}
diff --git a/src/main/java/org/torproject/descriptor/impl/ParseHelper.java b/src/main/java/org/torproject/descriptor/impl/ParseHelper.java
index dec33f5..1c7aa93 100644
--- a/src/main/java/org/torproject/descriptor/impl/ParseHelper.java
+++ b/src/main/java/org/torproject/descriptor/impl/ParseHelper.java
@@ -21,6 +21,15 @@ import java.util.regex.Pattern;
import javax.xml.bind.DatatypeConverter;
+/**
+ * Parse helper for descriptor contents.
+ *
+ * <p>Naming convention: methods starting with {@code parse} return valid and
+ * sometimes changed data items (e.g. string to port int), methods starting with
+ * {@code verify} only check data items without returning them, and methods
+ * starting with {@code convert} return checked and explicitly changed data
+ * items (e.g. base64 to hex encoding).</p>
+ */
public class ParseHelper {
private static Pattern keywordPattern = Pattern.compile("^[A-Za-z0-9-]+$");
@@ -281,7 +290,7 @@ public class ParseHelper {
private static Pattern twentyByteBase64Pattern =
Pattern.compile("^[0-9a-zA-Z+/]{27}$");
- protected static void parseTwentyByteBase64String(String line,
+ protected static void verifyTwentyByteBase64String(String line,
String base64String) throws DescriptorParseException {
convertTwentyByteBase64StringToHex(line, base64String);
}
@@ -301,7 +310,7 @@ public class ParseHelper {
private static Pattern thirtyTwoByteBase64Pattern =
Pattern.compile("^[0-9a-zA-Z+/]{43}$");
- protected static void parseThirtyTwoByteBase64String(String line,
+ protected static void verifyThirtyTwoByteBase64String(String line,
String base64String) throws DescriptorParseException {
convertThirtyTwoByteBase64StringToHex(line, base64String);
}
diff --git a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java
index 428b2f0..d438702 100644
--- a/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java
+++ b/src/main/java/org/torproject/descriptor/impl/ServerDescriptorImpl.java
@@ -467,7 +467,7 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl
this.extraInfoDigest = ParseHelper.parseTwentyByteHexString(line,
partsNoOpt[1]);
if (partsNoOpt.length >= 3) {
- ParseHelper.parseThirtyTwoByteBase64String(line, partsNoOpt[2]);
+ ParseHelper.verifyThirtyTwoByteBase64String(line, partsNoOpt[2]);
this.extraInfoDigestSha256 = partsNoOpt[2];
}
}
@@ -671,7 +671,7 @@ public abstract class ServerDescriptorImpl extends DescriptorImpl
if (partsNoOpt.length != 2) {
throw new DescriptorParseException("Illegal line '" + line + "'.");
}
- ParseHelper.parseThirtyTwoByteBase64String(line, partsNoOpt[1]);
+ ParseHelper.verifyThirtyTwoByteBase64String(line, partsNoOpt[1]);
this.setDigestSha256Base64(partsNoOpt[1]);
}
1
0