commit fec3b3bb9389d9d9f927c370d560fc50ec65c93b
Author: Nick Mathewson <nickm(a)torproject.org>
Date: Sun Sep 30 18:15:00 2018 -0500
Extract logic for dumping unparseable junk from routerparse.c
---
src/core/include.am | 2 +
src/feature/dirparse/routerparse.c | 582 +-----------------------------------
src/feature/dirparse/routerparse.h | 19 --
src/feature/dirparse/unparseable.c | 591 +++++++++++++++++++++++++++++++++++++
src/feature/dirparse/unparseable.h | 44 +++
src/test/fuzz/fuzz_consensus.c | 1 +
src/test/fuzz/fuzz_descriptor.c | 1 +
src/test/fuzz/fuzz_extrainfo.c | 1 +
src/test/fuzz/fuzz_hsdescv2.c | 1 +
src/test/fuzz/fuzz_hsdescv3.c | 1 +
src/test/fuzz/fuzz_iptsv2.c | 1 +
src/test/fuzz/fuzz_microdesc.c | 1 +
src/test/fuzz/fuzz_vrs.c | 1 +
src/test/test_dir.c | 2 +
14 files changed, 651 insertions(+), 597 deletions(-)
diff --git a/src/core/include.am b/src/core/include.am
index aa8e29ad3..45f4cb5c4 100644
--- a/src/core/include.am
+++ b/src/core/include.am
@@ -74,6 +74,7 @@ LIBTOR_APP_A_SOURCES = \
src/feature/dircommon/voting_schedule.c \
src/feature/dirparse/parsecommon.c \
src/feature/dirparse/routerparse.c \
+ src/feature/dirparse/unparseable.c \
src/feature/hibernate/hibernate.c \
src/feature/hs/hs_cache.c \
src/feature/hs/hs_cell.c \
@@ -287,6 +288,7 @@ noinst_HEADERS += \
src/feature/dircommon/voting_schedule.h \
src/feature/dirparse/parsecommon.h \
src/feature/dirparse/routerparse.h \
+ src/feature/dirparse/unparseable.h \
src/feature/hibernate/hibernate.h \
src/feature/hs/hs_cache.h \
src/feature/hs/hs_cell.h \
diff --git a/src/feature/dirparse/routerparse.c b/src/feature/dirparse/routerparse.c
index ea5ac6f00..83890cdb9 100644
--- a/src/feature/dirparse/routerparse.c
+++ b/src/feature/dirparse/routerparse.c
@@ -82,6 +82,7 @@
#include "lib/crypt_ops/crypto_util.h"
#include "lib/memarea/memarea.h"
#include "lib/sandbox/sandbox.h"
+#include "feature/dirparse/unparseable.h"
#include "feature/dirauth/dirvote.h"
@@ -107,9 +108,9 @@
#undef log
#include <math.h>
-#ifdef HAVE_SYS_STAT_H
-#include <sys/stat.h>
-#endif
+//#ifdef HAVE_SYS_STAT_H
+//#include <sys/stat.h>
+//#endif
/****************************************************************************/
@@ -423,581 +424,6 @@ static int check_signature_token(const char *digest,
#define DUMP_AREA(a,name) STMT_NIL
#endif /* defined(DEBUG_AREA_ALLOC) */
-/* Dump mechanism for unparseable descriptors */
-
-/** List of dumped descriptors for FIFO cleanup purposes */
-STATIC smartlist_t *descs_dumped = NULL;
-/** Total size of dumped descriptors for FIFO cleanup */
-STATIC uint64_t len_descs_dumped = 0;
-/** Directory to stash dumps in */
-static int have_dump_desc_dir = 0;
-static int problem_with_dump_desc_dir = 0;
-
-#define DESC_DUMP_DATADIR_SUBDIR "unparseable-descs"
-#define DESC_DUMP_BASE_FILENAME "unparseable-desc"
-
-/** Find the dump directory and check if we'll be able to create it */
-static void
-dump_desc_init(void)
-{
- char *dump_desc_dir;
-
- dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
-
- /*
- * We just check for it, don't create it at this point; we'll
- * create it when we need it if it isn't already there.
- */
- if (check_private_dir(dump_desc_dir, CPD_CHECK, get_options()->User) < 0) {
- /* Error, log and flag it as having a problem */
- log_notice(LD_DIR,
- "Doesn't look like we'll be able to create descriptor dump "
- "directory %s; dumps will be disabled.",
- dump_desc_dir);
- problem_with_dump_desc_dir = 1;
- tor_free(dump_desc_dir);
- return;
- }
-
- /* Check if it exists */
- switch (file_status(dump_desc_dir)) {
- case FN_DIR:
- /* We already have a directory */
- have_dump_desc_dir = 1;
- break;
- case FN_NOENT:
- /* Nothing, we'll need to create it later */
- have_dump_desc_dir = 0;
- break;
- case FN_ERROR:
- /* Log and flag having a problem */
- log_notice(LD_DIR,
- "Couldn't check whether descriptor dump directory %s already"
- " exists: %s",
- dump_desc_dir, strerror(errno));
- problem_with_dump_desc_dir = 1;
- break;
- case FN_FILE:
- case FN_EMPTY:
- default:
- /* Something else was here! */
- log_notice(LD_DIR,
- "Descriptor dump directory %s already exists and isn't a "
- "directory",
- dump_desc_dir);
- problem_with_dump_desc_dir = 1;
- }
-
- if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
- dump_desc_populate_fifo_from_directory(dump_desc_dir);
- }
-
- tor_free(dump_desc_dir);
-}
-
-/** Create the dump directory if needed and possible */
-static void
-dump_desc_create_dir(void)
-{
- char *dump_desc_dir;
-
- /* If the problem flag is set, skip it */
- if (problem_with_dump_desc_dir) return;
-
- /* Do we need it? */
- if (!have_dump_desc_dir) {
- dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
-
- if (check_private_dir(dump_desc_dir, CPD_CREATE,
- get_options()->User) < 0) {
- log_notice(LD_DIR,
- "Failed to create descriptor dump directory %s",
- dump_desc_dir);
- problem_with_dump_desc_dir = 1;
- }
-
- /* Okay, we created it */
- have_dump_desc_dir = 1;
-
- tor_free(dump_desc_dir);
- }
-}
-
-/** Dump desc FIFO/cleanup; take ownership of the given filename, add it to
- * the FIFO, and clean up the oldest entries to the extent they exceed the
- * configured cap. If any old entries with a matching hash existed, they
- * just got overwritten right before this was called and we should adjust
- * the total size counter without deleting them.
- */
-static void
-dump_desc_fifo_add_and_clean(char *filename, const uint8_t *digest_sha256,
- size_t len)
-{
- dumped_desc_t *ent = NULL, *tmp;
- uint64_t max_len;
-
- tor_assert(filename != NULL);
- tor_assert(digest_sha256 != NULL);
-
- if (descs_dumped == NULL) {
- /* We better have no length, then */
- tor_assert(len_descs_dumped == 0);
- /* Make a smartlist */
- descs_dumped = smartlist_new();
- }
-
- /* Make a new entry to put this one in */
- ent = tor_malloc_zero(sizeof(*ent));
- ent->filename = filename;
- ent->len = len;
- ent->when = time(NULL);
- memcpy(ent->digest_sha256, digest_sha256, DIGEST256_LEN);
-
- /* Do we need to do some cleanup? */
- max_len = get_options()->MaxUnparseableDescSizeToLog;
- /* Iterate over the list until we've freed enough space */
- while (len > max_len - len_descs_dumped &&
- smartlist_len(descs_dumped) > 0) {
- /* Get the oldest thing on the list */
- tmp = (dumped_desc_t *)(smartlist_get(descs_dumped, 0));
-
- /*
- * Check if it matches the filename we just added, so we don't delete
- * something we just emitted if we get repeated identical descriptors.
- */
- if (strcmp(tmp->filename, filename) != 0) {
- /* Delete it and adjust the length counter */
- tor_unlink(tmp->filename);
- tor_assert(len_descs_dumped >= tmp->len);
- len_descs_dumped -= tmp->len;
- log_info(LD_DIR,
- "Deleting old unparseable descriptor dump %s due to "
- "space limits",
- tmp->filename);
- } else {
- /*
- * Don't delete, but do adjust the counter since we will bump it
- * later
- */
- tor_assert(len_descs_dumped >= tmp->len);
- len_descs_dumped -= tmp->len;
- log_info(LD_DIR,
- "Replacing old descriptor dump %s with new identical one",
- tmp->filename);
- }
-
- /* Free it and remove it from the list */
- smartlist_del_keeporder(descs_dumped, 0);
- tor_free(tmp->filename);
- tor_free(tmp);
- }
-
- /* Append our entry to the end of the list and bump the counter */
- smartlist_add(descs_dumped, ent);
- len_descs_dumped += len;
-}
-
-/** Check if we already have a descriptor for this hash and move it to the
- * head of the queue if so. Return 1 if one existed and 0 otherwise.
- */
-static int
-dump_desc_fifo_bump_hash(const uint8_t *digest_sha256)
-{
- dumped_desc_t *match = NULL;
-
- tor_assert(digest_sha256);
-
- if (descs_dumped) {
- /* Find a match if one exists */
- SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
- if (ent &&
- tor_memeq(ent->digest_sha256, digest_sha256, DIGEST256_LEN)) {
- /*
- * Save a pointer to the match and remove it from its current
- * position.
- */
- match = ent;
- SMARTLIST_DEL_CURRENT_KEEPORDER(descs_dumped, ent);
- break;
- }
- } SMARTLIST_FOREACH_END(ent);
-
- if (match) {
- /* Update the timestamp */
- match->when = time(NULL);
- /* Add it back at the end of the list */
- smartlist_add(descs_dumped, match);
-
- /* Indicate we found one */
- return 1;
- }
- }
-
- return 0;
-}
-
-/** Clean up on exit; just memory, leave the dumps behind
- */
-STATIC void
-dump_desc_fifo_cleanup(void)
-{
- if (descs_dumped) {
- /* Free each descriptor */
- SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
- tor_assert(ent);
- tor_free(ent->filename);
- tor_free(ent);
- } SMARTLIST_FOREACH_END(ent);
- /* Free the list */
- smartlist_free(descs_dumped);
- descs_dumped = NULL;
- len_descs_dumped = 0;
- }
-}
-
-/** Handle one file for dump_desc_populate_fifo_from_directory(); make sure
- * the filename is sensibly formed and matches the file content, and either
- * return a dumped_desc_t for it or remove the file and return NULL.
- */
-MOCK_IMPL(STATIC dumped_desc_t *,
-dump_desc_populate_one_file, (const char *dirname, const char *f))
-{
- dumped_desc_t *ent = NULL;
- char *path = NULL, *desc = NULL;
- const char *digest_str;
- char digest[DIGEST256_LEN], content_digest[DIGEST256_LEN];
- /* Expected prefix before digest in filenames */
- const char *f_pfx = DESC_DUMP_BASE_FILENAME ".";
- /*
- * Stat while reading; this is important in case the file
- * contains a NUL character.
- */
- struct stat st;
-
- /* Sanity-check args */
- tor_assert(dirname != NULL);
- tor_assert(f != NULL);
-
- /* Form the full path */
- tor_asprintf(&path, "%s" PATH_SEPARATOR "%s", dirname, f);
-
- /* Check that f has the form DESC_DUMP_BASE_FILENAME.<digest256> */
-
- if (!strcmpstart(f, f_pfx)) {
- /* It matches the form, but is the digest parseable as such? */
- digest_str = f + strlen(f_pfx);
- if (base16_decode(digest, DIGEST256_LEN,
- digest_str, strlen(digest_str)) != DIGEST256_LEN) {
- /* We failed to decode it */
- digest_str = NULL;
- }
- } else {
- /* No match */
- digest_str = NULL;
- }
-
- if (!digest_str) {
- /* We couldn't get a sensible digest */
- log_notice(LD_DIR,
- "Removing unrecognized filename %s from unparseable "
- "descriptors directory", f);
- tor_unlink(path);
- /* We're done */
- goto done;
- }
-
- /*
- * The filename has the form DESC_DUMP_BASE_FILENAME "." <digest256> and
- * we've decoded the digest. Next, check that we can read it and the
- * content matches this digest. We are relying on the fact that if the
- * file contains a '\0', read_file_to_str() will allocate space for and
- * read the entire file and return the correct size in st.
- */
- desc = read_file_to_str(path, RFTS_IGNORE_MISSING|RFTS_BIN, &st);
- if (!desc) {
- /* We couldn't read it */
- log_notice(LD_DIR,
- "Failed to read %s from unparseable descriptors directory; "
- "attempting to remove it.", f);
- tor_unlink(path);
- /* We're done */
- goto done;
- }
-
-#if SIZE_MAX > UINT64_MAX
- if (BUG((uint64_t)st.st_size > (uint64_t)SIZE_MAX)) {
- /* LCOV_EXCL_START
- * Should be impossible since RFTS above should have failed to read the
- * huge file into RAM. */
- goto done;
- /* LCOV_EXCL_STOP */
- }
-#endif /* SIZE_MAX > UINT64_MAX */
- if (BUG(st.st_size < 0)) {
- /* LCOV_EXCL_START
- * Should be impossible, since the OS isn't supposed to be b0rken. */
- goto done;
- /* LCOV_EXCL_STOP */
- }
- /* (Now we can be sure that st.st_size is safe to cast to a size_t.) */
-
- /*
- * We got one; now compute its digest and check that it matches the
- * filename.
- */
- if (crypto_digest256((char *)content_digest, desc, (size_t) st.st_size,
- DIGEST_SHA256) < 0) {
- /* Weird, but okay */
- log_info(LD_DIR,
- "Unable to hash content of %s from unparseable descriptors "
- "directory", f);
- tor_unlink(path);
- /* We're done */
- goto done;
- }
-
- /* Compare the digests */
- if (tor_memneq(digest, content_digest, DIGEST256_LEN)) {
- /* No match */
- log_info(LD_DIR,
- "Hash of %s from unparseable descriptors directory didn't "
- "match its filename; removing it", f);
- tor_unlink(path);
- /* We're done */
- goto done;
- }
-
- /* Okay, it's a match, we should prepare ent */
- ent = tor_malloc_zero(sizeof(dumped_desc_t));
- ent->filename = path;
- memcpy(ent->digest_sha256, digest, DIGEST256_LEN);
- ent->len = (size_t) st.st_size;
- ent->when = st.st_mtime;
- /* Null out path so we don't free it out from under ent */
- path = NULL;
-
- done:
- /* Free allocations if we had them */
- tor_free(desc);
- tor_free(path);
-
- return ent;
-}
-
-/** Sort helper for dump_desc_populate_fifo_from_directory(); compares
- * the when field of dumped_desc_ts in a smartlist to put the FIFO in
- * the correct order after reconstructing it from the directory.
- */
-static int
-dump_desc_compare_fifo_entries(const void **a_v, const void **b_v)
-{
- const dumped_desc_t **a = (const dumped_desc_t **)a_v;
- const dumped_desc_t **b = (const dumped_desc_t **)b_v;
-
- if ((a != NULL) && (*a != NULL)) {
- if ((b != NULL) && (*b != NULL)) {
- /* We have sensible dumped_desc_ts to compare */
- if ((*a)->when < (*b)->when) {
- return -1;
- } else if ((*a)->when == (*b)->when) {
- return 0;
- } else {
- return 1;
- }
- } else {
- /*
- * We shouldn't see this, but what the hell, NULLs precede everythin
- * else
- */
- return 1;
- }
- } else {
- return -1;
- }
-}
-
-/** Scan the contents of the directory, and update FIFO/counters; this will
- * consistency-check descriptor dump filenames against hashes of descriptor
- * dump file content, and remove any inconsistent/unreadable dumps, and then
- * reconstruct the dump FIFO as closely as possible for the last time the
- * tor process shut down. If a previous dump was repeated more than once and
- * moved ahead in the FIFO, the mtime will not have been updated and the
- * reconstructed order will be wrong, but will always be a permutation of
- * the original.
- */
-STATIC void
-dump_desc_populate_fifo_from_directory(const char *dirname)
-{
- smartlist_t *files = NULL;
- dumped_desc_t *ent = NULL;
-
- tor_assert(dirname != NULL);
-
- /* Get a list of files */
- files = tor_listdir(dirname);
- if (!files) {
- log_notice(LD_DIR,
- "Unable to get contents of unparseable descriptor dump "
- "directory %s",
- dirname);
- return;
- }
-
- /*
- * Iterate through the list and decide which files should go in the
- * FIFO and which should be purged.
- */
-
- SMARTLIST_FOREACH_BEGIN(files, char *, f) {
- /* Try to get a FIFO entry */
- ent = dump_desc_populate_one_file(dirname, f);
- if (ent) {
- /*
- * We got one; add it to the FIFO. No need for duplicate checking
- * here since we just verified the name and digest match.
- */
-
- /* Make sure we have a list to add it to */
- if (!descs_dumped) {
- descs_dumped = smartlist_new();
- len_descs_dumped = 0;
- }
-
- /* Add it and adjust the counter */
- smartlist_add(descs_dumped, ent);
- len_descs_dumped += ent->len;
- }
- /*
- * If we didn't, we will have unlinked the file if necessary and
- * possible, and emitted a log message about it, so just go on to
- * the next.
- */
- } SMARTLIST_FOREACH_END(f);
-
- /* Did we get anything? */
- if (descs_dumped != NULL) {
- /* Sort the FIFO in order of increasing timestamp */
- smartlist_sort(descs_dumped, dump_desc_compare_fifo_entries);
-
- /* Log some stats */
- log_info(LD_DIR,
- "Reloaded unparseable descriptor dump FIFO with %d dump(s) "
- "totaling %"PRIu64 " bytes",
- smartlist_len(descs_dumped), (len_descs_dumped));
- }
-
- /* Free the original list */
- SMARTLIST_FOREACH(files, char *, f, tor_free(f));
- smartlist_free(files);
-}
-
-/** For debugging purposes, dump unparseable descriptor *<b>desc</b> of
- * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more
- * than one descriptor to disk per minute. If there is already such a
- * file in the data directory, overwrite it. */
-MOCK_IMPL(STATIC void,
-dump_desc,(const char *desc, const char *type))
-{
- tor_assert(desc);
- tor_assert(type);
- size_t len;
- /* The SHA256 of the string */
- uint8_t digest_sha256[DIGEST256_LEN];
- char digest_sha256_hex[HEX_DIGEST256_LEN+1];
- /* Filename to log it to */
- char *debugfile, *debugfile_base;
-
- /* Get the hash for logging purposes anyway */
- len = strlen(desc);
- if (crypto_digest256((char *)digest_sha256, desc, len,
- DIGEST_SHA256) < 0) {
- log_info(LD_DIR,
- "Unable to parse descriptor of type %s, and unable to even hash"
- " it!", type);
- goto err;
- }
-
- base16_encode(digest_sha256_hex, sizeof(digest_sha256_hex),
- (const char *)digest_sha256, sizeof(digest_sha256));
-
- /*
- * We mention type and hash in the main log; don't clutter up the files
- * with anything but the exact dump.
- */
- tor_asprintf(&debugfile_base,
- DESC_DUMP_BASE_FILENAME ".%s", digest_sha256_hex);
- debugfile = get_datadir_fname2(DESC_DUMP_DATADIR_SUBDIR, debugfile_base);
-
- /*
- * Check if the sandbox is active or will become active; see comment
- * below at the log message for why.
- */
- if (!(sandbox_is_active() || get_options()->Sandbox)) {
- if (len <= get_options()->MaxUnparseableDescSizeToLog) {
- if (!dump_desc_fifo_bump_hash(digest_sha256)) {
- /* Create the directory if needed */
- dump_desc_create_dir();
- /* Make sure we've got it */
- if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
- /* Write it, and tell the main log about it */
- write_str_to_file(debugfile, desc, 1);
- log_info(LD_DIR,
- "Unable to parse descriptor of type %s with hash %s and "
- "length %lu. See file %s in data directory for details.",
- type, digest_sha256_hex, (unsigned long)len,
- debugfile_base);
- dump_desc_fifo_add_and_clean(debugfile, digest_sha256, len);
- /* Since we handed ownership over, don't free debugfile later */
- debugfile = NULL;
- } else {
- /* Problem with the subdirectory */
- log_info(LD_DIR,
- "Unable to parse descriptor of type %s with hash %s and "
- "length %lu. Descriptor not dumped because we had a "
- "problem creating the " DESC_DUMP_DATADIR_SUBDIR
- " subdirectory",
- type, digest_sha256_hex, (unsigned long)len);
- /* We do have to free debugfile in this case */
- }
- } else {
- /* We already had one with this hash dumped */
- log_info(LD_DIR,
- "Unable to parse descriptor of type %s with hash %s and "
- "length %lu. Descriptor not dumped because one with that "
- "hash has already been dumped.",
- type, digest_sha256_hex, (unsigned long)len);
- /* We do have to free debugfile in this case */
- }
- } else {
- /* Just log that it happened without dumping */
- log_info(LD_DIR,
- "Unable to parse descriptor of type %s with hash %s and "
- "length %lu. Descriptor not dumped because it exceeds maximum"
- " log size all by itself.",
- type, digest_sha256_hex, (unsigned long)len);
- /* We do have to free debugfile in this case */
- }
- } else {
- /*
- * Not logging because the sandbox is active and seccomp2 apparently
- * doesn't have a sensible way to allow filenames according to a pattern
- * match. (If we ever figure out how to say "allow writes to /regex/",
- * remove this checK).
- */
- log_info(LD_DIR,
- "Unable to parse descriptor of type %s with hash %s and "
- "length %lu. Descriptor not dumped because the sandbox is "
- "configured",
- type, digest_sha256_hex, (unsigned long)len);
- }
-
- tor_free(debugfile_base);
- tor_free(debugfile);
-
- err:
- return;
-}
-
/** Set <b>digest</b> to the SHA-1 digest of the hash of the directory in
* <b>s</b>. Return 0 on success, -1 on failure.
*/
diff --git a/src/feature/dirparse/routerparse.h b/src/feature/dirparse/routerparse.h
index 87c2a75aa..51d39c617 100644
--- a/src/feature/dirparse/routerparse.h
+++ b/src/feature/dirparse/routerparse.h
@@ -114,28 +114,10 @@ void routerparse_init(void);
void routerparse_free_all(void);
#ifdef ROUTERPARSE_PRIVATE
-/*
- * One entry in the list of dumped descriptors; filename dumped to, length,
- * SHA-256 and timestamp.
- */
-
-typedef struct {
- char *filename;
- size_t len;
- uint8_t digest_sha256[DIGEST256_LEN];
- time_t when;
-} dumped_desc_t;
-
-EXTERN(uint64_t, len_descs_dumped)
-EXTERN(smartlist_t *, descs_dumped)
STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str,
networkstatus_t *vote,
vote_routerstatus_t *vote_rs,
routerstatus_t *rs);
-MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file,
- (const char *dirname, const char *f));
-STATIC void dump_desc_populate_fifo_from_directory(const char *dirname);
-STATIC void dump_desc_fifo_cleanup(void);
struct memarea_t;
STATIC routerstatus_t *routerstatus_parse_entry_from_string(
struct memarea_t *area,
@@ -144,7 +126,6 @@ STATIC routerstatus_t *routerstatus_parse_entry_from_string(
vote_routerstatus_t *vote_rs,
int consensus_method,
consensus_flavor_t flav);
-MOCK_DECL(STATIC void,dump_desc,(const char *desc, const char *type));
MOCK_DECL(STATIC int, router_compute_hash_final,(char *digest,
const char *start, size_t len,
digest_algorithm_t alg));
diff --git a/src/feature/dirparse/unparseable.c b/src/feature/dirparse/unparseable.c
new file mode 100644
index 000000000..80e38d070
--- /dev/null
+++ b/src/feature/dirparse/unparseable.c
@@ -0,0 +1,591 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define UNPARSEABLE_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/dirparse/unparseable.h"
+#include "lib/sandbox/sandbox.h"
+
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+/* Dump mechanism for unparseable descriptors */
+
+/** List of dumped descriptors for FIFO cleanup purposes */
+STATIC smartlist_t *descs_dumped = NULL;
+/** Total size of dumped descriptors for FIFO cleanup */
+STATIC uint64_t len_descs_dumped = 0;
+/** Directory to stash dumps in */
+static int have_dump_desc_dir = 0;
+static int problem_with_dump_desc_dir = 0;
+
+#define DESC_DUMP_DATADIR_SUBDIR "unparseable-descs"
+#define DESC_DUMP_BASE_FILENAME "unparseable-desc"
+
+/** Find the dump directory and check if we'll be able to create it */
+void
+dump_desc_init(void)
+{
+ char *dump_desc_dir;
+
+ dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
+
+ /*
+ * We just check for it, don't create it at this point; we'll
+ * create it when we need it if it isn't already there.
+ */
+ if (check_private_dir(dump_desc_dir, CPD_CHECK, get_options()->User) < 0) {
+ /* Error, log and flag it as having a problem */
+ log_notice(LD_DIR,
+ "Doesn't look like we'll be able to create descriptor dump "
+ "directory %s; dumps will be disabled.",
+ dump_desc_dir);
+ problem_with_dump_desc_dir = 1;
+ tor_free(dump_desc_dir);
+ return;
+ }
+
+ /* Check if it exists */
+ switch (file_status(dump_desc_dir)) {
+ case FN_DIR:
+ /* We already have a directory */
+ have_dump_desc_dir = 1;
+ break;
+ case FN_NOENT:
+ /* Nothing, we'll need to create it later */
+ have_dump_desc_dir = 0;
+ break;
+ case FN_ERROR:
+ /* Log and flag having a problem */
+ log_notice(LD_DIR,
+ "Couldn't check whether descriptor dump directory %s already"
+ " exists: %s",
+ dump_desc_dir, strerror(errno));
+ problem_with_dump_desc_dir = 1;
+ break;
+ case FN_FILE:
+ case FN_EMPTY:
+ default:
+ /* Something else was here! */
+ log_notice(LD_DIR,
+ "Descriptor dump directory %s already exists and isn't a "
+ "directory",
+ dump_desc_dir);
+ problem_with_dump_desc_dir = 1;
+ }
+
+ if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
+ dump_desc_populate_fifo_from_directory(dump_desc_dir);
+ }
+
+ tor_free(dump_desc_dir);
+}
+
+/** Create the dump directory if needed and possible */
+static void
+dump_desc_create_dir(void)
+{
+ char *dump_desc_dir;
+
+ /* If the problem flag is set, skip it */
+ if (problem_with_dump_desc_dir) return;
+
+ /* Do we need it? */
+ if (!have_dump_desc_dir) {
+ dump_desc_dir = get_datadir_fname(DESC_DUMP_DATADIR_SUBDIR);
+
+ if (check_private_dir(dump_desc_dir, CPD_CREATE,
+ get_options()->User) < 0) {
+ log_notice(LD_DIR,
+ "Failed to create descriptor dump directory %s",
+ dump_desc_dir);
+ problem_with_dump_desc_dir = 1;
+ }
+
+ /* Okay, we created it */
+ have_dump_desc_dir = 1;
+
+ tor_free(dump_desc_dir);
+ }
+}
+
+/** Dump desc FIFO/cleanup; take ownership of the given filename, add it to
+ * the FIFO, and clean up the oldest entries to the extent they exceed the
+ * configured cap. If any old entries with a matching hash existed, they
+ * just got overwritten right before this was called and we should adjust
+ * the total size counter without deleting them.
+ */
+static void
+dump_desc_fifo_add_and_clean(char *filename, const uint8_t *digest_sha256,
+ size_t len)
+{
+ dumped_desc_t *ent = NULL, *tmp;
+ uint64_t max_len;
+
+ tor_assert(filename != NULL);
+ tor_assert(digest_sha256 != NULL);
+
+ if (descs_dumped == NULL) {
+ /* We better have no length, then */
+ tor_assert(len_descs_dumped == 0);
+ /* Make a smartlist */
+ descs_dumped = smartlist_new();
+ }
+
+ /* Make a new entry to put this one in */
+ ent = tor_malloc_zero(sizeof(*ent));
+ ent->filename = filename;
+ ent->len = len;
+ ent->when = time(NULL);
+ memcpy(ent->digest_sha256, digest_sha256, DIGEST256_LEN);
+
+ /* Do we need to do some cleanup? */
+ max_len = get_options()->MaxUnparseableDescSizeToLog;
+ /* Iterate over the list until we've freed enough space */
+ while (len > max_len - len_descs_dumped &&
+ smartlist_len(descs_dumped) > 0) {
+ /* Get the oldest thing on the list */
+ tmp = (dumped_desc_t *)(smartlist_get(descs_dumped, 0));
+
+ /*
+ * Check if it matches the filename we just added, so we don't delete
+ * something we just emitted if we get repeated identical descriptors.
+ */
+ if (strcmp(tmp->filename, filename) != 0) {
+ /* Delete it and adjust the length counter */
+ tor_unlink(tmp->filename);
+ tor_assert(len_descs_dumped >= tmp->len);
+ len_descs_dumped -= tmp->len;
+ log_info(LD_DIR,
+ "Deleting old unparseable descriptor dump %s due to "
+ "space limits",
+ tmp->filename);
+ } else {
+ /*
+ * Don't delete, but do adjust the counter since we will bump it
+ * later
+ */
+ tor_assert(len_descs_dumped >= tmp->len);
+ len_descs_dumped -= tmp->len;
+ log_info(LD_DIR,
+ "Replacing old descriptor dump %s with new identical one",
+ tmp->filename);
+ }
+
+ /* Free it and remove it from the list */
+ smartlist_del_keeporder(descs_dumped, 0);
+ tor_free(tmp->filename);
+ tor_free(tmp);
+ }
+
+ /* Append our entry to the end of the list and bump the counter */
+ smartlist_add(descs_dumped, ent);
+ len_descs_dumped += len;
+}
+
+/** Check if we already have a descriptor for this hash and move it to the
+ * head of the queue if so. Return 1 if one existed and 0 otherwise.
+ */
+static int
+dump_desc_fifo_bump_hash(const uint8_t *digest_sha256)
+{
+ dumped_desc_t *match = NULL;
+
+ tor_assert(digest_sha256);
+
+ if (descs_dumped) {
+ /* Find a match if one exists */
+ SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
+ if (ent &&
+ tor_memeq(ent->digest_sha256, digest_sha256, DIGEST256_LEN)) {
+ /*
+ * Save a pointer to the match and remove it from its current
+ * position.
+ */
+ match = ent;
+ SMARTLIST_DEL_CURRENT_KEEPORDER(descs_dumped, ent);
+ break;
+ }
+ } SMARTLIST_FOREACH_END(ent);
+
+ if (match) {
+ /* Update the timestamp */
+ match->when = time(NULL);
+ /* Add it back at the end of the list */
+ smartlist_add(descs_dumped, match);
+
+ /* Indicate we found one */
+ return 1;
+ }
+ }
+
+ return 0;
+}
+
+/** Clean up on exit; just memory, leave the dumps behind
+ */
+void
+dump_desc_fifo_cleanup(void)
+{
+ if (descs_dumped) {
+ /* Free each descriptor */
+ SMARTLIST_FOREACH_BEGIN(descs_dumped, dumped_desc_t *, ent) {
+ tor_assert(ent);
+ tor_free(ent->filename);
+ tor_free(ent);
+ } SMARTLIST_FOREACH_END(ent);
+ /* Free the list */
+ smartlist_free(descs_dumped);
+ descs_dumped = NULL;
+ len_descs_dumped = 0;
+ }
+}
+
+/** Handle one file for dump_desc_populate_fifo_from_directory(); make sure
+ * the filename is sensibly formed and matches the file content, and either
+ * return a dumped_desc_t for it or remove the file and return NULL.
+ */
+MOCK_IMPL(STATIC dumped_desc_t *,
+dump_desc_populate_one_file, (const char *dirname, const char *f))
+{
+ dumped_desc_t *ent = NULL;
+ char *path = NULL, *desc = NULL;
+ const char *digest_str;
+ char digest[DIGEST256_LEN], content_digest[DIGEST256_LEN];
+ /* Expected prefix before digest in filenames */
+ const char *f_pfx = DESC_DUMP_BASE_FILENAME ".";
+ /*
+ * Stat while reading; this is important in case the file
+ * contains a NUL character.
+ */
+ struct stat st;
+
+ /* Sanity-check args */
+ tor_assert(dirname != NULL);
+ tor_assert(f != NULL);
+
+ /* Form the full path */
+ tor_asprintf(&path, "%s" PATH_SEPARATOR "%s", dirname, f);
+
+ /* Check that f has the form DESC_DUMP_BASE_FILENAME.<digest256> */
+
+ if (!strcmpstart(f, f_pfx)) {
+ /* It matches the form, but is the digest parseable as such? */
+ digest_str = f + strlen(f_pfx);
+ if (base16_decode(digest, DIGEST256_LEN,
+ digest_str, strlen(digest_str)) != DIGEST256_LEN) {
+ /* We failed to decode it */
+ digest_str = NULL;
+ }
+ } else {
+ /* No match */
+ digest_str = NULL;
+ }
+
+ if (!digest_str) {
+ /* We couldn't get a sensible digest */
+ log_notice(LD_DIR,
+ "Removing unrecognized filename %s from unparseable "
+ "descriptors directory", f);
+ tor_unlink(path);
+ /* We're done */
+ goto done;
+ }
+
+ /*
+ * The filename has the form DESC_DUMP_BASE_FILENAME "." <digest256> and
+ * we've decoded the digest. Next, check that we can read it and the
+ * content matches this digest. We are relying on the fact that if the
+ * file contains a '\0', read_file_to_str() will allocate space for and
+ * read the entire file and return the correct size in st.
+ */
+ desc = read_file_to_str(path, RFTS_IGNORE_MISSING|RFTS_BIN, &st);
+ if (!desc) {
+ /* We couldn't read it */
+ log_notice(LD_DIR,
+ "Failed to read %s from unparseable descriptors directory; "
+ "attempting to remove it.", f);
+ tor_unlink(path);
+ /* We're done */
+ goto done;
+ }
+
+#if SIZE_MAX > UINT64_MAX
+ if (BUG((uint64_t)st.st_size > (uint64_t)SIZE_MAX)) {
+ /* LCOV_EXCL_START
+ * Should be impossible since RFTS above should have failed to read the
+ * huge file into RAM. */
+ goto done;
+ /* LCOV_EXCL_STOP */
+ }
+#endif /* SIZE_MAX > UINT64_MAX */
+ if (BUG(st.st_size < 0)) {
+ /* LCOV_EXCL_START
+ * Should be impossible, since the OS isn't supposed to be b0rken. */
+ goto done;
+ /* LCOV_EXCL_STOP */
+ }
+ /* (Now we can be sure that st.st_size is safe to cast to a size_t.) */
+
+ /*
+ * We got one; now compute its digest and check that it matches the
+ * filename.
+ */
+ if (crypto_digest256((char *)content_digest, desc, (size_t) st.st_size,
+ DIGEST_SHA256) < 0) {
+ /* Weird, but okay */
+ log_info(LD_DIR,
+ "Unable to hash content of %s from unparseable descriptors "
+ "directory", f);
+ tor_unlink(path);
+ /* We're done */
+ goto done;
+ }
+
+ /* Compare the digests */
+ if (tor_memneq(digest, content_digest, DIGEST256_LEN)) {
+ /* No match */
+ log_info(LD_DIR,
+ "Hash of %s from unparseable descriptors directory didn't "
+ "match its filename; removing it", f);
+ tor_unlink(path);
+ /* We're done */
+ goto done;
+ }
+
+ /* Okay, it's a match, we should prepare ent */
+ ent = tor_malloc_zero(sizeof(dumped_desc_t));
+ ent->filename = path;
+ memcpy(ent->digest_sha256, digest, DIGEST256_LEN);
+ ent->len = (size_t) st.st_size;
+ ent->when = st.st_mtime;
+ /* Null out path so we don't free it out from under ent */
+ path = NULL;
+
+ done:
+ /* Free allocations if we had them */
+ tor_free(desc);
+ tor_free(path);
+
+ return ent;
+}
+
+/** Sort helper for dump_desc_populate_fifo_from_directory(); compares
+ * the when field of dumped_desc_ts in a smartlist to put the FIFO in
+ * the correct order after reconstructing it from the directory.
+ */
+static int
+dump_desc_compare_fifo_entries(const void **a_v, const void **b_v)
+{
+ const dumped_desc_t **a = (const dumped_desc_t **)a_v;
+ const dumped_desc_t **b = (const dumped_desc_t **)b_v;
+
+ if ((a != NULL) && (*a != NULL)) {
+ if ((b != NULL) && (*b != NULL)) {
+ /* We have sensible dumped_desc_ts to compare */
+ if ((*a)->when < (*b)->when) {
+ return -1;
+ } else if ((*a)->when == (*b)->when) {
+ return 0;
+ } else {
+ return 1;
+ }
+ } else {
+ /*
+ * We shouldn't see this, but what the hell, NULLs precede everythin
+ * else
+ */
+ return 1;
+ }
+ } else {
+ return -1;
+ }
+}
+
+/** Scan the contents of the directory, and update FIFO/counters; this will
+ * consistency-check descriptor dump filenames against hashes of descriptor
+ * dump file content, and remove any inconsistent/unreadable dumps, and then
+ * reconstruct the dump FIFO as closely as possible for the last time the
+ * tor process shut down. If a previous dump was repeated more than once and
+ * moved ahead in the FIFO, the mtime will not have been updated and the
+ * reconstructed order will be wrong, but will always be a permutation of
+ * the original.
+ */
+STATIC void
+dump_desc_populate_fifo_from_directory(const char *dirname)
+{
+ smartlist_t *files = NULL;
+ dumped_desc_t *ent = NULL;
+
+ tor_assert(dirname != NULL);
+
+ /* Get a list of files */
+ files = tor_listdir(dirname);
+ if (!files) {
+ log_notice(LD_DIR,
+ "Unable to get contents of unparseable descriptor dump "
+ "directory %s",
+ dirname);
+ return;
+ }
+
+ /*
+ * Iterate through the list and decide which files should go in the
+ * FIFO and which should be purged.
+ */
+
+ SMARTLIST_FOREACH_BEGIN(files, char *, f) {
+ /* Try to get a FIFO entry */
+ ent = dump_desc_populate_one_file(dirname, f);
+ if (ent) {
+ /*
+ * We got one; add it to the FIFO. No need for duplicate checking
+ * here since we just verified the name and digest match.
+ */
+
+ /* Make sure we have a list to add it to */
+ if (!descs_dumped) {
+ descs_dumped = smartlist_new();
+ len_descs_dumped = 0;
+ }
+
+ /* Add it and adjust the counter */
+ smartlist_add(descs_dumped, ent);
+ len_descs_dumped += ent->len;
+ }
+ /*
+ * If we didn't, we will have unlinked the file if necessary and
+ * possible, and emitted a log message about it, so just go on to
+ * the next.
+ */
+ } SMARTLIST_FOREACH_END(f);
+
+ /* Did we get anything? */
+ if (descs_dumped != NULL) {
+ /* Sort the FIFO in order of increasing timestamp */
+ smartlist_sort(descs_dumped, dump_desc_compare_fifo_entries);
+
+ /* Log some stats */
+ log_info(LD_DIR,
+ "Reloaded unparseable descriptor dump FIFO with %d dump(s) "
+ "totaling %"PRIu64 " bytes",
+ smartlist_len(descs_dumped), (len_descs_dumped));
+ }
+
+ /* Free the original list */
+ SMARTLIST_FOREACH(files, char *, f, tor_free(f));
+ smartlist_free(files);
+}
+
+/** For debugging purposes, dump unparseable descriptor *<b>desc</b> of
+ * type *<b>type</b> to file $DATADIR/unparseable-desc. Do not write more
+ * than one descriptor to disk per minute. If there is already such a
+ * file in the data directory, overwrite it. */
+MOCK_IMPL(void,
+dump_desc,(const char *desc, const char *type))
+{
+ tor_assert(desc);
+ tor_assert(type);
+ size_t len;
+ /* The SHA256 of the string */
+ uint8_t digest_sha256[DIGEST256_LEN];
+ char digest_sha256_hex[HEX_DIGEST256_LEN+1];
+ /* Filename to log it to */
+ char *debugfile, *debugfile_base;
+
+ /* Get the hash for logging purposes anyway */
+ len = strlen(desc);
+ if (crypto_digest256((char *)digest_sha256, desc, len,
+ DIGEST_SHA256) < 0) {
+ log_info(LD_DIR,
+ "Unable to parse descriptor of type %s, and unable to even hash"
+ " it!", type);
+ goto err;
+ }
+
+ base16_encode(digest_sha256_hex, sizeof(digest_sha256_hex),
+ (const char *)digest_sha256, sizeof(digest_sha256));
+
+ /*
+ * We mention type and hash in the main log; don't clutter up the files
+ * with anything but the exact dump.
+ */
+ tor_asprintf(&debugfile_base,
+ DESC_DUMP_BASE_FILENAME ".%s", digest_sha256_hex);
+ debugfile = get_datadir_fname2(DESC_DUMP_DATADIR_SUBDIR, debugfile_base);
+
+ /*
+ * Check if the sandbox is active or will become active; see comment
+ * below at the log message for why.
+ */
+ if (!(sandbox_is_active() || get_options()->Sandbox)) {
+ if (len <= get_options()->MaxUnparseableDescSizeToLog) {
+ if (!dump_desc_fifo_bump_hash(digest_sha256)) {
+ /* Create the directory if needed */
+ dump_desc_create_dir();
+ /* Make sure we've got it */
+ if (have_dump_desc_dir && !problem_with_dump_desc_dir) {
+ /* Write it, and tell the main log about it */
+ write_str_to_file(debugfile, desc, 1);
+ log_info(LD_DIR,
+ "Unable to parse descriptor of type %s with hash %s and "
+ "length %lu. See file %s in data directory for details.",
+ type, digest_sha256_hex, (unsigned long)len,
+ debugfile_base);
+ dump_desc_fifo_add_and_clean(debugfile, digest_sha256, len);
+ /* Since we handed ownership over, don't free debugfile later */
+ debugfile = NULL;
+ } else {
+ /* Problem with the subdirectory */
+ log_info(LD_DIR,
+ "Unable to parse descriptor of type %s with hash %s and "
+ "length %lu. Descriptor not dumped because we had a "
+ "problem creating the " DESC_DUMP_DATADIR_SUBDIR
+ " subdirectory",
+ type, digest_sha256_hex, (unsigned long)len);
+ /* We do have to free debugfile in this case */
+ }
+ } else {
+ /* We already had one with this hash dumped */
+ log_info(LD_DIR,
+ "Unable to parse descriptor of type %s with hash %s and "
+ "length %lu. Descriptor not dumped because one with that "
+ "hash has already been dumped.",
+ type, digest_sha256_hex, (unsigned long)len);
+ /* We do have to free debugfile in this case */
+ }
+ } else {
+ /* Just log that it happened without dumping */
+ log_info(LD_DIR,
+ "Unable to parse descriptor of type %s with hash %s and "
+ "length %lu. Descriptor not dumped because it exceeds maximum"
+ " log size all by itself.",
+ type, digest_sha256_hex, (unsigned long)len);
+ /* We do have to free debugfile in this case */
+ }
+ } else {
+ /*
+ * Not logging because the sandbox is active and seccomp2 apparently
+ * doesn't have a sensible way to allow filenames according to a pattern
+ * match. (If we ever figure out how to say "allow writes to /regex/",
+ * remove this checK).
+ */
+ log_info(LD_DIR,
+ "Unable to parse descriptor of type %s with hash %s and "
+ "length %lu. Descriptor not dumped because the sandbox is "
+ "configured",
+ type, digest_sha256_hex, (unsigned long)len);
+ }
+
+ tor_free(debugfile_base);
+ tor_free(debugfile);
+
+ err:
+ return;
+}
diff --git a/src/feature/dirparse/unparseable.h b/src/feature/dirparse/unparseable.h
new file mode 100644
index 000000000..831ab6777
--- /dev/null
+++ b/src/feature/dirparse/unparseable.h
@@ -0,0 +1,44 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2018, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file unparseable.h
+ * \brief Header file for unparseable.c.
+ **/
+
+#ifndef TOR_UNPARSEABLE_H
+#define TOR_UNPARSEABLE_H
+
+#include "lib/cc/torint.h"
+
+MOCK_DECL(void,dump_desc,(const char *desc, const char *type));
+void dump_desc_fifo_cleanup(void);
+void dump_desc_init(void);
+
+#ifdef UNPARSEABLE_PRIVATE
+
+/*
+ * One entry in the list of dumped descriptors; filename dumped to, length,
+ * SHA-256 and timestamp.
+ */
+
+typedef struct {
+ char *filename;
+ size_t len;
+ uint8_t digest_sha256[DIGEST256_LEN];
+ time_t when;
+} dumped_desc_t;
+struct smartlist_t;
+
+EXTERN(uint64_t, len_descs_dumped)
+EXTERN(struct smartlist_t *, descs_dumped)
+
+MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file,
+ (const char *dirname, const char *f));
+STATIC void dump_desc_populate_fifo_from_directory(const char *dirname);
+#endif
+
+#endif /* !defined(TOR_UNPARSEABLE_H) */
diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c
index c368b58d0..1b3f01986 100644
--- a/src/test/fuzz/fuzz_consensus.c
+++ b/src/test/fuzz/fuzz_consensus.c
@@ -3,6 +3,7 @@
#define ROUTERPARSE_PRIVATE
#include "core/or/or.h"
#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/unparseable.h"
#include "feature/nodelist/networkstatus.h"
#include "lib/crypt_ops/crypto_ed25519.h"
#include "feature/nodelist/networkstatus_st.h"
diff --git a/src/test/fuzz/fuzz_descriptor.c b/src/test/fuzz/fuzz_descriptor.c
index f7d4a7911..8087e1639 100644
--- a/src/test/fuzz/fuzz_descriptor.c
+++ b/src/test/fuzz/fuzz_descriptor.c
@@ -3,6 +3,7 @@
#define ROUTERPARSE_PRIVATE
#include "core/or/or.h"
#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/unparseable.h"
#include "feature/nodelist/routerlist.h"
#include "feature/nodelist/torcert.h"
#include "feature/keymgt/loadkey.h"
diff --git a/src/test/fuzz/fuzz_extrainfo.c b/src/test/fuzz/fuzz_extrainfo.c
index 56fd6b759..3ec2baf1e 100644
--- a/src/test/fuzz/fuzz_extrainfo.c
+++ b/src/test/fuzz/fuzz_extrainfo.c
@@ -3,6 +3,7 @@
#define ROUTERPARSE_PRIVATE
#include "core/or/or.h"
#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/unparseable.h"
#include "feature/nodelist/routerlist.h"
#include "feature/relay/routerkeys.h"
#include "test/fuzz/fuzzing.h"
diff --git a/src/test/fuzz/fuzz_hsdescv2.c b/src/test/fuzz/fuzz_hsdescv2.c
index ebab5c6ce..1963114ca 100644
--- a/src/test/fuzz/fuzz_hsdescv2.c
+++ b/src/test/fuzz/fuzz_hsdescv2.c
@@ -3,6 +3,7 @@
#define ROUTERPARSE_PRIVATE
#include "core/or/or.h"
#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/unparseable.h"
#include "feature/rend/rendcommon.h"
#include "lib/crypt_ops/crypto_ed25519.h"
#include "test/fuzz/fuzzing.h"
diff --git a/src/test/fuzz/fuzz_hsdescv3.c b/src/test/fuzz/fuzz_hsdescv3.c
index 4d630bf9b..ce6dfb177 100644
--- a/src/test/fuzz/fuzz_hsdescv3.c
+++ b/src/test/fuzz/fuzz_hsdescv3.c
@@ -9,6 +9,7 @@
#include "lib/crypt_ops/crypto_ed25519.h"
#include "feature/hs/hs_descriptor.h"
#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/unparseable.h"
#include "test/fuzz/fuzzing.h"
diff --git a/src/test/fuzz/fuzz_iptsv2.c b/src/test/fuzz/fuzz_iptsv2.c
index 6fdfec3c9..c2147c8ba 100644
--- a/src/test/fuzz/fuzz_iptsv2.c
+++ b/src/test/fuzz/fuzz_iptsv2.c
@@ -3,6 +3,7 @@
#define ROUTERPARSE_PRIVATE
#include "core/or/or.h"
#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/unparseable.h"
#include "feature/rend/rendcommon.h"
#include "lib/crypt_ops/crypto_ed25519.h"
diff --git a/src/test/fuzz/fuzz_microdesc.c b/src/test/fuzz/fuzz_microdesc.c
index 15e8b004c..789e522af 100644
--- a/src/test/fuzz/fuzz_microdesc.c
+++ b/src/test/fuzz/fuzz_microdesc.c
@@ -3,6 +3,7 @@
#define ROUTERPARSE_PRIVATE
#include "core/or/or.h"
#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/unparseable.h"
#include "feature/nodelist/microdesc.h"
#include "lib/crypt_ops/crypto_ed25519.h"
diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c
index 6b22077ae..8fb6e6ef2 100644
--- a/src/test/fuzz/fuzz_vrs.c
+++ b/src/test/fuzz/fuzz_vrs.c
@@ -4,6 +4,7 @@
#define NETWORKSTATUS_PRIVATE
#include "core/or/or.h"
#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/unparseable.h"
#include "lib/memarea/memarea.h"
#include "feature/nodelist/microdesc.h"
#include "feature/nodelist/networkstatus.h"
diff --git a/src/test/test_dir.c b/src/test/test_dir.c
index 23c8c4a1d..9c189ba6b 100644
--- a/src/test/test_dir.c
+++ b/src/test/test_dir.c
@@ -21,6 +21,7 @@
#define ROUTERLIST_PRIVATE
#define ROUTERPARSE_PRIVATE
#define ROUTER_PRIVATE
+#define UNPARSEABLE_PRIVATE
#define VOTEFLAGS_PRIVATE
#include "core/or/or.h"
@@ -52,6 +53,7 @@
#include "feature/nodelist/node_select.h"
#include "feature/nodelist/routerlist.h"
#include "feature/dirparse/routerparse.h"
+#include "feature/dirparse/unparseable.h"
#include "feature/nodelist/routerset.h"
#include "feature/nodelist/torcert.h"
#include "feature/relay/router.h"
