commit 5d4bb6f61f360ddcbee83476ff2f93a13f3a19c0 Merge: bd22ad1 ac9b0a3 Author: Nick Mathewson <nickm@torproject.org> Date: Fri Jan 23 09:36:00 2015 -0500 Merge remote-tracking branch 'public/ticket9969' Conflicts: src/or/directory.c src/or/routerlist.c src/or/routerlist.h src/test/include.am src/test/test.c src/or/directory.c | 104 ++++++++-------- src/or/directory.h | 11 +- src/or/or.h | 11 +- src/or/routerlist.c | 288 ++++++++++++++++++++++++++------------------ src/or/routerlist.h | 4 + src/test/include.am | 21 ++-- src/test/test.c | 98 +++++++-------- src/test/test_dir.c | 26 ++++ src/test/test_nodelist.c | 2 +- src/test/test_routerlist.c | 103 ++++++++++++++++ 10 files changed, 437 insertions(+), 231 deletions(-) diff --cc src/or/directory.c index 4f24f84,7f272ed..348d971 --- a/src/or/directory.c +++ b/src/or/directory.c @@@ -198,9 -197,49 +196,50 @@@ dir_conn_purpose_to_string(int purpose return "(unknown)"; } + /** Return the requisite directory information types. */ + STATIC dirinfo_type_t + dir_fetch_type(int dir_purpose, int router_purpose, const char *resource) + { + dirinfo_type_t type; + switch (dir_purpose) { + case DIR_PURPOSE_FETCH_EXTRAINFO: + type = EXTRAINFO_DIRINFO; + if (router_purpose == ROUTER_PURPOSE_BRIDGE) + type |= BRIDGE_DIRINFO; + else + type |= V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_SERVERDESC: + if (router_purpose == ROUTER_PURPOSE_BRIDGE) + type = BRIDGE_DIRINFO; + else + type = V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_STATUS_VOTE: + case DIR_PURPOSE_FETCH_DETACHED_SIGNATURES: + case DIR_PURPOSE_FETCH_CERTIFICATE: + type = V3_DIRINFO; + break; + case DIR_PURPOSE_FETCH_CONSENSUS: + type = V3_DIRINFO; + if (resource && !strcmp(resource, "microdesc")) + type |= MICRODESC_DIRINFO; + break; + case DIR_PURPOSE_FETCH_MICRODESC: + type = MICRODESC_DIRINFO; + break; + default: + log_warn(LD_BUG, "Unexpected purpose %d", (int)dir_purpose); + type = NO_DIRINFO; + break; + } + return type; + } + -/** Return true iff <b>identity_digest</b> is the digest of a router we - * believe to support extrainfo downloads. (If <b>is_authority</b> we do - * additional checking that's only valid for authorities.) */ ++ +/** Return true iff <b>identity_digest</b> is the digest of a router which + * says that it caches extrainfos. (If <b>is_authority</b> we always + * believe that to be true.) */ int router_supports_extrainfo(const char *identity_digest, int is_authority) { @@@ -526,25 -518,21 +539,21 @@@ MOCK_IMPL(void, directory_get_from_dirs /* */ rs = directory_pick_generic_dirserver(type, pds_flags, dir_purpose); - if (!rs) { - /*XXXX024 I'm pretty sure this can never do any good, since - * rs isn't set. */ + if (!rs) get_via_tor = 1; /* last resort: try routing it via Tor */ - } } } - } else { /* get_via_tor */ + } + + if (get_via_tor) { /* Never use fascistfirewall; we're going via Tor. */ - if (1) { - /* anybody with a non-zero dirport will do. Disregard firewalls. */ - pds_flags |= PDS_IGNORE_FASCISTFIREWALL; - rs = router_pick_directory_server(type, pds_flags); - } + pds_flags |= PDS_IGNORE_FASCISTFIREWALL; + rs = router_pick_directory_server(type, pds_flags); - /* If we have any hope of building an indirect conn, we know some router - * descriptors. If (rs==NULL), we can't build circuits anyway, so - * there's no point in falling back to the authorities in this case. */ } + /* If we have any hope of building an indirect conn, we know some router + * descriptors. If (rs==NULL), we can't build circuits anyway, so + * there's no point in falling back to the authorities in this case. */ if (rs) { const dir_indirection_t indirection = get_via_tor ? DIRIND_ANONYMOUS : DIRIND_ONEHOP; diff --cc src/or/routerlist.c index b5e58a4,6b1baca..9f389a4 --- a/src/or/routerlist.c +++ b/src/or/routerlist.c @@@ -1288,7 -1239,10 +1288,8 @@@ router_get_fallback_dir_servers(void const routerstatus_t * router_pick_directory_server(dirinfo_type_t type, int flags) { + int busy = 0; const routerstatus_t *choice; - if (get_options()->PreferTunneledDirConns) - flags |= PDS_PREFER_TUNNELED_DIR_CONNS_; if (!routerlist) return NULL; @@@ -1413,11 -1378,19 +1423,15 @@@ router_pick_dirserver_generic(smartlist #define DIR_503_TIMEOUT (60*60) /** Pick a random running valid directory server/mirror from our -- * routerlist. Arguments are as for router_pick_directory_server(), except - * that RETRY_IF_NO_SERVERS is ignored. - * that RETRY_IF_NO_SERVERS is ignored, and: - * - * If the PDS_PREFER_TUNNELED_DIR_CONNS_ flag is set, prefer directory servers - * that we can use with BEGINDIR. ++ * routerlist. Arguments are as for router_pick_directory_server(), except: + * + * If <b>n_busy_out</b> is provided, set *<b>n_busy_out</b> to the number of + * directories that we excluded for no other reason than + * PDS_NO_EXISTING_SERVERDESC_FETCH or PDS_NO_EXISTING_MICRODESC_FETCH. */ static const routerstatus_t * - router_pick_directory_server_impl(dirinfo_type_t type, int flags) + router_pick_directory_server_impl(dirinfo_type_t type, int flags, + int *n_busy_out) { const or_options_t *options = get_options(); const node_t *result; @@@ -1426,10 -1399,13 +1440,12 @@@ smartlist_t *overloaded_direct, *overloaded_tunnel; time_t now = time(NULL); const networkstatus_t *consensus = networkstatus_get_latest_consensus(); - int requireother = ! (flags & PDS_ALLOW_SELF); - int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); - int for_guard = (flags & PDS_FOR_GUARD); - int try_excluding = 1, n_excluded = 0; + const int requireother = ! (flags & PDS_ALLOW_SELF); + const int fascistfirewall = ! (flags & PDS_IGNORE_FASCISTFIREWALL); - const int prefer_tunnel = (flags & PDS_PREFER_TUNNELED_DIR_CONNS_); + const int no_serverdesc_fetching = (flags & PDS_NO_EXISTING_SERVERDESC_FETCH); + const int no_microdesc_fetching = (flags & PDS_NO_EXISTING_MICRODESC_FETCH); + const int for_guard = (flags & PDS_FOR_GUARD); + int try_excluding = 1, n_excluded = 0, n_busy = 0; if (!consensus) return NULL; @@@ -4247,14 -4358,29 +4289,28 @@@ MOCK_IMPL(STATIC void, initiate_descrip tor_free(resource); } - /** Max amount of hashes to download per request. - * Since squid does not like URLs >= 4096 bytes we limit it to 96. - * 4096 - strlen(http://255.255.255.255/tor/server/d/.z) == 4058 - * 4058/41 (40 for the hash and 1 for the + that separates them) => 98 - * So use 96 because it's a nice number. + /** Return the max number of hashes to put in a URL for a given request. */ - #define MAX_DL_PER_REQUEST 96 - #define MAX_MICRODESC_DL_PER_REQUEST 92 + static int + max_dl_per_request(const or_options_t *options, int purpose) + { + /* Since squid does not like URLs >= 4096 bytes we limit it to 96. + * 4096 - strlen(http://255.255.255.255/tor/server/d/.z) == 4058 + * 4058/41 (40 for the hash and 1 for the + that separates them) => 98 + * So use 96 because it's a nice number. + */ + int max = 96; + if (purpose == DIR_PURPOSE_FETCH_MICRODESC) { + max = 92; + } + /* If we're going to tunnel our connections, we can ask for a lot more + * in a request. */ - if (options->TunnelDirConns && - !directory_fetches_from_authorities(options)) { ++ if (!directory_fetches_from_authorities(options)) { + max = 500; + } + return max; + } + /** Don't split our requests so finely that we are requesting fewer than * this number per server. */ #define MIN_DL_PER_REQUEST 4 diff --cc src/or/routerlist.h index 0d22543,1e8b7c9..f106ca2 --- a/src/or/routerlist.h +++ b/src/or/routerlist.h @@@ -223,12 -211,10 +223,16 @@@ STATIC int choose_array_element_by_weig int n_entries); STATIC void scale_array_elements_to_u64(u64_dbl_t *entries, int n_entries, uint64_t *total_out); + +MOCK_DECL(int, router_descriptor_is_older_than, (const routerinfo_t *router, + int seconds)); +MOCK_DECL(STATIC was_router_added_t, extrainfo_insert, + (routerlist_t *rl, extrainfo_t *ei, int warn_if_incompatible)); + + MOCK_DECL(STATIC void, initiate_descriptor_downloads, + (const routerstatus_t *source, int purpose, smartlist_t *digests, + int lo, int hi, int pds_flags)); + #endif #endif diff --cc src/test/include.am index 3c59a8b,74ac526..f07c78b --- a/src/test/include.am +++ b/src/test/include.am @@@ -17,44 -17,32 +17,45 @@@ src_test_AM_CPPFLAGS = -DSHARE_DATADIR= src_test_test_SOURCES = \ src/test/test.c \ ++ src/test/test_accounting.c \ src/test/test_addr.c \ src/test/test_buffers.c \ src/test/test_cell_formats.c \ ++ src/test/test_cell_queue.c \ + src/test/test_channel.c \ + src/test/test_channeltls.c \ ++ src/test/test_checkdir.c \ src/test/test_circuitlist.c \ src/test/test_circuitmux.c \ + src/test/test_config.c \ src/test/test_containers.c \ src/test/test_controller_events.c \ src/test/test_crypto.c \ -- src/test/test_cell_queue.c \ src/test/test_data.c \ src/test/test_dir.c \ - src/test/test_checkdir.c \ + src/test/test_entryconn.c \ + src/test/test_entrynodes.c \ src/test/test_extorport.c \ + src/test/test_hs.c \ src/test/test_introduce.c \ src/test/test_logging.c \ src/test/test_microdesc.c \ + src/test/test_nodelist.c \ src/test/test_oom.c \ - src/test/test_accounting.c \ src/test/test_options.c \ ++ src/test/test_policy.c \ src/test/test_pt.c \ - src/test/test_relaycell.c \ + src/test/test_relay.c \ ++ src/test/test_relaycell.c \ src/test/test_replay.c \ src/test/test_routerkeys.c \ + src/test/test_routerlist.c \ ++ src/test/test_routerset.c \ + src/test/test_scheduler.c \ src/test/test_socks.c \ ++ src/test/test_status.c \ + src/test/test_threads.c \ src/test/test_util.c \ - src/test/test_config.c \ - src/test/test_hs.c \ - src/test/test_nodelist.c \ - src/test/test_policy.c \ - src/test/test_status.c \ - src/test/test_routerset.c \ src/ext/tinytest.c src_test_test_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS) diff --cc src/test/test.c index fc5290f,8b53754..debee0a --- a/src/test/test.c +++ b/src/test/test.c @@@ -1294,84 -1612,60 +1294,86 @@@ static struct testcase_t test_array[] END_OF_TESTCASES }; ++extern struct testcase_t accounting_tests[]; extern struct testcase_t addr_tests[]; extern struct testcase_t buffer_tests[]; --extern struct testcase_t crypto_tests[]; --extern struct testcase_t container_tests[]; --extern struct testcase_t util_tests[]; --extern struct testcase_t dir_tests[]; - extern struct testcase_t checkdir_tests[]; --extern struct testcase_t microdesc_tests[]; --extern struct testcase_t pt_tests[]; --extern struct testcase_t config_tests[]; --extern struct testcase_t introduce_tests[]; --extern struct testcase_t replaycache_tests[]; - extern struct testcase_t relaycell_tests[]; extern struct testcase_t cell_format_tests[]; ++extern struct testcase_t cell_queue_tests[]; ++extern struct testcase_t channel_tests[]; ++extern struct testcase_t channeltls_tests[]; ++extern struct testcase_t checkdir_tests[]; extern struct testcase_t circuitlist_tests[]; extern struct testcase_t circuitmux_tests[]; --extern struct testcase_t cell_queue_tests[]; --extern struct testcase_t options_tests[]; --extern struct testcase_t socks_tests[]; -extern struct testcase_t extorport_tests[]; ++extern struct testcase_t config_tests[]; ++extern struct testcase_t container_tests[]; + extern struct testcase_t controller_event_tests[]; -extern struct testcase_t logging_tests[]; -extern struct testcase_t backtrace_tests[]; ++extern struct testcase_t crypto_tests[]; ++extern struct testcase_t dir_tests[]; ++extern struct testcase_t entryconn_tests[]; +extern struct testcase_t entrynodes_tests[]; - extern struct testcase_t thread_tests[]; +extern struct testcase_t extorport_tests[]; - extern struct testcase_t controller_event_tests[]; - extern struct testcase_t logging_tests[]; extern struct testcase_t hs_tests[]; ++extern struct testcase_t introduce_tests[]; ++extern struct testcase_t logging_tests[]; ++extern struct testcase_t microdesc_tests[]; extern struct testcase_t nodelist_tests[]; --extern struct testcase_t routerkeys_tests[]; extern struct testcase_t oom_tests[]; - extern struct testcase_t accounting_tests[]; ++extern struct testcase_t options_tests[]; +extern struct testcase_t policy_tests[]; - extern struct testcase_t status_tests[]; - extern struct testcase_t routerset_tests[]; - extern struct testcase_t router_tests[]; - extern struct testcase_t channel_tests[]; - extern struct testcase_t channeltls_tests[]; ++extern struct testcase_t pt_tests[]; +extern struct testcase_t relay_tests[]; ++extern struct testcase_t relaycell_tests[]; ++extern struct testcase_t replaycache_tests[]; ++extern struct testcase_t router_tests[]; ++extern struct testcase_t routerkeys_tests[]; + extern struct testcase_t routerlist_tests[]; ++extern struct testcase_t routerset_tests[]; +extern struct testcase_t scheduler_tests[]; - extern struct testcase_t entryconn_tests[]; ++extern struct testcase_t socks_tests[]; ++extern struct testcase_t status_tests[]; ++extern struct testcase_t thread_tests[]; ++extern struct testcase_t util_tests[]; static struct testgroup_t testgroups[] = { { "", test_array }, - { "buffer/", buffer_tests }, - { "socks/", socks_tests }, ++ { "accounting/", accounting_tests }, { "addr/", addr_tests }, - { "crypto/", crypto_tests }, - { "container/", container_tests }, - { "util/", util_tests }, - { "util/logging/", logging_tests }, - { "util/thread/", thread_tests }, + { "buffer/", buffer_tests }, { "cellfmt/", cell_format_tests }, { "cellqueue/", cell_queue_tests }, - { "dir/", dir_tests }, ++ { "channel/", channel_tests }, ++ { "channeltls/", channeltls_tests }, + { "checkdir/", checkdir_tests }, - { "dir/md/", microdesc_tests }, - { "pt/", pt_tests }, - { "config/", config_tests }, - { "replaycache/", replaycache_tests }, - { "relaycell/", relaycell_tests }, - { "introduce/", introduce_tests }, { "circuitlist/", circuitlist_tests }, { "circuitmux/", circuitmux_tests }, - { "options/", options_tests }, - { "entrynodes/", entrynodes_tests }, + { "config/", config_tests }, + { "container/", container_tests }, + { "control/", controller_event_tests }, + { "crypto/", crypto_tests }, + { "dir/", dir_tests }, + { "dir/md/", microdesc_tests }, + { "entryconn/", entryconn_tests }, ++ { "entrynodes/", entrynodes_tests }, { "extorport/", extorport_tests }, - { "control/", controller_event_tests }, { "hs/", hs_tests }, + { "introduce/", introduce_tests }, { "nodelist/", nodelist_tests }, - { "routerkeys/", routerkeys_tests }, { "oom/", oom_tests }, - { "accounting/", accounting_tests }, + { "options/", options_tests }, + { "policy/" , policy_tests }, - { "status/" , status_tests }, - { "routerset/" , routerset_tests }, - { "channel/", channel_tests }, - { "channeltls/", channeltls_tests }, + { "pt/", pt_tests }, + { "relay/" , relay_tests }, ++ { "relaycell/", relaycell_tests }, ++ { "replaycache/", replaycache_tests }, + { "routerkeys/", routerkeys_tests }, + { "routerlist/", routerlist_tests }, - { "replaycache/", replaycache_tests }, ++ { "routerset/" , routerset_tests }, + { "scheduler/", scheduler_tests }, + { "socks/", socks_tests }, ++ { "status/" , status_tests }, + { "util/", util_tests }, + { "util/logging/", logging_tests }, ++ { "util/thread/", thread_tests }, END_OF_GROUPS }; diff --cc src/test/test_dir.c index a8c6c6f,c7f2480..991e613 --- a/src/test/test_dir.c +++ b/src/test/test_dir.c @@@ -2930,8 -2357,32 +2930,32 @@@ test_dir_http_handling(void *args tor_free(url); } + static void + test_dir_purpose_needs_anonymity(void *arg) + { + (void)arg; + tt_int_op(1, ==, purpose_needs_anonymity(0, ROUTER_PURPOSE_BRIDGE)); + tt_int_op(1, ==, purpose_needs_anonymity(0, ROUTER_PURPOSE_GENERAL)); + tt_int_op(0, ==, purpose_needs_anonymity(DIR_PURPOSE_FETCH_MICRODESC, + ROUTER_PURPOSE_GENERAL)); + done: ; + } + + static void + test_dir_fetch_type(void *arg) + { + (void)arg; + tt_assert(dir_fetch_type(DIR_PURPOSE_FETCH_MICRODESC, ROUTER_PURPOSE_GENERAL, + NULL) == MICRODESC_DIRINFO); + tt_assert(dir_fetch_type(DIR_PURPOSE_FETCH_SERVERDESC, ROUTER_PURPOSE_BRIDGE, + NULL) == BRIDGE_DIRINFO); + tt_assert(dir_fetch_type(DIR_PURPOSE_FETCH_CONSENSUS, ROUTER_PURPOSE_GENERAL, + "microdesc") == (V3_DIRINFO | MICRODESC_DIRINFO)); + done: ; + } + #define DIR_LEGACY(name) \ - { #name, legacy_test_helper, TT_FORK, &legacy_setup, test_dir_ ## name } + { #name, test_dir_ ## name , TT_FORK, NULL, NULL } #define DIR(name,flags) \ { #name, test_dir_##name, (flags), NULL, NULL } diff --cc src/test/test_routerlist.c index 0000000,1c2b1f8..390b691 mode 000000,100644..100644 --- a/src/test/test_routerlist.c +++ b/src/test/test_routerlist.c @@@ -1,0 -1,104 +1,103 @@@ + /* Copyright (c) 2014, The Tor Project, Inc. */ + /* See LICENSE for licensing information */ + + #define ROUTERLIST_PRIVATE + #include "or.h" + #include "routerlist.h" + #include "directory.h" + #include "test.h" + + + /* 4 digests + 3 sep + pre + post + NULL */ + static char output[4*BASE64_DIGEST256_LEN+3+2+2+1]; + + static void + mock_get_from_dirserver(uint8_t dir_purpose, uint8_t router_purpose, + const char *resource, int pds_flags) + { + (void)dir_purpose; + (void)router_purpose; + (void)pds_flags; - test_assert(resource); ++ tt_assert(resource); + strlcpy(output, resource, sizeof(output)); + done: + ; + } + + static void + test_routerlist_initiate_descriptor_downloads(void *arg) + { + const char *prose = "unhurried and wise, we perceive."; + smartlist_t *digests = smartlist_new(); + (void)arg; + + for (int i = 0; i < 20; i++) { + smartlist_add(digests, (char*)prose); + } + + MOCK(directory_get_from_dirserver, mock_get_from_dirserver); + initiate_descriptor_downloads(NULL, DIR_PURPOSE_FETCH_MICRODESC, + digests, 3, 7, 0); + UNMOCK(directory_get_from_dirserver); + - test_streq(output, "d/" \ - "dW5odXJyaWVkIGFuZCB3aXNlLCB3ZSBwZXJjZWl2ZS4-" \ - "dW5odXJyaWVkIGFuZCB3aXNlLCB3ZSBwZXJjZWl2ZS4-" \ - "dW5odXJyaWVkIGFuZCB3aXNlLCB3ZSBwZXJjZWl2ZS4-" \ - "dW5odXJyaWVkIGFuZCB3aXNlLCB3ZSBwZXJjZWl2ZS4" \ - ".z"); ++ tt_str_op(output, OP_EQ, "d/" ++ "dW5odXJyaWVkIGFuZCB3aXNlLCB3ZSBwZXJjZWl2ZS4-" ++ "dW5odXJyaWVkIGFuZCB3aXNlLCB3ZSBwZXJjZWl2ZS4-" ++ "dW5odXJyaWVkIGFuZCB3aXNlLCB3ZSBwZXJjZWl2ZS4-" ++ "dW5odXJyaWVkIGFuZCB3aXNlLCB3ZSBwZXJjZWl2ZS4" ++ ".z"); + + done: + smartlist_free(digests); + } + + static int count = 0; + + static void + mock_initiate_descriptor_downloads(const routerstatus_t *source, + int purpose, smartlist_t *digests, + int lo, int hi, int pds_flags) + { + (void)source; + (void)purpose; + (void)digests; + (void)pds_flags; + (void)hi; + (void)lo; + count += 1; + } + + static void + test_routerlist_launch_descriptor_downloads(void *arg) + { + smartlist_t *downloadable = smartlist_new(); + time_t now = time(NULL); + char *cp; + (void)arg; + + for (int i = 0; i < 100; i++) { + cp = tor_malloc(DIGEST256_LEN); - test_assert(cp); ++ tt_assert(cp); + crypto_rand(cp, DIGEST256_LEN); + smartlist_add(downloadable, cp); + } + + MOCK(initiate_descriptor_downloads, mock_initiate_descriptor_downloads); + launch_descriptor_downloads(DIR_PURPOSE_FETCH_MICRODESC, downloadable, + NULL, now); + tt_int_op(3, ==, count); + UNMOCK(initiate_descriptor_downloads); + + done: + SMARTLIST_FOREACH(downloadable, char *, cp1, tor_free(cp1)); + smartlist_free(downloadable); + } + + #define NODE(name, flags) \ + { #name, test_routerlist_##name, (flags), NULL, NULL } + + struct testcase_t routerlist_tests[] = { + NODE(initiate_descriptor_downloads, 0), + NODE(launch_descriptor_downloads, 0), + END_OF_TESTCASES + }; -