December 2015 - tor-commits - lists.torproject.org

[tor/master] Prop210: Check fallback directories and authorities work as expected
by nickm＠torproject.org 17 Dec '15

17 Dec '15

commit e7e61ec7ec32b4aa874a38991358812438405d96 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Thu Dec 17 11:39:40 2015 +1100 Prop210: Check fallback directories and authorities work as expected Also clarify comments. --- src/or/networkstatus.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/or/networkstatus.c b/src/or/networkstatus.c index 584569a..0cf8873 100644 --- a/src/or/networkstatus.c +++ b/src/or/networkstatus.c @@ -1297,14 +1297,18 @@ networkstatus_consensus_can_use_multiple_directories( } /** Check if we can use fallback directory mirrors for a consensus download. - * Only clients that have a list of additional fallbacks can use fallbacks. */ + * If we have fallbacks and don't want to fetch from the authorities, + * we can use them. */ int networkstatus_consensus_can_use_extra_fallbacks(const or_options_t *options) { - /* If we are a client, and we have additional mirrors, we can use them. - * The list length comparisons are a quick way to check if we have any + /* The list length comparisons are a quick way to check if we have any * non-authority fallback directories. If we ever have any authorities that * aren't fallback directories, we will need to change this code. */ + tor_assert(smartlist_len(router_get_fallback_dir_servers()) + >= smartlist_len(router_get_trusted_dir_servers())); + /* If we don't fetch from the authorities, and we have additional mirrors, + * we can use them. */ return (!directory_fetches_from_authorities(options) && (smartlist_len(router_get_fallback_dir_servers()) > smartlist_len(router_get_trusted_dir_servers())));

1 0

[tor/master] Prop210: Fix directory fetch tests
by nickm＠torproject.org 17 Dec '15

17 Dec '15

commit 1b704979488fad24bc439b52e7392e6936955577 Author: teor (Tim Wilson-Brown) <teor2345(a)gmail.com> Date: Thu Dec 17 11:40:49 2015 +1100 Prop210: Fix directory fetch tests Check that directory fetches behave as expected under Prop 210. --- src/or/router.c | 8 +-- src/or/router.h | 5 +- src/test/test_config.c | 169 ++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 172 insertions(+), 10 deletions(-) diff --git a/src/or/router.c b/src/or/router.c index 49e2e31..c35f629 100644 --- a/src/or/router.c +++ b/src/or/router.c @@ -1457,8 +1457,8 @@ static int server_is_advertised=0; /** Return true iff we have published our descriptor lately. */ -int -advertised_server_mode(void) +MOCK_IMPL(int, +advertised_server_mode,(void)) { return server_is_advertised; } @@ -1820,8 +1820,8 @@ static int router_guess_address_from_dir_headers(uint32_t *guess); * it's configured in torrc, or because we've learned it from * dirserver headers. Place the answer in *<b>addr</b> and return * 0 on success, else return -1 if we have no guess. */ -int -router_pick_published_address(const or_options_t *options, uint32_t *addr) +MOCK_IMPL(int, +router_pick_published_address,(const or_options_t *options, uint32_t *addr)) { *addr = get_last_resolved_addr(); if (!*addr && diff --git a/src/or/router.h b/src/or/router.h index 85f43d8..a4b3e96 100644 --- a/src/or/router.h +++ b/src/or/router.h @@ -67,7 +67,7 @@ uint16_t router_get_advertised_dir_port(const or_options_t *options, MOCK_DECL(int, server_mode, (const or_options_t *options)); MOCK_DECL(int, public_server_mode, (const or_options_t *options)); -int advertised_server_mode(void); +MOCK_DECL(int, advertised_server_mode, (void)); int proxy_mode(const or_options_t *options); void consider_publishable_server(int force); int should_refuse_unknown_exits(const or_options_t *options); @@ -89,7 +89,8 @@ int router_digest_is_me(const char *digest); const uint8_t *router_get_my_id_digest(void); int router_extrainfo_digest_is_me(const char *digest); int router_is_me(const routerinfo_t *router); -int router_pick_published_address(const or_options_t *options, uint32_t *addr); +MOCK_DECL(int,router_pick_published_address,(const or_options_t *options, + uint32_t *addr)); int router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e); int router_rebuild_descriptor(int force); char *router_dump_router_to_string(routerinfo_t *router, diff --git a/src/test/test_config.c b/src/test/test_config.c index 53fc693..098b0a8 100644 --- a/src/test/test_config.c +++ b/src/test/test_config.c @@ -19,6 +19,8 @@ #include "transports.h" #include "routerlist.h" #include "networkstatus.h" +#include "router.h" +#include "dirserv.h" static void test_config_addressmap(void *arg) @@ -3450,22 +3452,74 @@ test_config_default_dir_servers(void *arg) or_options_free(opts); } +static int mock_router_pick_published_address_result = 0; + +static int +mock_router_pick_published_address(const or_options_t *options, uint32_t *addr) +{ + (void)options; + (void)addr; + return mock_router_pick_published_address_result; +} + +static int mock_router_my_exit_policy_is_reject_star_result = 0; + +static int +mock_router_my_exit_policy_is_reject_star(void) +{ + return mock_router_my_exit_policy_is_reject_star_result; +} + +static int mock_advertised_server_mode_result = 0; + +static int +mock_advertised_server_mode(void) +{ + return mock_advertised_server_mode_result; +} + +static routerinfo_t *mock_router_get_my_routerinfo_result = NULL; + +static const routerinfo_t * +mock_router_get_my_routerinfo(void) +{ + return mock_router_get_my_routerinfo_result; +} + static void -test_config_use_multiple_directories(void *arg) +test_config_directory_fetch(void *arg) { (void)arg; + /* Test Setup */ or_options_t *options = tor_malloc_zero(sizeof(or_options_t)); + routerinfo_t routerinfo; + memset(&routerinfo, 0, sizeof(routerinfo)); + mock_router_pick_published_address_result = -1; + mock_router_my_exit_policy_is_reject_star_result = 1; + mock_advertised_server_mode_result = 0; + mock_router_get_my_routerinfo_result = NULL; + MOCK(router_pick_published_address, mock_router_pick_published_address); + MOCK(router_my_exit_policy_is_reject_star, + mock_router_my_exit_policy_is_reject_star); + MOCK(advertised_server_mode, mock_advertised_server_mode); + MOCK(router_get_my_routerinfo, mock_router_get_my_routerinfo); /* Clients can use multiple directory mirrors for bootstrap */ memset(options, 0, sizeof(or_options_t)); options->ClientOnly = 1; + tt_assert(server_mode(options) == 0); + tt_assert(public_server_mode(options) == 0); + tt_assert(directory_fetches_from_authorities(options) == 0); tt_assert(networkstatus_consensus_can_use_multiple_directories(options) == 1); /* Bridge Clients can use multiple directory mirrors for bootstrap */ memset(options, 0, sizeof(or_options_t)); options->UseBridges = 1; + tt_assert(server_mode(options) == 0); + tt_assert(public_server_mode(options) == 0); + tt_assert(directory_fetches_from_authorities(options) == 0); tt_assert(networkstatus_consensus_can_use_multiple_directories(options) == 1); @@ -3473,23 +3527,130 @@ test_config_use_multiple_directories(void *arg) * directory mirrors for bootstrap */ memset(options, 0, sizeof(or_options_t)); options->BridgeRelay = 1; + options->ORPort_set = 1; + tt_assert(server_mode(options) == 1); + tt_assert(public_server_mode(options) == 0); + tt_assert(directory_fetches_from_authorities(options) == 0); tt_assert(networkstatus_consensus_can_use_multiple_directories(options) == 1); - /* Clients set to FetchDirInfoEarly must fetch it from the authorities */ + /* Clients set to FetchDirInfoEarly must fetch it from the authorities, + * but can use multiple authorities for bootstrap */ memset(options, 0, sizeof(or_options_t)); options->FetchDirInfoEarly = 1; + tt_assert(server_mode(options) == 0); + tt_assert(public_server_mode(options) == 0); + tt_assert(directory_fetches_from_authorities(options) == 1); + tt_assert(networkstatus_consensus_can_use_multiple_directories(options) + == 1); + + /* OR servers only fetch the consensus from the authorities when they don't + * know their own address, but never use multiple directories for bootstrap + */ + memset(options, 0, sizeof(or_options_t)); + options->ORPort_set = 1; + + mock_router_pick_published_address_result = -1; + tt_assert(server_mode(options) == 1); + tt_assert(public_server_mode(options) == 1); + tt_assert(directory_fetches_from_authorities(options) == 1); + tt_assert(networkstatus_consensus_can_use_multiple_directories(options) + == 0); + + mock_router_pick_published_address_result = 0; + tt_assert(server_mode(options) == 1); + tt_assert(public_server_mode(options) == 1); + tt_assert(directory_fetches_from_authorities(options) == 0); + tt_assert(networkstatus_consensus_can_use_multiple_directories(options) + == 0); + + /* Exit OR servers only fetch the consensus from the authorities when they + * refuse unknown exits, but never use multiple directories for bootstrap + */ + memset(options, 0, sizeof(or_options_t)); + options->ORPort_set = 1; + options->ExitRelay = 1; + mock_router_pick_published_address_result = 0; + mock_router_my_exit_policy_is_reject_star_result = 0; + mock_advertised_server_mode_result = 1; + mock_router_get_my_routerinfo_result = &routerinfo; + + options->RefuseUnknownExits = 1; + tt_assert(server_mode(options) == 1); + tt_assert(public_server_mode(options) == 1); + tt_assert(directory_fetches_from_authorities(options) == 1); + tt_assert(networkstatus_consensus_can_use_multiple_directories(options) + == 0); + + options->RefuseUnknownExits = 0; + mock_router_pick_published_address_result = 0; + tt_assert(server_mode(options) == 1); + tt_assert(public_server_mode(options) == 1); + tt_assert(directory_fetches_from_authorities(options) == 0); tt_assert(networkstatus_consensus_can_use_multiple_directories(options) == 0); - /* OR servers must fetch the consensus from the authorities */ + /* Dir servers fetch the consensus from the authorities, unless they are not + * advertising themselves (hibernating) or have no routerinfo or are not + * advertising their dirport, and never use multiple directories for + * bootstrap. This only applies if they are also OR servers. + * (We don't care much about the behaviour of non-OR directory servers.) */ memset(options, 0, sizeof(or_options_t)); + options->DirPort_set = 1; options->ORPort_set = 1; + mock_router_pick_published_address_result = 0; + mock_router_my_exit_policy_is_reject_star_result = 1; + + mock_advertised_server_mode_result = 1; + routerinfo.dir_port = 1; + mock_router_get_my_routerinfo_result = &routerinfo; + tt_assert(server_mode(options) == 1); + tt_assert(public_server_mode(options) == 1); + tt_assert(directory_fetches_from_authorities(options) == 1); + tt_assert(networkstatus_consensus_can_use_multiple_directories(options) + == 0); + + mock_advertised_server_mode_result = 0; + routerinfo.dir_port = 1; + mock_router_get_my_routerinfo_result = &routerinfo; + tt_assert(server_mode(options) == 1); + tt_assert(public_server_mode(options) == 1); + tt_assert(directory_fetches_from_authorities(options) == 0); + tt_assert(networkstatus_consensus_can_use_multiple_directories(options) + == 0); + + mock_advertised_server_mode_result = 1; + mock_router_get_my_routerinfo_result = NULL; + tt_assert(server_mode(options) == 1); + tt_assert(public_server_mode(options) == 1); + tt_assert(directory_fetches_from_authorities(options) == 0); + tt_assert(networkstatus_consensus_can_use_multiple_directories(options) + == 0); + + mock_advertised_server_mode_result = 1; + routerinfo.dir_port = 0; + mock_router_get_my_routerinfo_result = &routerinfo; + tt_assert(server_mode(options) == 1); + tt_assert(public_server_mode(options) == 1); + tt_assert(directory_fetches_from_authorities(options) == 0); + tt_assert(networkstatus_consensus_can_use_multiple_directories(options) + == 0); + + mock_advertised_server_mode_result = 1; + routerinfo.dir_port = 1; + mock_router_get_my_routerinfo_result = &routerinfo; + tt_assert(server_mode(options) == 1); + tt_assert(public_server_mode(options) == 1); + tt_assert(directory_fetches_from_authorities(options) == 1); tt_assert(networkstatus_consensus_can_use_multiple_directories(options) == 0); done: tor_free(options); + UNMOCK(router_pick_published_address); + UNMOCK(router_get_my_routerinfo); + UNMOCK(advertised_server_mode); + UNMOCK(router_my_exit_policy_is_reject_star); } static void @@ -3539,7 +3700,7 @@ struct testcase_t config_tests[] = { CONFIG_TEST(check_or_create_data_subdir, TT_FORK), CONFIG_TEST(write_to_data_subdir, TT_FORK), CONFIG_TEST(fix_my_family, 0), - CONFIG_TEST(use_multiple_directories, 0), + CONFIG_TEST(directory_fetch, 0), END_OF_TESTCASES };

1 0

[tor/master] More emergency-check code for un-removed pending entry conns
by nickm＠torproject.org 17 Dec '15

17 Dec '15

commit e6be486aea1a102d73c4432340c533eea80740e1 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Dec 16 19:16:07 2015 -0500 More emergency-check code for un-removed pending entry conns This might also be what #17752 needs. --- src/or/connection.c | 7 +++++++ src/or/connection_edge.c | 21 ++++++++++++++++----- src/or/connection_edge.h | 4 ++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/or/connection.c b/src/or/connection.c index d6e5fcb..b4cd4cd 100644 --- a/src/or/connection.c +++ b/src/or/connection.c @@ -678,6 +678,13 @@ connection_free,(connection_t *conn)) if (conn->type == CONN_TYPE_CONTROL) { connection_control_closed(TO_CONTROL_CONN(conn)); } +#if 1 + /* DEBUGGING */ + if (conn->type == CONN_TYPE_AP) { + connection_ap_warn_and_unmark_if_pending_circ(TO_ENTRY_CONN(conn), + "connection_free"); + } +#endif connection_unregister_events(conn); connection_free_(conn); } diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index d4ef064..8a3beb3 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -566,11 +566,9 @@ connection_ap_about_to_close(entry_connection_t *entry_conn) #if 1 /* Check to make sure that this isn't in pending_entry_connections if it * didn't actually belong there. */ - if (TO_CONN(edge_conn)->type == CONN_TYPE_AP && - smartlist_contains(pending_entry_connections, entry_conn)) { - log_warn(LD_BUG, "What was %p doing in pending_entry_connections???", - entry_conn); - smartlist_remove(pending_entry_connections, entry_conn); + if (TO_CONN(edge_conn)->type == CONN_TYPE_AP) { + connection_ap_warn_and_unmark_if_pending_circ(entry_conn, + "about_to_close"); } #endif @@ -903,6 +901,19 @@ connection_ap_mark_as_non_pending_circuit(entry_connection_t *entry_conn) smartlist_remove(pending_entry_connections, entry_conn); } +/** DOCDOC */ +void +connection_ap_warn_and_unmark_if_pending_circ(entry_connection_t *entry_conn, + const char *where) +{ + if (pending_entry_connections && + smartlist_contains(pending_entry_connections, entry_conn)) { + log_warn(LD_BUG, "What was %p doing in pending_entry_connections in %s?", + entry_conn, where); + connection_ap_mark_as_non_pending_circuit(entry_conn); + } +} + /** Tell any AP streams that are waiting for a one-hop tunnel to * <b>failed_digest</b> that they are going to fail. */ /* XXX024 We should get rid of this function, and instead attach diff --git a/src/or/connection_edge.h b/src/or/connection_edge.h index 6da51eb..2aba801 100644 --- a/src/or/connection_edge.h +++ b/src/or/connection_edge.h @@ -115,6 +115,10 @@ streamid_t get_unique_stream_id_by_circ(origin_circuit_t *circ); void connection_edge_free_all(void); +void connection_ap_warn_and_unmark_if_pending_circ( + entry_connection_t *entry_conn, + const char *where); + /** @name Begin-cell flags * * These flags are used in RELAY_BEGIN cells to change the default behavior

1 0

[tor/master] Move pending-connection code into connection_ap_about_to_close
by nickm＠torproject.org 16 Dec '15

16 Dec '15

commit 613e0e1c1ac3e44bad7a876147c49bc232460df2 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Dec 16 18:49:23 2015 -0500 Move pending-connection code into connection_ap_about_to_close It is AP-specific, so that's where it belongs. This shouldn't have caused a bug, but due to #17876, we were never actually calling connection_edge_about_to_close from connection_ap_about_to_close, causing bug #17874 (aka bug #17752). --- src/or/connection_edge.c | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index 961e49a..fdb16a2 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -527,27 +527,6 @@ connection_edge_about_to_close(edge_connection_t *edge_conn) conn->marked_for_close_file, conn->marked_for_close); tor_fragile_assert(); } - - if (TO_CONN(edge_conn)->type != CONN_TYPE_AP || - PREDICT_UNLIKELY(NULL == pending_entry_connections)) - return; - - entry_connection_t *entry_conn = EDGE_TO_ENTRY_CONN(edge_conn); - - if (TO_CONN(edge_conn)->state == AP_CONN_STATE_CIRCUIT_WAIT) { - smartlist_remove(pending_entry_connections, entry_conn); - } - -#if 1 - /* Check to make sure that this isn't in pending_entry_connections if it - * didn't actually belong there. */ - if (TO_CONN(edge_conn)->type == CONN_TYPE_AP && - smartlist_contains(pending_entry_connections, entry_conn)) { - log_warn(LD_BUG, "What was %p doing in pending_entry_connections???", - entry_conn); - smartlist_remove(pending_entry_connections, entry_conn); - } -#endif } /** Called when we're about to finally unlink and free an AP (client) @@ -577,6 +556,22 @@ connection_ap_about_to_close(entry_connection_t *entry_conn) conn->marked_for_close_file, conn->marked_for_close); dnsserv_reject_request(entry_conn); } + + if (TO_CONN(edge_conn)->state == AP_CONN_STATE_CIRCUIT_WAIT) { + smartlist_remove(pending_entry_connections, entry_conn); + } + +#if 1 + /* Check to make sure that this isn't in pending_entry_connections if it + * didn't actually belong there. */ + if (TO_CONN(edge_conn)->type == CONN_TYPE_AP && + smartlist_contains(pending_entry_connections, entry_conn)) { + log_warn(LD_BUG, "What was %p doing in pending_entry_connections???", + entry_conn); + smartlist_remove(pending_entry_connections, entry_conn); + } +#endif + control_event_stream_bandwidth(edge_conn); control_event_stream_status(entry_conn, STREAM_EVENT_CLOSED, edge_conn->end_reason);

1 0

[tor/master] Add an edge_about_to_close() call to ap_about_to_close().
by nickm＠torproject.org 16 Dec '15

16 Dec '15

commit 24fcb6adbb3896395edda38d6ecccb6ad53bddbd Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Dec 16 18:52:34 2015 -0500 Add an edge_about_to_close() call to ap_about_to_close(). Fixes #17876 --- changes/bug17876 | 5 +++++ src/or/connection_edge.c | 2 ++ 2 files changed, 7 insertions(+) diff --git a/changes/bug17876 b/changes/bug17876 new file mode 100644 index 0000000..1bd3dd7 --- /dev/null +++ b/changes/bug17876 @@ -0,0 +1,5 @@ + o Minor bugfixes: + - When closing an entry connection, generate a warning if we should + have sent an end cell for it but we haven't. Fixes bug 17876; + bugfix on 0.2.3.2-alpha. + diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index fdb16a2..d4ef064 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -538,6 +538,8 @@ connection_ap_about_to_close(entry_connection_t *entry_conn) edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn); connection_t *conn = ENTRY_TO_CONN(entry_conn); + connection_edge_about_to_close(edge_conn); + if (entry_conn->socks_request->has_finished == 0) { /* since conn gets removed right after this function finishes, * there's no point trying to send back a reply at this point. */

1 0

[translation/tails-persistence-setup_completed] Update translations for tails-persistence-setup_completed
by translation＠torproject.org 16 Dec '15

16 Dec '15

commit 0c5ffdacca3bbf364627b9fffc5b11292b8ed419 Author: Translation commit bot <translation(a)torproject.org> Date: Wed Dec 16 22:45:27 2015 +0000 Update translations for tails-persistence-setup_completed --- pt_BR/pt_BR.po | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pt_BR/pt_BR.po b/pt_BR/pt_BR.po index 7c32201..2a81585 100644 --- a/pt_BR/pt_BR.po +++ b/pt_BR/pt_BR.po @@ -20,8 +20,8 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: Tails developers <tails(a)boum.org>\n" "POT-Creation-Date: 2015-12-14 21:35+0100\n" -"PO-Revision-Date: 2015-12-15 09:27+0000\n" -"Last-Translator: carolyn <carolyn(a)anhalt.org>\n" +"PO-Revision-Date: 2015-12-16 22:43+0000\n" +"Last-Translator: Communia <ameaneantie(a)riseup.net>\n" "Language-Team: Portuguese (Brazil) (http://www.transifex.com/otf/torproject/language/pt_BR/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -123,7 +123,7 @@ msgstr "Icedove" #: ../lib/Tails/Persistence/Configuration/Presets.pm:160 msgid "Icedove profiles and locally stored email" -msgstr "Perfis Icedove e email armazenado localmente" +msgstr "Perfis de Icedove e e-mail armazenado localmente" #: ../lib/Tails/Persistence/Configuration/Presets.pm:168 msgid "Dotfiles"

1 0

[translation/tails-persistence-setup] Update translations for tails-persistence-setup
by translation＠torproject.org 16 Dec '15

16 Dec '15

commit 1d4a12bedbcec985eac1946a5666894d6b576066 Author: Translation commit bot <translation(a)torproject.org> Date: Wed Dec 16 22:45:22 2015 +0000 Update translations for tails-persistence-setup --- pt_BR/pt_BR.po | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pt_BR/pt_BR.po b/pt_BR/pt_BR.po index 7c32201..2a81585 100644 --- a/pt_BR/pt_BR.po +++ b/pt_BR/pt_BR.po @@ -20,8 +20,8 @@ msgstr "" "Project-Id-Version: The Tor Project\n" "Report-Msgid-Bugs-To: Tails developers <tails(a)boum.org>\n" "POT-Creation-Date: 2015-12-14 21:35+0100\n" -"PO-Revision-Date: 2015-12-15 09:27+0000\n" -"Last-Translator: carolyn <carolyn(a)anhalt.org>\n" +"PO-Revision-Date: 2015-12-16 22:43+0000\n" +"Last-Translator: Communia <ameaneantie(a)riseup.net>\n" "Language-Team: Portuguese (Brazil) (http://www.transifex.com/otf/torproject/language/pt_BR/)\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -123,7 +123,7 @@ msgstr "Icedove" #: ../lib/Tails/Persistence/Configuration/Presets.pm:160 msgid "Icedove profiles and locally stored email" -msgstr "Perfis Icedove e email armazenado localmente" +msgstr "Perfis de Icedove e e-mail armazenado localmente" #: ../lib/Tails/Persistence/Configuration/Presets.pm:168 msgid "Dotfiles"

1 0

[gettor/develop] RESTful API
by ilv＠torproject.org 16 Dec '15

16 Dec '15

commit 6b12e964082feb9ffc1d5b7da84ead87378b8ae9 Author: ilv <ilv(a)users.noreply.github.com> Date: Wed Dec 16 16:55:25 2015 -0300 RESTful API --- gettor/http.py | 493 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ process_http.py | 13 ++ 2 files changed, 506 insertions(+) diff --git a/gettor/http.py b/gettor/http.py new file mode 100644 index 0000000..c1441c6 --- /dev/null +++ b/gettor/http.py @@ -0,0 +1,493 @@ +# -*- coding: utf-8 -*- +# +# This file is part of GetTor. +# +# :authors: Israel Leiva <ilv(a)torproject.org> +# see also AUTHORS file +# +# :copyright: (c) 2008-2015, The Tor Project, Inc. +# (c) 2015, Israel Leiva +# +# :license: This is Free Software. See LICENSE for license information. + +import os +import re +import json +import codecs +import urllib2 +import ConfigParser + +from time import gmtime, strftime + +from flask import Flask +from flask_restful import Api, Resource, reqparse + +import core +import utils + +"""GetTor RESTful API""" + +# currently supported locales for Tor Browser +LC = ['ar', 'de', 'en-US', 'es-ES', 'fa', 'fr', 'it', 'ko', 'nl', 'pl', + 'pt-PT', 'ru', 'tr', 'vi', 'zh-CN'] + +# https://gitweb.tpo/tor-browser-spec.git/tree/processes/VersionNumbers +# does not say anything about operating systems, so it's possible the +# notation might change in the future. We should always use the same three +# strings though: linux, windows, osx. +OS = { + 'Linux': 'linux', + 'Windows': 'windows', + 'MacOS': 'osx' +} + +# based on +# https://gitweb.tpo.org/tor-browser-spec.git/tree/processes/VersionNumbers +# except for the first one, which is based on current RecommendedTBBVersions +RE = { + 'os': '(.*)-(\w+)', + 'alpha': '\d\.\d(\.\d)*a\d+', + 'beta': '\d\.\d(\.\d)*b\d+', + 'stable': '\d\.\d(\.\d)*' +} + +# strings to build names of packages depending on OS. +PKG = { + 'windows': 'torbrowser-install-%s_%s.exe', + 'linux': 'tor-browser-linux%s-%s_%s.tar.xz', + 'osx': 'TorBrowser-%s-osx64_%s.dmg' +} + +# bin and asc are used to build the download links for each version, os and lc +URL = { + 'version': 'https://www.torproject.org/projects/torbrowser/RecommendedTBBVersions', + 'bin': 'https://www.torproject.org/dist/torbrowser/%s/%s', + 'asc': 'https://www.torproject.org/dist/torbrowser/%s/%s.asc' +} + +# filters for API resources +parser = reqparse.RequestParser() +parser.add_argument('os', type=str, help='Operating System') +parser.add_argument('lc', type=str, help='Locale') + + +class ConfigError(Exception): + pass + + +class InternalError(Exception): + pass + + +class HTTP(object): + """ Provide useful resources via RESTful API. """ + def __init__(self, cfg=None): + """ Create new object by reading a configuration file. + + :param: cfg (string) path of the configuration file. + + """ + default_cfg = 'http.cfg' + config = ConfigParser.ConfigParser() + + if cfg is None or not os.path.isfile(cfg): + cfg = default_cfg + + try: + with open(cfg) as f: + config.readfp(f) + except IOError: + raise ConfigError("File %s not found!" % cfg) + + try: + # server that provides the RESTful API + self.server = config.get('general', 'server') + # path to the links files + self.links_path = config.get('general', 'links') + # path to mirrors in json + self.mirrors_path = config.get('general', 'mirrors') + + # we will ask gettor.core for the links + core_cfg = config.get('general', 'core') + self.core = core.Core(core_cfg) + + except ConfigParser.Error as e: + raise ConfigError("Configuration error: %s" % str(e)) + except core.ConfigError as e: + raise InternalError("HTTP error: %s" % str(e)) + + def _is_json(self, my_json): + """ Check if json generated is valid. + + :param: my_json (string) data to ve verified. + + :return: (bool) true if data is json-valid, false otherwise. + + """ + try: + json_object = json.loads(my_json) + except ValueError, e: + return False + return True + + def _get_provider_name(self, p): + """ Return simplified version of provider's name. + + :param: p (string) provider's name. + + :return: (string) provider's name in lowercase and without spaces. + + """ + p = p.replace(' ', '-') + return p.lower() + + def _add_links(self, lv, release, version, os): + """ Add link for all locales in LC depending on given OS. + + :param: lv (dict) latest version data structure. + :param: release (string) release to which add the links. + :param: version (string) version obtained from tpo. + :param: os (string) operating system. + + """ + for lc in LC: + if os == 'linux': + pkg32 = PKG['linux'] % ('32', version, lc) + link_bin32 = URL['bin'] % (version, pkg32) + link_asc32 = URL['asc'] % (version, pkg32) + + pkg64 = PKG['linux'] % ('64', version, lc) + link_bin64 = URL['bin'] % (version, pkg64) + link_asc64 = URL['asc'] % (version, pkg64) + + lv[release]['downloads'][os][lc] = { + 'binary32': link_bin32, + 'signature32': link_asc32, + 'binary64': link_bin64, + 'signature64': link_asc64, + } + else: + if os == 'windows': + pkg = PKG['windows'] % (version, lc) + + elif os == 'osx': + pkg = PKG['osx'] % (version, lc) + + else: + continue + + link_bin = URL['bin'] % (version, pkg) + link_asc = URL['asc'] % (version, pkg) + lv[release]['downloads'][os][lc] = { + 'binary': link_bin, + 'signature': link_asc + } + + def _load_latest_version(self): + """ Load latest version data. """ + response = urllib2.urlopen(URL['version']) + json_response = json.load(response) + + lv = { + 'stable': { + 'latest_version': '', + 'downloads': {} + }, + 'alpha': { + 'latest_version': '', + 'downloads': {} + }, + 'beta': { + 'latest_version': '', + 'downloads': {} + } + } + + self.releases = { + 'alpha': '%s/latest/alpha' % self.server, + 'beta': '%s/latest/beta' % self.server, + 'stable': '%s/latest/stable' % self.server, + 'updated_at': strftime("%Y-%m-%d %H:%M:%S", gmtime()) + } + + # one iteration to find the latest version for each release + for v in json_response: + # latest version for each release + if not re.match(RE['os'], v): + if re.match(RE['alpha'], v): + if v > lv['alpha']['latest_version']: + # we'll use the latest one + lv['alpha']['latest_version'] = v + + elif re.match(RE['beta'], v): + if v > lv['beta']['latest_version']: + # we'll use the latest one + lv['beta']['latest_version'] = v + + elif re.match(RE['stable'], v): + if v > lv['stable']['latest_version']: + # we'll use the latest one + lv['stable']['latest_version'] = v + + latest_alpha = lv['alpha']['latest_version'] + latest_beta = lv['beta']['latest_version'] + latest_stable = lv['stable']['latest_version'] + + # another iteration to add the links + for v in json_response: + # based on current RecommendedTBBVersions scheme + # for each release and for each os we build links for all locales + if re.match(RE['os'], v): + m = re.match(RE['os'], v) + version = m.group(1) + osys = m.group(2) + + if osys in OS: + if latest_alpha and version == latest_alpha \ + and re.match(RE['alpha'], version): + lv['alpha']['downloads'][OS[osys]] = {} + self._add_links(lv, 'alpha', version, OS[osys]) + + elif latest_beta and version == latest_beta \ + and re.match(RE['beta'], version): + lv['beta']['downloads'][OS[osys]] = {} + self._add_links(lv, 'beta', version, OS[osys]) + + elif latest_stable and version == latest_stable \ + and re.match(RE['stable'], version): + lv['stable']['downloads'][OS[osys]] = {} + self._add_links(lv, 'stable', version, OS[osys]) + + lv['updated_at'] = strftime("%Y-%m-%d %H:%M:%S", gmtime()) + self.lv = lv + + def _load_links(self): + """ Load links and providers data. """ + links_files = [] + + # look for files ending with .links in links_path + p = re.compile('.*\.links$') + for name in os.listdir(self.links_path): + path = os.path.abspath(os.path.join(self.links_path, name)) + if os.path.isfile(path) and p.match(path): + links_files.append(path) + + links = {} + providers = {} + supported_os = self.core.get_supported_os() + supported_lc = self.core.get_supported_lc() + + for name in links_files: + config = ConfigParser.ConfigParser() + try: + with open(name) as f: + config.readfp(f) + except IOError: + raise InternalError("File %s not found!" % name) + + try: + pname = config.get('provider', 'name') + pname = self._get_provider_name(pname) + + # build providers dict + providers[pname] = '%s/providers/%s' % (self.server, pname) + providers['updated_at'] = strftime( + "%Y-%m-%d %H:%M:%S", gmtime() + ) + + self.providers = providers + links[pname] = {} + + # build links data. + for osys in supported_os: + links[pname][osys] = {} + for lc in supported_lc: + links[pname][osys][lc] = {} + + for osys in supported_os: + for lc in supported_lc: + l_str = config.get(osys, lc) + + # linux has 32 and 64 bit packages + if osys == 'linux': + l32_str, l64_str = l_str.split(',') + + link32, sig32, sha32 = [ + l for l in l32_str.split("$") if l + ] + + link64, sig64, sha64 = [ + l for l in l64_str.split("$") if l + ] + link64 = link64.lstrip() + + links[pname][osys][lc]['binary32'] = link32 + links[pname][osys][lc]['signature32'] = sig32 + links[pname][osys][lc]['sha256-32'] = sha32 + links[pname][osys][lc]['binary64'] = link64 + links[pname][osys][lc]['signature64'] = sig64 + links[pname][osys][lc]['sha256-64'] = sha64 + + else: + link, sig, sha = [l for l in l_str.split("$") if l] + links[pname][osys][lc]['binary'] = link + links[pname][osys][lc]['signature'] = sig + links[pname][osys][lc]['sha256'] = sha + + except ConfigParser.Error as e: + raise InternalError("%s" % str(e)) + + links['updated_at'] = strftime("%Y-%m-%d %H:%M:%S", gmtime()) + self.links = links + + def _load_mirrors(self): + """ Load mirrors data. """ + mirrors = [] + + # json of mirrors should be obtained from get_mirrors.py + json_data = open(self.mirrors_path).read() + mirrors = json.loads(json_data) + + self.mirrors = mirrors + + def _load_resources(self): + """ Load available resources data. """ + + self.resources = { + 'providers': '%s/providers' % self.server, + 'mirrors': '%s/mirrors' % self.server, + 'latest_version': '%s/latest' % self.server, + 'updated_at': strftime("%Y-%m-%d %H:%M:%S", gmtime()) + } + + def load_data(self): + """ Load all data. + + Since data is not frequently updated, we load all data before + running the RESTful API. Every time the links/mirrors/version + data is updated we should restart the API. + + """ + self._load_links() + self._load_mirrors() + self._load_resources() + self._load_latest_version() + + def run(self): + """ Run RESTful API. """ + app = Flask(__name__) + api = Api(app) + + api.add_resource( + AvailableResources, + '/', + resource_class_kwargs={ + 'resources': self.resources + } + ) + + api.add_resource( + Providers, + '/providers', + '/providers/<string:provider>', + resource_class_kwargs={ + 'links': self.links, + 'providers': self.providers + } + ) + + api.add_resource( + LatestVersion, + '/latest', + '/latest/<string:release>', + resource_class_kwargs={ + 'latest_version': self.lv, + 'releases': self.releases + } + ) + + api.add_resource( + Mirrors, + '/mirrors', + resource_class_kwargs={ + 'mirrors': self.mirrors + } + ) + + app.run(debug=True) + + +class AvailableResources(Resource): + def __init__(self, resources): + """ Set initial data. """ + self.resources = resources + + def get(self): + """ Return available resources on the API. """ + return self.resources + + +class Providers(Resource): + def __init__(self, providers, links): + """ Set initial data. """ + self.providers = providers + self.links = links + + def get(self, provider=None): + """ Return providers and links data. """ + + # we use arg to filter results by os and lc (in that order) + arg = parser.parse_args() + + if provider: + if arg['os']: + if arg['lc']: + # links by provider, os, and lc (in that order) + return self.links[provider][arg['os']][arg['lc']] + else: + # links by provider and os (in that order) + return self.links[provider][arg['os']] + else: + # links by provider + return self.links[provider] + else: + # list of providers + return self.providers + + +class LatestVersion(Resource): + def __init__(self, latest_version, releases): + """ Set initial data. """ + self.lv = latest_version + self.releases = releases + + def get(self, release=None): + """ Return latest version data. """ + + # we use arg to filter results by os and lc (in that order) + arg = parser.parse_args() + + if release: + if arg['os']: + if arg['lc']: + # tpo links by release, os and lc (in that order) + return self.lv[release]['downloads'][arg['os']][arg['lc']] + else: + # tpo links by release and os (in that order) + return self.lv[release]['downloads'][arg['os']] + else: + # version and tpo links by release + return self.lv[release] + else: + # list of releases + return self.releases + + +class Mirrors(Resource): + def __init__(self, mirrors): + """ Set initial data. """ + self.mirrors = mirrors + + def get(self): + """ Return mirrors data. """ + return self.mirrors diff --git a/process_http.py b/process_http.py new file mode 100644 index 0000000..31475f1 --- /dev/null +++ b/process_http.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import gettor.http + + +def main(): + api = gettor.http.HTTP('http.cfg') + api.load_data() + api.run() + +if __name__ == '__main__': + main()

1 0

[tor/release-0.2.7] Fix backtrace compilation on FreeBSD
by nickm＠torproject.org 16 Dec '15

16 Dec '15

commit 07cca627eaab800d4874f3d0914d3cf7eaa601a9 Author: cypherpunks <cypherpunks(a)torproject.org> Date: Tue Dec 15 16:30:04 2015 +0100 Fix backtrace compilation on FreeBSD On FreeBSD backtrace(3) uses size_t instead of int (as glibc does). This causes integer precision loss errors when we used int to store its results. The issue is fixed by using size_t to store the results of backtrace(3). The manual page of glibc does not mention that backtrace(3) returns negative values. Therefore, no unsigned integer wrapping occurs when its result is stored in an unsigned data type. --- changes/bug17827 | 3 +++ src/common/backtrace.c | 16 ++++++++-------- src/common/backtrace.h | 2 +- src/common/sandbox.c | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/changes/bug17827 b/changes/bug17827 new file mode 100644 index 0000000..04cd3b5 --- /dev/null +++ b/changes/bug17827 @@ -0,0 +1,3 @@ + o Minor bugfixes (compilation): + - Fix backtrace compilation on FreeBSD. Fixes bug 17827; bugfix on + tor-0.2.5.2-alpha. diff --git a/src/common/backtrace.c b/src/common/backtrace.c index a2d5378..8d544ed 100644 --- a/src/common/backtrace.c +++ b/src/common/backtrace.c @@ -62,16 +62,16 @@ static tor_mutex_t cb_buf_mutex; * ucontext_t structure. */ void -clean_backtrace(void **stack, int depth, const ucontext_t *ctx) +clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx) { #ifdef PC_FROM_UCONTEXT #if defined(__linux__) - const int n = 1; + const size_t n = 1; #elif defined(__darwin__) || defined(__APPLE__) || defined(__OpenBSD__) \ || defined(__FreeBSD__) - const int n = 2; + const size_t n = 2; #else - const int n = 1; + const size_t n = 1; #endif if (depth <= n) return; @@ -89,9 +89,9 @@ clean_backtrace(void **stack, int depth, const ucontext_t *ctx) void log_backtrace(int severity, int domain, const char *msg) { - int depth; + size_t depth; char **symbols; - int i; + size_t i; tor_mutex_acquire(&cb_buf_mutex); @@ -120,7 +120,7 @@ static void crash_handler(int sig, siginfo_t *si, void *ctx_) { char buf[40]; - int depth; + size_t depth; ucontext_t *ctx = (ucontext_t *) ctx_; int n_fds, i; const int *fds = NULL; @@ -174,7 +174,7 @@ install_bt_handler(void) * libc has pre-loaded the symbols we need to dump things, so that later * reads won't be denied by the sandbox code */ char **symbols; - int depth = backtrace(cb_buf, MAX_DEPTH); + size_t depth = backtrace(cb_buf, MAX_DEPTH); symbols = backtrace_symbols(cb_buf, depth); if (symbols) free(symbols); diff --git a/src/common/backtrace.h b/src/common/backtrace.h index a9151d7..838e18e 100644 --- a/src/common/backtrace.h +++ b/src/common/backtrace.h @@ -13,7 +13,7 @@ void clean_up_backtrace_handler(void); #ifdef EXPOSE_CLEAN_BACKTRACE #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) -void clean_backtrace(void **stack, int depth, const ucontext_t *ctx); +void clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx); #endif #endif diff --git a/src/common/sandbox.c b/src/common/sandbox.c index b995762..3a9f2a1 100644 --- a/src/common/sandbox.c +++ b/src/common/sandbox.c @@ -1598,7 +1598,7 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context) const char *syscall_name; int syscall; #ifdef USE_BACKTRACE - int depth; + size_t depth; int n_fds, i; const int *fds = NULL; #endif

1 0

[tor/release-0.2.7] ... and fix another backtrace_symbols_fd call in sandbox.c
by nickm＠torproject.org 16 Dec '15

16 Dec '15

commit 784e9fff9baa4fb63712de5dd9cc0ac530e2eb23 Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Dec 16 09:05:49 2015 -0500 ... and fix another backtrace_symbols_fd call in sandbox.c --- src/common/sandbox.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/sandbox.c b/src/common/sandbox.c index 3a9f2a1..950a92f 100644 --- a/src/common/sandbox.c +++ b/src/common/sandbox.c @@ -1630,7 +1630,7 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context) #ifdef USE_BACKTRACE n_fds = tor_log_get_sigsafe_err_fds(&fds); for (i=0; i < n_fds; ++i) - backtrace_symbols_fd(syscall_cb_buf, depth, fds[i]); + backtrace_symbols_fd(syscall_cb_buf, (int)depth, fds[i]); #endif #if defined(DEBUGGING_CLOSE)

1 0