April 2015 - tor-commits - lists.torproject.org

[tor/master] Refactor rend_cache_lookup_entry() and how it's used
by nickm＠torproject.org 15 Apr '15

15 Apr '15

commit 91009dce971655f6b18b5eba5f0c0b48dbd8a737 Author: David Goulet <dgoulet(a)ev0ke.net> Date: Mon Jan 26 17:42:42 2015 -0500 Refactor rend_cache_lookup_entry() and how it's used Here is why: 1) v0 descriptors are deprecated since 0.2.2.1 and not suppose to be alive in the network anymore. This function should only serve v2 version for now as the default. 2) It should return different error code depending on what's the actual error is. Right now, there is no distinction between a cache entry not found and an invalid query. 3) This function should NOT test if the intro points are usable or not. This adds some load on a function that should be "O(1)" and do one job. Furthermore, multiple callsites actually already test that doing twice the job... 4) While adding control event, it would be useful to be able to lookup a cache entry without having it checking the intro points. There are also places in the code that do want to lookup the cache entry without doing that. Fixes #14391 Signed-off-by: David Goulet <dgoulet(a)ev0ke.net> --- src/or/connection_edge.c | 59 +++++++++++++++++++++++++----------------- src/or/rendclient.c | 56 ++++++++++++++++++++++++++-------------- src/or/rendcommon.c | 64 +++++++++++++++++++++++++++++----------------- 3 files changed, 113 insertions(+), 66 deletions(-) diff --git a/src/or/connection_edge.c b/src/or/connection_edge.c index f541249..3833d16 100644 --- a/src/or/connection_edge.c +++ b/src/or/connection_edge.c @@ -1502,23 +1502,45 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, log_info(LD_REND,"Got a hidden service request for ID '%s'", safe_str_client(rend_data->onion_address)); - /* see if we already have a hidden service descriptor cached for this - * address. */ + /* Lookup the given onion address. If invalid, stop right now else we + * might have it in the cache or not, it will be tested later on. */ + unsigned int refetch_desc = 0; rend_cache_entry_t *entry = NULL; const int rend_cache_lookup_result = rend_cache_lookup_entry(rend_data->onion_address, -1, &entry); if (rend_cache_lookup_result < 0) { - /* We should already have rejected this address! */ - log_warn(LD_BUG,"Invalid service name '%s'", - safe_str_client(rend_data->onion_address)); - connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); - return -1; + switch (-rend_cache_lookup_result) { + case EINVAL: + /* We should already have rejected this address! */ + log_warn(LD_BUG,"Invalid service name '%s'", + safe_str_client(rend_data->onion_address)); + connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL); + return -1; + case ENOENT: + refetch_desc = 1; + break; + default: + log_warn(LD_BUG, "Unknown cache lookup error %d", + rend_cache_lookup_result); + return -1; + } } /* Help predict this next time. We're not sure if it will need * a stable circuit yet, but we know we'll need *something*. */ rep_hist_note_used_internal(now, 0, 1); + /* Now we have a descriptor but is it usable or not? If not, refetch. + * Also, a fetch could have been requested if the onion address was not + * found in the cache previously. */ + if (refetch_desc || !rend_client_any_intro_points_usable(entry)) { + base_conn->state = AP_CONN_STATE_RENDDESC_WAIT; + log_info(LD_REND, "Unknown descriptor %s. Fetching.", + safe_str_client(rend_data->onion_address)); + rend_client_refetch_v2_renddesc(rend_data); + return 0; + } + /* Look up if we have client authorization configured for this hidden * service. If we do, associate it with the rend_data. */ rend_service_authorization_t *client_auth = @@ -1532,22 +1554,13 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn, rend_data->auth_type = client_auth->auth_type; } - /* Now, we either launch an attempt to connect to the hidden service, - * or we launch an attempt to look up its descriptor, depending on - * whether we had the descriptor. */ - if (rend_cache_lookup_result == 0) { - base_conn->state = AP_CONN_STATE_RENDDESC_WAIT; - log_info(LD_REND, "Unknown descriptor %s. Fetching.", - safe_str_client(rend_data->onion_address)); - rend_client_refetch_v2_renddesc(rend_data); - } else { /* rend_cache_lookup_result > 0 */ - base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT; - log_info(LD_REND, "Descriptor is here. Great."); - if (connection_ap_handshake_attach_circuit(conn) < 0) { - if (!base_conn->marked_for_close) - connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); - return -1; - } + /* We have the descriptor so launch a connection to the HS. */ + base_conn->state = AP_CONN_STATE_CIRCUIT_WAIT; + log_info(LD_REND, "Descriptor is here. Great."); + if (connection_ap_handshake_attach_circuit(conn) < 0) { + if (!base_conn->marked_for_close) + connection_mark_unattached_ap(conn, END_STREAM_REASON_CANT_ATTACH); + return -1; } return 0; } diff --git a/src/or/rendclient.c b/src/or/rendclient.c index 0c02243..350cd2e 100644 --- a/src/or/rendclient.c +++ b/src/or/rendclient.c @@ -141,7 +141,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc, int r, v3_shift = 0; char payload[RELAY_PAYLOAD_SIZE]; char tmp[RELAY_PAYLOAD_SIZE]; - rend_cache_entry_t *entry; + rend_cache_entry_t *entry = NULL; crypt_path_t *cpath; off_t dh_offset; crypto_pk_t *intro_key = NULL; @@ -158,8 +158,13 @@ rend_client_send_introduction(origin_circuit_t *introcirc, tor_assert(!(rendcirc->build_state->onehop_tunnel)); #endif - if (rend_cache_lookup_entry(introcirc->rend_data->onion_address, -1, - &entry) < 1) { + r = rend_cache_lookup_entry(introcirc->rend_data->onion_address, -1, + &entry); + /* An invalid onion address is not possible else we have a big issue. */ + tor_assert(r != -EINVAL); + if (r < 0 || !rend_client_any_intro_points_usable(entry)) { + /* If the descriptor is not found or the intro points are not usable + * anymore, trigger a fetch. */ log_info(LD_REND, "query %s didn't have valid rend desc in cache. " "Refetching descriptor.", @@ -734,7 +739,7 @@ rend_client_refetch_v2_renddesc(const rend_data_t *rend_query) return; } /* Before fetching, check if we already have a usable descriptor here. */ - if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) > 0 && + if (rend_cache_lookup_entry(rend_query->onion_address, -1, &e) == 0 && rend_client_any_intro_points_usable(e)) { log_info(LD_REND, "We would fetch a v2 rendezvous descriptor, but we " "already have a usable descriptor here. Not fetching."); @@ -848,17 +853,26 @@ rend_client_report_intro_point_failure(extend_info_t *failed_intro, connection_t *conn; r = rend_cache_lookup_entry(rend_query->onion_address, -1, &ent); - if (r<0) { - log_warn(LD_BUG, "Malformed service ID %s.", - escaped_safe_str_client(rend_query->onion_address)); - return -1; - } - if (r==0) { - log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.", - escaped_safe_str_client(rend_query->onion_address)); - rend_client_refetch_v2_renddesc(rend_query); - return 0; + if (r < 0) { + /* Either invalid onion address or cache entry not found. */ + switch (-r) { + case EINVAL: + log_warn(LD_BUG, "Malformed service ID %s.", + escaped_safe_str_client(rend_query->onion_address)); + return -1; + case ENOENT: + log_info(LD_REND, "Unknown service %s. Re-fetching descriptor.", + escaped_safe_str_client(rend_query->onion_address)); + rend_client_refetch_v2_renddesc(rend_query); + return 0; + default: + log_warn(LD_BUG, "Unknown cache lookup returned code: %d", r); + return -1; + } } + /* The intro points are not checked here if they are usable or not because + * this is called when an intro point circuit is closed thus there must be + * at least one intro point that is usable and is about to be flagged. */ for (i = 0; i < smartlist_len(ent->parsed->intro_nodes); i++) { rend_intro_point_t *intro = smartlist_get(ent->parsed->intro_nodes, i); @@ -1057,7 +1071,7 @@ rend_client_desc_trynow(const char *query) continue; assert_connection_ok(base_conn, now); if (rend_cache_lookup_entry(rend_data->onion_address, -1, - &entry) == 1 && + &entry) == 0 && rend_client_any_intro_points_usable(entry)) { /* either this fetch worked, or it failed but there was a * valid entry from before which we should reuse */ @@ -1121,13 +1135,17 @@ rend_client_note_connection_attempt_ended(const char *onion_address) extend_info_t * rend_client_get_random_intro(const rend_data_t *rend_query) { + int ret; extend_info_t *result; rend_cache_entry_t *entry; - if (rend_cache_lookup_entry(rend_query->onion_address, -1, &entry) < 1) { - log_warn(LD_REND, - "Query '%s' didn't have valid rend desc in cache. Failing.", - safe_str_client(rend_query->onion_address)); + ret = rend_cache_lookup_entry(rend_query->onion_address, -1, &entry); + if (ret < 0 || !rend_client_any_intro_points_usable(entry)) { + log_warn(LD_REND, + "Query '%s' didn't have valid rend desc in cache. Failing.", + safe_str_client(rend_query->onion_address)); + /* XXX: Should we refetch the descriptor here if the IPs are not usable + * anymore ?. */ return NULL; } diff --git a/src/or/rendcommon.c b/src/or/rendcommon.c index 866f4fb..0317455 100644 --- a/src/or/rendcommon.c +++ b/src/or/rendcommon.c @@ -920,36 +920,52 @@ rend_valid_service_id(const char *query) return 1; } -/** If we have a cached rend_cache_entry_t for the service ID query - * with version, set *e to that entry and return 1. - * Else return 0. If version is nonnegative, only return an entry - * in that descriptor format version. Otherwise (if version is - * negative), return the most recent format we have. - */ +/** Lookup in the client cache the given service ID query for + * version. + * + * Return 0 if found and if e is non NULL, set it with the entry + * found. Else, a negative value is returned and e is untouched. + * -EINVAL means that query is not a valid service id. + * -ENOENT means that no entry in the cache was found. */ int rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e) { - char key[REND_SERVICE_ID_LEN_BASE32+2]; /* <version><query>\0 */ + int ret = 0; + char key[REND_SERVICE_ID_LEN_BASE32 + 2]; /* <version><query>\0 */ + rend_cache_entry_t *entry = NULL; + static const int default_version = 2; + tor_assert(rend_cache); - if (!rend_valid_service_id(query)) - return -1; - *e = NULL; - if (version != 0) { - tor_snprintf(key, sizeof(key), "2%s", query); - *e = strmap_get_lc(rend_cache, key); + tor_assert(query); + + if (!rend_valid_service_id(query)) { + ret = -EINVAL; + goto end; } - if (!*e && version != 2) { - tor_snprintf(key, sizeof(key), "0%s", query); - *e = strmap_get_lc(rend_cache, key); + + switch (version) { + case 0: + log_warn(LD_REND, "Cache lookup of a v0 renddesc is deprecated."); + break; + case 2: + /* Default is version 2. */ + default: + tor_snprintf(key, sizeof(key), "%d%s", default_version, query); + entry = strmap_get_lc(rend_cache, key); + break; } - if (!*e) - return 0; - tor_assert((*e)->parsed && (*e)->parsed->intro_nodes); - /* XXX023 hack for now, to return "not found" if there are no intro - * points remaining. See bug 997. */ - if (! rend_client_any_intro_points_usable(*e)) - return 0; - return 1; + if (!entry) { + ret = -ENOENT; + goto end; + } + tor_assert(entry->parsed && entry->parsed->intro_nodes); + + if (e) { + *e = entry; + } + +end: + return ret; } /** Lookup the v2 service descriptor with base32-encoded desc_id and

1 0

[tor/master] Merge remote-tracking branch 'dgoulet/bug14391_026_v2'
by nickm＠torproject.org 15 Apr '15

15 Apr '15

commit 8837cc266e44f8033f32a7d1a05020e585f034c7 Merge: 7e6437b 91009dc Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Apr 15 10:33:04 2015 -0400 Merge remote-tracking branch 'dgoulet/bug14391_026_v2' src/or/connection_edge.c | 59 +++++++++++++++++++++++++----------------- src/or/rendclient.c | 56 ++++++++++++++++++++++++++-------------- src/or/rendcommon.c | 64 +++++++++++++++++++++++++++++----------------- 3 files changed, 113 insertions(+), 66 deletions(-)

1 0

[tor/master] changes file for 14391
by nickm＠torproject.org 15 Apr '15

15 Apr '15

commit 7e1fa9d63e7ef6d41b78280a73f86a0b17110dfd Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Apr 15 10:34:19 2015 -0400 changes file for 14391 --- changes/bug14391 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changes/bug14391 b/changes/bug14391 new file mode 100644 index 0000000..5907372 --- /dev/null +++ b/changes/bug14391 @@ -0,0 +1,5 @@ + + o Code simplification and refactoring: + - Refactor hidden service client-side cache lookup to intelligently + report its various failure cases, and disentangle failure cases + involving a lack of introduction points. Closes ticket 14391.

1 0

[orbot/master] fixing flag typo
by n8fr8＠torproject.org 15 Apr '15

15 Apr '15

commit 84c6645220dfc570c332967e419e9e4abdff82b9 Author: Nathan Freitas <nathan(a)freitas.net> Date: Wed Apr 15 10:15:38 2015 -0400 fixing flag typo --- external/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/Makefile b/external/Makefile index c7f5269..b9cd735 100644 --- a/external/Makefile +++ b/external/Makefile @@ -52,7 +52,7 @@ STRIP := $(NDK_TOOLCHAIN_BASE)/bin/$(HOST)-strip \ --strip-unneeded -R .note -R .comment --strip-debug # PIEFLAGS for SDK 16/Android L must be set to -fPIE -pie, but can override for earlier targets -PIEFLAGS =? -fPIE -pie +PIEFLAGS ?= -fPIE -pie CFLAGS = -DANDROID $(TARGET_CFLAGS) $(PIEFLAGS) LDFLAGS = -llog $(TARGET_LDFLAGS) $(PIEFLAGS)

1 0

[orbot/master] fix NPE crash if VPN prepare Intent is null on boot
by n8fr8＠torproject.org 15 Apr '15

15 Apr '15

commit ec2442891590366b2c8a625551a92e74e9947393 Author: Nathan Freitas <nathan(a)freitas.net> Date: Mon Apr 13 10:24:03 2015 -0400 fix NPE crash if VPN prepare Intent is null on boot --- src/org/torproject/android/service/OnBootReceiver.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/org/torproject/android/service/OnBootReceiver.java b/src/org/torproject/android/service/OnBootReceiver.java index f87d2bb..a97091e 100644 --- a/src/org/torproject/android/service/OnBootReceiver.java +++ b/src/org/torproject/android/service/OnBootReceiver.java @@ -33,15 +33,15 @@ public class OnBootReceiver extends BroadcastReceiver { @SuppressLint("NewApi") public void startVpnService (Context context) - { - Intent intent = VpnService.prepare(context); - intent.setFlags(Intent.FLAG_ACTIVITY_NEW_TASK); - - if (intent != null) { - context.startActivity(intent); - } + { + Intent intent = VpnService.prepare(context); + + if (intent != null) { + intent.setFlags(Intent.FLAG_ACTIVITY_NEW_TASK); + context.startActivity(intent); + } - } + } private void startService (String action, Context context) {

1 0

[orbot/master] handle NPE on number format instance
by n8fr8＠torproject.org 15 Apr '15

15 Apr '15

commit 1bc427b50e91c30bdb0f75a2b31f9f8c7f9c6403 Author: Nathan Freitas <nathan(a)freitas.net> Date: Mon Apr 13 10:25:58 2015 -0400 handle NPE on number format instance --- src/org/torproject/android/service/TorService.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/org/torproject/android/service/TorService.java b/src/org/torproject/android/service/TorService.java index 1a48c46..50792b8 100644 --- a/src/org/torproject/android/service/TorService.java +++ b/src/org/torproject/android/service/TorService.java @@ -604,6 +604,7 @@ public class TorService extends Service implements TorServiceConstants, OrbotCon { try { + mNumberFormat = NumberFormat.getInstance(Locale.getDefault()); //localized numbers! if (mNotificationManager == null) { @@ -636,7 +637,6 @@ public class TorService extends Service implements TorServiceConstants, OrbotCon } }).start(); - mNumberFormat = NumberFormat.getInstance(Locale.getDefault()); //localized numbers! if (OrbotVpnService.mSocksProxyPort == -1) OrbotVpnService.mSocksProxyPort = (int)((Math.random()*1000)+10000); @@ -1619,10 +1619,13 @@ public class TorService extends Service implements TorServiceConstants, OrbotCon // Converts the supplied argument into a string. // Under 2Mb, returns "xxx.xKb" // Over 2Mb, returns "xxx.xxMb" - if (count < 1e6) - return mNumberFormat.format(Math.round((float)((int)(count*10/1024))/10)) + "Kbps"; - else - return mNumberFormat.format(Math.round((float)((int)(count*100/1024/1024))/100)) + "Mbps"; + if (mNumberFormat != null) + if (count < 1e6) + return mNumberFormat.format(Math.round((float)((int)(count*10/1024))/10)) + "Kbps"; + else + return mNumberFormat.format(Math.round((float)((int)(count*100/1024/1024))/100)) + "Mbps"; + else + return ""; //return count+" kB"; }

1 0

[orbot/master] fix PIE_FLAGS variable to allow for override
by n8fr8＠torproject.org 15 Apr '15

15 Apr '15

commit dbe3024dfc048cefd846426463e685a58a7eb56e Author: Nathan Freitas <nathan(a)freitas.net> Date: Wed Apr 15 09:58:02 2015 -0400 fix PIE_FLAGS variable to allow for override also, standardize use of STRIP command --- external/Makefile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/external/Makefile b/external/Makefile index 5b13fb5..c7f5269 100644 --- a/external/Makefile +++ b/external/Makefile @@ -49,10 +49,10 @@ LD := $(NDK_TOOLCHAIN_BASE)/bin/$(HOST)-ld AR := $(NDK_TOOLCHAIN_BASE)/bin/$(HOST)-ar RANLIB := $(NDK_TOOLCHAIN_BASE)/bin/$(HOST)-ranlib STRIP := $(NDK_TOOLCHAIN_BASE)/bin/$(HOST)-strip \ - --strip-unneeded -R .note -R .comment + --strip-unneeded -R .note -R .comment --strip-debug -# PIEFLAGS for SDK 16/Android L must be set to -fPIE -pie -PIEFLAGS = -fPIE -pie +# PIEFLAGS for SDK 16/Android L must be set to -fPIE -pie, but can override for earlier targets +PIEFLAGS =? -fPIE -pie CFLAGS = -DANDROID $(TARGET_CFLAGS) $(PIEFLAGS) LDFLAGS = -llog $(TARGET_LDFLAGS) $(PIEFLAGS) @@ -258,11 +258,14 @@ assets: tor polipo jtorctl iptables install -d ../libs install bin/jtorctl.jar ../libs install -d ../assets/$(APP_ABI) - -$(NDK_TOOLCHAIN_BASE)/bin/$(HOST)-strip --strip-debug bin/polipo +## -$(NDK_TOOLCHAIN_BASE)/bin/$(HOST)-strip --strip-debug + -$(STRIP) bin/polipo -zip ../assets/$(APP_ABI)/polipo.mp3 bin/polipo - -$(NDK_TOOLCHAIN_BASE)/bin/$(HOST)-strip --strip-debug bin/tor +## -$(NDK_TOOLCHAIN_BASE)/bin/$(HOST)-strip --strip-debug bin/tor + -$(STRIP) bin/tor -zip ../assets/$(APP_ABI)/tor.mp3 bin/tor - -$(NDK_TOOLCHAIN_BASE)/bin/$(HOST)-strip --strip-debug bin/xtables +## -$(NDK_TOOLCHAIN_BASE)/bin/$(HOST)-strip --strip-debug bin/xtables + -$(STRIP) bin/xtables -zip ../assets/$(APP_ABI)/xtables.mp3 bin/xtables assets-clean:

1 0

[tor/master] Make --hash-password imply --hush to prevent unnecessary noise.
by nickm＠torproject.org 15 Apr '15

15 Apr '15

commit 59e753a4a6751a38c9caa0731c883cd1d8ac07c2 Author: cypherpunks <cypherpunks(a)torproject.org> Date: Fri Apr 10 16:10:44 2015 +0200 Make --hash-password imply --hush to prevent unnecessary noise. --- changes/ticket15542 | 3 +++ src/or/main.c | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/changes/ticket15542 b/changes/ticket15542 new file mode 100644 index 0000000..fcf293e --- /dev/null +++ b/changes/ticket15542 @@ -0,0 +1,3 @@ + o Minor features (command-line interface): + - Make --hash-password imply --hush to prevent unnecessary noise. Closes + ticket 15542. diff --git a/src/or/main.c b/src/or/main.c index a3da148..07d5ba8 100644 --- a/src/or/main.c +++ b/src/or/main.c @@ -2484,10 +2484,11 @@ tor_init(int argc, char *argv[]) if (!strcmp(cl->key, "--quiet") || !strcmp(cl->key, "--dump-config")) quiet = 2; - /* --version, --digests, and --help imply --hush */ + /* The following options imply --hush */ if (!strcmp(cl->key, "--version") || !strcmp(cl->key, "--digests") || !strcmp(cl->key, "--list-torrc-options") || !strcmp(cl->key, "--library-versions") || + !strcmp(cl->key, "--hash-password") || !strcmp(cl->key, "-h") || !strcmp(cl->key, "--help")) { if (quiet < 1) quiet = 1;

1 0

[tor/master] chagnes file for 15542
by nickm＠torproject.org 15 Apr '15

15 Apr '15

commit 7e6437babcea32c98cb028796bff1f81675b2c3b Author: Nick Mathewson <nickm(a)torproject.org> Date: Wed Apr 15 09:40:27 2015 -0400 chagnes file for 15542 --- changes/15542 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changes/15542 b/changes/15542 new file mode 100644 index 0000000..bf289c9 --- /dev/null +++ b/changes/15542 @@ -0,0 +1,3 @@ + o Minor features: + - The "--hash-password" option now implies "--hush" to avoid needless + noise. Closes ticket 15542. Patch from "cypherpunks".

1 0

[tor-browser-bundle/master] Bug 15670: add support for the no_update_for option in config.yml
by gk＠torproject.org 15 Apr '15

15 Apr '15

commit 7d7faa30b4443f0cc76474738a36fd3f45e875f4 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Apr 13 19:53:11 2015 +0200 Bug 15670: add support for the no_update_for option in config.yml --- tools/update-responses/update_responses | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index 07efc7c..6c497ef 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -283,6 +283,10 @@ sub write_htaccess { my $version = $config->{channels}{$channel}; my $files = $config->{versions}{$version}{files}; $htaccess .= "RewriteRule ^[^\/]+/$version/ no-update.xml $flags\n"; + my $noup_for = $config->{no_update_for}{$channel}; + for my $noup_ver (ref $noup_for eq 'ARRAY' ? @$noup_for : ()) { + $htaccess .= "RewriteRule ^[^\/]+/$noup_ver/ no-update.xml $flags\n"; + } foreach my $os (sort keys %$files) { my $bt = build_target_by_os($os); foreach my $lang (sort keys %{$files->{$os}}) {

1 0