- tbb-commits - lists.torproject.org

[tor-browser-build/master] Fix typo in README.MAKEFILE
by gk＠torproject.org 28 Aug '17

28 Aug '17

commit 971d6fe6d6a79c3ef2a448638435bdad24a8e329 Author: Georg Koppen <gk(a)torproject.org> Date: Mon Aug 28 08:11:39 2017 +0000 Fix typo in README.MAKEFILE --- README.MAKEFILE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.MAKEFILE b/README.MAKEFILE index cd4bf2f..98137b2 100644 --- a/README.MAKEFILE +++ b/README.MAKEFILE @@ -14,7 +14,7 @@ Same as release, for the selected platform only. alpha ----- Build Tor Browser for the alpha channel, for all supported platforms. -The resulting build can be found in directoryp alpha/$version. +The resulting build can be found in directory alpha/$version. alpha-{linux-x86_64,linux-i686,windows-i686,osx-x86_64} -------------------------------------------------------

1 0

[tor-browser-bundle/maint-7.0] Add Felix's obfs4 bridges to the built-in bridges.
by gk＠torproject.org 24 Aug '17

24 Aug '17

commit a0aace68fe848963408de4af7b1c659f2f446fc2 Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 23 19:27:44 2017 +0000 Add Felix's obfs4 bridges to the built-in bridges. * FIXES #23166: https://bugs.torproject.org/23166 --- Bundle-Data/PTConfigs/bridge_prefs.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Bundle-Data/PTConfigs/bridge_prefs.js b/Bundle-Data/PTConfigs/bridge_prefs.js index 03ccb07..a76985b 100644 --- a/Bundle-Data/PTConfigs/bridge_prefs.js +++ b/Bundle-Data/PTConfigs/bridge_prefs.js @@ -37,6 +37,8 @@ pref("extensions.torlauncher.default_bridge.obfs4.20", "obfs4 38.229.1.78:80 C8C pref("extensions.torlauncher.default_bridge.obfs4.22", "obfs4 [2001:470:b381:bfff:216:3eff:fe23:d6c3]:443 CDF2E852BF539B82BD10E27E9115A31734E378C2 cert=qUVQ0srL1JI/vO6V6m/24anYXiJD3QP2HgzUKQtQ7GRqqUvs7P+tG43RtAqdhLOALP7DJQ iat-mode=1"); pref("extensions.torlauncher.default_bridge.obfs4.23", "obfs4 37.218.240.34:40035 88CD36D45A35271963EF82E511C8827A24730913 cert=eGXYfWODcgqIdPJ+rRupg4GGvVGfh25FWaIXZkit206OSngsp7GAIiGIXOJJROMxEqFKJg iat-mode=1"); pref("extensions.torlauncher.default_bridge.obfs4.24", "obfs4 37.218.245.14:38224 D9A82D2F9C2F65A18407B1D2B764F130847F8B5D cert=bjRaMrr1BRiAW8IE9U5z27fQaYgOhX1UCmOpg2pFpoMvo6ZgQMzLsaTzzQNTlm7hNcb+Sg iat-mode=0"); +pref("extensions.torlauncher.default_bridge.obfs4.25", "obfs4 85.31.186.98:443 011F2599C0E9B27EE74B353155E244813763C3E5 cert=ayq0XzCwhpdysn5o0EyDUbmSOx3X/oTEbzDMvczHOdBJKlvIdHHLJGkZARtT4dcBFArPPg iat-mode=0"); +pref("extensions.torlauncher.default_bridge.obfs4.26", "obfs4 85.31.186.26:443 91A6354697E6B02A386312F68D82CF86824D3606 cert=PBwr+S8JTVZo6MPdHnkTwXJPILWADLqfMGoVvhZClMq/Urndyd42BwX9YFJHZnBB3H0XCw iat-mode=0"); pref("extensions.torlauncher.default_bridge.meek-amazon.1", "meek 0.0.2.0:2 B9E7141C594AF25699E0079C1F0146F409495296 url=https://d2cly7j4zqgua7.cloudfront.net/ front=a0.awsstatic.com"); pref("extensions.torlauncher.default_bridge.meek-azure.1", "meek 0.0.2.0:3 97700DFE9F483596DDA6264C4D7DF7641E1E39CE url=https://meek.azureedge.net/ front=ajax.aspnetcdn.com");

1 0

[tor-browser-bundle/master] Add Felix's obfs4 bridges to the built-in bridges.
by gk＠torproject.org 24 Aug '17

24 Aug '17

commit fc2449fc0ef8bebfeaaa1a44c23215a86f64d75a Author: Isis Lovecruft <isis(a)torproject.org> Date: Wed Aug 23 19:27:44 2017 +0000 Add Felix's obfs4 bridges to the built-in bridges. * FIXES #23166: https://bugs.torproject.org/23166 --- Bundle-Data/PTConfigs/bridge_prefs.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Bundle-Data/PTConfigs/bridge_prefs.js b/Bundle-Data/PTConfigs/bridge_prefs.js index 2f68a1f..258586f 100644 --- a/Bundle-Data/PTConfigs/bridge_prefs.js +++ b/Bundle-Data/PTConfigs/bridge_prefs.js @@ -37,6 +37,8 @@ pref("extensions.torlauncher.default_bridge.obfs4.20", "obfs4 38.229.1.78:80 C8C pref("extensions.torlauncher.default_bridge.obfs4.22", "obfs4 [2001:470:b381:bfff:216:3eff:fe23:d6c3]:443 CDF2E852BF539B82BD10E27E9115A31734E378C2 cert=qUVQ0srL1JI/vO6V6m/24anYXiJD3QP2HgzUKQtQ7GRqqUvs7P+tG43RtAqdhLOALP7DJQ iat-mode=1"); pref("extensions.torlauncher.default_bridge.obfs4.23", "obfs4 37.218.240.34:40035 88CD36D45A35271963EF82E511C8827A24730913 cert=eGXYfWODcgqIdPJ+rRupg4GGvVGfh25FWaIXZkit206OSngsp7GAIiGIXOJJROMxEqFKJg iat-mode=1"); pref("extensions.torlauncher.default_bridge.obfs4.24", "obfs4 37.218.245.14:38224 D9A82D2F9C2F65A18407B1D2B764F130847F8B5D cert=bjRaMrr1BRiAW8IE9U5z27fQaYgOhX1UCmOpg2pFpoMvo6ZgQMzLsaTzzQNTlm7hNcb+Sg iat-mode=0"); +pref("extensions.torlauncher.default_bridge.obfs4.25", "obfs4 85.31.186.98:443 011F2599C0E9B27EE74B353155E244813763C3E5 cert=ayq0XzCwhpdysn5o0EyDUbmSOx3X/oTEbzDMvczHOdBJKlvIdHHLJGkZARtT4dcBFArPPg iat-mode=0"); +pref("extensions.torlauncher.default_bridge.obfs4.26", "obfs4 85.31.186.26:443 91A6354697E6B02A386312F68D82CF86824D3606 cert=PBwr+S8JTVZo6MPdHnkTwXJPILWADLqfMGoVvhZClMq/Urndyd42BwX9YFJHZnBB3H0XCw iat-mode=0"); pref("extensions.torlauncher.default_bridge.meek-amazon.1", "meek 0.0.2.0:2 B9E7141C594AF25699E0079C1F0146F409495296 url=https://d2cly7j4zqgua7.cloudfront.net/ front=a0.awsstatic.com"); pref("extensions.torlauncher.default_bridge.meek-azure.1", "meek 0.0.2.0:3 97700DFE9F483596DDA6264C4D7DF7641E1E39CE url=https://meek.azureedge.net/ front=ajax.aspnetcdn.com");

1 0

[torbutton/master] Bug 20375: Warn users after entering fullscreen mode.
by gk＠torproject.org 23 Aug '17

23 Aug '17

commit f6b52b8d7f26277569211946248307996d12099c Author: Arthur Edelstein <arthuredelstein(a)gmail.com> Date: Tue Aug 22 12:06:21 2017 -0700 Bug 20375: Warn users after entering fullscreen mode. When the browser window is maximized, we show users a notification warning that maximizing the screen is fingerprintable. We should show this warning when the window enters fullscreen mode as well. This change is especially important for macOS, which uses window.STATE_FULLSCREEN instead of window.STATE_MAXIMIZED when the user clicks the "maximize" button on the window title bar. --- src/chrome/content/torbutton.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/chrome/content/torbutton.js b/src/chrome/content/torbutton.js index 11f4b61..db6b694 100644 --- a/src/chrome/content/torbutton.js +++ b/src/chrome/content/torbutton.js @@ -2126,7 +2126,8 @@ var torbutton_resizelistener = m_tb_resize_handler = async function() { // Wait for end of execution queue to ensure we have correct windowState. await new Promise(resolve => setTimeout(resolve, 0)); - if (window.windowState === 1) { + if (window.windowState === window.STATE_MAXIMIZED || + window.windowState === window.STATE_FULLSCREEN) { if (m_tb_prefs. getIntPref("extensions.torbutton.maximize_warnings_remaining") > 0) {

1 0

[torbutton/maint-1.9.7] Bug 22989: Fix dimensions of new windows on macOS.
by gk＠torproject.org 23 Aug '17

23 Aug '17

commit 6aa6d6ef1355179d7198f0eb09405ac0968ba4e7 Author: Arthur Edelstein <arthuredelstein(a)gmail.com> Date: Mon Aug 14 15:56:00 2017 -0700 Bug 22989: Fix dimensions of new windows on macOS. Before this patch, a new Tor Browser window, non-maximized, was showing a "don't maximize this window" notification briefly as soon as it was first displayed. Worse, the notification box that appears causes the window height to increase by 10 or 11 pixels and thus breaks our desired window dimension rounding. Our intended behavior is that this notification only be shown when the window is maximized by the user. I discovered that, on macOS, windowState === window.STATE_MAXIMIZED for a short time when the window is first created. Then it rapidly changes to windowState === window.STATE_NORMAL. So I added a `setTimeout` promise to postpone checking the window size until after window creation "settles" and that seems to be enough to ensure we avoid showing the spurious notification box at window creation. Also fixes Bug 22543, "Tor Browser 7.0 shows window resize warning on every new window for a few seconds". And fixes issues raised in comments 2 and 3 in that ticket as well. --- src/chrome/content/torbutton.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/chrome/content/torbutton.js b/src/chrome/content/torbutton.js index 0c297aa..11f4b61 100644 --- a/src/chrome/content/torbutton.js +++ b/src/chrome/content/torbutton.js @@ -2123,7 +2123,9 @@ var torbutton_resizelistener = onLocationChange: function(aProgress, aRequest, aURI) {}, onStateChange: function(aProgress, aRequest, aFlag, aStatus) { if (aFlag & Ci.nsIWebProgressListener.STATE_STOP) { - m_tb_resize_handler = function() { + m_tb_resize_handler = async function() { + // Wait for end of execution queue to ensure we have correct windowState. + await new Promise(resolve => setTimeout(resolve, 0)); if (window.windowState === 1) { if (m_tb_prefs. getIntPref("extensions.torbutton.maximize_warnings_remaining") > 0) {

1 0

[torbutton/master] Bug 22989: Fix dimensions of new windows on macOS.
by gk＠torproject.org 23 Aug '17

23 Aug '17

commit 97380447e4ab66a44abed4ec0aeb5eeabf86404e Author: Arthur Edelstein <arthuredelstein(a)gmail.com> Date: Mon Aug 14 15:56:00 2017 -0700 Bug 22989: Fix dimensions of new windows on macOS. Before this patch, a new Tor Browser window, non-maximized, was showing a "don't maximize this window" notification briefly as soon as it was first displayed. Worse, the notification box that appears causes the window height to increase by 10 or 11 pixels and thus breaks our desired window dimension rounding. Our intended behavior is that this notification only be shown when the window is maximized by the user. I discovered that, on macOS, windowState === window.STATE_MAXIMIZED for a short time when the window is first created. Then it rapidly changes to windowState === window.STATE_NORMAL. So I added a `setTimeout` promise to postpone checking the window size until after window creation "settles" and that seems to be enough to ensure we avoid showing the spurious notification box at window creation. Also fixes Bug 22543, "Tor Browser 7.0 shows window resize warning on every new window for a few seconds". And fixes issues raised in comments 2 and 3 in that ticket as well. --- src/chrome/content/torbutton.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/chrome/content/torbutton.js b/src/chrome/content/torbutton.js index 0c297aa..11f4b61 100644 --- a/src/chrome/content/torbutton.js +++ b/src/chrome/content/torbutton.js @@ -2123,7 +2123,9 @@ var torbutton_resizelistener = onLocationChange: function(aProgress, aRequest, aURI) {}, onStateChange: function(aProgress, aRequest, aFlag, aStatus) { if (aFlag & Ci.nsIWebProgressListener.STATE_STOP) { - m_tb_resize_handler = function() { + m_tb_resize_handler = async function() { + // Wait for end of execution queue to ensure we have correct windowState. + await new Promise(resolve => setTimeout(resolve, 0)); if (window.windowState === 1) { if (m_tb_prefs. getIntPref("extensions.torbutton.maximize_warnings_remaining") > 0) {

1 0

[tor-browser/tor-browser-52.3.0esr-7.5-2] Bug 23230: Fix build error on Windows 64
by gk＠torproject.org 22 Aug '17

22 Aug '17

commit f36d2a2dd30368c97594923092a43dd74c61846c Author: Nicolas Vigier <boklm(a)torproject.org> Date: Sat Aug 12 20:42:59 2017 +0200 Bug 23230: Fix build error on Windows 64 Fix the following build error we have when building for Windows 64: /var/tmp/build/firefox-af0ca6a886c7/media/libcubeb/src/cubeb_wasapi.cpp:1596:46: error: cast from cubeb_devid {aka void*} to int loses precision [-fpermissive] We fix that by using %p instead of %x to remove the need for a cast. --- media/libcubeb/src/cubeb_wasapi.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/media/libcubeb/src/cubeb_wasapi.cpp b/media/libcubeb/src/cubeb_wasapi.cpp index e88d6becd861..212cc263c0b6 100644 --- a/media/libcubeb/src/cubeb_wasapi.cpp +++ b/media/libcubeb/src/cubeb_wasapi.cpp @@ -1575,7 +1575,7 @@ int setup_wasapi_stream(cubeb_stream * stm) XASSERT((!stm->output_client || !stm->input_client) && "WASAPI stream already setup, close it first."); if (has_input(stm)) { - LOG("Setup capture: device=%x", (int)stm->input_device); + LOG("Setup capture: device=%p", stm->input_device); rv = setup_wasapi_stream_one_side(stm, &stm->input_stream_params, stm->input_device, @@ -1593,7 +1593,7 @@ int setup_wasapi_stream(cubeb_stream * stm) } if (has_output(stm)) { - LOG("Setup render: device=%x", (int)stm->output_device); + LOG("Setup render: device=%p", stm->output_device); rv = setup_wasapi_stream_one_side(stm, &stm->output_stream_params, stm->output_device,

1 0

[tor-browser-build/master] Include mar-tools zip in the output files
by boklm＠torproject.org 21 Aug '17

21 Aug '17

commit f3fed13feabaead01099d009f099120a8677c650 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Aug 21 18:10:03 2017 +0200 Include mar-tools zip in the output files --- projects/tor-browser/build | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/tor-browser/build b/projects/tor-browser/build index 77a02ea..31c2d6d 100644 --- a/projects/tor-browser/build +++ b/projects/tor-browser/build @@ -275,6 +275,8 @@ cd $distdir [% END %] rm -rf $distdir/${PKG_DIR} +cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/ + # If we did not create a multi-lingual package above, create a package for # each locale. [% IF ! c("var/multi_lingual") %]

1 0

[tor-browser-build/master] Bug 17381: create tar containing update responses xml files
by boklm＠torproject.org 21 Aug '17

21 Aug '17

commit 93bb526421830f185c431d9064043e54c38d6bab Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Aug 21 19:01:16 2017 +0200 Bug 17381: create tar containing update responses xml files Create a tar file in directory {alpha,release}/update-responses/, to make it a little easier to copy them to the host where they will be deployed. --- Makefile | 4 ++-- README.MAKEFILE | 3 ++- projects/release/config | 5 +++++ projects/release/create_update_responses_tar | 9 +++++++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1229cfa..ce63e54 100644 --- a/Makefile +++ b/Makefile @@ -97,11 +97,11 @@ incrementals-alpha: submodule-update update_responses-release: submodule-update $(rbm) build release --step update_responses_config --target release --target signed - tools/update-responses/update_responses release + $(rbm) build release --step create_update_responses_tar --target release --target signed update_responses-alpha: submodule-update $(rbm) build release --step update_responses_config --target alpha --target signed - tools/update-responses/update_responses alpha + $(rbm) build release --step create_update_responses_tar --target alpha --target signed dmg2mar-release: submodule-update $(rbm) build release --step update_responses_config --target release --target signed diff --git a/README.MAKEFILE b/README.MAKEFILE index 9e28864..cd4bf2f 100644 --- a/README.MAKEFILE +++ b/README.MAKEFILE @@ -91,5 +91,6 @@ signing the OSX dmg files. update_responses-{release,alpha} -------------------------------- Create update responses xml files for a signed build in the release or -alpha channel. +alpha channel. The files can be found in a tar in the directory +{release,alpha}/update-responses. diff --git a/projects/release/config b/projects/release/config index 0a4276a..4ecbc0a 100644 --- a/projects/release/config +++ b/projects/release/config @@ -122,6 +122,11 @@ steps: debug: 0 input_files: [] update_responses_config: '[% INCLUDE update_responses_config %]' + create_update_responses_tar: + build_log: '-' + debug: 0 + input_files: [] + create_update_responses_tar: '[% INCLUDE create_update_responses_tar %]' hash_incrementals: build_log: '-' debug: 0 diff --git a/projects/release/create_update_responses_tar b/projects/release/create_update_responses_tar new file mode 100644 index 0000000..ee1180f --- /dev/null +++ b/projects/release/create_update_responses_tar @@ -0,0 +1,9 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +[% shell_quote(c("basedir")) %]/tools/update-responses/update_responses +mkdir -p [% shell_quote(path(dest_dir)) %]/update-responses +[% SET channel = pc('firefox', 'var/torbrowser_update_channel') -%] +mv [% shell_quote(c("basedir")) %]/tools/update-responses/htdocs/[% channel %] [% channel %] +chmod 775 [% channel %] +chmod 664 [% channel %]/.htaccess [% channel %]/* +tar cf [% shell_quote(path(dest_dir)) %]/update-responses/update-responses-[% channel %]-[% c("version") %].tar [% channel %]

1 0

[tor-browser-build/master] Bug 17381: add and adapt the update_responses script
by boklm＠torproject.org 21 Aug '17

21 Aug '17

commit de3f97f325df5465fa49b7c8ca2ff229b670b207 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Thu Aug 17 13:32:46 2017 +0200 Bug 17381: add and adapt the update_responses script Add the update_responses and incrementals makefile rules. The update_responses script is identical to what we have in tor-browser-bundle.git, however the config.yml file it is using is now generated using the infos from rbm. --- Makefile | 20 + README.MAKEFILE | 10 + keyring/torbrowser.gpg | Bin 0 -> 49267 bytes projects/firefox/config | 3 +- projects/release/config | 21 +- projects/release/hash_incrementals | 4 + projects/release/update_responses_config | 11 + projects/release/update_responses_config.yml | 40 ++ rbm.conf | 3 + tools/update-responses/.gitignore | 2 + tools/update-responses/README.md | 50 ++ .../check_update_responses_deployement | 1 + tools/update-responses/download_missing_versions | 1 + tools/update-responses/gen_incrementals | 1 + tools/update-responses/get_channel_version | 1 + tools/update-responses/update_responses | 643 +++++++++++++++++++++ 16 files changed, 807 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c45b0a1..d49d670 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,26 @@ signtag-release: submodule-update signtag-alpha: submodule-update $(rbm) build release --step signtag --target alpha +incrementals-release: submodule-update + $(rbm) build release --step update_responses_config --target release + tools/update-responses/download_missing_versions release + tools/update-responses/gen_incrementals release + $(rbm) build release --step hash_incrementals --target release + +incrementals-alpha: submodule-update + $(rbm) build release --step update_responses_config --target alpha + tools/update-responses/download_missing_versions alpha + tools/update-responses/gen_incrementals alpha + $(rbm) build release --step hash_incrementals --target alpha + +update_responses-release: submodule-update + $(rbm) build release --step update_responses_config --target release --target signed + tools/update-responses/update_responses release + +update_responses-alpha: submodule-update + $(rbm) build release --step update_responses_config --target alpha --target signed + tools/update-responses/update_responses alpha + submodule-update: git submodule update --init diff --git a/README.MAKEFILE b/README.MAKEFILE index ef33904..167f5d5 100644 --- a/README.MAKEFILE +++ b/README.MAKEFILE @@ -77,3 +77,13 @@ signtag-{release,alpha} Create a git signed tag for the selected channel, using the version and build number defined as var/torbrowser_version and var/torbrowser_build. +incrementals-{release,alpha} +---------------------------- +Create incremental mar files for an unsigned build in the release or +alpha channel. + +update_responses-{release,alpha} +-------------------------------- +Create update responses xml files for a signed build in the release or +alpha channel. + diff --git a/keyring/torbrowser.gpg b/keyring/torbrowser.gpg new file mode 100644 index 0000000..dfdee06 Binary files /dev/null and b/keyring/torbrowser.gpg differ diff --git a/projects/firefox/config b/projects/firefox/config index 1360772..034d40f 100644 --- a/projects/firefox/config +++ b/projects/firefox/config @@ -7,7 +7,8 @@ git_url: https://git.torproject.org/tor-browser.git gpg_keyring: torbutton.gpg var: - firefox_version: 52.3.0esr + firefox_platform_version: 52.3.0 + firefox_version: '[% c("var/firefox_platform_version") %]esr' torbrowser_branch: 7.5 torbrowser_update_channel: alpha copyright_year: '[% exec("git show -s --format=%ci").remove("-.*") %]' diff --git a/projects/release/config b/projects/release/config index b785878..3c771bf 100644 --- a/projects/release/config +++ b/projects/release/config @@ -1,10 +1,11 @@ # vim: filetype=yaml sw=2 version: '[% c("var/torbrowser_version") %]' -output_dir: 'release/unsigned' +output_dir: release var: + signed_status: unsigned today: '[% USE date; date.format(format = "%Y-%m-%d") %]' - publish_dir: '[% c("version") %]-[% c("var/torbrowser_build") %]' + publish_dir: '[% c("var/signed_status") %]/[% c("version") %]-[% c("var/torbrowser_build") %]' targets: torbrowser-all: @@ -50,7 +51,7 @@ targets: publish_dir: '[% c("var/today") %]' alpha: - output_dir: 'alpha/unsigned' + output_dir: alpha var: build_target: alpha @@ -60,6 +61,10 @@ targets: build_target: torbrowser-testbuild publish_dir: '' + signed: + var: + signed_status: signed + input_files: # Release @@ -112,3 +117,13 @@ steps: debug: 0 input_files: [] signtag: '[% INCLUDE signtag %]' + update_responses_config: + build_log: '-' + debug: 0 + input_files: [] + update_responses_config: '[% INCLUDE update_responses_config %]' + hash_incrementals: + build_log: '-' + debug: 0 + input_files: [] + hash_incrementals: '[% INCLUDE hash_incrementals %]' diff --git a/projects/release/hash_incrementals b/projects/release/hash_incrementals new file mode 100644 index 0000000..9cf1b27 --- /dev/null +++ b/projects/release/hash_incrementals @@ -0,0 +1,4 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +cd [% shell_quote(path(dest_dir)) %]/[% c("var/signed_status") %]/[% c("version") %]-[% c("var/torbrowser_build") %] +sha256sum `ls -1 | grep '\.incremental\.mar$' | sort` > sha256sums-[% c("var/signed_status") %]-build.incrementals.txt diff --git a/projects/release/update_responses_config b/projects/release/update_responses_config new file mode 100644 index 0000000..4ff470d --- /dev/null +++ b/projects/release/update_responses_config @@ -0,0 +1,11 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +cat > [% shell_quote(c("basedir")) %]/tools/update-responses/config.yml << 'EOF' +[% INCLUDE update_responses_config.yml -%] +EOF + +# Update / create symlink $torbrowser_version -> $torbrowser_version-buildN +versiondir=[% shell_quote(path(dest_dir)) _ '/' _ c("var/signed_status") _ '/' _ c("version") %] +test -L "$versiondir" && rm -f "$versiondir" +test -d "$versiondir" || ln -s [% shell_quote(c("version")) %]-[% shell_quote(c("var/torbrowser_build")) %] "$versiondir" + diff --git a/projects/release/update_responses_config.yml b/projects/release/update_responses_config.yml new file mode 100644 index 0000000..f0d9dbf --- /dev/null +++ b/projects/release/update_responses_config.yml @@ -0,0 +1,40 @@ +--- +appname_marfile: tor-browser +appname_bundle_osx: TorBrowser +appname_bundle_linux: tor-browser +appname_bundle_win: torbrowser-install +releases_dir: [% path(c('output_dir')) %]/[% c("var/signed_status") %] +download: + archive_url: https://archive.torproject.org/tor-package-archive/torbrowser + gpg_keyring: ../../keyring/torbrowser.gpg + bundles_url: https://dist.torproject.org/torbrowser + mars_url: https://cdn.torproject.org/aus1/torbrowser +build_targets: + linux32: Linux_x86-gcc3 + linux64: Linux_x86_64-gcc3 + win32: + - WINNT_x86-gcc3 + - WINNT_x86-gcc3-x86 + - WINNT_x86-gcc3-x64 + osx32: Darwin_x86-gcc3 + osx64: Darwin_x86_64-gcc3 +channels: + [% pc('firefox', 'var/torbrowser_update_channel') %]: [% c("var/torbrowser_version") %] +versions: + [% c("var/torbrowser_version") %]: + platformVersion: [% pc('firefox', 'var/firefox_platform_version') %] + detailsURL: https://blog.torproject.org/blog/tor-browser-[% c("var/torbrowser_version") FILTER remove('\.') %]-released + incremental_from: +[% FOREACH v IN c("var/torbrowser_incremental_from") -%] + - [% v %] +[% END -%] + migrate_archs: + osx32: osx64 + migrate_langs: + pt-PT: pt-BR + win32: + minSupportedInstructionSet: SSE2 + osx32: + minSupportedOSVersion: 13.0.0 + osx64: + minSupportedOSVersion: 13.0.0 diff --git a/rbm.conf b/rbm.conf index 5e1e66d..44fc9f5 100644 --- a/rbm.conf +++ b/rbm.conf @@ -16,6 +16,9 @@ buildconf: var: torbrowser_version: '7.5a4' torbrowser_build: 'build2' + torbrowser_incremental_from: + - 7.5a2 + - 7.5a3 project_name: tor-browser multi_lingual: 0 build_mar: 1 diff --git a/tools/update-responses/.gitignore b/tools/update-responses/.gitignore new file mode 100644 index 0000000..5947fca --- /dev/null +++ b/tools/update-responses/.gitignore @@ -0,0 +1,2 @@ +/htdocs +/config.yml diff --git a/tools/update-responses/README.md b/tools/update-responses/README.md new file mode 100644 index 0000000..5209ed5 --- /dev/null +++ b/tools/update-responses/README.md @@ -0,0 +1,50 @@ +Tor Browser Update Responses script +=================================== + +This repository contains a script to generate responses for Tor Browser +updater. + +See ticket [#12622](https://trac.torproject.org/projects/tor/ticket/12622) +for details. + + +Dependencies +------------ + +The following perl modules need to be installed to run the script: + FindBin YAML File::Slurp Digest::SHA XML::Writer File::Temp + IO::CaptureOutput Parallel::ForkManager XML::LibXML LWP JSON + +On Debian / Ubuntu you can install them with: + +``` + # apt-get install libfindbin-libs-perl libyaml-perl libfile-slurp-perl \ + libdigest-sha-perl libxml-writer-perl \ + libio-captureoutput-perl libparallel-forkmanager-perl \ + libxml-libxml-perl libwww-perl libjson-perl +``` + +On Red Hat / Fedora you can install them with: + +``` + # for module in FindBin YAML File::Slurp Digest::SHA XML::Writer \ + File::Temp IO::CaptureOutput Parallel::ForkManager \ + XML::LibXML LWP JSON + do yum install "perl($module)"; done +``` + + +URL Format +---------- + +The URL format is: + https://something/$channel/$build_target/$tb_version/$lang?force=1 + +'build_target' is the OS for which the browser was built. The correspo +ndance between the build target and the OS name that we use in archive +files is defined in the config.yml file. + +'tb_version' is the Tor Browser version. + +'lang' is the locale. + diff --git a/tools/update-responses/check_update_responses_deployement b/tools/update-responses/check_update_responses_deployement new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/check_update_responses_deployement @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/download_missing_versions b/tools/update-responses/download_missing_versions new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/download_missing_versions @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/gen_incrementals b/tools/update-responses/gen_incrementals new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/gen_incrementals @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/get_channel_version b/tools/update-responses/get_channel_version new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/get_channel_version @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses new file mode 100755 index 0000000..0ed19c1 --- /dev/null +++ b/tools/update-responses/update_responses @@ -0,0 +1,643 @@ +#!/usr/bin/perl -w + +use strict; +use feature "state"; +use English; +use FindBin; +use YAML qw(LoadFile); +use File::Slurp; +use Digest::SHA qw(sha256_hex); +use XML::Writer; +use Cwd; +use File::Copy; +use File::Temp; +use File::Find; +use POSIX qw(setlocale LC_ALL); +use IO::CaptureOutput qw(capture_exec); +use Parallel::ForkManager; +use File::Basename; +use XML::LibXML '1.70'; +use LWP::Simple; +use JSON; + +# Set umask and locale to provide a consistent environment for MAR file +# generation, etc. +umask(0022); +$ENV{"LC_ALL"} = "C"; +setlocale(LC_ALL, "C"); + +my $htdocsdir = "$FindBin::Bin/htdocs"; +my $config = LoadFile("$FindBin::Bin/config.yml"); +my %htdocsfiles; +my $releases_dir = $config->{releases_dir}; +$releases_dir = "$FindBin::Bin/$releases_dir" unless $releases_dir =~ m/^\//; +my @check_errors; +my $initPATH = $ENV{PATH}; +my $initLD_LIBRARY_PATH = $ENV{LD_LIBRARY_PATH}; + +sub exit_error { + print STDERR "Error: ", $_[0], "\n"; + chdir '/'; + exit (exists $_[1] ? $_[1] : 1); +} + +sub build_targets_by_os { + return ($_[0]) unless $config->{build_targets}{$_[0]}; + my $r = $config->{build_targets}{$_[0]}; + return ref $r eq 'ARRAY' ? @$r : ($r); +} + +sub get_nbprocs { + return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS}; + if (-f '/proc/cpuinfo') { + return scalar grep { m/^processor\s+:\s/ } read_file '/proc/cpuinfo'; + } + return 4; +} + +sub write_htdocs { + my ($channel, $file, $content) = @_; + mkdir $htdocsdir unless -d $htdocsdir; + mkdir "$htdocsdir/$channel" unless -d "$htdocsdir/$channel"; + write_file("$htdocsdir/$channel/$file", $content); + $htdocsfiles{$channel}->{$file} = 1; +} + +sub clean_htdocs { + my (@channels) = @_; + foreach my $channel (@channels) { + opendir(my $d, "$htdocsdir/$channel"); + my @files = grep { ! $htdocsfiles{$channel}->{$_} } readdir $d; + closedir $d; + unlink map { "$htdocsdir/$channel/$_" } @files; + } +} + +sub get_sha512_hex_of_file { + my ($file) = @_; + my $sha = Digest::SHA->new("512"); + $sha->addfile($file); + return $sha->hexdigest; +} + +sub get_version_files { + my ($config, $version) = @_; + return if $config->{versions}{$version}{files}; + my $appname = $config->{appname_marfile}; + my $files = {}; + my $vdir = "$releases_dir/$version"; + my $download_url = "$config->{download}{mars_url}/$version"; + opendir(my $d, $vdir) or exit_error "Error opening directory $vdir"; + foreach my $file (readdir $d) { + next unless -f "$vdir/$file"; + if ($file =~ m/^$appname-([^-]+)-${version}_(.+)\.mar$/) { + my ($os, $lang) = ($1, $2); + $files->{$os}{$lang}{complete} = { + type => 'complete', + URL => "$download_url/$file", + size => -s "$vdir/$file", + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file("$vdir/$file"), + }; + next; + } + if ($file =~ m/^$appname-([^-]+)-(.+)-${version}_(.+)\.incremental\.mar$/) { + my ($os, $from_version, $lang) = ($1, $2, $3); + $files->{$os}{$lang}{partial}{$from_version} = { + type => 'partial', + URL => "$download_url/$file", + size => -s "$vdir/$file", + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file("$vdir/$file"), + } + } + } + closedir $d; + $config->{versions}{$version}{files} = $files; +} + +sub get_version_downloads { + my ($config, $version) = @_; + my $downloads = {}; + my $vdir = "$releases_dir/$version"; + my $download_url = "$config->{download}{bundles_url}/$version"; + opendir(my $d, $vdir) or exit_error "Error opening directory $vdir"; + foreach my $file (readdir $d) { + next unless -f "$vdir/$file"; + my ($os, $lang); + if ($file =~ m/^$config->{appname_bundle_osx}-$version-osx64_(.+).dmg$/) { + ($os, $lang) = ('osx64', $1); + } elsif ($file =~ m/^$config->{appname_bundle_linux}-(linux32|linux64)-${version}_(.+).tar.xz$/) { + ($os, $lang) = ($1, $2); + } elsif ($file =~ m/^$config->{appname_bundle_win}-${version}_(.+).exe$/) { + ($os, $lang) = ('win32', $1); + } else { + next; + } + $downloads->{$os}{$lang} = { + binary => "$download_url/$file", + sig => "$download_url/$file.asc", + }; + } + closedir $d; + $config->{versions}{$version}{downloads} = $downloads; +} + +sub extract_mar { + my ($mar_file, $dest_dir) = @_; + my $old_cwd = getcwd; + mkdir $dest_dir; + chdir $dest_dir or exit_error "Cannot enter $dest_dir"; + my $res = system('mar', '-x', $mar_file); + exit_error "Error extracting $mar_file" if $res; + my $bunzip_file = sub { + return unless -f $File::Find::name; + rename $File::Find::name, "$File::Find::name.bz2"; + system('bunzip2', "$File::Find::name.bz2") == 0 + || exit_error "Error decompressing $File::Find::name"; + }; + find($bunzip_file, $dest_dir); + my $manifest = -f 'updatev3.manifest' ? 'updatev3.manifest' + : 'updatev2.manifest'; + my @lines = read_file($manifest) if -f $manifest; + foreach my $line (@lines) { + if ($line =~ m/^addsymlink "(.+)" "(.+)"$/) { + exit_error "$mar_file: Could not create symlink $1 -> $2" + unless symlink $2, $1; + } + } + chdir $old_cwd; +} + +sub mar_filename { + my ($appname, $version, $os, $lang) = @_; + "$releases_dir/$version/$appname-$os-${version}_$lang.mar"; +} + +sub create_incremental_mar { + my ($config, $pm, $from_version, $new_version, $os, $lang) = @_; + my $appname = $config->{appname_marfile}; + my $mar_file = "$appname-$os-${from_version}-${new_version}_$lang.incremental.mar"; + my $mar_file_path = "$releases_dir/$new_version/$mar_file"; + if ($ENV{MAR_SKIP_EXISTING} && -f $mar_file_path) { + print "Skipping $mar_file\n"; + return; + } + print "Starting $mar_file\n"; + my $download_url = "$config->{download}{mars_url}/$new_version"; + my $finished_file = sub { + exit_error "Error creating $mar_file" unless $_[1] == 0; + print "Finished $mar_file\n"; + $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = { + type => 'partial', + URL => "$download_url/$mar_file", + size => -s $mar_file_path, + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file($mar_file_path), + }; + }; + return if $pm->start($finished_file); + my $tmpdir = File::Temp->newdir(); + extract_mar(mar_filename($appname, $from_version, $os, $lang), "$tmpdir/A"); + extract_mar(mar_filename($appname, $new_version, $os, $lang), "$tmpdir/B"); + if ($ENV{CHECK_CODESIGNATURE_EXISTS}) { + unless (-f "$tmpdir/A/Contents/_CodeSignature/CodeResources" + && -f "$tmpdir/B/Contents/_CodeSignature/CodeResources") { + exit_error "Missing code signature while creating $mar_file"; + } + } + my ($out, $err, $success) = capture_exec('make_incremental_update.sh', + $mar_file_path, "$tmpdir/A", "$tmpdir/B"); + if (!$success) { + unlink $mar_file_path if -f $mar_file_path; + exit_error "making incremental mar:\n" . $err; + } + $pm->finish; +} + +sub create_incremental_mars_for_version { + my ($config, $version) = @_; + my $pm = Parallel::ForkManager->new(get_nbprocs); + $pm->run_on_finish(sub { $_[2]->(@_) }); + my $v = $config->{versions}{$version}; + foreach my $from_version (@{$v->{incremental_from}}) { + $config->{versions}{$from_version} //= {}; + get_version_files($config, $from_version); + my $from_v = $config->{versions}{$from_version}; + foreach my $os (keys %{$v->{files}}) { + foreach my $lang (keys %{$v->{files}{$os}}) { + next unless defined $from_v->{files}{$os}{$lang}{complete}; + create_incremental_mar($config, $pm, $from_version, $version, $os, $lang); + } + } + } + $pm->wait_all_children; +} + +sub get_config { + my ($config, $version, $os, $name) = @_; + return $config->{versions}{$version}{$os}{$name} + // $config->{versions}{$version}{$name} + // $config->{$name}; +} + +sub channel_to_version { + my ($config, @channels) = @_; + return values %{$config->{channels}} unless @channels; + foreach my $channel (@channels) { + exit_error "Unknown channel $channel" + unless $config->{channels}{$channel}; + } + return map { $config->{channels}{$_} } @channels; +} + +sub get_buildinfos { + my ($config, $version) = @_; + return if exists $config->{versions}{$version}{buildID}; + extract_martools($version); + my $files = $config->{versions}{$version}{files}; + foreach my $os (keys %$files) { + foreach my $lang (keys %{$files->{$os}}) { + next unless $files->{$os}{$lang}{complete}; + my $tmpdir = File::Temp->newdir(); + extract_mar( + mar_filename($config->{appname_marfile}, $version, $os, $lang), + "$tmpdir"); + my $appfile = "$tmpdir/application.ini" if -f "$tmpdir/application.ini"; + $appfile = "$tmpdir/Contents/Resources/application.ini" + if -f "$tmpdir/Contents/Resources/application.ini"; + exit_error "Could not find application.ini" unless $appfile; + foreach my $line (read_file($appfile)) { + if ($line =~ m/^BuildID=(.*)$/) { + $config->{versions}{$version}{buildID} = $1; + return; + } + } + exit_error "Could not extract buildID from application.ini"; + } + } +} + +sub get_response { + my ($config, $version, $os, @patches) = @_; + my $res; + my $writer = XML::Writer->new(OUTPUT => \$res, ENCODING => 'UTF-8'); + $writer->xmlDecl; + $writer->startTag('updates'); + if (get_config($config, $version, $os, 'unsupported')) { + $writer->startTag('update', + unsupported => 'true', + detailsURL => get_config($config, $version, $os, 'detailsURL'), + ); + goto CLOSETAGS; + } + my $minversion = get_config($config, $version, $os, 'minSupportedOSVersion'); + my $mininstruc = get_config($config, $version, $os, 'minSupportedInstructionSet'); + $writer->startTag('update', + type => 'minor', + displayVersion => $version, + appVersion => $version, + platformVersion => get_config($config, $version, $os, 'platformVersion'), + buildID => get_config($config, $version, $os, 'buildID'), + detailsURL => get_config($config, $version, $os, 'detailsURL'), + actions => 'showURL', + openURL => get_config($config, $version, $os, 'detailsURL'), + defined $minversion ? ( minSupportedOSVersion => $minversion ) : (), + defined $mininstruc ? ( minSupportedInstructionSet => $mininstruc ) : (), + ); + foreach my $patch (@patches) { + my @sorted_patch = map { $_ => $patch->{$_} } sort keys %$patch; + $writer->startTag('patch', @sorted_patch); + $writer->endTag('patch'); + } + CLOSETAGS: + $writer->endTag('update'); + $writer->endTag('updates'); + $writer->end; + return $res; +} + +sub write_responses { + my ($config, @channels) = @_; + @channels = keys %{$config->{channels}} unless @channels; + foreach my $channel (@channels) { + my $version = $config->{channels}{$channel}; + get_version_files($config, $version); + get_buildinfos($config, $version); + my $files = $config->{versions}{$version}{files}; + my $migrate_archs = $config->{versions}{$version}{migrate_archs} // {}; + foreach my $old_os (keys %$migrate_archs) { + my $new_os = $migrate_archs->{$old_os}; + foreach my $lang (keys %{$files->{$new_os}}) { + $files->{$old_os}{$lang}{complete} = + $files->{$new_os}{$lang}{complete}; + } + } + foreach my $os (keys %$files) { + foreach my $lang (keys %{$files->{$os}}) { + my $resp = get_response($config, $version, $os, + $files->{$os}{$lang}{complete}); + write_htdocs($channel, "$version-$os-$lang.xml", $resp); + foreach my $from_version (keys %{$files->{$os}{$lang}{partial}}) { + $resp = get_response($config, $version, $os, + $files->{$os}{$lang}{complete}, + $files->{$os}{$lang}{partial}{$from_version}); + write_htdocs($channel, "$from_version-$version-$os-$lang.xml", $resp); + } + } + } + write_htdocs($channel, 'no-update.xml', + '<?xml version="1.0" encoding="UTF-8"?>' + . "\n<updates></updates>\n"); + } +} + +sub write_htaccess { + my ($config, @channels) = @_; + @channels = keys %{$config->{channels}} unless @channels; + my $flags = "[last]"; + foreach my $channel (@channels) { + my $htaccess = "RewriteEngine On\n"; + my $version = $config->{channels}{$channel}; + my $migrate_langs = $config->{versions}{$version}{migrate_langs} // {}; + my $files = $config->{versions}{$version}{files}; + $htaccess .= "RewriteRule ^[^\/]+/$version/ no-update.xml $flags\n"; + foreach my $os (sort keys %$files) { + foreach my $bt (build_targets_by_os($os)) { + foreach my $lang (sort keys %{$files->{$os}}) { + foreach my $from_version (sort keys %{$files->{$os}{$lang}{partial}}) { + $htaccess .= "RewriteRule ^$bt/$from_version/$lang " + . "$from_version-$version-$os-$lang.xml $flags\n"; + } + $htaccess .= "RewriteRule ^$bt/[^\/]+/$lang " + . "$version-$os-$lang.xml $flags\n"; + } + foreach my $lang (sort keys %$migrate_langs) { + $htaccess .= "RewriteRule ^$bt/[^\/]+/$lang " + . "$version-$os-$migrate_langs->{$lang}.xml $flags\n"; + } + $htaccess .= "RewriteRule ^$bt/ $version-$os-en-US.xml $flags\n"; + } + } + write_htdocs($channel, '.htaccess', $htaccess); + } +} + +sub write_downloads_json { + my ($config, @channels) = @_; + @channels = keys %{$config->{channels}} unless @channels; + foreach my $channel (@channels) { + my $version = $config->{channels}{$channel}; + my $data = { + version => $version, + downloads => get_version_downloads($config, $version), + }; + write_htdocs($channel, 'downloads.json', + JSON->new->utf8->canonical->encode($data)); + } +} + +sub osname { + my ($osname) = capture_exec('uname', '-s'); + my ($arch) = capture_exec('uname', '-m'); + chomp($osname, $arch); + if ($osname eq 'Linux' && $arch eq 'x86_64') { + return 'linux64'; + } + if ($osname eq 'Linux' && $arch =~ m/^i.86$/) { + return 'linux32'; + } + exit_error 'Unknown OS'; +} + +my $martools_tmpdir; +sub extract_martools { + my ($version) = @_; + my $osname = osname; + my $marzip = "$releases_dir/$version/mar-tools-$osname.zip"; + $martools_tmpdir = File::Temp->newdir(); + my $old_cwd = getcwd; + chdir $martools_tmpdir; + my (undef, undef, $success) = capture_exec('unzip', $marzip); + chdir $old_cwd; + exit_error "Error extracting $marzip" unless $success; + $ENV{PATH} = "$martools_tmpdir/mar-tools:$initPATH"; + if ($initLD_LIBRARY_PATH) { + $ENV{LD_LIBRARY_PATH} = "$initLD_LIBRARY_PATH:$martools_tmpdir/mar-tools"; + } else { + $ENV{LD_LIBRARY_PATH} = "$martools_tmpdir/mar-tools"; + } +} + +sub log_step { + my ($url, $step, $status, $details) = @_; + state $u; + if (!defined $u || $url ne $u) { + print "\n" if $u; + print "$url\n"; + $u = $url; + } + print ' ', $step, $status ? ': OK' : ': ERROR', + $details ? " - $details\n" : "\n"; + return if $status; + push @check_errors, { url => $url, step => $step, details => $details }; +} + +sub get_remote_xml { + my ($url) = @_; + my $content = get $url; + log_step($url, 'get', defined $content); + return undef unless defined $content; + my $dom = eval { XML::LibXML->load_xml(string => $content) }; + log_step($url, 'parse_xml', defined $dom, $@); + return $dom; +} + +sub check_get_version { + my ($dom) = @_; + my @updates = $dom->documentElement()->getChildrenByLocalName('update'); + return undef unless @updates; + return $updates[0]->getAttribute('appVersion'); +} + +sub check_no_update { + my ($dom) = @_; + my @updates = $dom->documentElement()->getChildrenByLocalName('update'); + return @updates == 0; +} + +sub check_has_incremental { + my ($dom) = @_; + my @updates = $dom->documentElement()->getChildrenByLocalName('update'); + return undef unless @updates; + my @patches = $updates[0]->getChildrenByLocalName('patch'); + foreach my $patch (@patches) { + return 1 if $patch->getAttribute('type') eq 'partial'; + } + return undef; +} + +sub build_targets_list { + map { ref $_ eq 'ARRAY' ? @$_ : $_ } values %{$config->{build_targets}}; +} + +sub check_update_responses_channel { + my ($config, $base_url, $channel) = @_; + my $channel_version = $config->{channels}{$channel}; + foreach my $build_target (build_targets_list()) { + foreach my $lang (qw(en-US de)) { + my $url = "$base_url/$channel/$build_target/1.0/$lang"; + my $dom = get_remote_xml($url); + if ($dom) { + my $version = check_get_version($dom); + log_step($url, 'version', $version eq $channel_version, + "expected: $channel_version received: $version"); + } + $url = "$base_url/$channel/$build_target/$channel_version/$lang"; + $dom = get_remote_xml($url); + log_step($url, 'no_update', check_no_update($dom)) if $dom; + my @inc = @{$config->{versions}{$channel_version}{incremental_from}} + if $config->{versions}{$channel_version}{incremental_from}; + foreach my $inc_from (@inc) { + my $url = "$base_url/$channel/$build_target/$inc_from/$lang"; + $dom = get_remote_xml($url); + next unless $dom; + my $version = check_get_version($dom); + log_step($url, 'version', $version eq $channel_version, + "expected: $channel_version received: $version"); + log_step($url, 'has_incremental', check_has_incremental($dom)); + } + } + } +} + +sub download_version { + my ($config, $version) = @_; + my $tmpdir = File::Temp->newdir(); + my $destdir = "$releases_dir/$version"; + my $urldir = "$config->{download}{archive_url}/$version"; + print "Downloading version $version\n"; + foreach my $file (qw(sha256sums-unsigned-build.txt sha256sums-unsigned-build.txt.asc)) { + if (getstore("$urldir/$file", "$tmpdir/$file") != 200) { + exit_error "Error downloading $urldir/$file"; + } + } + if (system('gpg', '--no-default-keyring', '--keyring', + "$FindBin::Bin/$config->{download}{gpg_keyring}", '--verify', + "$tmpdir/sha256sums-unsigned-build.txt.asc", + "$tmpdir/sha256sums-unsigned-build.txt")) { + exit_error "Error checking gpg signature for version $version"; + } + mkdir $destdir; + move "$tmpdir/sha256sums-unsigned-build.txt.asc", "$destdir/sha256sums-unsigned-build.txt.asc"; + move "$tmpdir/sha256sums-unsigned-build.txt", "$destdir/sha256sums-unsigned-build.txt"; + my %sums = map { chomp; reverse split ' ', $_ } + read_file "$destdir/sha256sums-unsigned-build.txt"; + + my $martools = 'mar-tools-' . osname . '.zip'; + exit_error "Error downloading $urldir/$martools\n" + unless getstore("$urldir/$martools", "$tmpdir/$martools") == 200; + exit_error "Error downloading $urldir/$martools.asc\n" + unless getstore("$urldir/$martools.asc", "$tmpdir/$martools.asc") == 200; + if (system('gpg', '--no-default-keyring', '--keyring', + "$FindBin::Bin/$config->{download}{gpg_keyring}", '--verify', + "$tmpdir/$martools.asc", "$tmpdir/$martools")) { + exit_error "Error checking gpg signature for $version/$martools"; + } + exit_error "Wrong checksum for $version/$martools" + unless $sums{$martools} eq sha256_hex(read_file("$tmpdir/$martools")); + move "$tmpdir/$martools", "$destdir/$martools"; + move "$tmpdir/$martools.asc", "$destdir/$martools.asc"; + extract_martools($version); + + foreach my $file (sort grep { $_ =~ m/\.mar$/ } keys %sums) { + print "Downloading $file\n"; + exit_error "Error downloading $urldir/$file\n" + unless getstore("$urldir/$file", "$tmpdir/$file") == 200; + if ($sums{$file} ne sha256_hex(read_file("$tmpdir/$file"))) { + exit_error "Error unsigning $file" + if system('signmar', '-r', "$tmpdir/$file", "$tmpdir/$file.u"); + exit_error "Wrong checksum for $file" + unless $sums{$file} eq sha256_hex(read_file("$tmpdir/$file.u")); + move "$tmpdir/$file.u", "$tmpdir/$file"; + } + move "$tmpdir/$file", "$destdir/$file"; + } +} + +sub download_missing_versions { + my ($config, @channels) = @_; + foreach my $channel (@channels) { + exit_error "Unknown channel $channel" + unless $config->{channels}{$channel}; + my $cversion = $config->{channels}{$channel}; + next unless $config->{versions}{$cversion}{incremental_from}; + foreach my $version (@{$config->{versions}{$cversion}{incremental_from}}) { + next if -d "$releases_dir/$version"; + download_version($config, $version); + } + } +} + +sub check_update_responses { + my ($config) = @_; + exit_error "usage: $PROGRAM_NAME <base_url> [channels...]" unless @ARGV; + my ($base_url, @channels) = @ARGV; + foreach my $channel (@channels ? @channels : keys %{$config->{channels}}) { + check_update_responses_channel($config, $base_url, $channel); + } + if (!@check_errors) { + print "\n\nNo errors\n"; + return; + } + print "\n\nErrors list:\n"; + my $url = ''; + foreach my $error (@check_errors) { + if ($url ne $error->{url}) { + $url = $error->{url}; + print "$url\n"; + } + print " $error->{step}", + $error->{details} ? " - $error->{details}\n" : "\n"; + } +} + +my %actions = ( + update_responses => sub { + my ($config) = @_; + my @channels = @ARGV ? @ARGV : keys %{$config->{channels}}; + foreach my $channel (@channels) { + exit_error "Unknown channel $channel" + unless $config->{channels}{$channel}; + $htdocsfiles{$channel} = { '.' => 1, '..' => 1 }; + } + write_responses($config, @channels); + write_htaccess($config, @channels); + write_downloads_json($config, @channels); + clean_htdocs(@channels); + }, + gen_incrementals => sub { + my ($config) = @_; + foreach my $version (channel_to_version($config, @ARGV)) { + extract_martools($version); + get_version_files($config, $version); + create_incremental_mars_for_version($config, $version); + } + }, + download_missing_versions => sub { + my ($config) = @_; + my @channels = @ARGV ? @ARGV : keys %{$config->{channels}}; + download_missing_versions($config, @channels); + }, + check_update_responses_deployement => \&check_update_responses, + get_channel_version => sub { + my ($config) = @_; + exit_error "Wrong arguments" unless @ARGV == 1; + exit_error "Unknown channel" unless $config->{channels}{$ARGV[0]}; + print $config->{channels}{$ARGV[0]}, "\n"; + }, +); + +my $action = fileparse($PROGRAM_NAME); +exit_error "Unknown action $action" unless $actions{$action}; +$actions{$action}->($config);

1 0