- tbb-commits - lists.torproject.org

[tor-browser-bundle/master] Update gitian/*.sh scripts for incremental mars
by mikeperry＠torproject.org 25 Oct '14

25 Oct '14

commit f1472e59d24945d3f05a740925a05c4bfac4ccc7 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Oct 20 18:35:50 2014 +0200 Update gitian/*.sh scripts for incremental mars Running tools/update-responses/update-responses will generate incremental mar files in the gitian/$release directory. hash-bundles.sh has been updated to exclude the incremental mars from sha256sums.txt and include them in sha256sums.incrementals.txt. Some makefile rules (hash, hash-alpha, hash-beta, hash-nightly) to run hash-bundles.sh have been added. upload-signature.sh has been updated to sign and upload sha256sums.incrementals.txt if present. check-match.sh has been updated to match sha256sums.incrementals.txt if present. An 'incrementals' makefile rule has also been added to run the update_responses script to generate incremental mars. --- gitian/Makefile | 15 +++++++++++++++ gitian/check-match.sh | 44 ++++++++++++++++++++++++++++++++++++++++++-- gitian/hash-bundles.sh | 8 ++++++-- gitian/upload-signature.sh | 9 ++++++++- 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/gitian/Makefile b/gitian/Makefile index 3967795..2c17656 100644 --- a/gitian/Makefile +++ b/gitian/Makefile @@ -32,6 +32,21 @@ build-beta: ./mkbundle-mac.sh versions.beta ./hash-bundles.sh versions.beta +incrementals: + ../tools/update-responses/update_responses + +hash: + ./hash-bundles.sh versions + +hash-alpha: + ./hash-bundles.sh versions.alpha + +hash-beta: + ./hash-bundles.sh versions.beta + +hash-nightly: + ./hash-bundles.sh versions.nightly + sign: $(TORSOCKS) ./upload-signature.sh versions diff --git a/gitian/check-match.sh b/gitian/check-match.sh index 09bf179..43d1ef1 100755 --- a/gitian/check-match.sh +++ b/gitian/check-match.sh @@ -26,6 +26,7 @@ fi . $VERSIONS_FILE VALID="" +VALID_incrementals="" for u in $USERS do @@ -49,14 +50,53 @@ do VALID="$u $VALID" done +cd ../.. +if [ -f $TORBROWSER_VERSION/sha256sums.incrementals.txt ] +then + for u in $USERS + do + cd $WRAPPER_DIR + + # XXX: Is there a better way to store these and rename them? + mkdir -p $TORBROWSER_VERSION/$u + cd $TORBROWSER_VERSION/$u + + wget -U "" -N https://$HOST/~$u/builds/$TORBROWSER_VERSION/sha256sums.incrementals.txt || continue + wget -U "" -N https://$HOST/~$u/builds/$TORBROWSER_VERSION/sha256sums.incrementals.txt.asc || continue + + keyring="../../gpg/$u.gpg" + + # XXX: Remove this dir + gpghome=$(mktemp -d) + GNUPGHOME="$gpghome" gpg --import "$keyring" + GNUPGHOME="$gpghome" gpg sha256sums.incrementals.txt.asc || exit 1 + + diff -u ../sha256sums.incrementals.txt sha256sums.incrementals.txt || exit 1 + + VALID_incrementals="$u $VALID_incrementals" + done +fi +cd ../.. + +exit_val=0 if [ -z "$VALID" ]; then echo "No bundle hashes or sigs published for $TORBROWSER_VERSION." echo - exit 1 + exit_val=1 else echo "Matching bundles exist from the following users: $VALID" - exit 0 fi +if [ -f $TORBROWSER_VERSION/sha256sums.incrementals.txt ] +then + if [ -z "$VALID_incrementals" ] + then + echo "No incremental mars hashes or sigs published for $TORBROWSER_VERSION." + else + echo "Matching incremental mars exist from the following users: $VALID_incrementals" + fi +fi + +exit $exit_val diff --git a/gitian/hash-bundles.sh b/gitian/hash-bundles.sh index 20a26d0..2684158 100755 --- a/gitian/hash-bundles.sh +++ b/gitian/hash-bundles.sh @@ -18,8 +18,12 @@ fi export LC_ALL=C cd $TORBROWSER_VERSION -rm -f sha256sums.txt -sha256sum `ls -1 | sort` > sha256sums.txt +rm -f sha256sums.txt sha256sums.incrementals.txt +sha256sum `ls -1 | grep -v '\.incremental\.mar$' | sort` > sha256sums.txt +if ls -1 | grep -q '\.incremental\.mar$' +then + sha256sum `ls -1 | grep '\.incremental\.mar$' | sort` > sha256sums.incrementals.txt +fi echo echo "If this is an official build, you should now sign your result with: " diff --git a/gitian/upload-signature.sh b/gitian/upload-signature.sh index ffb97fe..58059aa 100755 --- a/gitian/upload-signature.sh +++ b/gitian/upload-signature.sh @@ -28,7 +28,14 @@ then popd fi +if [ -f $TORBROWSER_VERSION/sha256sums.incrementals.txt ] \ + && [ ! -f $TORBROWSER_VERSION/sha256sums.incrementals.txt.asc ] +then + pushd $TORBROWSER_VERSION && gpg -abs sha256sums.incrementals.txt + popd +fi + ssh $HOST "mkdir -p $BASE_DIR/$TORBROWSER_VERSION" -scp $TORBROWSER_VERSION/sha256sums.txt* $HOST:$BASE_DIR/$TORBROWSER_VERSION/ +scp $TORBROWSER_VERSION/sha256sums*.txt* $HOST:$BASE_DIR/$TORBROWSER_VERSION/ ssh $HOST "chmod 755 $BASE_DIR/$TORBROWSER_VERSION && chmod 644 $BASE_DIR/$TORBROWSER_VERSION/*"

1 0

[tor-browser-bundle/master] Bug #13324: Address comments from mcs's review
by mikeperry＠torproject.org 25 Oct '14

25 Oct '14

commit 8ad38fa9eed101c4e44ce6c7951579cf13fd708b Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Fri Oct 24 16:49:41 2014 -0700 Bug #13324: Address comments from mcs's review --- gitian/check-match.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitian/check-match.sh b/gitian/check-match.sh index 43d1ef1..0622512 100755 --- a/gitian/check-match.sh +++ b/gitian/check-match.sh @@ -52,6 +52,7 @@ do done cd ../.. +# XXX: We should refactor this code into a shared function if [ -f $TORBROWSER_VERSION/sha256sums.incrementals.txt ] then for u in $USERS @@ -94,6 +95,7 @@ then if [ -z "$VALID_incrementals" ] then echo "No incremental mars hashes or sigs published for $TORBROWSER_VERSION." + exit_val=1 else echo "Matching incremental mars exist from the following users: $VALID_incrementals" fi

1 0

[tor-browser-bundle/master] Bump NoScript and Tor versions.
by mikeperry＠torproject.org 25 Oct '14

25 Oct '14

commit 45180fdb5a647c13f7192e91e8d63370e860d4b9 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Fri Oct 24 16:54:30 2014 -0700 Bump NoScript and Tor versions. --- Bundle-Data/Docs/ChangeLog.txt | 2 ++ gitian/versions | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Bundle-Data/Docs/ChangeLog.txt b/Bundle-Data/Docs/ChangeLog.txt index 92a060b..fe53b58 100644 --- a/Bundle-Data/Docs/ChangeLog.txt +++ b/Bundle-Data/Docs/ChangeLog.txt @@ -1,5 +1,7 @@ Tor Browser 4.0.1 -- Oct 30 2014 * All Platforms + * Update Tor to 0.2.5.10 + * Update NoScript to 2.6.9.3 * Bug 13301: Prevent extensions incompatibility error after upgrades * Bug 13460: Fix MSVC compilation issue * Windows diff --git a/gitian/versions b/gitian/versions index a24f492..5db0e5d 100755 --- a/gitian/versions +++ b/gitian/versions @@ -9,7 +9,7 @@ FIREFOX_VERSION=31.2.0esr TORBROWSER_UPDATE_CHANNEL=release TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-4.x-1-build1 -TOR_TAG=tor-0.2.5.8-rc +TOR_TAG=tor-0.2.5.10 TORLAUNCHER_TAG=0.2.7.0.1 TORBUTTON_TAG=1.7.0.1 HTTPSE_TAG=3.5.3 # XXX: HTTPSE_VER is used instead, pending #11630 @@ -53,7 +53,7 @@ GO_VER=1.3.3 ## File names for the source packages OPENSSL_PACKAGE=openssl-${OPENSSL_VER}.tar.gz GMP_PACKAGE=gmp-${GMP_VER}.tar.bz2 -NOSCRIPT_PACKAGE=noscript_security_suite-2.6.9.1-fn+fx+sm.xpi +NOSCRIPT_PACKAGE=noscript_security_suite-2.6.9.3-sm+fn+fx.xpi HTTPSE_PACKAGE=https-everywhere-${HTTPSE_VER}.xpi TOOLCHAIN4_PACKAGE=x86_64-apple-darwin10.tar.xz TOOLCHAIN4_OLD_PACKAGE=multiarch-darwin11-cctools127.2-gcc42-5666.3-llvmgcc42-2336.1-Linux-120724.tar.xz @@ -81,7 +81,7 @@ GMP_HASH=752079520b4690531171d0f4532e40f08600215feefede70b24fabdc6f1ab160 OSXSDK_HASH=6602d8d5ddb371fbc02e2a5967d9bd0cd7358d46f9417753c8234b923f2ea6fc TOOLCHAIN4_HASH=7b71bfe02820409b994c5c33a7eab81a81c72550f5da85ff7af70da3da244645 TOOLCHAIN4_OLD_HASH=65c1b2d302358a6b95a26c6828a66908a199276193bb0b268f2dcc1a997731e9 -NOSCRIPT_HASH=92bfa1f8a5673ffc82fe54db43c5f410c1cf403e38b82780f01eeeeae8233d0b +NOSCRIPT_HASH=d399d724953fe032ccca758e577c2136b8fd6c377cea0e3f42ed999b796d290a HTTPSE_HASH=cd811b1d424ff24939249de1e2b745ba86cbd7e6804699a9460c631a0b3474fe MSVCR100_HASH=1221a09484964a6f38af5e34ee292b9afefccb3dc6e55435fd3aaf7c235d9067 PYCRYPTO_HASH=f2ce1e989b272cfcb677616763e0a2e7ec659effa67a88aa92b3a65528f60a3c

1 0

[tor-browser-bundle/master] update_responses: rename incremental files to *.incremental.mar
by mikeperry＠torproject.org 25 Oct '14

25 Oct '14

commit f11faec46052d5996f8316fce515fbfefecf5491 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Oct 13 19:21:02 2014 +0200 update_responses: rename incremental files to *.incremental.mar This should make them easier to grep. --- tools/update-responses/update_responses | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index 7f1df2d..a42e700 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -76,7 +76,7 @@ sub get_version_files { }; next; } - if ($file =~ m/^tor-browser-([^-]+)-(.+)-${version}_(.+)\.mar$/) { + if ($file =~ m/^tor-browser-([^-]+)-(.+)-${version}_(.+)\.incremental\.mar$/) { my ($os, $from_version, $lang) = ($1, $2, $3); $files->{$os}{$lang}{partial}{$from_version} = { type => 'partial', @@ -124,7 +124,7 @@ sub mar_filename { sub create_incremental_mar { my ($config, $pm, $from_version, $new_version, $os, $lang) = @_; - my $mar_file = "tor-browser-$os-${from_version}-${new_version}_$lang.mar"; + my $mar_file = "tor-browser-$os-${from_version}-${new_version}_$lang.incremental.mar"; print "Starting $mar_file\n"; my $mar_file_path = "$releases_dir/$new_version/$mar_file"; my $finished_file = sub {

1 0

[tor-browser-bundle/master] Bump Tor Browser tag.
by mikeperry＠torproject.org 25 Oct '14

25 Oct '14

commit 3511160eff94afce904590940eee59dd6dd6685a Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Fri Oct 24 16:55:55 2014 -0700 Bump Tor Browser tag. --- gitian/versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitian/versions b/gitian/versions index 5db0e5d..8d8c4c9 100755 --- a/gitian/versions +++ b/gitian/versions @@ -8,7 +8,7 @@ FIREFOX_VERSION=31.2.0esr TORBROWSER_UPDATE_CHANNEL=release -TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-4.x-1-build1 +TORBROWSER_TAG=tor-browser-${FIREFOX_VERSION}-4.x-1-build2 TOR_TAG=tor-0.2.5.10 TORLAUNCHER_TAG=0.2.7.0.1 TORBUTTON_TAG=1.7.0.1

1 0

[tor-browser-bundle/master] Revert "The poodle hotfix."
by mikeperry＠torproject.org 25 Oct '14

25 Oct '14

commit 77e6fb571a69e4b146909dfda8f7552b34ab34d8 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Fri Oct 24 16:29:43 2014 -0700 Revert "The poodle hotfix." This reverts commit fd7e8eef09b813c445e31239bcb674de7a03d191. We set this pref in our browser preferences now. --- gitian/descriptors/linux/gitian-bundle.yml | 4 ---- gitian/descriptors/mac/gitian-bundle.yml | 4 ---- gitian/descriptors/windows/gitian-bundle.yml | 4 ---- 3 files changed, 12 deletions(-) diff --git a/gitian/descriptors/linux/gitian-bundle.yml b/gitian/descriptors/linux/gitian-bundle.yml index 3de0d49..19233b6 100644 --- a/gitian/descriptors/linux/gitian-bundle.yml +++ b/gitian/descriptors/linux/gitian-bundle.yml @@ -146,8 +146,6 @@ script: | cp defaults/preferences/000-tor-browser.js ~/build/ # Set the locale of the bundle. echo "pref(\"general.useragent.locale\", \"en-US\");" >> defaults/preferences/000-tor-browser.js - # The poodle hotfix - echo "pref(\"security.tls.version.min\", 1);" >> defaults/preferences/000-tor-browser.js zip -Xm omni.ja defaults/preferences/000-tor-browser.js rm -rf defaults popd @@ -178,8 +176,6 @@ script: | cp ~/build/000-tor-browser.js defaults/preferences/ # Set the locale of the bundle. echo "pref(\"general.useragent.locale\", \"$LANG\");" >> defaults/preferences/000-tor-browser.js - # The poodle hotfix - echo "pref(\"security.tls.version.min\", 1);" >> defaults/preferences/000-tor-browser.js zip -Xm omni.ja defaults/preferences/000-tor-browser.js rm -rf defaults popd diff --git a/gitian/descriptors/mac/gitian-bundle.yml b/gitian/descriptors/mac/gitian-bundle.yml index 9e78429..82b6595 100644 --- a/gitian/descriptors/mac/gitian-bundle.yml +++ b/gitian/descriptors/mac/gitian-bundle.yml @@ -166,8 +166,6 @@ script: | cp defaults/preferences/000-tor-browser.js ~/build/ # Set the locale of the bundle. echo "pref(\"general.useragent.locale\", \"en-US\");" >> defaults/preferences/000-tor-browser.js - # The poodle hotfix - echo "pref(\"security.tls.version.min\", 1);" >> defaults/preferences/000-tor-browser.js zip -Xm omni.ja defaults/preferences/000-tor-browser.js rm -rf defaults popd @@ -223,8 +221,6 @@ script: | cp ~/build/000-tor-browser.js defaults/preferences/ # Set the locale of the bundle. echo "pref(\"general.useragent.locale\", \"$LANG\");" >> defaults/preferences/000-tor-browser.js - # The poodle hotfix - echo "pref(\"security.tls.version.min\", 1);" >> defaults/preferences/000-tor-browser.js zip -Xm omni.ja defaults/preferences/000-tor-browser.js rm -rf defaults popd diff --git a/gitian/descriptors/windows/gitian-bundle.yml b/gitian/descriptors/windows/gitian-bundle.yml index a3c4b8a..76f5c67 100644 --- a/gitian/descriptors/windows/gitian-bundle.yml +++ b/gitian/descriptors/windows/gitian-bundle.yml @@ -126,8 +126,6 @@ script: | cp defaults/preferences/000-tor-browser.js ~/build/ # Set the locale bundle. echo "pref(\"general.useragent.locale\", \"en-US\");" >> defaults/preferences/000-tor-browser.js - # The poodle hotfix - echo "pref(\"security.tls.version.min\", 1);" >> defaults/preferences/000-tor-browser.js zip -Xm omni.ja defaults/preferences/000-tor-browser.js rm -rf defaults popd @@ -158,8 +156,6 @@ script: | mkdir -p defaults/preferences cp ~/build/000-tor-browser.js defaults/preferences/ echo "pref(\"general.useragent.locale\", \"$LANG\");" >> defaults/preferences/000-tor-browser.js - # The poodle hotfix - echo "pref(\"security.tls.version.min\", 1);" >> defaults/preferences/000-tor-browser.js zip -Xm omni.ja defaults/preferences/000-tor-browser.js rm -rf defaults popd

1 0

[tor-browser-bundle/master] update_responses: extract mar-tools in a temporary directory
by mikeperry＠torproject.org 25 Oct '14

25 Oct '14

commit d5459fbac32c7be88e70093b18aad138dd620b7c Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Oct 13 12:31:33 2014 +0200 update_responses: extract mar-tools in a temporary directory Extract the gitian-builder/inputs/mar-tools-linux*.zip file that was created during the last firefox build. --- tools/update-responses/update_responses | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index 55ebfb9..7f1df2d 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -268,6 +268,33 @@ sub check_deps { } } +sub osname { + my ($osname) = capture_exec('uname', '-s'); + my ($arch) = capture_exec('uname', '-m'); + chomp($osname, $arch); + if ($osname eq 'Linux' && $arch eq 'x86_64') { + return 'linux64'; + } + if ($osname eq 'Linux' && $arch =~ m/^i.86$/) { + return 'linux32'; + } + exit_error 'Unknown OS'; +} + +my $martools_tmpdir; +sub extract_martools { + my $osname = osname; + my $marzip = "$FindBin::Bin/../../../gitian-builder/inputs/mar-tools-$osname.zip"; + $martools_tmpdir = File::Temp->newdir(); + my $old_cwd = getcwd; + chdir $martools_tmpdir; + my (undef, undef, $success) = capture_exec('unzip', $marzip); + chdir $old_cwd; + exit_error "Error extracting $marzip" unless $success; + $ENV{PATH} .= ":$martools_tmpdir/mar-tools"; +} + +extract_martools; check_deps; write_responses($config); write_htaccess($config);

1 0

[tor-browser-bundle/master] Add changelog for 4.0.1.
by mikeperry＠torproject.org 25 Oct '14

25 Oct '14

commit adcf0528cb53e2054d46871b36289b28f7e52498 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Fri Oct 24 16:39:31 2014 -0700 Add changelog for 4.0.1. --- Bundle-Data/Docs/ChangeLog.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Bundle-Data/Docs/ChangeLog.txt b/Bundle-Data/Docs/ChangeLog.txt index 7d8470b..92a060b 100644 --- a/Bundle-Data/Docs/ChangeLog.txt +++ b/Bundle-Data/Docs/ChangeLog.txt @@ -1,3 +1,10 @@ +Tor Browser 4.0.1 -- Oct 30 2014 + * All Platforms + * Bug 13301: Prevent extensions incompatibility error after upgrades + * Bug 13460: Fix MSVC compilation issue + * Windows + * Bug 13443: Disable DirectShow to prevent crashes on many sites + Tor Browser 4.0 -- Oct 15 2014 * All Platforms * Update Firefox to 31.2.0esr @@ -26,7 +33,6 @@ Tor Browser 4.0 -- Oct 15 2014 * Bug 13186: Disable DOM Performance timers * Bug 13028: Defense-in-depth checks for OCSP/Cert validation proxy usage - Tor Browser 4.0-alpha-3 -- Sep 24 2014 * All Platforms * Update Tor to 0.2.5.8-rc

1 0

[tor-browser-bundle/master] update_responses: create symlinks when extracting mars
by mikeperry＠torproject.org 25 Oct '14

25 Oct '14

commit cf83610e79951c0676212dca49d7144bf5df0f21 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Sun Oct 12 15:03:43 2014 +0200 update_responses: create symlinks when extracting mars Find the addsymlink instructions in the updatev3.manifest file and create corresponding symbolic links. If no updatev3.manifest file is present, we do it with the updatev2.manifest file if it is present. --- tools/update-responses/update_responses | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index 3590227..55ebfb9 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -105,6 +105,15 @@ sub extract_mar { || exit_error "Error decompressing $File::Find::name"; }; find($bunzip_file, $dest_dir); + my $manifest = -f 'updatev3.manifest' ? 'updatev3.manifest' + : 'updatev2.manifest'; + my @lines = read_file($manifest) if -f $manifest; + foreach my $line (@lines) { + if ($line =~ m/^addsymlink "(.+)" "(.+)"$/) { + exit_error "$mar_file: Could not create symlink $1 -> $2" + unless symlink $2, $1; + } + } chdir $old_cwd; }

1 0

[tor-browser-bundle/master] update_responses: generate multiple MAR files in parallel
by mikeperry＠torproject.org 25 Oct '14

25 Oct '14

commit 49c5f2e28f172e85c66f0ba3cd63645a7373858e Author: Nicolas Vigier <boklm(a)mars-attacks.org> Date: Thu Oct 2 18:35:08 2014 +0200 update_responses: generate multiple MAR files in parallel Generating incremental MAR files takes a lot of time but only uses one thread. To make that faster, we will be creating multiple MAR files in parallel. The number of files we generate in parallel will be equal to the number of CPU cores, unless the NUM_PROCS environment variable is set. --- tools/update-responses/README.md | 8 +++--- tools/update-responses/update_responses | 43 +++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/tools/update-responses/README.md b/tools/update-responses/README.md index c7ecdd4..206bd7a 100644 --- a/tools/update-responses/README.md +++ b/tools/update-responses/README.md @@ -13,21 +13,23 @@ Dependencies The following perl modules need to be installed to run the script: FindBin YAML File::Slurp Digest::SHA XML::Writer File::Temp - IO::CaptureOutput File::Which + IO::CaptureOutput File::Which Parallel::ForkManager On Debian / Ubuntu you can install them with: ``` # apt-get install libfindbin-libs-perl libyaml-perl libfile-slurp-perl \ libdigest-sha-perl libxml-writer-perl \ - libio-captureoutput-perl libfile-which-perl + libio-captureoutput-perl libfile-which-perl \ + libparallel-forkmanager-perl ``` On Red Hat / Fedora you can install them with: ``` # for module in FindBin YAML File::Slurp Digest::SHA XML::Writer \ - File::Temp IO::CaptureOutput File::Which + File::Temp IO::CaptureOutput File::Which \ + Parallel::ForkManager do yum install "perl($module)"; done ``` diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index 1946278..3590227 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -11,6 +11,7 @@ use File::Temp; use File::Find; use File::Which; use IO::CaptureOutput qw(capture_exec); +use Parallel::ForkManager; my $htdocsdir = "$FindBin::Bin/htdocs"; my $config = LoadFile("$FindBin::Bin/config.yml"); @@ -27,6 +28,14 @@ sub build_target_by_os { $config->{build_targets}{$_[0]} ? $config->{build_targets}{$_[0]} : $_[0]; } +sub get_nbprocs { + return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS}; + if (-f '/proc/cpuinfo') { + return scalar grep { m/^processor\s+:\s/ } read_file '/proc/cpuinfo'; + } + return 4; +} + sub write_htdocs { my ($file, $content) = @_; mkdir $htdocsdir unless -d $htdocsdir; @@ -105,31 +114,38 @@ sub mar_filename { } sub create_incremental_mar { - my ($config, $from_version, $new_version, $os, $lang) = @_; - print "Creating incremental mar file ", - "$from_version -> $new_version ($os $lang)\n"; + my ($config, $pm, $from_version, $new_version, $os, $lang) = @_; + my $mar_file = "tor-browser-$os-${from_version}-${new_version}_$lang.mar"; + print "Starting $mar_file\n"; + my $mar_file_path = "$releases_dir/$new_version/$mar_file"; + my $finished_file = sub { + exit_error "Error creating $mar_file" unless $_[1] == 0; + print "Finished $mar_file\n"; + $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = { + type => 'partial', + URL => "$config->{versions}{$new_version}{download_url}/$mar_file", + size => -s $mar_file_path, + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file($mar_file_path), + }; + }; + return if $pm->start($finished_file); my $tmpdir = File::Temp->newdir(); extract_mar(mar_filename($from_version, $os, $lang), "$tmpdir/A"); extract_mar(mar_filename($new_version, $os, $lang), "$tmpdir/B"); - my $mar_file = "tor-browser-$os-${from_version}-${new_version}_$lang.mar"; - my $mar_file_path = "$releases_dir/$new_version/$mar_file"; my ($out, $err, $success) = capture_exec('make_incremental_update.sh', $mar_file_path, "$tmpdir/A", "$tmpdir/B"); if (!$success) { unlink $mar_file_path if -f $mar_file_path; exit_error "making incremental mar:\n" . $err; } - $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = { - type => 'partial', - URL => "$config->{versions}{$new_version}{download_url}/$mar_file", - size => -s $mar_file_path, - hashFunction => 'SHA512', - hashValue => get_sha512_hex_of_file($mar_file_path), - }; + $pm->finish; } sub create_missing_incremental_mars { my ($config, $version) = @_; + my $pm = Parallel::ForkManager->new(get_nbprocs); + $pm->run_on_finish(sub { $_[2]->(@_) }); my $v = $config->{versions}{$version}; foreach my $from_version (@{$v->{incremental_from}}) { $config->{versions}{$from_version} //= { download_url => '' }; @@ -139,10 +155,11 @@ sub create_missing_incremental_mars { foreach my $lang (keys %{$v->{files}{$os}}) { next if defined $v->{files}{$os}{$lang}{partial}{$from_version}; next unless defined $from_v->{files}{$os}{$lang}{complete}; - create_incremental_mar($config, $from_version, $version, $os, $lang); + create_incremental_mar($config, $pm, $from_version, $version, $os, $lang); } } } + $pm->wait_all_children; } sub get_config {

1 0