commit 2d129f5c6b282c6e9e915339f7b4dfb0f54b5a80 Author: Nicolas Vigier boklm@torproject.org Date: Wed Oct 10 01:33:59 2018 +0200
Bug 27218: generate multiple bundles in parallel --- projects/tor-browser/build | 28 ++++++++++++++++++++++------ projects/tor-browser/config | 16 ++++++++++------ projects/tor-browser/ddmg.sh | 10 +++++----- projects/tor-browser/run_scripts | 30 ++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 17 deletions(-)
diff --git a/projects/tor-browser/build b/projects/tor-browser/build index 4d51ec6..68050f9 100644 --- a/projects/tor-browser/build +++ b/projects/tor-browser/build @@ -5,6 +5,9 @@ export TORBROWSER_VERSION='[% c("version") %]'
mkdir -p $distdir
+scripts_dir=/var/tmp/build_scripts +mkdir -p "$scripts_dir" + OUTDIR='[% dest_dir _ "/" _ c("filename") %]' mkdir -p $OUTDIR
@@ -252,8 +255,6 @@ popd mv $distdir/tbb-windows-installer ${TB_STAGE_DIR} [% END %]
-cp -a ${TB_STAGE_DIR} $distdir/$PKG_DIR - [% IF c("var/windows") %] TBDIR="$distdir/$PKG_DIR/Tor Browser/Browser" [% ELSIF c("var/osx") %] @@ -262,6 +263,11 @@ cp -a ${TB_STAGE_DIR} $distdir/$PKG_DIR TBDIR="$distdir/$PKG_DIR/Browser" [% END %]
+cat > "$scripts_dir/create-$PKG_DIR" << SCRIPT_EOF +#!/bin/bash +set -e +cp -a ${TB_STAGE_DIR} $distdir/$PKG_DIR + pushd "$TBDIR[% IF c("var/osx") %]/Contents/Resources/[% END %]" rm -f precomplete python $MARTOOLS/createprecomplete.py @@ -271,8 +277,8 @@ cd $distdir
[% IF c("var/build_mar") -%] # Create full MAR file and compressed package. - MAR_FILE=tor-browser-[% c("var/mar_osname") %]-[% c("var/torbrowser_version") %]_${PKG_LOCALE}.mar - MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/$MAR_FILE "$TBDIR" + [% SET mar_file = 'tor-browser-' _ c("var/mar_osname") _ '-' _ c("var/torbrowser_version") _ '_${PKG_LOCALE}.mar' %] + MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/[% mar_file %] "$TBDIR" [% END -%]
[% IF c("var/linux") %] @@ -298,6 +304,7 @@ cd $distdir popd [% END %] rm -rf $distdir/${PKG_DIR} +SCRIPT_EOF
cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/ [% IF c("var/linux-x86_64") -%] @@ -314,6 +321,7 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/ [% SET lang = tmpl(lang); SET xpi = '$rootdir/' _ c('input_files_by_name/firefox-langpacks') _ '/' _ lang _ '.xpi'; SET tbdir = '$distdir/tor-browser_' _ lang; + SET mar_file = 'tor-browser-' _ c("var/mar_osname") _ '-' _ c("var/torbrowser_version") _ '_' _ lang _ '.mar'; IF c("var/osx"); SET browserdir = tbdir _ '/Tor Browser.app'; ELSIF c("var/windows"); @@ -322,6 +330,9 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/ SET browserdir = tbdir _ '/Browser'; END; %] + cat > "$scripts_dir/create-tor-browser_[% lang %]" << SCRIPT_EOF +#!/bin/bash + set -e cp -a ${TB_STAGE_DIR} [% tbdir %] cp [% xpi %] "[% browserdir %]/$EXTSPATH/langpack-[% lang %]@firefox.mozilla.org.xpi"
@@ -349,9 +360,10 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/ python $MARTOOLS/createprecomplete.py popd
+ cd $distdir + # Create full MAR file and compressed package for this locale. - MAR_FILE=tor-browser-[% c("var/mar_osname") %]-[% c("var/torbrowser_version") %]_[% lang %].mar - MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/$MAR_FILE "[% browserdir %]" + MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/[% mar_file %] "[% browserdir %]" [% IF c("var/linux") %] [% SET tardir = 'tor-browser_' _ lang; c('tar', { @@ -380,5 +392,9 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/ popd [% END %] rm -rf [% tbdir %] +SCRIPT_EOF [% END %] [% END %] + +chmod 775 $rootdir/run_scripts "$scripts_dir"/* +$rootdir/run_scripts [% c("buildconf/num_procs") %] "$scripts_dir" diff --git a/projects/tor-browser/config b/projects/tor-browser/config index 50d67e0..bb1c259 100644 --- a/projects/tor-browser/config +++ b/projects/tor-browser/config @@ -6,6 +6,14 @@ var: container: use_container: 1 ddmg: '[% INCLUDE ddmg.sh %]' + deps: + - python + - libparallel-forkmanager-perl + - libfile-slurp-perl + - zip + - unzip + - bzip2 + - xz-utils
targets: linux-i686: @@ -17,14 +25,9 @@ targets: osx-x86_64: var: mar_osname: osx64 - deps: + arch_deps: - genisoimage - - zip - - unzip - - python - - bzip2 - faketime - - xz-utils windows: var: arch_deps: @@ -41,6 +44,7 @@ targets:
input_files: - project: container-image + - filename: run_scripts - project: firefox name: firefox - project: tor diff --git a/projects/tor-browser/ddmg.sh b/projects/tor-browser/ddmg.sh index a715300..97fddb1 100755 --- a/projects/tor-browser/ddmg.sh +++ b/projects/tor-browser/ddmg.sh @@ -4,8 +4,8 @@ find [% src %] ! -executable -exec chmod 0644 {} ;
find [% src %] -exec [% c("var/touch") %] {} ;
-dmg_tmpdir=$(mktemp -d) -[% SET filelist = '"$dmg_tmpdir/filelist.txt"' %] +dmg_tmpdir=$(mktemp -d) +[% SET filelist = '"$dmg_tmpdir/filelist.txt"' %] pushd [% src %] find . -type f | sed -e 's/^.///' | sort | xargs -i echo "{}={}" > [% filelist %] find . -type l | sed -e 's/^.///' | sort | xargs -i echo "{}={}" >> [% filelist %] @@ -13,9 +13,9 @@ find . -type l | sed -e 's/^.///' | sort | xargs -i echo "{}={}" >> [% filelis export LD_PRELOAD=[% c("var/faketime_path") %] export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]"
-genisoimage -D -V "Tor Browser" -no-pad -R -apple -o "$dmg_tmpdir/tbb-uncompressed.dmg" -path-list [% filelist %] -graft-points -gid 20 -dir-mode 0755 -new-dir-mode 0755 +genisoimage -D -V "Tor Browser" -no-pad -R -apple -o "$dmg_tmpdir/tbb-uncompressed.dmg" -path-list [% filelist %] -graft-points -gid 20 -dir-mode 0755 -new-dir-mode 0755
-dmg dmg "$dmg_tmpdir/tbb-uncompressed.dmg" [% c('dmg_out', { error_if_undef => 1 }) %] +dmg dmg "$dmg_tmpdir/tbb-uncompressed.dmg" [% c('dmg_out', { error_if_undef => 1 }) %] popd
-rm -Rf "$dmg_tmpdir" +rm -Rf "$dmg_tmpdir" diff --git a/projects/tor-browser/run_scripts b/projects/tor-browser/run_scripts new file mode 100755 index 0000000..4b85b50 --- /dev/null +++ b/projects/tor-browser/run_scripts @@ -0,0 +1,30 @@ +#!/usr/bin/perl -w +use strict; + +use Parallel::ForkManager; +use File::Slurp; + +sub exit_error { + print STDERR "Error: ", $_[0], "\n"; + chdir '/'; + exit (exists $_[1] ? $_[1] : 1); +} + +exit_error "Wrong number of arguments" unless @ARGV == 2; +my ($nb_threads, $scripts_dir) = @ARGV; +exit_error "Could not enter $scripts_dir" unless chdir $scripts_dir; +my $pm = Parallel::ForkManager->new($nb_threads); +$pm->run_on_finish( + sub { + my ($pid, $exit, $id) = @_; + exit_error "Error running $id" unless $exit == 0; + print "Finished $id\n"; + } +); +foreach my $script (sort(read_dir($scripts_dir))) { + $pm->start($script) and next; + print "Running $script\n"; + exit_error "Error running $script" unless system("./$script") == 0; + $pm->finish; +} +$pm->wait_all_children;