[tor-browser-build/maint-8.0] Bug 27218: generate multiple bundles in parallel

5 Dec 2018

commit 2d129f5c6b282c6e9e915339f7b4dfb0f54b5a80
Author: Nicolas Vigier <boklm@torproject.org>
Date:   Wed Oct 10 01:33:59 2018 +0200

    Bug 27218: generate multiple bundles in parallel
---
 projects/tor-browser/build       | 28 ++++++++++++++++++++++------
 projects/tor-browser/config      | 16 ++++++++++------
 projects/tor-browser/ddmg.sh     | 10 +++++-----
 projects/tor-browser/run_scripts | 30 ++++++++++++++++++++++++++++++
 4 files changed, 67 insertions(+), 17 deletions(-)

diff --git a/projects/tor-browser/build b/projects/tor-browser/build
index 4d51ec6..68050f9 100644
--- a/projects/tor-browser/build
+++ b/projects/tor-browser/build
@@ -5,6 +5,9 @@ export TORBROWSER_VERSION='[% c("version") %]'
 
 mkdir -p $distdir
 
+scripts_dir=/var/tmp/build_scripts
+mkdir -p "$scripts_dir"
+
 OUTDIR='[% dest_dir _ "/" _ c("filename") %]'
 mkdir -p $OUTDIR
 
@@ -252,8 +255,6 @@ popd
   mv $distdir/tbb-windows-installer ${TB_STAGE_DIR}
 [% END %]
 
-cp -a ${TB_STAGE_DIR} $distdir/$PKG_DIR
-
 [% IF c("var/windows") %]
   TBDIR="$distdir/$PKG_DIR/Tor Browser/Browser"
 [% ELSIF c("var/osx") %]
@@ -262,6 +263,11 @@ cp -a ${TB_STAGE_DIR} $distdir/$PKG_DIR
   TBDIR="$distdir/$PKG_DIR/Browser"
 [% END %]
 
+cat > "$scripts_dir/create-$PKG_DIR" << SCRIPT_EOF
+#!/bin/bash
+set -e
+cp -a ${TB_STAGE_DIR} $distdir/$PKG_DIR
+
 pushd "$TBDIR[% IF c("var/osx") %]/Contents/Resources/[% END %]"
 rm -f precomplete
 python $MARTOOLS/createprecomplete.py
@@ -271,8 +277,8 @@ cd $distdir
 
 [% IF c("var/build_mar") -%]
   # Create full MAR file and compressed package.
-  MAR_FILE=tor-browser-[% c("var/mar_osname") %]-[% c("var/torbrowser_version") %]_${PKG_LOCALE}.mar
-  MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/$MAR_FILE "$TBDIR"
+  [% SET mar_file = 'tor-browser-' _ c("var/mar_osname") _ '-' _ c("var/torbrowser_version") _ '_${PKG_LOCALE}.mar' %]
+  MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/[% mar_file %] "$TBDIR"
 [% END -%]
 
 [% IF c("var/linux") %]
@@ -298,6 +304,7 @@ cd $distdir
   popd
 [% END %]
 rm -rf $distdir/${PKG_DIR}
+SCRIPT_EOF
 
 cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
 [% IF c("var/linux-x86_64") -%]
@@ -314,6 +321,7 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
     [% SET lang = tmpl(lang);
        SET xpi = '$rootdir/' _ c('input_files_by_name/firefox-langpacks') _ '/' _ lang _ '.xpi';
        SET tbdir = '$distdir/tor-browser_' _ lang;
+       SET mar_file = 'tor-browser-' _ c("var/mar_osname") _ '-' _ c("var/torbrowser_version") _ '_' _ lang _ '.mar';
        IF c("var/osx");
          SET browserdir = tbdir _ '/Tor Browser.app';
        ELSIF c("var/windows");
@@ -322,6 +330,9 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
          SET browserdir = tbdir _ '/Browser';
        END;
     %]
+      cat > "$scripts_dir/create-tor-browser_[% lang %]" << SCRIPT_EOF
+#!/bin/bash
+      set -e
       cp -a ${TB_STAGE_DIR} [% tbdir %]
       cp [% xpi %] "[% browserdir %]/$EXTSPATH/langpack-[% lang %]@firefox.mozilla.org.xpi"
 
@@ -349,9 +360,10 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
       python $MARTOOLS/createprecomplete.py
       popd
 
+      cd $distdir
+
       # Create full MAR file and compressed package for this locale.
-      MAR_FILE=tor-browser-[% c("var/mar_osname") %]-[% c("var/torbrowser_version") %]_[% lang %].mar
-      MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/$MAR_FILE "[% browserdir %]"
+      MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/[% mar_file %] "[% browserdir %]"
       [% IF c("var/linux") %]
         [% SET tardir = 'tor-browser_' _ lang;
           c('tar', {
@@ -380,5 +392,9 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
         popd
       [% END %]
       rm -rf [% tbdir %]
+SCRIPT_EOF
   [% END %]
 [% END %]
+
+chmod 775 $rootdir/run_scripts "$scripts_dir"/*
+$rootdir/run_scripts [% c("buildconf/num_procs") %] "$scripts_dir"
diff --git a/projects/tor-browser/config b/projects/tor-browser/config
index 50d67e0..bb1c259 100644
--- a/projects/tor-browser/config
+++ b/projects/tor-browser/config
@@ -6,6 +6,14 @@ var:
   container:
     use_container: 1
   ddmg: '[% INCLUDE ddmg.sh %]'
+  deps:
+    - python
+    - libparallel-forkmanager-perl
+    - libfile-slurp-perl
+    - zip
+    - unzip
+    - bzip2
+    - xz-utils
 
 targets:
   linux-i686:
@@ -17,14 +25,9 @@ targets:
   osx-x86_64:
     var:
       mar_osname: osx64
-      deps:
+      arch_deps:
         - genisoimage
-        - zip
-        - unzip
-        - python
-        - bzip2
         - faketime
-        - xz-utils
   windows:
     var:
       arch_deps:
@@ -41,6 +44,7 @@ targets:
 
 input_files:
   - project: container-image
+  - filename: run_scripts
   - project: firefox
     name: firefox
   - project: tor
diff --git a/projects/tor-browser/ddmg.sh b/projects/tor-browser/ddmg.sh
index a715300..97fddb1 100755
--- a/projects/tor-browser/ddmg.sh
+++ b/projects/tor-browser/ddmg.sh
@@ -4,8 +4,8 @@ find [% src %] ! -executable -exec chmod 0644 {} \;
 
 find [% src %] -exec [% c("var/touch") %] {} \;
 
-dmg_tmpdir=$(mktemp -d)
-[% SET filelist = '"$dmg_tmpdir/filelist.txt"' %]
+dmg_tmpdir=\$(mktemp -d)
+[% SET filelist = '"\$dmg_tmpdir/filelist.txt"' %]
 pushd [% src %] 
 find . -type f | sed -e 's/^\.\///' | sort | xargs -i echo "{}={}" > [% filelist %]
 find . -type l | sed -e 's/^\.\///' | sort | xargs -i echo "{}={}" >> [% filelist %]
@@ -13,9 +13,9 @@ find . -type l | sed -e 's/^\.\///' | sort | xargs -i echo "{}={}" >> [% filelis
 export LD_PRELOAD=[% c("var/faketime_path") %]
 export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]"
 
-genisoimage -D -V "Tor Browser" -no-pad -R -apple -o "$dmg_tmpdir/tbb-uncompressed.dmg" -path-list [% filelist %] -graft-points -gid 20 -dir-mode 0755 -new-dir-mode 0755
+genisoimage -D -V "Tor Browser" -no-pad -R -apple -o "\$dmg_tmpdir/tbb-uncompressed.dmg" -path-list [% filelist %] -graft-points -gid 20 -dir-mode 0755 -new-dir-mode 0755
 
-dmg dmg "$dmg_tmpdir/tbb-uncompressed.dmg" [% c('dmg_out', { error_if_undef => 1 }) %]
+dmg dmg "\$dmg_tmpdir/tbb-uncompressed.dmg" [% c('dmg_out', { error_if_undef => 1 }) %]
 popd
 
-rm -Rf "$dmg_tmpdir"
+rm -Rf "\$dmg_tmpdir"
diff --git a/projects/tor-browser/run_scripts b/projects/tor-browser/run_scripts
new file mode 100755
index 0000000..4b85b50
--- /dev/null
+++ b/projects/tor-browser/run_scripts
@@ -0,0 +1,30 @@
+#!/usr/bin/perl -w
+use strict;
+
+use Parallel::ForkManager;
+use File::Slurp;
+
+sub exit_error {
+    print STDERR "Error: ", $_[0], "\n";
+    chdir '/';
+    exit (exists $_[1] ? $_[1] : 1);
+}
+
+exit_error "Wrong number of arguments" unless @ARGV == 2;
+my ($nb_threads, $scripts_dir) = @ARGV;
+exit_error "Could not enter $scripts_dir" unless chdir $scripts_dir;
+my $pm = Parallel::ForkManager->new($nb_threads);
+$pm->run_on_finish(
+    sub {
+        my ($pid, $exit, $id) = @_;
+        exit_error "Error running $id" unless $exit == 0;
+        print "Finished $id\n";
+    }
+);
+foreach my $script (sort(read_dir($scripts_dir))) {
+    $pm->start($script) and next;
+    print "Running $script\n";
+    exit_error "Error running $script" unless system("./$script") == 0;
+    $pm->finish;
+}
+$pm->wait_all_children;

    

gk＠torproject.org

tags

participants (1)