[tor-commits] [tor-browser-build/master] Bug 27218: generate multiple bundles in parallel
gk at torproject.org
gk at torproject.org
Tue Oct 16 19:04:00 UTC 2018
commit dfa0cc46fc0ef70f78313a651bffe178b4d1c4ee
Author: Nicolas Vigier <boklm at torproject.org>
Date: Wed Oct 10 01:33:59 2018 +0200
Bug 27218: generate multiple bundles in parallel
---
projects/tor-browser/build | 28 ++++++++++++++++++++++------
projects/tor-browser/config | 16 ++++++++++------
projects/tor-browser/ddmg.sh | 10 +++++-----
projects/tor-browser/run_scripts | 30 ++++++++++++++++++++++++++++++
4 files changed, 67 insertions(+), 17 deletions(-)
diff --git a/projects/tor-browser/build b/projects/tor-browser/build
index 9e8cddb..250a0e9 100644
--- a/projects/tor-browser/build
+++ b/projects/tor-browser/build
@@ -5,6 +5,9 @@ export TORBROWSER_VERSION='[% c("version") %]'
mkdir -p $distdir
+scripts_dir=/var/tmp/build_scripts
+mkdir -p "$scripts_dir"
+
OUTDIR='[% dest_dir _ "/" _ c("filename") %]'
mkdir -p $OUTDIR
@@ -260,8 +263,6 @@ popd
mv $distdir/tbb-windows-installer ${TB_STAGE_DIR}
[% END %]
-cp -a ${TB_STAGE_DIR} $distdir/$PKG_DIR
-
[% IF c("var/windows") %]
TBDIR="$distdir/$PKG_DIR/Tor Browser/Browser"
[% ELSIF c("var/osx") %]
@@ -270,6 +271,11 @@ cp -a ${TB_STAGE_DIR} $distdir/$PKG_DIR
TBDIR="$distdir/$PKG_DIR/Browser"
[% END %]
+cat > "$scripts_dir/create-$PKG_DIR" << SCRIPT_EOF
+#!/bin/bash
+set -e
+cp -a ${TB_STAGE_DIR} $distdir/$PKG_DIR
+
pushd "$TBDIR[% IF c("var/osx") %]/Contents/Resources/[% END %]"
rm -f precomplete
python $MARTOOLS/createprecomplete.py
@@ -279,8 +285,8 @@ cd $distdir
[% IF c("var/build_mar") -%]
# Create full MAR file and compressed package.
- MAR_FILE=tor-browser-[% c("var/mar_osname") %]-[% c("var/torbrowser_version") %]_${PKG_LOCALE}.mar
- MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/$MAR_FILE "$TBDIR"
+ [% SET mar_file = 'tor-browser-' _ c("var/mar_osname") _ '-' _ c("var/torbrowser_version") _ '_${PKG_LOCALE}.mar' %]
+ MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/[% mar_file %] "$TBDIR"
[% END -%]
[% IF c("var/linux") %]
@@ -306,6 +312,7 @@ cd $distdir
popd
[% END %]
rm -rf $distdir/${PKG_DIR}
+SCRIPT_EOF
cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
[% IF c("var/linux-x86_64") -%]
@@ -322,6 +329,7 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
[% SET lang = tmpl(lang);
SET xpi = '$rootdir/' _ c('input_files_by_name/firefox-langpacks') _ '/' _ lang _ '.xpi';
SET tbdir = '$distdir/tor-browser_' _ lang;
+ SET mar_file = 'tor-browser-' _ c("var/mar_osname") _ '-' _ c("var/torbrowser_version") _ '_' _ lang _ '.mar';
IF c("var/osx");
SET browserdir = tbdir _ '/Tor Browser.app';
ELSIF c("var/windows");
@@ -330,6 +338,9 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
SET browserdir = tbdir _ '/Browser';
END;
%]
+ cat > "$scripts_dir/create-tor-browser_[% lang %]" << SCRIPT_EOF
+#!/bin/bash
+ set -e
cp -a ${TB_STAGE_DIR} [% tbdir %]
cp [% xpi %] "[% browserdir %]/$EXTSPATH/langpack-[% lang %]@firefox.mozilla.org.xpi"
@@ -357,9 +368,10 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
python $MARTOOLS/createprecomplete.py
popd
+ cd $distdir
+
# Create full MAR file and compressed package for this locale.
- MAR_FILE=tor-browser-[% c("var/mar_osname") %]-[% c("var/torbrowser_version") %]_[% lang %].mar
- MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/$MAR_FILE "[% browserdir %]"
+ MAR=$MARTOOLS/mar MBSDIFF=$MARTOOLS/mbsdiff $MARTOOLS/make_full_update.sh -q $OUTDIR/[% mar_file %] "[% browserdir %]"
[% IF c("var/linux") %]
[% SET tardir = 'tor-browser_' _ lang;
c('tar', {
@@ -388,5 +400,9 @@ cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/
popd
[% END %]
rm -rf [% tbdir %]
+SCRIPT_EOF
[% END %]
[% END %]
+
+chmod 775 $rootdir/run_scripts "$scripts_dir"/*
+$rootdir/run_scripts [% c("buildconf/num_procs") %] "$scripts_dir"
diff --git a/projects/tor-browser/config b/projects/tor-browser/config
index 1647b77..7b44d8f 100644
--- a/projects/tor-browser/config
+++ b/projects/tor-browser/config
@@ -6,6 +6,14 @@ var:
container:
use_container: 1
ddmg: '[% INCLUDE ddmg.sh %]'
+ deps:
+ - python
+ - libparallel-forkmanager-perl
+ - libfile-slurp-perl
+ - zip
+ - unzip
+ - bzip2
+ - xz-utils
targets:
linux-i686:
@@ -17,14 +25,9 @@ targets:
osx-x86_64:
var:
mar_osname: osx64
- deps:
+ arch_deps:
- genisoimage
- - zip
- - unzip
- - python
- - bzip2
- faketime
- - xz-utils
windows:
var:
arch_deps:
@@ -41,6 +44,7 @@ targets:
input_files:
- project: container-image
+ - filename: run_scripts
- project: firefox
name: firefox
- project: tor
diff --git a/projects/tor-browser/ddmg.sh b/projects/tor-browser/ddmg.sh
index a715300..97fddb1 100755
--- a/projects/tor-browser/ddmg.sh
+++ b/projects/tor-browser/ddmg.sh
@@ -4,8 +4,8 @@ find [% src %] ! -executable -exec chmod 0644 {} \;
find [% src %] -exec [% c("var/touch") %] {} \;
-dmg_tmpdir=$(mktemp -d)
-[% SET filelist = '"$dmg_tmpdir/filelist.txt"' %]
+dmg_tmpdir=\$(mktemp -d)
+[% SET filelist = '"\$dmg_tmpdir/filelist.txt"' %]
pushd [% src %]
find . -type f | sed -e 's/^\.\///' | sort | xargs -i echo "{}={}" > [% filelist %]
find . -type l | sed -e 's/^\.\///' | sort | xargs -i echo "{}={}" >> [% filelist %]
@@ -13,9 +13,9 @@ find . -type l | sed -e 's/^\.\///' | sort | xargs -i echo "{}={}" >> [% filelis
export LD_PRELOAD=[% c("var/faketime_path") %]
export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]"
-genisoimage -D -V "Tor Browser" -no-pad -R -apple -o "$dmg_tmpdir/tbb-uncompressed.dmg" -path-list [% filelist %] -graft-points -gid 20 -dir-mode 0755 -new-dir-mode 0755
+genisoimage -D -V "Tor Browser" -no-pad -R -apple -o "\$dmg_tmpdir/tbb-uncompressed.dmg" -path-list [% filelist %] -graft-points -gid 20 -dir-mode 0755 -new-dir-mode 0755
-dmg dmg "$dmg_tmpdir/tbb-uncompressed.dmg" [% c('dmg_out', { error_if_undef => 1 }) %]
+dmg dmg "\$dmg_tmpdir/tbb-uncompressed.dmg" [% c('dmg_out', { error_if_undef => 1 }) %]
popd
-rm -Rf "$dmg_tmpdir"
+rm -Rf "\$dmg_tmpdir"
diff --git a/projects/tor-browser/run_scripts b/projects/tor-browser/run_scripts
new file mode 100755
index 0000000..4b85b50
--- /dev/null
+++ b/projects/tor-browser/run_scripts
@@ -0,0 +1,30 @@
+#!/usr/bin/perl -w
+use strict;
+
+use Parallel::ForkManager;
+use File::Slurp;
+
+sub exit_error {
+ print STDERR "Error: ", $_[0], "\n";
+ chdir '/';
+ exit (exists $_[1] ? $_[1] : 1);
+}
+
+exit_error "Wrong number of arguments" unless @ARGV == 2;
+my ($nb_threads, $scripts_dir) = @ARGV;
+exit_error "Could not enter $scripts_dir" unless chdir $scripts_dir;
+my $pm = Parallel::ForkManager->new($nb_threads);
+$pm->run_on_finish(
+ sub {
+ my ($pid, $exit, $id) = @_;
+ exit_error "Error running $id" unless $exit == 0;
+ print "Finished $id\n";
+ }
+);
+foreach my $script (sort(read_dir($scripts_dir))) {
+ $pm->start($script) and next;
+ print "Running $script\n";
+ exit_error "Error running $script" unless system("./$script") == 0;
+ $pm->finish;
+}
+$pm->wait_all_children;
More information about the tor-commits
mailing list