- tbb-commits - lists.torproject.org

[tor-browser-build/master] Don't show detailed build logs on stdout
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit 36a1538a2d119aac1e2f7b3ce21260d0258ccc88 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Wed Feb 15 00:11:07 2017 +0100 Don't show detailed build logs on stdout To provide a more "high level" view of progress, don't output build logs of each componets on stdout, but write them in files in the logs/ directory. This idea was suggested by mcs and Kathy in this comment: https://trac.torproject.org/projects/tor/ticket/17379#comment:12 --- .gitignore | 1 + rbm | 2 +- rbm.conf | 1 + rbm.local.conf.example | 5 +++++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 98554a3..e38b069 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ alpha_nightly nightly hardened rbm.local.conf +logs diff --git a/rbm b/rbm index 3fdeb23..b0382cc 160000 --- a/rbm +++ b/rbm @@ -1 +1 @@ -Subproject commit 3fdeb23bc4ab4dc65e69ff0fac49db603c4efb08 +Subproject commit b0382ccc58b9286e066bf5980f80cf041e287067 diff --git a/rbm.conf b/rbm.conf index 47ffbfb..802bb48 100644 --- a/rbm.conf +++ b/rbm.conf @@ -2,6 +2,7 @@ debug: '[% GET ! ENV.RBM_NO_DEBUG %]' compress_tar: gz output_dir: "out/[% project %]" +build_log: 'logs/[% project %]-[% c("var/osname") %].log' pkg_type: build diff --git a/rbm.local.conf.example b/rbm.local.conf.example index 20f081f..ac54935 100644 --- a/rbm.local.conf.example +++ b/rbm.local.conf.example @@ -22,6 +22,11 @@ ### use the same docker image names. #docker_image_prefix: tor-browser_XXXXX +### The build_log option defines in which file the build logs of each +### component are stored. If you set it to '-' the logs are output on +### stdout and stderr. +#build_log: '-' + var: local_conf: 1

1 0

[rbm/master] Add the build_log option
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit b0382ccc58b9286e066bf5980f80cf041e287067 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 23:55:10 2017 +0100 Add the build_log option --- NEWS | 2 ++ doc/rbm_cli.asc | 4 ++++ doc/rbm_config.asc | 4 ++++ lib/RBM.pm | 24 ++++++++++++++++++++++-- lib/RBM/DefaultConfig.pm | 1 + rbm | 2 +- 6 files changed, 34 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 1a1789f..663a281 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,5 @@ +- Add the build_log option + - Add support for optional local configuration file rbm.local.conf - add the git_submodule option diff --git a/doc/rbm_cli.asc b/doc/rbm_cli.asc index 66f6025..adb8b27 100644 --- a/doc/rbm_cli.asc +++ b/doc/rbm_cli.asc @@ -42,6 +42,10 @@ The following options can be set on the command line : --output-dir=<directory>:: Set the output directory + --build-log=<filename>:: + Set the file where the build logs will be written. If the value is `-` + (the default), the logs will be output on stdout and stderr. + --version=<version>:: Set the package version diff --git a/doc/rbm_config.asc b/doc/rbm_config.asc index e8decd9..a461b3d 100644 --- a/doc/rbm_config.asc +++ b/doc/rbm_config.asc @@ -140,6 +140,10 @@ output_dir:: The directory where output files (tarballs, spec files or packages) are created. The default value is +out+. +build_log:: + The file where the build logs will be written. If the value is `-` (the + default), the logs will be output on stdout and stderr. + fetch:: The value should be 0 or 1, depending on whether the commits from the remote git or hg repository should be fetched diff --git a/lib/RBM.pm b/lib/RBM.pm index 4b5bfe9..2248b41 100644 --- a/lib/RBM.pm +++ b/lib/RBM.pm @@ -884,6 +884,18 @@ sub input_files { return \%res_getfnames if $action eq 'getfnames'; } +sub system_log { + my ($log_file, @args) = @_; + return system(@args) if $log_file eq '-'; + if (my $pid = fork) { + waitpid($pid, 0); + return ${^CHILD_ERROR_NATIVE}; + } + exit_error "Could not open $log_file" unless open(STDOUT, '>>', $log_file); + open(STDERR, '>&', *STDOUT); + exec(@args); +} + sub build_run { my ($project, $script_name, $options) = @_; my $old_step = $config->{step}; @@ -959,6 +971,14 @@ sub build_run { write_file("$srcdir/$s", $build_script{$s}); chmod 0700, "$srcdir/$s"; } + my $build_log = project_config($project, "build_log", $options); + if ($build_log ne '-') { + $build_log = path($build_log); + make_path(dirname($build_log)); + my $now = localtime; + write_file($build_log, {append => 1}, "Starting build: $now\n"); + print "Build log: $build_log\n"; + } chdir $srcdir; my $res; if ($remote_tmp_src && $remote_tmp_dst) { @@ -982,7 +1002,7 @@ sub build_run { exec_name => $s, exec_as_root => $scripts_root{$s}, }); - if (run_script($project, $cmd, sub { system(@_) }) != 0) { + if (run_script($project, $cmd, sub { system_log($build_log, @_) }) != 0) { $error = "Error running $script_name"; if (project_config($project, 'debug', $options)) { print STDERR $error, "\nOpening debug shell\n"; @@ -1019,7 +1039,7 @@ sub build_run { foreach my $s (@scripts) { my $cmd = $scripts_root{$s} ? project_config($project, 'suexec', { suexec_cmd => "$srcdir/$s" }) : "$srcdir/$s"; - if (system($cmd) != 0) { + if (system_log($build_log, $cmd) != 0) { $error = "Error running $script_name"; if (project_config($project, 'debug', $options)) { print STDERR $error, "\nOpening debug shell\n"; diff --git a/lib/RBM/DefaultConfig.pm b/lib/RBM/DefaultConfig.pm index aad9451..ec03ff0 100644 --- a/lib/RBM/DefaultConfig.pm +++ b/lib/RBM/DefaultConfig.pm @@ -112,6 +112,7 @@ our %default_config = ( fetch => 'if_needed', rpmspec => '[% SET tmpl = project _ ".spec"; INCLUDE $tmpl -%]', build => '[% INCLUDE build -%]', + build_log => '-', notmpl => [ qw(projects_dir) ], describe => \&git_describe, abbrev_lenght => '12', diff --git a/rbm b/rbm index c618ada..050cff5 100755 --- a/rbm +++ b/rbm @@ -95,7 +95,7 @@ sub set_options { gpg-keyring=s gpg-keyring-dir=s gpg-args=s gpg-bin=s sysconf-file=s debsign-keyid=s use-pbuilder! step=s target=s@ publish-src-dir=s debug! hg-clone-dir=s - hg-hash=s localconf-file=s); + hg-hash=s localconf-file=s build-log=s); my %val; Getopt::Long::GetOptionsFromArray(\@_, \%val, @options) || exit 1; foreach my $k (keys %val) {

1 0

[tor-browser-build/master] projects/tor-browser: fix Docs path
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit 0e1862403ebef5f91e3a2f2e3253baf74a339621 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 18:21:35 2017 +0100 projects/tor-browser: fix Docs path --- projects/tor-browser/build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/tor-browser/build b/projects/tor-browser/build index d19b6b7..caf0cf4 100644 --- a/projects/tor-browser/build +++ b/projects/tor-browser/build @@ -105,7 +105,8 @@ tar -C $TBDIR[% IF ! c("var/osx") %]/TorBrowser[% END %] -xf [% c('input_files_b tar -C Bundle-Data/mac-applications.dmg -c . | tar -C $TB_STAGE_DIR -x [% END %] -cp -a Bundle-Data/Docs $TBDIR/$DOCSPATH +mkdir -p $TBDIR/$DOCSPATH +cp -a Bundle-Data/Docs/* $TBDIR/$DOCSPATH tar -C Bundle-Data/[% bundledata_osname %] -c . | tar -C $TBDIR[% IF ! c("var/osx") %]/TorBrowser[% END %] -x cat Bundle-Data/PTConfigs/[% bundledata_osname %]/torrc-defaults-appendix >> $TBDIR/$TORCONFIGPATH/torrc-defaults

1 0

[tor-browser-build/master] projects/tor-browser: add missing TORCONFIGPATH var on Linux/Windows
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit 55b71bbf54bb402918b4b102894f0c7a09cf68c7 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 17:54:13 2017 +0100 projects/tor-browser: add missing TORCONFIGPATH var on Linux/Windows --- projects/tor-browser/build | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/tor-browser/build b/projects/tor-browser/build index e25a1d8..d19b6b7 100644 --- a/projects/tor-browser/build +++ b/projects/tor-browser/build @@ -34,6 +34,7 @@ TB_STAGE_DIR=$distdir/tor-browser-stage DOCSPATH=TorBrowser/Docs EXTSPATH=TorBrowser/Data/Browser/profile.default/extensions + TORCONFIGPATH=TorBrowser/Data/Tor EXTOVERRIDESPATH=TorBrowser/Data/Browser/profile.default/preferences/extension-overrides.js MEEKPROFILEPATH=TorBrowser/Data/Browser/profile.meek-http-helper

1 0

[tor-browser-build/master] projects/meek: only include *.1 files from doc directory
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit fc068986a71b27e896c9a727754bbd09dcf64a8c Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 18:02:55 2017 +0100 projects/meek: only include *.1 files from doc directory --- projects/meek/build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/meek/build b/projects/meek/build index d195e90..283066a 100644 --- a/projects/meek/build +++ b/projects/meek/build @@ -34,7 +34,7 @@ cp -a meek-client-torbrowser[% IF c("var/windows") %].exe[% END %] $PTDIR [% END %] cd .. -cp -a README doc/* $DOCSDIR +cp -a README doc/*.1 $DOCSDIR cd firefox [% c('zip', {

1 0

[tor-browser-build/master] Update rbm: set timestamp on files in zip wrapper
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit e45112bd7ce3a59fc57c42432ce8ec4d23d176e8 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 16:00:14 2017 +0100 Update rbm: set timestamp on files in zip wrapper --- rbm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rbm b/rbm index a57b484..3fdeb23 160000 --- a/rbm +++ b/rbm @@ -1 +1 @@ -Subproject commit a57b4846c8aba5697ce1f197996e2962ffb5b534 +Subproject commit 3fdeb23bc4ab4dc65e69ff0fac49db603c4efb08

1 0

[rbm/master] Set timestamp on files in zip wrapper
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit 3fdeb23bc4ab4dc65e69ff0fac49db603c4efb08 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 11:35:10 2017 +0100 Set timestamp on files in zip wrapper --- lib/RBM/DefaultConfig.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/RBM/DefaultConfig.pm b/lib/RBM/DefaultConfig.pm index 052cf16..aad9451 100644 --- a/lib/RBM/DefaultConfig.pm +++ b/lib/RBM/DefaultConfig.pm @@ -569,6 +569,8 @@ TAR_END #### zip => <<ZIP_END, [%- SET src = c('zip_src', { error_if_undef => 1 }) -%] +[% USE date -%] +find [% src.join(' ') %] -exec touch -m -t [% date.format(c('timestamp'), format = '%Y%m%d%H%M') %] -- {} + find [% src.join(' ') %] [% IF c('gnu_utils') %]-executable[% ELSE %]-perm +0111[% END %] -exec chmod 700 {} \\; find [% src.join(' ') %] ! [% IF c('gnu_utils') %]-executable[% ELSE %]-perm +0111[% END %] -exec chmod 600 {} \\; find [% src.join(' ') %] | sort | \

1 0

[tor-browser-build/master] projects/tor: use faketime on OSX
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit de8a0de76cf4a07bf2e9138e9c39bd4af8a93446 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 13:04:22 2017 +0100 projects/tor: use faketime on OSX --- projects/tor/build | 7 +++++++ projects/tor/config | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/projects/tor/build b/projects/tor/build index bd86928..a568512 100644 --- a/projects/tor/build +++ b/projects/tor/build @@ -58,8 +58,15 @@ cd /var/tmp/build/[% project %]-[% c('version') %] ./configure --disable-asciidoc --with-libevent-dir="$libeventdir" --with-openssl-dir="$openssldir" \ [% IF c("var/windows") %]--with-zlib-dir="$zlibdir"[% END %] \ --prefix="$distdir" [% c("var/configure_opt") %] +[% IF c("var/osx") -%] + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 + export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]" +[% END -%] make -j4 make install +[% IF c("var/osx") -%] + unset LD_PRELOAD +[% END -%] cp $distdir/share/tor/geoip $TORCONFIGDIR cp $distdir/share/tor/geoip6 $TORCONFIGDIR diff --git a/projects/tor/config b/projects/tor/config index b73a447..2fbbf24 100644 --- a/projects/tor/config +++ b/projects/tor/config @@ -36,6 +36,10 @@ targets: - libtool - zlib1g-dev - hardening-wrapper + osx-x86_64: + var: + arch_deps: + - faketime expert-bundle: var: expert_bundle: 1

1 0

[tor-browser-build/master] Add docker_image_prefix option to rbm.local.conf
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit e30719ad9b5ad2c63156d88e63a5d4a082021455 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 01:31:22 2017 +0100 Add docker_image_prefix option to rbm.local.conf --- README | 12 ++++++++++++ projects/docker-image/config | 1 - rbm.conf | 1 + rbm.local.conf.example | 6 ++++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/README b/README index 956e162..e3672ec 100644 --- a/README +++ b/README @@ -169,6 +169,18 @@ There will be a script to clean old build files and containers that are no longer used, but it has not been added yet. +Multiple build directories on the same host +------------------------------------------- + +You can do multiple builds of Tor Browser in different directories on +the same host. However the docker images namespace is global, so you +may have some conflicts with the same image names used by the +different builds. By default, the docker images are prefixed with +tor-browser_$USER. You can change this prefix by defining the +docker_image_prefix option in rbm.local.conf, using a different prefix +for each of your build directories. + + Common Build Errors ------------------- diff --git a/projects/docker-image/config b/projects/docker-image/config index 27d2654..1968b77 100644 --- a/projects/docker-image/config +++ b/projects/docker-image/config @@ -1,7 +1,6 @@ # vim: filetype=yaml sw=2 filename: '[% sha256(c("pre")).substr(0, 12) %]' remote_docker: 1 -docker_image_prefix: '[% GET c("var/project_name") ? c("var/project_name") : "rbm-build" %]_[% GET ENV.RBM_BUILDNAME ? ENV.RBM_BUILDNAME : ENV.USER ? ENV.USER : c("uid") %]' docker_save_image: '[% c("docker_image_prefix") %]:[% c("filename") %]' pkg_type: build diff --git a/rbm.conf b/rbm.conf index 5dfdc30..47ffbfb 100644 --- a/rbm.conf +++ b/rbm.conf @@ -155,6 +155,7 @@ targets: docker_image: '[% pc("docker-image", "docker_save_image") %]' +docker_image_prefix: '[% GET c("var/project_name") ? c("var/project_name") : "rbm-build" %]_[% GET ENV.RBM_BUILDNAME ? ENV.RBM_BUILDNAME : ENV.USER ? ENV.USER : c("uid") %]' # change the default gpg_wrapper to allow git tag signed using an # expired key. diff --git a/rbm.local.conf.example b/rbm.local.conf.example index 42bf431..20f081f 100644 --- a/rbm.local.conf.example +++ b/rbm.local.conf.example @@ -16,6 +16,12 @@ ### this. #debug: 0 +### If you are doing multiple builds in different directories on the +### same host, you should define docker_image_prefix with a different +### value for each build directory, so that the different builds don't +### use the same docker image names. +#docker_image_prefix: tor-browser_XXXXX + var: local_conf: 1

1 0

[tor-browser-build/master] Add support for optional local configuration file rbm.local.conf
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit 99f69d0aa0665e57a5036f66e16dbdcd0fea94d0 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 00:40:49 2017 +0100 Add support for optional local configuration file rbm.local.conf Update rbm and add example rbm.local.conf file. --- .gitignore | 1 + README | 2 ++ README.BUILD_ERRORS | 2 ++ rbm | 2 +- rbm.local.conf.example | 22 ++++++++++++++++++++++ 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c755fcf..98554a3 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ alpha alpha_nightly nightly hardened +rbm.local.conf diff --git a/README b/README index 775f782..001a692 100644 --- a/README +++ b/README @@ -89,6 +89,8 @@ the RBM_NO_DEBUG environment variable to 1: export RBM_NO_DEBUG=1 +Or set the debug option to 0 in the rbm.local.conf file. + If you want to select the output directory, you can use rbm's --output-dir option. You can look at the Makefile to find the rbm command for what you want to build, and add the --output-dir option. For example if you diff --git a/README.BUILD_ERRORS b/README.BUILD_ERRORS index c1ee70c..6be4319 100644 --- a/README.BUILD_ERRORS +++ b/README.BUILD_ERRORS @@ -42,3 +42,5 @@ $ mkdir /home/user/tmp $ export TMPDIR=/home/user/tmp ---- +You can also define the tmp_dir option in the rbm.local.conf file. + diff --git a/rbm b/rbm index af970a5..a57b484 160000 --- a/rbm +++ b/rbm @@ -1 +1 @@ -Subproject commit af970a558f122acaba080e0b015e3293f74509fb +Subproject commit a57b4846c8aba5697ce1f197996e2962ffb5b534 diff --git a/rbm.local.conf.example b/rbm.local.conf.example new file mode 100644 index 0000000..fab3213 --- /dev/null +++ b/rbm.local.conf.example @@ -0,0 +1,22 @@ +--- +### This file is used to override options from rbm.conf to adapt them +### to your local setup. +### +### Copy this file as rbm.local.conf to enable it, and uncomment the +### options you want to modify. + + +### The tmp_dir option defines where temporary files are stored. +### You should modify this if your /tmp partition is small. +#tmp_dir: /tmp + +### The debug option defines whether a debugging shell should be opened +### automatically in the build directory/container in case of build +### failure. If you are doing automated builds, you might want to disable +### this. +#debug: 0 + +var: + local_conf: 1 + +# vim: filetype=yaml sw=2

1 0