February 2017 - tbb-commits - lists.torproject.org

[tor-browser-build/master] projects/tor: use faketime on OSX
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit de8a0de76cf4a07bf2e9138e9c39bd4af8a93446 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 13:04:22 2017 +0100 projects/tor: use faketime on OSX --- projects/tor/build | 7 +++++++ projects/tor/config | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/projects/tor/build b/projects/tor/build index bd86928..a568512 100644 --- a/projects/tor/build +++ b/projects/tor/build @@ -58,8 +58,15 @@ cd /var/tmp/build/[% project %]-[% c('version') %] ./configure --disable-asciidoc --with-libevent-dir="$libeventdir" --with-openssl-dir="$openssldir" \ [% IF c("var/windows") %]--with-zlib-dir="$zlibdir"[% END %] \ --prefix="$distdir" [% c("var/configure_opt") %] +[% IF c("var/osx") -%] + export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1 + export FAKETIME="[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]" +[% END -%] make -j4 make install +[% IF c("var/osx") -%] + unset LD_PRELOAD +[% END -%] cp $distdir/share/tor/geoip $TORCONFIGDIR cp $distdir/share/tor/geoip6 $TORCONFIGDIR diff --git a/projects/tor/config b/projects/tor/config index b73a447..2fbbf24 100644 --- a/projects/tor/config +++ b/projects/tor/config @@ -36,6 +36,10 @@ targets: - libtool - zlib1g-dev - hardening-wrapper + osx-x86_64: + var: + arch_deps: + - faketime expert-bundle: var: expert_bundle: 1

1 0

[tor-browser-build/master] Add build signing options to rbm.local.conf
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit 8c36b34c471ddcc7ebb0ff54e41bb5c30fe15594 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 01:02:52 2017 +0100 Add build signing options to rbm.local.conf --- README | 4 +++- projects/release/config | 4 ++-- rbm.conf | 3 +++ rbm.local.conf.example | 9 +++++++++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/README b/README index 001a692..956e162 100644 --- a/README +++ b/README @@ -157,7 +157,9 @@ Signing builds If the environment variable RBM_SIGN_BUILD is set to 1, the sha256sums-unsigned-build.txt file will be signed with gpg. You can use the RBM_GPG_OPTS environment variable to add some options -to the gpg command used to sign the file. +to the gpg command used to sign the file. You can also set the +var/sign_build and var/sign_build_gpg_opts options in the rbm.local.conf +file. Cleaning obsolete files and containers images diff --git a/projects/release/config b/projects/release/config index daa78dd..0b3727f 100644 --- a/projects/release/config +++ b/projects/release/config @@ -125,7 +125,7 @@ build: | [% END -%] cd "$destdir" sha256sum $(ls -1 *.exe *.tar.xz *.dmg *.mar *.zip | grep -v '\.incremental\.mar$' | sort) > sha256sums-unsigned-build.txt - [% IF ENV.RBM_SIGN_BUILD -%] - gpg -abs [% ENV.RBM_GPG_OPTS %] sha256sums-unsigned-build.txt + [% IF c("var/sign_build") -%] + gpg -abs [% c("var/sign_build_gpg_opts") %] sha256sums-unsigned-build.txt [% END -%] cat sha256sums-unsigned-build.txt diff --git a/rbm.conf b/rbm.conf index 05e45a6..5dfdc30 100644 --- a/rbm.conf +++ b/rbm.conf @@ -49,6 +49,9 @@ var: - vi - zh-CN + sign_build: '[% ENV.RBM_SIGN_BUILD %]' + sign_build_gpg_opts: '[% ENV.RBM_GPG_OPTS %]' + targets: notarget: linux-x86_64 diff --git a/rbm.local.conf.example b/rbm.local.conf.example index fab3213..42bf431 100644 --- a/rbm.local.conf.example +++ b/rbm.local.conf.example @@ -19,4 +19,13 @@ var: local_conf: 1 + ### The var/sign_build option defines if you want to sign the + ### sha256sums-unsigned-build.txt file with gpg. + #sign_build: 1 + + ### The var/sign_build_gpg_opts option can be used to define some gpg + ### options to select the key to use to sign the sha256sums-unsigned-build.txt + ### file. + #sign_build_gpg_opts: '--local-user XXXXXXXX' + # vim: filetype=yaml sw=2

1 0

[tor-browser-build/master] Add support for optional local configuration file rbm.local.conf
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit 99f69d0aa0665e57a5036f66e16dbdcd0fea94d0 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 00:40:49 2017 +0100 Add support for optional local configuration file rbm.local.conf Update rbm and add example rbm.local.conf file. --- .gitignore | 1 + README | 2 ++ README.BUILD_ERRORS | 2 ++ rbm | 2 +- rbm.local.conf.example | 22 ++++++++++++++++++++++ 5 files changed, 28 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c755fcf..98554a3 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ alpha alpha_nightly nightly hardened +rbm.local.conf diff --git a/README b/README index 775f782..001a692 100644 --- a/README +++ b/README @@ -89,6 +89,8 @@ the RBM_NO_DEBUG environment variable to 1: export RBM_NO_DEBUG=1 +Or set the debug option to 0 in the rbm.local.conf file. + If you want to select the output directory, you can use rbm's --output-dir option. You can look at the Makefile to find the rbm command for what you want to build, and add the --output-dir option. For example if you diff --git a/README.BUILD_ERRORS b/README.BUILD_ERRORS index c1ee70c..6be4319 100644 --- a/README.BUILD_ERRORS +++ b/README.BUILD_ERRORS @@ -42,3 +42,5 @@ $ mkdir /home/user/tmp $ export TMPDIR=/home/user/tmp ---- +You can also define the tmp_dir option in the rbm.local.conf file. + diff --git a/rbm b/rbm index af970a5..a57b484 160000 --- a/rbm +++ b/rbm @@ -1 +1 @@ -Subproject commit af970a558f122acaba080e0b015e3293f74509fb +Subproject commit a57b4846c8aba5697ce1f197996e2962ffb5b534 diff --git a/rbm.local.conf.example b/rbm.local.conf.example new file mode 100644 index 0000000..fab3213 --- /dev/null +++ b/rbm.local.conf.example @@ -0,0 +1,22 @@ +--- +### This file is used to override options from rbm.conf to adapt them +### to your local setup. +### +### Copy this file as rbm.local.conf to enable it, and uncomment the +### options you want to modify. + + +### The tmp_dir option defines where temporary files are stored. +### You should modify this if your /tmp partition is small. +#tmp_dir: /tmp + +### The debug option defines whether a debugging shell should be opened +### automatically in the build directory/container in case of build +### failure. If you are doing automated builds, you might want to disable +### this. +#debug: 0 + +var: + local_conf: 1 + +# vim: filetype=yaml sw=2

1 0

[tor-browser-build/master] Add docker_image_prefix option to rbm.local.conf
by boklm＠torproject.org 14 Feb '17

14 Feb '17

commit e30719ad9b5ad2c63156d88e63a5d4a082021455 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 01:31:22 2017 +0100 Add docker_image_prefix option to rbm.local.conf --- README | 12 ++++++++++++ projects/docker-image/config | 1 - rbm.conf | 1 + rbm.local.conf.example | 6 ++++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/README b/README index 956e162..e3672ec 100644 --- a/README +++ b/README @@ -169,6 +169,18 @@ There will be a script to clean old build files and containers that are no longer used, but it has not been added yet. +Multiple build directories on the same host +------------------------------------------- + +You can do multiple builds of Tor Browser in different directories on +the same host. However the docker images namespace is global, so you +may have some conflicts with the same image names used by the +different builds. By default, the docker images are prefixed with +tor-browser_$USER. You can change this prefix by defining the +docker_image_prefix option in rbm.local.conf, using a different prefix +for each of your build directories. + + Common Build Errors ------------------- diff --git a/projects/docker-image/config b/projects/docker-image/config index 27d2654..1968b77 100644 --- a/projects/docker-image/config +++ b/projects/docker-image/config @@ -1,7 +1,6 @@ # vim: filetype=yaml sw=2 filename: '[% sha256(c("pre")).substr(0, 12) %]' remote_docker: 1 -docker_image_prefix: '[% GET c("var/project_name") ? c("var/project_name") : "rbm-build" %]_[% GET ENV.RBM_BUILDNAME ? ENV.RBM_BUILDNAME : ENV.USER ? ENV.USER : c("uid") %]' docker_save_image: '[% c("docker_image_prefix") %]:[% c("filename") %]' pkg_type: build diff --git a/rbm.conf b/rbm.conf index 5dfdc30..47ffbfb 100644 --- a/rbm.conf +++ b/rbm.conf @@ -155,6 +155,7 @@ targets: docker_image: '[% pc("docker-image", "docker_save_image") %]' +docker_image_prefix: '[% GET c("var/project_name") ? c("var/project_name") : "rbm-build" %]_[% GET ENV.RBM_BUILDNAME ? ENV.RBM_BUILDNAME : ENV.USER ? ENV.USER : c("uid") %]' # change the default gpg_wrapper to allow git tag signed using an # expired key. diff --git a/rbm.local.conf.example b/rbm.local.conf.example index 42bf431..20f081f 100644 --- a/rbm.local.conf.example +++ b/rbm.local.conf.example @@ -16,6 +16,12 @@ ### this. #debug: 0 +### If you are doing multiple builds in different directories on the +### same host, you should define docker_image_prefix with a different +### value for each build directory, so that the different builds don't +### use the same docker image names. +#docker_image_prefix: tor-browser_XXXXX + var: local_conf: 1

1 0

[rbm/master] Add support for optional local configuration file rbm.local.conf
by boklm＠torproject.org 13 Feb '17

13 Feb '17

commit a57b4846c8aba5697ce1f197996e2962ffb5b534 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Feb 14 00:28:45 2017 +0100 Add support for optional local configuration file rbm.local.conf --- NEWS | 2 ++ doc/rbm_cli.asc | 3 +++ doc/rbm_config.asc | 29 ++++++++++++++++++++++++++--- lib/RBM.pm | 10 +++++++++- lib/RBM/DefaultConfig.pm | 1 + rbm | 3 ++- 6 files changed, 43 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index 82890a5..1a1789f 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,5 @@ +- Add support for optional local configuration file rbm.local.conf + - add the git_submodule option - add the hg_opt option diff --git a/doc/rbm_cli.asc b/doc/rbm_cli.asc index 92b84ef..66f6025 100644 --- a/doc/rbm_cli.asc +++ b/doc/rbm_cli.asc @@ -31,6 +31,9 @@ The following options can be set on the command line : --sysconf-file=<file>:: Set the optional system configuration file path + --localconf-file=<file>:: + Set the optional local configuration file path + --target=<target>:: Set the target. This option can be used multiple times to use more than one target. See link:rbm_targets.html[rbm_targets(7)] for diff --git a/doc/rbm_config.asc b/doc/rbm_config.asc index 27b8f7a..e8decd9 100644 --- a/doc/rbm_config.asc +++ b/doc/rbm_config.asc @@ -8,11 +8,13 @@ rbm_config - The rbm configuration DESCRIPTION ----------- -All configuration options can be defined in 3 different places : +All configuration options can be defined in 5 different places : -- in the main configuration in your working directory +- in the main configuration in your working directory (+rbm.conf+) -- in the global system configuration +- in the local configuration in your working directory (+rbm.local.conf+) + +- in the global system configuration (+/etc/rbm.conf+) - in a project configuration @@ -30,6 +32,14 @@ The option values are used with the following priority order : - project config +- local config for matching step and target + +- local config for matching step + +- local config for matching target + +- local config + - workspace config for matching step and target - workspace config for matching step @@ -62,6 +72,13 @@ This is where you will put configuration relevant to all projects under this working directory. All relative paths used in the configuration are relative from the +rbm.conf+ location. +The local configuration file is +rbm.local.conf+ (or the file defined +by the +localconf_file+ option), in the same directory as the +rbm.conf+ +file. If the file does not exists, it is ignored. The +rbm.local.conf+ +file is used to override some options from +rbm.conf+ relevent to your +local setup, whithout modifying the +rbm.conf+ file which is usually +maintained in a VCS. + An example +rbm.conf+ file will look like this : ---- @@ -89,6 +106,12 @@ sysconf_file:: is +/etc/rbm.conf+. This can also be set with the --sysconf-file command line parameter. +localconf_file:: + The path to an optional local configuration file. The default + is +rbm.local.conf+. If the path is relative, it is relative to + the directory where the +rbm.conf+ file is located. This can + also be set with the --localconf-file command line parameter. + projects_dir:: The directory containing the projects definitions. The default value is +projects+. diff --git a/lib/RBM.pm b/lib/RBM.pm index ad39526..4b5bfe9 100644 --- a/lib/RBM.pm +++ b/lib/RBM.pm @@ -61,6 +61,13 @@ sub load_system_config { $config->{system} = -f $cfile ? load_config_file($cfile) : {}; } +sub load_local_config { + my ($project) = @_; + my $cfile = project_config($project ? $project : 'undef', 'localconf_file'); + $cfile = path($cfile); + $config->{local} = -f $cfile ? load_config_file($cfile) : {}; +} + sub find_config_file { for (my $dir = getcwd; $dir ne '/'; $dir = dirname($dir)) { return "$dir/rbm.conf" if -f "$dir/rbm.conf"; @@ -209,7 +216,8 @@ sub project_config { my $opt_save = $config->{opt}; $config->{opt} = { %{$config->{opt}}, %$options } if $options; $res = config($project, $name, $options, ['opt'], ['run'], - ['projects', $project], [], ['system'], ['default']); + ['projects', $project], ['local'], [], + ['system'], ['default']); if (!$options->{no_tmpl} && defined($res) && !ref $res && !notmpl(confkey_str($name), $project)) { $res = process_template($project, $res, diff --git a/lib/RBM/DefaultConfig.pm b/lib/RBM/DefaultConfig.pm index 8172f04..052cf16 100644 --- a/lib/RBM/DefaultConfig.pm +++ b/lib/RBM/DefaultConfig.pm @@ -103,6 +103,7 @@ sub docker_version { our %default_config = ( sysconf_file => '/etc/rbm.conf', + localconf_file=> 'rbm.local.conf', tmp_dir => '[% GET ENV.TMPDIR ? ENV.TMPDIR : "/tmp"; %]', projects_dir => 'projects', output_dir => 'out', diff --git a/rbm b/rbm index 52f3076..c618ada 100755 --- a/rbm +++ b/rbm @@ -95,7 +95,7 @@ sub set_options { gpg-keyring=s gpg-keyring-dir=s gpg-args=s gpg-bin=s sysconf-file=s debsign-keyid=s use-pbuilder! step=s target=s@ publish-src-dir=s debug! hg-clone-dir=s - hg-hash=s); + hg-hash=s localconf-file=s); my %val; Getopt::Long::GetOptionsFromArray(\@_, \%val, @options) || exit 1; foreach my $k (keys %val) { @@ -108,6 +108,7 @@ sub set_options { $RBM::config->{run}{$l} = $val{$k}; } RBM::load_system_config(@_); + RBM::load_local_config(@_); return $RBM::config->{run}{args} = \@_; }

1 0

[tor-browser-build/master] Update rbm
by boklm＠torproject.org 12 Feb '17

12 Feb '17

commit 465a77590547aedce0f99d1fbb8e85fd411a6619 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Sun Feb 12 23:36:33 2017 +0100 Update rbm - Fix docker_remote_finish for docker >= 1.10.0 - If the TMPDIR env var exists, use it instead of /tmp --- rbm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rbm b/rbm index 9f78558..af970a5 160000 --- a/rbm +++ b/rbm @@ -1 +1 @@ -Subproject commit 9f78558d15854848d045a38b35666e1b78aad7db +Subproject commit af970a558f122acaba080e0b015e3293f74509fb

1 0

[tor-browser-build/master] Add a README.BUILD_ERRORS file listing some common errors
by boklm＠torproject.org 12 Feb '17

12 Feb '17

commit 88146b2f6e947d35d672f0b82e39283a46dfe0d4 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Feb 13 00:06:13 2017 +0100 Add a README.BUILD_ERRORS file listing some common errors --- README | 7 +++++++ README.BUILD_ERRORS | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/README b/README index 2c36d74..775f782 100644 --- a/README +++ b/README @@ -164,3 +164,10 @@ Cleaning obsolete files and containers images There will be a script to clean old build files and containers that are no longer used, but it has not been added yet. + +Common Build Errors +------------------- + +You can look at the README.BUILD_ERRORS file for a list of common build +errors and their solutions. + diff --git a/README.BUILD_ERRORS b/README.BUILD_ERRORS new file mode 100644 index 0000000..c1ee70c --- /dev/null +++ b/README.BUILD_ERRORS @@ -0,0 +1,44 @@ +This file lists some common build errors and their solutions. + + +Error starting remote +--------------------- + +If you have an error like this: + +---- +Error: Error starting remote: +/bin/sh: 1: adduser: not found +Segmentation fault (core dumped) +---- + +You might be having this issue: +https://github.com/docker/docker/issues/28705 + +When the kernel is configured with CONFIG_LEGACY_VSYSCALL_NONE, running +Debian Wheezy containers fails with a segfault. This should be fixed by +adding "vsyscall=emulate"to the kernel cmdline. + +If you are building inside Qubes, you can change the kernel cmdline for +the VM you are using with something like this in dom0: + +---- +$ qvm-pref --get [vmname] kernelopts +nopat +$ qvm-pref --set [vmname] kernelopts 'nopat vsyscall=emulate' +---- + + +tmp partition is full +--------------------- + +If your /tmp partition is small, you will get a 'No space left on device' +error during the build. To select an other directory with more space +available to store temporary files, you can define the TMPDIR environment +variable: + +---- +$ mkdir /home/user/tmp +$ export TMPDIR=/home/user/tmp +---- +

1 0

[rbm/master] Fix docker_remote_finish for docker >= 1.10.0
by boklm＠torproject.org 12 Feb '17

12 Feb '17

commit dbd2143279680104971dc2b08877f252a38a1b2c Author: Nicolas Vigier <boklm(a)torproject.org> Date: Sun Feb 12 23:11:13 2017 +0100 Fix docker_remote_finish for docker >= 1.10.0 With docker >= 1.10.0, the -f flag on 'docker tag' is deprecated: https://docs.docker.com/engine/deprecated/#/f-flag-on-docker-tag However we need it when using an older version of docker. To add it only when needed, we parse the output from 'docker version'. --- lib/RBM/DefaultConfig.pm | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/RBM/DefaultConfig.pm b/lib/RBM/DefaultConfig.pm index 100ad2c..8eaf9dd 100644 --- a/lib/RBM/DefaultConfig.pm +++ b/lib/RBM/DefaultConfig.pm @@ -86,6 +86,21 @@ sub get_arch { return $stdout; } +sub docker_version { + my ($stdout, $stderr, $success) + = capture_exec('docker', 'version', '--format', '{{.Client.Version}}'); + if ($success) { + chomp $stdout; + return $stdout; + } + ($stdout, $stderr, $success) = capture_exec('docker', 'version'); + RBM::exit_error("Error running 'docker version'") unless $success; + foreach my $line (split "\n", $stdout) { + return $1 if ($line =~ m/Client version: (.*)$/); + } + RBM::exit_error("Could not find docker version"); +} + our %default_config = ( sysconf_file => '/etc/rbm.conf', tmp_dir => '/tmp', @@ -382,6 +397,10 @@ OPT_END #### #### #### + docker_version => \&docker_version, +#### +#### +#### docker_build_image => 'rbm-[% sha256(c("build_id")).substr(0, 12) %]', #### #### @@ -408,7 +427,7 @@ OPT_END #!/bin/sh set -e [% IF c('docker_save_image') -%] -docker tag -f [% c('docker_build_image') %] [% c('docker_save_image') %] +docker tag [% IF versioncmp(c('docker_version'), '1.10.0') == -1; GET '-f'; END; %] [% c('docker_build_image') %] [% c('docker_save_image') %] [% END -%] docker rmi -f [% c('docker_build_image') %] > /dev/null OPT_END

1 0

[rbm/master] If the TMPDIR env var exists, use it instead of /tmp
by boklm＠torproject.org 12 Feb '17

12 Feb '17

commit af970a558f122acaba080e0b015e3293f74509fb Author: Nicolas Vigier <boklm(a)torproject.org> Date: Sun Feb 12 23:24:09 2017 +0100 If the TMPDIR env var exists, use it instead of /tmp --- lib/RBM/DefaultConfig.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/RBM/DefaultConfig.pm b/lib/RBM/DefaultConfig.pm index 8eaf9dd..8172f04 100644 --- a/lib/RBM/DefaultConfig.pm +++ b/lib/RBM/DefaultConfig.pm @@ -103,7 +103,7 @@ sub docker_version { our %default_config = ( sysconf_file => '/etc/rbm.conf', - tmp_dir => '/tmp', + tmp_dir => '[% GET ENV.TMPDIR ? ENV.TMPDIR : "/tmp"; %]', projects_dir => 'projects', output_dir => 'out', git_clone_dir => 'git_clones',

1 0

[tor-browser-build/master] Use OpenSSL 1.0.2k in nightlies
by boklm＠torproject.org 09 Feb '17

09 Feb '17

commit cf76fe40cbf2b58d69cde9613fcf60527550d48c Author: Nicolas Vigier <boklm(a)torproject.org> Date: Thu Feb 9 12:29:46 2017 +0100 Use OpenSSL 1.0.2k in nightlies --- projects/openssl/config | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/projects/openssl/config b/projects/openssl/config index dcdac50..637f3d1 100644 --- a/projects/openssl/config +++ b/projects/openssl/config @@ -24,9 +24,15 @@ targets: pre_pkginst: dpkg --add-architecture i386 compiler: macosx-toolchain-gcc + nightly: + version: 1.0.2k + input_files: - project: docker-image - name: '[% c("var/compiler") %]' project: '[% c("var/compiler") %]' - URL: 'https://www.openssl.org/source/openssl-[% c("version") %].tar.gz' sha256sum: e7aff292be21c259c6af26469c7a9b3ba26e9abaaffd325e3dccc9785256c431 + targets: + nightly: + sha256sum: 6b3977c61f2aedf0f96367dcfb5c6e578cf37e7b8d913b4ecb6643c3cb88d8c0

1 0