- tbb-commits - lists.torproject.org

[tor-browser/tor-browser-52.3.0esr-7.5-2] Bug 23230: Fix build error on Windows 64
by gk＠torproject.org 22 Aug '17

22 Aug '17

commit f36d2a2dd30368c97594923092a43dd74c61846c Author: Nicolas Vigier <boklm(a)torproject.org> Date: Sat Aug 12 20:42:59 2017 +0200 Bug 23230: Fix build error on Windows 64 Fix the following build error we have when building for Windows 64: /var/tmp/build/firefox-af0ca6a886c7/media/libcubeb/src/cubeb_wasapi.cpp:1596:46: error: cast from cubeb_devid {aka void*} to int loses precision [-fpermissive] We fix that by using %p instead of %x to remove the need for a cast. --- media/libcubeb/src/cubeb_wasapi.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/media/libcubeb/src/cubeb_wasapi.cpp b/media/libcubeb/src/cubeb_wasapi.cpp index e88d6becd861..212cc263c0b6 100644 --- a/media/libcubeb/src/cubeb_wasapi.cpp +++ b/media/libcubeb/src/cubeb_wasapi.cpp @@ -1575,7 +1575,7 @@ int setup_wasapi_stream(cubeb_stream * stm) XASSERT((!stm->output_client || !stm->input_client) && "WASAPI stream already setup, close it first."); if (has_input(stm)) { - LOG("Setup capture: device=%x", (int)stm->input_device); + LOG("Setup capture: device=%p", stm->input_device); rv = setup_wasapi_stream_one_side(stm, &stm->input_stream_params, stm->input_device, @@ -1593,7 +1593,7 @@ int setup_wasapi_stream(cubeb_stream * stm) } if (has_output(stm)) { - LOG("Setup render: device=%x", (int)stm->output_device); + LOG("Setup render: device=%p", stm->output_device); rv = setup_wasapi_stream_one_side(stm, &stm->output_stream_params, stm->output_device,

1 0

[tor-browser-build/master] Bug 17381: add dmg2mar
by boklm＠torproject.org 21 Aug '17

21 Aug '17

commit a510614a08cedcb99a03329acb8fa0bd165314e6 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Aug 21 14:41:34 2017 +0200 Bug 17381: add dmg2mar Add the dmg2mar makefile rules. --- Makefile | 12 ++++ README.MAKEFILE | 6 ++ projects/release/config | 5 ++ projects/release/dmg2mar | 4 ++ tools/dmg2mar | 168 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 195 insertions(+) diff --git a/Makefile b/Makefile index d49d670..1229cfa 100644 --- a/Makefile +++ b/Makefile @@ -103,6 +103,18 @@ update_responses-alpha: submodule-update $(rbm) build release --step update_responses_config --target alpha --target signed tools/update-responses/update_responses alpha +dmg2mar-release: submodule-update + $(rbm) build release --step update_responses_config --target release --target signed + $(rbm) build release --step dmg2mar --target release --target signed + tools/update-responses/download_missing_versions release + CHECK_CODESIGNATURE_EXISTS=1 MAR_SKIP_EXISTING=1 tools/update-responses/gen_incrementals release + +dmg2mar-alpha: submodule-update + $(rbm) build release --step update_responses_config --target alpha --target signed + $(rbm) build release --step dmg2mar --target alpha --target signed + tools/update-responses/download_missing_versions alpha + CHECK_CODESIGNATURE_EXISTS=1 MAR_SKIP_EXISTING=1 tools/update-responses/gen_incrementals alpha + submodule-update: git submodule update --init diff --git a/README.MAKEFILE b/README.MAKEFILE index 167f5d5..9e28864 100644 --- a/README.MAKEFILE +++ b/README.MAKEFILE @@ -82,6 +82,12 @@ incrementals-{release,alpha} Create incremental mar files for an unsigned build in the release or alpha channel. +dmg2mar-{release,alpha) +----------------------- +Generate updated mar files for the OSX bundles, from the dmg files, then +regenerate the OSX incremental mar files. You should run this after +signing the OSX dmg files. + update_responses-{release,alpha} -------------------------------- Create update responses xml files for a signed build in the release or diff --git a/projects/release/config b/projects/release/config index 3c771bf..0a4276a 100644 --- a/projects/release/config +++ b/projects/release/config @@ -127,3 +127,8 @@ steps: debug: 0 input_files: [] hash_incrementals: '[% INCLUDE hash_incrementals %]' + dmg2mar: + build_log: '-' + debug: 0 + input_files: [] + dmg2mar: '[% INCLUDE dmg2mar %]' diff --git a/projects/release/dmg2mar b/projects/release/dmg2mar new file mode 100644 index 0000000..d6175f5 --- /dev/null +++ b/projects/release/dmg2mar @@ -0,0 +1,4 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +cd [% shell_quote(path(dest_dir)) %]/[% c("var/signed_status") %]/[% c("version") %] +[% shell_quote(c("basedir")) %]/tools/dmg2mar diff --git a/tools/dmg2mar b/tools/dmg2mar new file mode 100755 index 0000000..51d6acb --- /dev/null +++ b/tools/dmg2mar @@ -0,0 +1,168 @@ +#!/usr/bin/perl -w +# +# This script converts all dmg files from the current directory and +# listed in the sha256sums-unsigned-build.txt file to full update +# mar files. After code signing the dmg files, this script can be used +# to update the mar files. +# +# A recent version of p7zip is required to extract the dmg files, such +# as 15.14. The version in Debian Jessie (9.20) is not recent enough. +# It is possible to install the p7zip-full package from Debian testing, +# or build p7zip from sources: +# $ p7zipdir=/some_directory/p7zip +# $ mkdir $p7zipdir +# $ cd $p7zipdir +# $ wget http://snapshot.debian.org/archive/debian/20160417T044336Z/pool/main/p/p7zi… +# $ echo 'e9e696e2fa77b00445a4d85fa07506debeae01943fdc1bee1472152d7d1386af p7zip_15.14.1+dfsg.orig.tar.xz' | sha256sum -c +# $ wget http://snapshot.debian.org/archive/debian/20160515T161830Z/pool/main/p/p7zi… +# $ echo 'f4db6803535fc30b6ae9db5aabfd9f57a851c6773d72073847ec5e3731b7af37 p7zip_15.14.1+dfsg-2.debian.tar.xz' | sha256sum -c +# $ tar xvf p7zip_15.14.1+dfsg-2.debian.tar.xz +# $ tar xvf p7zip_15.14.1+dfsg.orig.tar.xz +# $ cd p7zip_15.14.1/ +# $ for patch in $(cat ../debian/patches/series ); do patch -p1 < ../debian/patches/$patch; done +# $ make 7z +# $ mkdir $p7zipdir/bin +# $ echo '#!/bin/sh' > $p7zipdir/bin/7z +# $ echo "export LD_LIBRARY_PATH=$PWD/bin" >> $p7zipdir/bin/7z +# $ echo "exec $PWD/bin/7z "'"$@"' >> $p7zipdir/bin/7z +# $ chmod +x $p7zipdir/bin/7z +# $ export "PATH=$p7zipdir/bin:$PATH" + +use strict; +use IO::CaptureOutput qw(capture_exec); +use File::Slurp; +use File::Find; +use Parallel::ForkManager; +use Cwd; + +# If the application is not TorBrowser (for instance, TorMessenger) +# set the application name in the TOR_APPNAME_BUNDLE_OSX and in +# the TOR_APPNAME_MARFILE environment variables +my $appname = $ENV{TOR_APPNAME_BUNDLE_OSX} // 'TorBrowser'; +my $appname_mar = $ENV{TOR_APPNAME_MARFILE} // 'tor-browser'; + +sub exit_error { + print STDERR "Error: ", $_[0], "\n"; + chdir '/'; + exit (exists $_[1] ? $_[1] : 1); +} + +sub osname { + my ($osname) = capture_exec('uname', '-s'); + my ($arch) = capture_exec('uname', '-m'); + chomp($osname, $arch); + if ($osname eq 'Linux' && $arch eq 'x86_64') { + return 'linux64'; + } + if ($osname eq 'Linux' && $arch =~ m/^i.86$/) { + return 'linux32'; + } + exit_error 'Unknown OS'; +} + +my $martools_tmpdir; +sub extract_martools { + my $osname = osname; + my $marzip = getcwd . "/mar-tools-$osname.zip"; + $martools_tmpdir = File::Temp->newdir(); + my $old_cwd = getcwd; + chdir $martools_tmpdir; + my (undef, undef, $success) = capture_exec('unzip', $marzip); + chdir $old_cwd; + exit_error "Error extracting $marzip" unless $success; + $ENV{PATH} = "$martools_tmpdir/mar-tools:$ENV{PATH}"; + if ($ENV{LD_LIBRARY_PATH}) { + $ENV{LD_LIBRARY_PATH} .= ":$martools_tmpdir/mar-tools"; + } else { + $ENV{LD_LIBRARY_PATH} = "$martools_tmpdir/mar-tools"; + } + $ENV{MAR} = "$martools_tmpdir/mar-tools/mar"; + $ENV{MSBDIFF} = "$martools_tmpdir/mar-tools/mbsdiff"; +} + +sub get_nbprocs { + return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS}; + if (-f '/proc/cpuinfo') { + return scalar grep { m/^processor\s+:\s/ } read_file '/proc/cpuinfo'; + } + return 4; +} + +sub get_dmg_files_from_sha256sums { + exit_error "Missing sha256sums-unsigned-build.txt file" + unless -f 'sha256sums-unsigned-build.txt'; + my @files; + foreach my $line (read_file('sha256sums-unsigned-build.txt')) { + my (undef, $filename) = split ' ', $line; + chomp $filename; + next unless $filename =~ m/^$appname-(.+)-osx64_(.+)\.dmg$/; + push @files, { filename => $filename, version => $1, lang => $2 }; + } + return @files; +} + +sub convert_files { + my $pm = Parallel::ForkManager->new(get_nbprocs); + $pm->run_on_finish(sub { print "Finished $_[2]\n" }); + foreach my $file (get_dmg_files_from_sha256sums) { + # The 'ja' locale is a special case: it is called 'ja-JP-mac' + # internally on OSX, but the dmg file still uses 'ja' to avoid + # confusing users. + my $mar_lang = $file->{lang} eq 'ja' ? 'ja-JP-mac' : $file->{lang}; + my $output = "$appname_mar-osx64-$file->{version}_$mar_lang.mar"; + my $step_name = "$file->{filename} -> $output"; + print "Starting $step_name\n"; + $pm->start($step_name) and next; + my $tmpdir = File::Temp->newdir(); + my (undef, $err, $success) = capture_exec('7z', 'x', "-o$tmpdir", + $file->{filename}); + exit_error "Error extracting $file->{filename}: $err" unless $success; + + # 7z does not currently extract file permissions from the dmg files + # so we also extract the old mar file to copy the permissions + # https://trac.torproject.org/projects/tor/ticket/20210 + my $tmpdir_oldmar = File::Temp->newdir(); + my $oldmar = getcwd . '/' . $output; + exit_error "Error extracting $output" + unless system('mar', '-C', $tmpdir_oldmar, '-x', $oldmar) == 0; + my $wanted = sub { + my $file = $File::Find::name; + $file =~ s{^$tmpdir/$appname\.app/}{}; + if (-f "$tmpdir_oldmar/$file") { + my (undef, undef, $mode) = stat("$tmpdir_oldmar/$file"); + chmod $mode, $File::Find::name; + return; + } + chmod 0644, $File::Find::name if -f $File::Find::name; + chmod 0755, $File::Find::name if -d $File::Find::name; + }; + find($wanted, "$tmpdir/$appname.app"); + + unlink $output; + (undef, $err, $success) = capture_exec('make_full_update.sh', '-q', + $output, "$tmpdir/$appname.app"); + exit_error "Error updating $output: $err" unless $success; + $pm->finish; + } + $pm->wait_all_children; +} + +sub remove_incremental_mars { + exit_error "Missing sha256sums-unsigned-build.incrementals.txt file" + unless -f 'sha256sums-unsigned-build.incrementals.txt'; + foreach my $line (read_file('sha256sums-unsigned-build.incrementals.txt')) { + my (undef, $filename) = split ' ', $line; + chomp $filename; + next unless $filename =~ m/^$appname_mar-osx64.+\.incremental\.mar$/; + next unless -f $filename; + print "Removing $filename\n"; + unlink $filename; + } +} + +# Set LC_ALL=C to avoid reproducibility issues when creating mar files +$ENV{LC_ALL} = 'C'; + +extract_martools; +convert_files; +remove_incremental_mars;

1 0

[tor-browser-build/master] Bug 17381: add and adapt the update_responses script
by boklm＠torproject.org 21 Aug '17

21 Aug '17

commit de3f97f325df5465fa49b7c8ca2ff229b670b207 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Thu Aug 17 13:32:46 2017 +0200 Bug 17381: add and adapt the update_responses script Add the update_responses and incrementals makefile rules. The update_responses script is identical to what we have in tor-browser-bundle.git, however the config.yml file it is using is now generated using the infos from rbm. --- Makefile | 20 + README.MAKEFILE | 10 + keyring/torbrowser.gpg | Bin 0 -> 49267 bytes projects/firefox/config | 3 +- projects/release/config | 21 +- projects/release/hash_incrementals | 4 + projects/release/update_responses_config | 11 + projects/release/update_responses_config.yml | 40 ++ rbm.conf | 3 + tools/update-responses/.gitignore | 2 + tools/update-responses/README.md | 50 ++ .../check_update_responses_deployement | 1 + tools/update-responses/download_missing_versions | 1 + tools/update-responses/gen_incrementals | 1 + tools/update-responses/get_channel_version | 1 + tools/update-responses/update_responses | 643 +++++++++++++++++++++ 16 files changed, 807 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c45b0a1..d49d670 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,26 @@ signtag-release: submodule-update signtag-alpha: submodule-update $(rbm) build release --step signtag --target alpha +incrementals-release: submodule-update + $(rbm) build release --step update_responses_config --target release + tools/update-responses/download_missing_versions release + tools/update-responses/gen_incrementals release + $(rbm) build release --step hash_incrementals --target release + +incrementals-alpha: submodule-update + $(rbm) build release --step update_responses_config --target alpha + tools/update-responses/download_missing_versions alpha + tools/update-responses/gen_incrementals alpha + $(rbm) build release --step hash_incrementals --target alpha + +update_responses-release: submodule-update + $(rbm) build release --step update_responses_config --target release --target signed + tools/update-responses/update_responses release + +update_responses-alpha: submodule-update + $(rbm) build release --step update_responses_config --target alpha --target signed + tools/update-responses/update_responses alpha + submodule-update: git submodule update --init diff --git a/README.MAKEFILE b/README.MAKEFILE index ef33904..167f5d5 100644 --- a/README.MAKEFILE +++ b/README.MAKEFILE @@ -77,3 +77,13 @@ signtag-{release,alpha} Create a git signed tag for the selected channel, using the version and build number defined as var/torbrowser_version and var/torbrowser_build. +incrementals-{release,alpha} +---------------------------- +Create incremental mar files for an unsigned build in the release or +alpha channel. + +update_responses-{release,alpha} +-------------------------------- +Create update responses xml files for a signed build in the release or +alpha channel. + diff --git a/keyring/torbrowser.gpg b/keyring/torbrowser.gpg new file mode 100644 index 0000000..dfdee06 Binary files /dev/null and b/keyring/torbrowser.gpg differ diff --git a/projects/firefox/config b/projects/firefox/config index 1360772..034d40f 100644 --- a/projects/firefox/config +++ b/projects/firefox/config @@ -7,7 +7,8 @@ git_url: https://git.torproject.org/tor-browser.git gpg_keyring: torbutton.gpg var: - firefox_version: 52.3.0esr + firefox_platform_version: 52.3.0 + firefox_version: '[% c("var/firefox_platform_version") %]esr' torbrowser_branch: 7.5 torbrowser_update_channel: alpha copyright_year: '[% exec("git show -s --format=%ci").remove("-.*") %]' diff --git a/projects/release/config b/projects/release/config index b785878..3c771bf 100644 --- a/projects/release/config +++ b/projects/release/config @@ -1,10 +1,11 @@ # vim: filetype=yaml sw=2 version: '[% c("var/torbrowser_version") %]' -output_dir: 'release/unsigned' +output_dir: release var: + signed_status: unsigned today: '[% USE date; date.format(format = "%Y-%m-%d") %]' - publish_dir: '[% c("version") %]-[% c("var/torbrowser_build") %]' + publish_dir: '[% c("var/signed_status") %]/[% c("version") %]-[% c("var/torbrowser_build") %]' targets: torbrowser-all: @@ -50,7 +51,7 @@ targets: publish_dir: '[% c("var/today") %]' alpha: - output_dir: 'alpha/unsigned' + output_dir: alpha var: build_target: alpha @@ -60,6 +61,10 @@ targets: build_target: torbrowser-testbuild publish_dir: '' + signed: + var: + signed_status: signed + input_files: # Release @@ -112,3 +117,13 @@ steps: debug: 0 input_files: [] signtag: '[% INCLUDE signtag %]' + update_responses_config: + build_log: '-' + debug: 0 + input_files: [] + update_responses_config: '[% INCLUDE update_responses_config %]' + hash_incrementals: + build_log: '-' + debug: 0 + input_files: [] + hash_incrementals: '[% INCLUDE hash_incrementals %]' diff --git a/projects/release/hash_incrementals b/projects/release/hash_incrementals new file mode 100644 index 0000000..9cf1b27 --- /dev/null +++ b/projects/release/hash_incrementals @@ -0,0 +1,4 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +cd [% shell_quote(path(dest_dir)) %]/[% c("var/signed_status") %]/[% c("version") %]-[% c("var/torbrowser_build") %] +sha256sum `ls -1 | grep '\.incremental\.mar$' | sort` > sha256sums-[% c("var/signed_status") %]-build.incrementals.txt diff --git a/projects/release/update_responses_config b/projects/release/update_responses_config new file mode 100644 index 0000000..4ff470d --- /dev/null +++ b/projects/release/update_responses_config @@ -0,0 +1,11 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +cat > [% shell_quote(c("basedir")) %]/tools/update-responses/config.yml << 'EOF' +[% INCLUDE update_responses_config.yml -%] +EOF + +# Update / create symlink $torbrowser_version -> $torbrowser_version-buildN +versiondir=[% shell_quote(path(dest_dir)) _ '/' _ c("var/signed_status") _ '/' _ c("version") %] +test -L "$versiondir" && rm -f "$versiondir" +test -d "$versiondir" || ln -s [% shell_quote(c("version")) %]-[% shell_quote(c("var/torbrowser_build")) %] "$versiondir" + diff --git a/projects/release/update_responses_config.yml b/projects/release/update_responses_config.yml new file mode 100644 index 0000000..f0d9dbf --- /dev/null +++ b/projects/release/update_responses_config.yml @@ -0,0 +1,40 @@ +--- +appname_marfile: tor-browser +appname_bundle_osx: TorBrowser +appname_bundle_linux: tor-browser +appname_bundle_win: torbrowser-install +releases_dir: [% path(c('output_dir')) %]/[% c("var/signed_status") %] +download: + archive_url: https://archive.torproject.org/tor-package-archive/torbrowser + gpg_keyring: ../../keyring/torbrowser.gpg + bundles_url: https://dist.torproject.org/torbrowser + mars_url: https://cdn.torproject.org/aus1/torbrowser +build_targets: + linux32: Linux_x86-gcc3 + linux64: Linux_x86_64-gcc3 + win32: + - WINNT_x86-gcc3 + - WINNT_x86-gcc3-x86 + - WINNT_x86-gcc3-x64 + osx32: Darwin_x86-gcc3 + osx64: Darwin_x86_64-gcc3 +channels: + [% pc('firefox', 'var/torbrowser_update_channel') %]: [% c("var/torbrowser_version") %] +versions: + [% c("var/torbrowser_version") %]: + platformVersion: [% pc('firefox', 'var/firefox_platform_version') %] + detailsURL: https://blog.torproject.org/blog/tor-browser-[% c("var/torbrowser_version") FILTER remove('\.') %]-released + incremental_from: +[% FOREACH v IN c("var/torbrowser_incremental_from") -%] + - [% v %] +[% END -%] + migrate_archs: + osx32: osx64 + migrate_langs: + pt-PT: pt-BR + win32: + minSupportedInstructionSet: SSE2 + osx32: + minSupportedOSVersion: 13.0.0 + osx64: + minSupportedOSVersion: 13.0.0 diff --git a/rbm.conf b/rbm.conf index 5e1e66d..44fc9f5 100644 --- a/rbm.conf +++ b/rbm.conf @@ -16,6 +16,9 @@ buildconf: var: torbrowser_version: '7.5a4' torbrowser_build: 'build2' + torbrowser_incremental_from: + - 7.5a2 + - 7.5a3 project_name: tor-browser multi_lingual: 0 build_mar: 1 diff --git a/tools/update-responses/.gitignore b/tools/update-responses/.gitignore new file mode 100644 index 0000000..5947fca --- /dev/null +++ b/tools/update-responses/.gitignore @@ -0,0 +1,2 @@ +/htdocs +/config.yml diff --git a/tools/update-responses/README.md b/tools/update-responses/README.md new file mode 100644 index 0000000..5209ed5 --- /dev/null +++ b/tools/update-responses/README.md @@ -0,0 +1,50 @@ +Tor Browser Update Responses script +=================================== + +This repository contains a script to generate responses for Tor Browser +updater. + +See ticket [#12622](https://trac.torproject.org/projects/tor/ticket/12622) +for details. + + +Dependencies +------------ + +The following perl modules need to be installed to run the script: + FindBin YAML File::Slurp Digest::SHA XML::Writer File::Temp + IO::CaptureOutput Parallel::ForkManager XML::LibXML LWP JSON + +On Debian / Ubuntu you can install them with: + +``` + # apt-get install libfindbin-libs-perl libyaml-perl libfile-slurp-perl \ + libdigest-sha-perl libxml-writer-perl \ + libio-captureoutput-perl libparallel-forkmanager-perl \ + libxml-libxml-perl libwww-perl libjson-perl +``` + +On Red Hat / Fedora you can install them with: + +``` + # for module in FindBin YAML File::Slurp Digest::SHA XML::Writer \ + File::Temp IO::CaptureOutput Parallel::ForkManager \ + XML::LibXML LWP JSON + do yum install "perl($module)"; done +``` + + +URL Format +---------- + +The URL format is: + https://something/$channel/$build_target/$tb_version/$lang?force=1 + +'build_target' is the OS for which the browser was built. The correspo +ndance between the build target and the OS name that we use in archive +files is defined in the config.yml file. + +'tb_version' is the Tor Browser version. + +'lang' is the locale. + diff --git a/tools/update-responses/check_update_responses_deployement b/tools/update-responses/check_update_responses_deployement new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/check_update_responses_deployement @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/download_missing_versions b/tools/update-responses/download_missing_versions new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/download_missing_versions @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/gen_incrementals b/tools/update-responses/gen_incrementals new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/gen_incrementals @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/get_channel_version b/tools/update-responses/get_channel_version new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/get_channel_version @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses new file mode 100755 index 0000000..0ed19c1 --- /dev/null +++ b/tools/update-responses/update_responses @@ -0,0 +1,643 @@ +#!/usr/bin/perl -w + +use strict; +use feature "state"; +use English; +use FindBin; +use YAML qw(LoadFile); +use File::Slurp; +use Digest::SHA qw(sha256_hex); +use XML::Writer; +use Cwd; +use File::Copy; +use File::Temp; +use File::Find; +use POSIX qw(setlocale LC_ALL); +use IO::CaptureOutput qw(capture_exec); +use Parallel::ForkManager; +use File::Basename; +use XML::LibXML '1.70'; +use LWP::Simple; +use JSON; + +# Set umask and locale to provide a consistent environment for MAR file +# generation, etc. +umask(0022); +$ENV{"LC_ALL"} = "C"; +setlocale(LC_ALL, "C"); + +my $htdocsdir = "$FindBin::Bin/htdocs"; +my $config = LoadFile("$FindBin::Bin/config.yml"); +my %htdocsfiles; +my $releases_dir = $config->{releases_dir}; +$releases_dir = "$FindBin::Bin/$releases_dir" unless $releases_dir =~ m/^\//; +my @check_errors; +my $initPATH = $ENV{PATH}; +my $initLD_LIBRARY_PATH = $ENV{LD_LIBRARY_PATH}; + +sub exit_error { + print STDERR "Error: ", $_[0], "\n"; + chdir '/'; + exit (exists $_[1] ? $_[1] : 1); +} + +sub build_targets_by_os { + return ($_[0]) unless $config->{build_targets}{$_[0]}; + my $r = $config->{build_targets}{$_[0]}; + return ref $r eq 'ARRAY' ? @$r : ($r); +} + +sub get_nbprocs { + return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS}; + if (-f '/proc/cpuinfo') { + return scalar grep { m/^processor\s+:\s/ } read_file '/proc/cpuinfo'; + } + return 4; +} + +sub write_htdocs { + my ($channel, $file, $content) = @_; + mkdir $htdocsdir unless -d $htdocsdir; + mkdir "$htdocsdir/$channel" unless -d "$htdocsdir/$channel"; + write_file("$htdocsdir/$channel/$file", $content); + $htdocsfiles{$channel}->{$file} = 1; +} + +sub clean_htdocs { + my (@channels) = @_; + foreach my $channel (@channels) { + opendir(my $d, "$htdocsdir/$channel"); + my @files = grep { ! $htdocsfiles{$channel}->{$_} } readdir $d; + closedir $d; + unlink map { "$htdocsdir/$channel/$_" } @files; + } +} + +sub get_sha512_hex_of_file { + my ($file) = @_; + my $sha = Digest::SHA->new("512"); + $sha->addfile($file); + return $sha->hexdigest; +} + +sub get_version_files { + my ($config, $version) = @_; + return if $config->{versions}{$version}{files}; + my $appname = $config->{appname_marfile}; + my $files = {}; + my $vdir = "$releases_dir/$version"; + my $download_url = "$config->{download}{mars_url}/$version"; + opendir(my $d, $vdir) or exit_error "Error opening directory $vdir"; + foreach my $file (readdir $d) { + next unless -f "$vdir/$file"; + if ($file =~ m/^$appname-([^-]+)-${version}_(.+)\.mar$/) { + my ($os, $lang) = ($1, $2); + $files->{$os}{$lang}{complete} = { + type => 'complete', + URL => "$download_url/$file", + size => -s "$vdir/$file", + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file("$vdir/$file"), + }; + next; + } + if ($file =~ m/^$appname-([^-]+)-(.+)-${version}_(.+)\.incremental\.mar$/) { + my ($os, $from_version, $lang) = ($1, $2, $3); + $files->{$os}{$lang}{partial}{$from_version} = { + type => 'partial', + URL => "$download_url/$file", + size => -s "$vdir/$file", + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file("$vdir/$file"), + } + } + } + closedir $d; + $config->{versions}{$version}{files} = $files; +} + +sub get_version_downloads { + my ($config, $version) = @_; + my $downloads = {}; + my $vdir = "$releases_dir/$version"; + my $download_url = "$config->{download}{bundles_url}/$version"; + opendir(my $d, $vdir) or exit_error "Error opening directory $vdir"; + foreach my $file (readdir $d) { + next unless -f "$vdir/$file"; + my ($os, $lang); + if ($file =~ m/^$config->{appname_bundle_osx}-$version-osx64_(.+).dmg$/) { + ($os, $lang) = ('osx64', $1); + } elsif ($file =~ m/^$config->{appname_bundle_linux}-(linux32|linux64)-${version}_(.+).tar.xz$/) { + ($os, $lang) = ($1, $2); + } elsif ($file =~ m/^$config->{appname_bundle_win}-${version}_(.+).exe$/) { + ($os, $lang) = ('win32', $1); + } else { + next; + } + $downloads->{$os}{$lang} = { + binary => "$download_url/$file", + sig => "$download_url/$file.asc", + }; + } + closedir $d; + $config->{versions}{$version}{downloads} = $downloads; +} + +sub extract_mar { + my ($mar_file, $dest_dir) = @_; + my $old_cwd = getcwd; + mkdir $dest_dir; + chdir $dest_dir or exit_error "Cannot enter $dest_dir"; + my $res = system('mar', '-x', $mar_file); + exit_error "Error extracting $mar_file" if $res; + my $bunzip_file = sub { + return unless -f $File::Find::name; + rename $File::Find::name, "$File::Find::name.bz2"; + system('bunzip2', "$File::Find::name.bz2") == 0 + || exit_error "Error decompressing $File::Find::name"; + }; + find($bunzip_file, $dest_dir); + my $manifest = -f 'updatev3.manifest' ? 'updatev3.manifest' + : 'updatev2.manifest'; + my @lines = read_file($manifest) if -f $manifest; + foreach my $line (@lines) { + if ($line =~ m/^addsymlink "(.+)" "(.+)"$/) { + exit_error "$mar_file: Could not create symlink $1 -> $2" + unless symlink $2, $1; + } + } + chdir $old_cwd; +} + +sub mar_filename { + my ($appname, $version, $os, $lang) = @_; + "$releases_dir/$version/$appname-$os-${version}_$lang.mar"; +} + +sub create_incremental_mar { + my ($config, $pm, $from_version, $new_version, $os, $lang) = @_; + my $appname = $config->{appname_marfile}; + my $mar_file = "$appname-$os-${from_version}-${new_version}_$lang.incremental.mar"; + my $mar_file_path = "$releases_dir/$new_version/$mar_file"; + if ($ENV{MAR_SKIP_EXISTING} && -f $mar_file_path) { + print "Skipping $mar_file\n"; + return; + } + print "Starting $mar_file\n"; + my $download_url = "$config->{download}{mars_url}/$new_version"; + my $finished_file = sub { + exit_error "Error creating $mar_file" unless $_[1] == 0; + print "Finished $mar_file\n"; + $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = { + type => 'partial', + URL => "$download_url/$mar_file", + size => -s $mar_file_path, + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file($mar_file_path), + }; + }; + return if $pm->start($finished_file); + my $tmpdir = File::Temp->newdir(); + extract_mar(mar_filename($appname, $from_version, $os, $lang), "$tmpdir/A"); + extract_mar(mar_filename($appname, $new_version, $os, $lang), "$tmpdir/B"); + if ($ENV{CHECK_CODESIGNATURE_EXISTS}) { + unless (-f "$tmpdir/A/Contents/_CodeSignature/CodeResources" + && -f "$tmpdir/B/Contents/_CodeSignature/CodeResources") { + exit_error "Missing code signature while creating $mar_file"; + } + } + my ($out, $err, $success) = capture_exec('make_incremental_update.sh', + $mar_file_path, "$tmpdir/A", "$tmpdir/B"); + if (!$success) { + unlink $mar_file_path if -f $mar_file_path; + exit_error "making incremental mar:\n" . $err; + } + $pm->finish; +} + +sub create_incremental_mars_for_version { + my ($config, $version) = @_; + my $pm = Parallel::ForkManager->new(get_nbprocs); + $pm->run_on_finish(sub { $_[2]->(@_) }); + my $v = $config->{versions}{$version}; + foreach my $from_version (@{$v->{incremental_from}}) { + $config->{versions}{$from_version} //= {}; + get_version_files($config, $from_version); + my $from_v = $config->{versions}{$from_version}; + foreach my $os (keys %{$v->{files}}) { + foreach my $lang (keys %{$v->{files}{$os}}) { + next unless defined $from_v->{files}{$os}{$lang}{complete}; + create_incremental_mar($config, $pm, $from_version, $version, $os, $lang); + } + } + } + $pm->wait_all_children; +} + +sub get_config { + my ($config, $version, $os, $name) = @_; + return $config->{versions}{$version}{$os}{$name} + // $config->{versions}{$version}{$name} + // $config->{$name}; +} + +sub channel_to_version { + my ($config, @channels) = @_; + return values %{$config->{channels}} unless @channels; + foreach my $channel (@channels) { + exit_error "Unknown channel $channel" + unless $config->{channels}{$channel}; + } + return map { $config->{channels}{$_} } @channels; +} + +sub get_buildinfos { + my ($config, $version) = @_; + return if exists $config->{versions}{$version}{buildID}; + extract_martools($version); + my $files = $config->{versions}{$version}{files}; + foreach my $os (keys %$files) { + foreach my $lang (keys %{$files->{$os}}) { + next unless $files->{$os}{$lang}{complete}; + my $tmpdir = File::Temp->newdir(); + extract_mar( + mar_filename($config->{appname_marfile}, $version, $os, $lang), + "$tmpdir"); + my $appfile = "$tmpdir/application.ini" if -f "$tmpdir/application.ini"; + $appfile = "$tmpdir/Contents/Resources/application.ini" + if -f "$tmpdir/Contents/Resources/application.ini"; + exit_error "Could not find application.ini" unless $appfile; + foreach my $line (read_file($appfile)) { + if ($line =~ m/^BuildID=(.*)$/) { + $config->{versions}{$version}{buildID} = $1; + return; + } + } + exit_error "Could not extract buildID from application.ini"; + } + } +} + +sub get_response { + my ($config, $version, $os, @patches) = @_; + my $res; + my $writer = XML::Writer->new(OUTPUT => \$res, ENCODING => 'UTF-8'); + $writer->xmlDecl; + $writer->startTag('updates'); + if (get_config($config, $version, $os, 'unsupported')) { + $writer->startTag('update', + unsupported => 'true', + detailsURL => get_config($config, $version, $os, 'detailsURL'), + ); + goto CLOSETAGS; + } + my $minversion = get_config($config, $version, $os, 'minSupportedOSVersion'); + my $mininstruc = get_config($config, $version, $os, 'minSupportedInstructionSet'); + $writer->startTag('update', + type => 'minor', + displayVersion => $version, + appVersion => $version, + platformVersion => get_config($config, $version, $os, 'platformVersion'), + buildID => get_config($config, $version, $os, 'buildID'), + detailsURL => get_config($config, $version, $os, 'detailsURL'), + actions => 'showURL', + openURL => get_config($config, $version, $os, 'detailsURL'), + defined $minversion ? ( minSupportedOSVersion => $minversion ) : (), + defined $mininstruc ? ( minSupportedInstructionSet => $mininstruc ) : (), + ); + foreach my $patch (@patches) { + my @sorted_patch = map { $_ => $patch->{$_} } sort keys %$patch; + $writer->startTag('patch', @sorted_patch); + $writer->endTag('patch'); + } + CLOSETAGS: + $writer->endTag('update'); + $writer->endTag('updates'); + $writer->end; + return $res; +} + +sub write_responses { + my ($config, @channels) = @_; + @channels = keys %{$config->{channels}} unless @channels; + foreach my $channel (@channels) { + my $version = $config->{channels}{$channel}; + get_version_files($config, $version); + get_buildinfos($config, $version); + my $files = $config->{versions}{$version}{files}; + my $migrate_archs = $config->{versions}{$version}{migrate_archs} // {}; + foreach my $old_os (keys %$migrate_archs) { + my $new_os = $migrate_archs->{$old_os}; + foreach my $lang (keys %{$files->{$new_os}}) { + $files->{$old_os}{$lang}{complete} = + $files->{$new_os}{$lang}{complete}; + } + } + foreach my $os (keys %$files) { + foreach my $lang (keys %{$files->{$os}}) { + my $resp = get_response($config, $version, $os, + $files->{$os}{$lang}{complete}); + write_htdocs($channel, "$version-$os-$lang.xml", $resp); + foreach my $from_version (keys %{$files->{$os}{$lang}{partial}}) { + $resp = get_response($config, $version, $os, + $files->{$os}{$lang}{complete}, + $files->{$os}{$lang}{partial}{$from_version}); + write_htdocs($channel, "$from_version-$version-$os-$lang.xml", $resp); + } + } + } + write_htdocs($channel, 'no-update.xml', + '<?xml version="1.0" encoding="UTF-8"?>' + . "\n<updates></updates>\n"); + } +} + +sub write_htaccess { + my ($config, @channels) = @_; + @channels = keys %{$config->{channels}} unless @channels; + my $flags = "[last]"; + foreach my $channel (@channels) { + my $htaccess = "RewriteEngine On\n"; + my $version = $config->{channels}{$channel}; + my $migrate_langs = $config->{versions}{$version}{migrate_langs} // {}; + my $files = $config->{versions}{$version}{files}; + $htaccess .= "RewriteRule ^[^\/]+/$version/ no-update.xml $flags\n"; + foreach my $os (sort keys %$files) { + foreach my $bt (build_targets_by_os($os)) { + foreach my $lang (sort keys %{$files->{$os}}) { + foreach my $from_version (sort keys %{$files->{$os}{$lang}{partial}}) { + $htaccess .= "RewriteRule ^$bt/$from_version/$lang " + . "$from_version-$version-$os-$lang.xml $flags\n"; + } + $htaccess .= "RewriteRule ^$bt/[^\/]+/$lang " + . "$version-$os-$lang.xml $flags\n"; + } + foreach my $lang (sort keys %$migrate_langs) { + $htaccess .= "RewriteRule ^$bt/[^\/]+/$lang " + . "$version-$os-$migrate_langs->{$lang}.xml $flags\n"; + } + $htaccess .= "RewriteRule ^$bt/ $version-$os-en-US.xml $flags\n"; + } + } + write_htdocs($channel, '.htaccess', $htaccess); + } +} + +sub write_downloads_json { + my ($config, @channels) = @_; + @channels = keys %{$config->{channels}} unless @channels; + foreach my $channel (@channels) { + my $version = $config->{channels}{$channel}; + my $data = { + version => $version, + downloads => get_version_downloads($config, $version), + }; + write_htdocs($channel, 'downloads.json', + JSON->new->utf8->canonical->encode($data)); + } +} + +sub osname { + my ($osname) = capture_exec('uname', '-s'); + my ($arch) = capture_exec('uname', '-m'); + chomp($osname, $arch); + if ($osname eq 'Linux' && $arch eq 'x86_64') { + return 'linux64'; + } + if ($osname eq 'Linux' && $arch =~ m/^i.86$/) { + return 'linux32'; + } + exit_error 'Unknown OS'; +} + +my $martools_tmpdir; +sub extract_martools { + my ($version) = @_; + my $osname = osname; + my $marzip = "$releases_dir/$version/mar-tools-$osname.zip"; + $martools_tmpdir = File::Temp->newdir(); + my $old_cwd = getcwd; + chdir $martools_tmpdir; + my (undef, undef, $success) = capture_exec('unzip', $marzip); + chdir $old_cwd; + exit_error "Error extracting $marzip" unless $success; + $ENV{PATH} = "$martools_tmpdir/mar-tools:$initPATH"; + if ($initLD_LIBRARY_PATH) { + $ENV{LD_LIBRARY_PATH} = "$initLD_LIBRARY_PATH:$martools_tmpdir/mar-tools"; + } else { + $ENV{LD_LIBRARY_PATH} = "$martools_tmpdir/mar-tools"; + } +} + +sub log_step { + my ($url, $step, $status, $details) = @_; + state $u; + if (!defined $u || $url ne $u) { + print "\n" if $u; + print "$url\n"; + $u = $url; + } + print ' ', $step, $status ? ': OK' : ': ERROR', + $details ? " - $details\n" : "\n"; + return if $status; + push @check_errors, { url => $url, step => $step, details => $details }; +} + +sub get_remote_xml { + my ($url) = @_; + my $content = get $url; + log_step($url, 'get', defined $content); + return undef unless defined $content; + my $dom = eval { XML::LibXML->load_xml(string => $content) }; + log_step($url, 'parse_xml', defined $dom, $@); + return $dom; +} + +sub check_get_version { + my ($dom) = @_; + my @updates = $dom->documentElement()->getChildrenByLocalName('update'); + return undef unless @updates; + return $updates[0]->getAttribute('appVersion'); +} + +sub check_no_update { + my ($dom) = @_; + my @updates = $dom->documentElement()->getChildrenByLocalName('update'); + return @updates == 0; +} + +sub check_has_incremental { + my ($dom) = @_; + my @updates = $dom->documentElement()->getChildrenByLocalName('update'); + return undef unless @updates; + my @patches = $updates[0]->getChildrenByLocalName('patch'); + foreach my $patch (@patches) { + return 1 if $patch->getAttribute('type') eq 'partial'; + } + return undef; +} + +sub build_targets_list { + map { ref $_ eq 'ARRAY' ? @$_ : $_ } values %{$config->{build_targets}}; +} + +sub check_update_responses_channel { + my ($config, $base_url, $channel) = @_; + my $channel_version = $config->{channels}{$channel}; + foreach my $build_target (build_targets_list()) { + foreach my $lang (qw(en-US de)) { + my $url = "$base_url/$channel/$build_target/1.0/$lang"; + my $dom = get_remote_xml($url); + if ($dom) { + my $version = check_get_version($dom); + log_step($url, 'version', $version eq $channel_version, + "expected: $channel_version received: $version"); + } + $url = "$base_url/$channel/$build_target/$channel_version/$lang"; + $dom = get_remote_xml($url); + log_step($url, 'no_update', check_no_update($dom)) if $dom; + my @inc = @{$config->{versions}{$channel_version}{incremental_from}} + if $config->{versions}{$channel_version}{incremental_from}; + foreach my $inc_from (@inc) { + my $url = "$base_url/$channel/$build_target/$inc_from/$lang"; + $dom = get_remote_xml($url); + next unless $dom; + my $version = check_get_version($dom); + log_step($url, 'version', $version eq $channel_version, + "expected: $channel_version received: $version"); + log_step($url, 'has_incremental', check_has_incremental($dom)); + } + } + } +} + +sub download_version { + my ($config, $version) = @_; + my $tmpdir = File::Temp->newdir(); + my $destdir = "$releases_dir/$version"; + my $urldir = "$config->{download}{archive_url}/$version"; + print "Downloading version $version\n"; + foreach my $file (qw(sha256sums-unsigned-build.txt sha256sums-unsigned-build.txt.asc)) { + if (getstore("$urldir/$file", "$tmpdir/$file") != 200) { + exit_error "Error downloading $urldir/$file"; + } + } + if (system('gpg', '--no-default-keyring', '--keyring', + "$FindBin::Bin/$config->{download}{gpg_keyring}", '--verify', + "$tmpdir/sha256sums-unsigned-build.txt.asc", + "$tmpdir/sha256sums-unsigned-build.txt")) { + exit_error "Error checking gpg signature for version $version"; + } + mkdir $destdir; + move "$tmpdir/sha256sums-unsigned-build.txt.asc", "$destdir/sha256sums-unsigned-build.txt.asc"; + move "$tmpdir/sha256sums-unsigned-build.txt", "$destdir/sha256sums-unsigned-build.txt"; + my %sums = map { chomp; reverse split ' ', $_ } + read_file "$destdir/sha256sums-unsigned-build.txt"; + + my $martools = 'mar-tools-' . osname . '.zip'; + exit_error "Error downloading $urldir/$martools\n" + unless getstore("$urldir/$martools", "$tmpdir/$martools") == 200; + exit_error "Error downloading $urldir/$martools.asc\n" + unless getstore("$urldir/$martools.asc", "$tmpdir/$martools.asc") == 200; + if (system('gpg', '--no-default-keyring', '--keyring', + "$FindBin::Bin/$config->{download}{gpg_keyring}", '--verify', + "$tmpdir/$martools.asc", "$tmpdir/$martools")) { + exit_error "Error checking gpg signature for $version/$martools"; + } + exit_error "Wrong checksum for $version/$martools" + unless $sums{$martools} eq sha256_hex(read_file("$tmpdir/$martools")); + move "$tmpdir/$martools", "$destdir/$martools"; + move "$tmpdir/$martools.asc", "$destdir/$martools.asc"; + extract_martools($version); + + foreach my $file (sort grep { $_ =~ m/\.mar$/ } keys %sums) { + print "Downloading $file\n"; + exit_error "Error downloading $urldir/$file\n" + unless getstore("$urldir/$file", "$tmpdir/$file") == 200; + if ($sums{$file} ne sha256_hex(read_file("$tmpdir/$file"))) { + exit_error "Error unsigning $file" + if system('signmar', '-r', "$tmpdir/$file", "$tmpdir/$file.u"); + exit_error "Wrong checksum for $file" + unless $sums{$file} eq sha256_hex(read_file("$tmpdir/$file.u")); + move "$tmpdir/$file.u", "$tmpdir/$file"; + } + move "$tmpdir/$file", "$destdir/$file"; + } +} + +sub download_missing_versions { + my ($config, @channels) = @_; + foreach my $channel (@channels) { + exit_error "Unknown channel $channel" + unless $config->{channels}{$channel}; + my $cversion = $config->{channels}{$channel}; + next unless $config->{versions}{$cversion}{incremental_from}; + foreach my $version (@{$config->{versions}{$cversion}{incremental_from}}) { + next if -d "$releases_dir/$version"; + download_version($config, $version); + } + } +} + +sub check_update_responses { + my ($config) = @_; + exit_error "usage: $PROGRAM_NAME <base_url> [channels...]" unless @ARGV; + my ($base_url, @channels) = @ARGV; + foreach my $channel (@channels ? @channels : keys %{$config->{channels}}) { + check_update_responses_channel($config, $base_url, $channel); + } + if (!@check_errors) { + print "\n\nNo errors\n"; + return; + } + print "\n\nErrors list:\n"; + my $url = ''; + foreach my $error (@check_errors) { + if ($url ne $error->{url}) { + $url = $error->{url}; + print "$url\n"; + } + print " $error->{step}", + $error->{details} ? " - $error->{details}\n" : "\n"; + } +} + +my %actions = ( + update_responses => sub { + my ($config) = @_; + my @channels = @ARGV ? @ARGV : keys %{$config->{channels}}; + foreach my $channel (@channels) { + exit_error "Unknown channel $channel" + unless $config->{channels}{$channel}; + $htdocsfiles{$channel} = { '.' => 1, '..' => 1 }; + } + write_responses($config, @channels); + write_htaccess($config, @channels); + write_downloads_json($config, @channels); + clean_htdocs(@channels); + }, + gen_incrementals => sub { + my ($config) = @_; + foreach my $version (channel_to_version($config, @ARGV)) { + extract_martools($version); + get_version_files($config, $version); + create_incremental_mars_for_version($config, $version); + } + }, + download_missing_versions => sub { + my ($config) = @_; + my @channels = @ARGV ? @ARGV : keys %{$config->{channels}}; + download_missing_versions($config, @channels); + }, + check_update_responses_deployement => \&check_update_responses, + get_channel_version => sub { + my ($config) = @_; + exit_error "Wrong arguments" unless @ARGV == 1; + exit_error "Unknown channel" unless $config->{channels}{$ARGV[0]}; + print $config->{channels}{$ARGV[0]}, "\n"; + }, +); + +my $action = fileparse($PROGRAM_NAME); +exit_error "Unknown action $action" unless $actions{$action}; +$actions{$action}->($config);

1 0

[tor-browser-build/master] Include mar-tools zip in the output files
by boklm＠torproject.org 21 Aug '17

21 Aug '17

commit f3fed13feabaead01099d009f099120a8677c650 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Aug 21 18:10:03 2017 +0200 Include mar-tools zip in the output files --- projects/tor-browser/build | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/tor-browser/build b/projects/tor-browser/build index 77a02ea..31c2d6d 100644 --- a/projects/tor-browser/build +++ b/projects/tor-browser/build @@ -275,6 +275,8 @@ cd $distdir [% END %] rm -rf $distdir/${PKG_DIR} +cp $rootdir/[% c('input_files_by_name/firefox') %]/mar-tools-*.zip "$OUTDIR"/ + # If we did not create a multi-lingual package above, create a package for # each locale. [% IF ! c("var/multi_lingual") %]

1 0

[tor-browser-build/master] Bug 17381: create tar containing update responses xml files
by boklm＠torproject.org 21 Aug '17

21 Aug '17

commit 93bb526421830f185c431d9064043e54c38d6bab Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Aug 21 19:01:16 2017 +0200 Bug 17381: create tar containing update responses xml files Create a tar file in directory {alpha,release}/update-responses/, to make it a little easier to copy them to the host where they will be deployed. --- Makefile | 4 ++-- README.MAKEFILE | 3 ++- projects/release/config | 5 +++++ projects/release/create_update_responses_tar | 9 +++++++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1229cfa..ce63e54 100644 --- a/Makefile +++ b/Makefile @@ -97,11 +97,11 @@ incrementals-alpha: submodule-update update_responses-release: submodule-update $(rbm) build release --step update_responses_config --target release --target signed - tools/update-responses/update_responses release + $(rbm) build release --step create_update_responses_tar --target release --target signed update_responses-alpha: submodule-update $(rbm) build release --step update_responses_config --target alpha --target signed - tools/update-responses/update_responses alpha + $(rbm) build release --step create_update_responses_tar --target alpha --target signed dmg2mar-release: submodule-update $(rbm) build release --step update_responses_config --target release --target signed diff --git a/README.MAKEFILE b/README.MAKEFILE index 9e28864..cd4bf2f 100644 --- a/README.MAKEFILE +++ b/README.MAKEFILE @@ -91,5 +91,6 @@ signing the OSX dmg files. update_responses-{release,alpha} -------------------------------- Create update responses xml files for a signed build in the release or -alpha channel. +alpha channel. The files can be found in a tar in the directory +{release,alpha}/update-responses. diff --git a/projects/release/config b/projects/release/config index 0a4276a..4ecbc0a 100644 --- a/projects/release/config +++ b/projects/release/config @@ -122,6 +122,11 @@ steps: debug: 0 input_files: [] update_responses_config: '[% INCLUDE update_responses_config %]' + create_update_responses_tar: + build_log: '-' + debug: 0 + input_files: [] + create_update_responses_tar: '[% INCLUDE create_update_responses_tar %]' hash_incrementals: build_log: '-' debug: 0 diff --git a/projects/release/create_update_responses_tar b/projects/release/create_update_responses_tar new file mode 100644 index 0000000..ee1180f --- /dev/null +++ b/projects/release/create_update_responses_tar @@ -0,0 +1,9 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +[% shell_quote(c("basedir")) %]/tools/update-responses/update_responses +mkdir -p [% shell_quote(path(dest_dir)) %]/update-responses +[% SET channel = pc('firefox', 'var/torbrowser_update_channel') -%] +mv [% shell_quote(c("basedir")) %]/tools/update-responses/htdocs/[% channel %] [% channel %] +chmod 775 [% channel %] +chmod 664 [% channel %]/.htaccess [% channel %]/* +tar cf [% shell_quote(path(dest_dir)) %]/update-responses/update-responses-[% channel %]-[% c("version") %].tar [% channel %]

1 0

[tor-browser-build/master] Add a README.MAKEFILE file to describe makefile rules
by boklm＠torproject.org 15 Aug '17

15 Aug '17

commit f291e750afda75d68ba29bbcb998fba67e3f8012 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Aug 15 13:22:39 2017 +0200 Add a README.MAKEFILE file to describe makefile rules --- README | 6 +++++ README.MAKEFILE | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/README b/README index 751ccdd..de79d35 100644 --- a/README +++ b/README @@ -212,3 +212,9 @@ Hacking on the Tor Browser build The file README.HACKING tries to list the main things to know when making changes to the Tor Browser build. +Description of makefile rules +----------------------------- + +You can find a description of the makefile rules in the README.MAKEFILE +file. + diff --git a/README.MAKEFILE b/README.MAKEFILE new file mode 100644 index 0000000..ef33904 --- /dev/null +++ b/README.MAKEFILE @@ -0,0 +1,79 @@ +Description of makefile rules +============================= + + +release +------- +Build Tor Browser for the release channel, for all supported platforms. +The resulting build can be found in directory release/$version. + +release-{linux-x86_64,linux-i686,windows-i686,osx-x86_64} +--------------------------------------------------------- +Same as release, for the selected platform only. + +alpha +----- +Build Tor Browser for the alpha channel, for all supported platforms. +The resulting build can be found in directoryp alpha/$version. + +alpha-{linux-x86_64,linux-i686,windows-i686,osx-x86_64} +------------------------------------------------------- +Same as alpha, for the selected platform only. + +nightly +------- +Build Tor Browser for the nightly channel, for all supported platforms. +Many of the components will be based on the master branch. You might +want to run "make fetch" before, to fetch the latest commits from all +components. + +The resulting build can be found in directory nightly/$date + +nightly-{linux-x86_64,linux-i686,windows-i686,osx-x86_64} +--------------------------------------------------------- +Same as nightly, for the selected platform only. + +alpha_nightly +------------- +Build Tor Browser for the alpha channel, for all supported platforms. +This is the same as 'make alpha', but the output directory is different. +The resulting build can be found in directory alpha_nightly/$date + +alpha_nightly-{linux-x86_64,linux-i686,windows-i686,osx-x86_64} +--------------------------------------------------------------- +The same as alpha_nightly, for the selected platform only. + +testbuild +--------- +Build a Tor Browser test build. To make the build faster, only the en-US +locale is created, without mar files. By default it is based on the +alpha channel, but you can change that by editing rbm.conf.local. +The resulting build can be found in directory testbuild. + +fetch +----- +Fetch new commits from all components. This is useful when you want to +build the nightly channel. + +clean +----- +Clean files created during previous builds that are no longer used in +current builds. Before using this command, you first need to configure +in rbm.local.conf the channel(s) that you are building. + +In other software, the clean rule is used to remove all build files, +but this is not the case here, where only obsolete build files are +removed. It is usually not necessary to remove all build files before +starting a new build as the need for rebuilding components is detected +automatically. If you want to remove all build files, you can remove +the out/ directory. + +clean-dry-run +------------- +Print the files that would be removed when running 'make clean'. + +signtag-{release,alpha} +----------------------- +Create a git signed tag for the selected channel, using the version and +build number defined as var/torbrowser_version and var/torbrowser_build. +

1 0

[tor-browser-build/master] Update rbm for the build_log_append option
by boklm＠torproject.org 11 Aug '17

11 Aug '17

commit 89656909bd3d54722aa6ee1627b93858c5cdd93c Author: Nicolas Vigier <boklm(a)torproject.org> Date: Fri Aug 11 13:02:32 2017 +0200 Update rbm for the build_log_append option --- rbm | 2 +- rbm.local.conf.example | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/rbm b/rbm index 6902007..0f5857b 160000 --- a/rbm +++ b/rbm @@ -1 +1 @@ -Subproject commit 690200770631f4adcfbbe7988379eb39698e11ab +Subproject commit 0f5857bb0486187fcd35c553dec9ae6d3657a98c diff --git a/rbm.local.conf.example b/rbm.local.conf.example index e30d8a8..62d13b2 100644 --- a/rbm.local.conf.example +++ b/rbm.local.conf.example @@ -21,6 +21,11 @@ ### stdout and stderr. #build_log: '-' +### By default, the logs from previous builds are kept in the log files. +### If you set build_log_append to 0, then previous logs are cleaned +### when starting a new build. +#build_log_append: 0 + buildconf: buildconf: 1

1 0

[rbm/master] Add the build_log_append option
by boklm＠torproject.org 11 Aug '17

11 Aug '17

commit 0f5857bb0486187fcd35c553dec9ae6d3657a98c Author: Nicolas Vigier <boklm(a)torproject.org> Date: Fri Aug 11 12:56:25 2017 +0200 Add the build_log_append option --- NEWS | 2 ++ doc/rbm_config.asc | 5 +++++ lib/RBM.pm | 3 ++- lib/RBM/DefaultConfig.pm | 1 + 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index c43dc2d..d517840 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,5 @@ +- Add the build_log_append option + - Add the rbm_tmp_dir option - Add the input_files_paths option diff --git a/doc/rbm_config.asc b/doc/rbm_config.asc index dc3b5e4..1935cd3 100644 --- a/doc/rbm_config.asc +++ b/doc/rbm_config.asc @@ -149,6 +149,11 @@ build_log:: The file where the build logs will be written. If the value is `-` (the default), the logs will be output on stdout and stderr. +build_log_append:: + If you set to +build_log_append+ to 0, the log file (defined in + +build_log+) will be cleaned when starting a new build. The + default is 1. + fetch:: The value should be 0 or 1, depending on whether the commits from the remote git or hg repository should be fetched diff --git a/lib/RBM.pm b/lib/RBM.pm index c1240cd..df29a01 100644 --- a/lib/RBM.pm +++ b/lib/RBM.pm @@ -1020,10 +1020,11 @@ sub build_run { } my $build_log = project_config($project, "build_log", $options); if ($build_log ne '-') { + my $append = project_config($project, "build_log_append", $options); $build_log = path($build_log); make_path(dirname($build_log)); my $now = localtime; - write_file($build_log, {append => 1}, "Starting build: $now\n"); + write_file($build_log, {append => $append}, "Starting build: $now\n"); print "Build log: $build_log\n"; } chdir $srcdir; diff --git a/lib/RBM/DefaultConfig.pm b/lib/RBM/DefaultConfig.pm index 63021e9..b1f97cb 100644 --- a/lib/RBM/DefaultConfig.pm +++ b/lib/RBM/DefaultConfig.pm @@ -126,6 +126,7 @@ our %default_config = ( rpmspec => '[% SET tmpl = project _ ".spec"; INCLUDE $tmpl -%]', build => '[% INCLUDE build -%]', build_log => '-', + build_log_append => '1', notmpl => [ qw(projects_dir) ], describe => \&git_describe, abbrev_lenght => '12',

1 0

[tor-browser-build/master] Display container's path when opening debug shell
by boklm＠torproject.org 11 Aug '17

11 Aug '17

commit b9512ca6c0fddee7aa61e6c1672df59ddaa0aacc Author: Nicolas Vigier <boklm(a)torproject.org> Date: Fri Aug 11 12:22:54 2017 +0200 Display container's path when opening debug shell --- README.HACKING | 5 ++--- rbm.conf | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.HACKING b/README.HACKING index 8dde2f2..ea5d500 100644 --- a/README.HACKING +++ b/README.HACKING @@ -256,7 +256,6 @@ in the build containers, but vi can be used instead. When you press ctrl+d (twice if you opened a bash shell) the debug shell is closed and the container containing the failed build is removed. -If you want to make a backup of the container's rootfs with the failed -build to look at it later, you should be able to find it in -/tmp/$UID/rbm-containers. +The path to the container should be printed on the screen in case you +want to backup its rootfs to be able to look at it later. diff --git a/rbm.conf b/rbm.conf index 523a005..93c5e5b 100644 --- a/rbm.conf +++ b/rbm.conf @@ -314,6 +314,9 @@ runc: remote_exec: | #!/bin/sh set -e + [% IF c("interactive") -%] + echo Container directory: [% shell_quote(c("var/container/dir")) %] + [% END -%] mkdir -p '[% c("var/container/dir", { error_if_undef => 1 }) %]'/rootfs/rbm echo '#!/bin/sh' > '[% c("var/container/dir") %]'/rootfs/rbm/cmd echo [% shell_quote(c('exec_cmd')) %] >> '[% c("var/container/dir") %]'/rootfs/rbm/cmd

1 0

[tor-browser-build/master] Avoid using directory with predictable name in /tmp
by boklm＠torproject.org 11 Aug '17

11 Aug '17

commit 36296855fd4e553fb871c2f03a2fc62597d562a5 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Fri Aug 11 12:28:57 2017 +0200 Avoid using directory with predictable name in /tmp Update rbm with support for the rbm_tmp_dir option, and use it in the containers directory. --- rbm | 2 +- rbm.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rbm b/rbm index f494d16..6902007 160000 --- a/rbm +++ b/rbm @@ -1 +1 @@ -Subproject commit f494d16a1b86700661a029f5eca00c6216bf08d1 +Subproject commit 690200770631f4adcfbbe7988379eb39698e11ab diff --git a/rbm.conf b/rbm.conf index 93c5e5b..5e1e66d 100644 --- a/rbm.conf +++ b/rbm.conf @@ -36,7 +36,7 @@ var: build: [% c("build", { filename => 'f', output_dir => '/out' }) %] container: - dir: '[% c("tmp_dir") %]/[% c("uid") %]/rbm-containers/[% sha256(c("build_id")) %]' + dir: '[% c("rbm_tmp_dir") %]/rbm-containers/[% sha256(c("build_id")) %]' user: rbm input_files_list: | [% FOREACH file IN c("input_files_by_name").keys.sort -%]

1 0