commit a510614a08cedcb99a03329acb8fa0bd165314e6 Author: Nicolas Vigier boklm@torproject.org Date: Mon Aug 21 14:41:34 2017 +0200

Bug 17381: add dmg2mar

Add the dmg2mar makefile rules. --- Makefile | 12 ++++ README.MAKEFILE | 6 ++ projects/release/config | 5 ++ projects/release/dmg2mar | 4 ++ tools/dmg2mar | 168 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 195 insertions(+)

diff --git a/Makefile b/Makefile index d49d670..1229cfa 100644 --- a/Makefile +++ b/Makefile @@ -103,6 +103,18 @@ update_responses-alpha: submodule-update $(rbm) build release --step update_responses_config --target alpha --target signed tools/update-responses/update_responses alpha

+dmg2mar-release: submodule-update + $(rbm) build release --step update_responses_config --target release --target signed + $(rbm) build release --step dmg2mar --target release --target signed + tools/update-responses/download_missing_versions release + CHECK_CODESIGNATURE_EXISTS=1 MAR_SKIP_EXISTING=1 tools/update-responses/gen_incrementals release + +dmg2mar-alpha: submodule-update + $(rbm) build release --step update_responses_config --target alpha --target signed + $(rbm) build release --step dmg2mar --target alpha --target signed + tools/update-responses/download_missing_versions alpha + CHECK_CODESIGNATURE_EXISTS=1 MAR_SKIP_EXISTING=1 tools/update-responses/gen_incrementals alpha + submodule-update: git submodule update --init

diff --git a/README.MAKEFILE b/README.MAKEFILE index 167f5d5..9e28864 100644 --- a/README.MAKEFILE +++ b/README.MAKEFILE @@ -82,6 +82,12 @@ incrementals-{release,alpha} Create incremental mar files for an unsigned build in the release or alpha channel.

+dmg2mar-{release,alpha) +----------------------- +Generate updated mar files for the OSX bundles, from the dmg files, then +regenerate the OSX incremental mar files. You should run this after +signing the OSX dmg files. + update_responses-{release,alpha} -------------------------------- Create update responses xml files for a signed build in the release or diff --git a/projects/release/config b/projects/release/config index 3c771bf..0a4276a 100644 --- a/projects/release/config +++ b/projects/release/config @@ -127,3 +127,8 @@ steps: debug: 0 input_files: [] hash_incrementals: '[% INCLUDE hash_incrementals %]' + dmg2mar: + build_log: '-' + debug: 0 + input_files: [] + dmg2mar: '[% INCLUDE dmg2mar %]' diff --git a/projects/release/dmg2mar b/projects/release/dmg2mar new file mode 100644 index 0000000..d6175f5 --- /dev/null +++ b/projects/release/dmg2mar @@ -0,0 +1,4 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +cd [% shell_quote(path(dest_dir)) %]/[% c("var/signed_status") %]/[% c("version") %] +[% shell_quote(c("basedir")) %]/tools/dmg2mar diff --git a/tools/dmg2mar b/tools/dmg2mar new file mode 100755 index 0000000..51d6acb --- /dev/null +++ b/tools/dmg2mar @@ -0,0 +1,168 @@ +#!/usr/bin/perl -w +# +# This script converts all dmg files from the current directory and +# listed in the sha256sums-unsigned-build.txt file to full update +# mar files. After code signing the dmg files, this script can be used +# to update the mar files. +# +# A recent version of p7zip is required to extract the dmg files, such +# as 15.14. The version in Debian Jessie (9.20) is not recent enough. +# It is possible to install the p7zip-full package from Debian testing, +# or build p7zip from sources: +# $ p7zipdir=/some_directory/p7zip +# $ mkdir $p7zipdir +# $ cd $p7zipdir +# $ wget http://snapshot.debian.org/archive/debian/20160417T044336Z/pool/main/p/p7zip... +# $ echo 'e9e696e2fa77b00445a4d85fa07506debeae01943fdc1bee1472152d7d1386af p7zip_15.14.1+dfsg.orig.tar.xz' | sha256sum -c +# $ wget http://snapshot.debian.org/archive/debian/20160515T161830Z/pool/main/p/p7zip... +# $ echo 'f4db6803535fc30b6ae9db5aabfd9f57a851c6773d72073847ec5e3731b7af37 p7zip_15.14.1+dfsg-2.debian.tar.xz' | sha256sum -c +# $ tar xvf p7zip_15.14.1+dfsg-2.debian.tar.xz +# $ tar xvf p7zip_15.14.1+dfsg.orig.tar.xz +# $ cd p7zip_15.14.1/ +# $ for patch in $(cat ../debian/patches/series ); do patch -p1 < ../debian/patches/$patch; done +# $ make 7z +# $ mkdir $p7zipdir/bin +# $ echo '#!/bin/sh' > $p7zipdir/bin/7z +# $ echo "export LD_LIBRARY_PATH=$PWD/bin" >> $p7zipdir/bin/7z +# $ echo "exec $PWD/bin/7z "'"$@"' >> $p7zipdir/bin/7z +# $ chmod +x $p7zipdir/bin/7z +# $ export "PATH=$p7zipdir/bin:$PATH" + +use strict; +use IO::CaptureOutput qw(capture_exec); +use File::Slurp; +use File::Find; +use Parallel::ForkManager; +use Cwd; + +# If the application is not TorBrowser (for instance, TorMessenger) +# set the application name in the TOR_APPNAME_BUNDLE_OSX and in +# the TOR_APPNAME_MARFILE environment variables +my $appname = $ENV{TOR_APPNAME_BUNDLE_OSX} // 'TorBrowser'; +my $appname_mar = $ENV{TOR_APPNAME_MARFILE} // 'tor-browser'; + +sub exit_error { + print STDERR "Error: ", $_[0], "\n"; + chdir '/'; + exit (exists $_[1] ? $_[1] : 1); +} + +sub osname { + my ($osname) = capture_exec('uname', '-s'); + my ($arch) = capture_exec('uname', '-m'); + chomp($osname, $arch); + if ($osname eq 'Linux' && $arch eq 'x86_64') { + return 'linux64'; + } + if ($osname eq 'Linux' && $arch =~ m/^i.86$/) { + return 'linux32'; + } + exit_error 'Unknown OS'; +} + +my $martools_tmpdir; +sub extract_martools { + my $osname = osname; + my $marzip = getcwd . "/mar-tools-$osname.zip"; + $martools_tmpdir = File::Temp->newdir(); + my $old_cwd = getcwd; + chdir $martools_tmpdir; + my (undef, undef, $success) = capture_exec('unzip', $marzip); + chdir $old_cwd; + exit_error "Error extracting $marzip" unless $success; + $ENV{PATH} = "$martools_tmpdir/mar-tools:$ENV{PATH}"; + if ($ENV{LD_LIBRARY_PATH}) { + $ENV{LD_LIBRARY_PATH} .= ":$martools_tmpdir/mar-tools"; + } else { + $ENV{LD_LIBRARY_PATH} = "$martools_tmpdir/mar-tools"; + } + $ENV{MAR} = "$martools_tmpdir/mar-tools/mar"; + $ENV{MSBDIFF} = "$martools_tmpdir/mar-tools/mbsdiff"; +} + +sub get_nbprocs { + return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS}; + if (-f '/proc/cpuinfo') { + return scalar grep { m/^processor\s+:\s/ } read_file '/proc/cpuinfo'; + } + return 4; +} + +sub get_dmg_files_from_sha256sums { + exit_error "Missing sha256sums-unsigned-build.txt file" + unless -f 'sha256sums-unsigned-build.txt'; + my @files; + foreach my $line (read_file('sha256sums-unsigned-build.txt')) { + my (undef, $filename) = split ' ', $line; + chomp $filename; + next unless $filename =~ m/^$appname-(.+)-osx64_(.+).dmg$/; + push @files, { filename => $filename, version => $1, lang => $2 }; + } + return @files; +} + +sub convert_files { + my $pm = Parallel::ForkManager->new(get_nbprocs); + $pm->run_on_finish(sub { print "Finished $_[2]\n" }); + foreach my $file (get_dmg_files_from_sha256sums) { + # The 'ja' locale is a special case: it is called 'ja-JP-mac' + # internally on OSX, but the dmg file still uses 'ja' to avoid + # confusing users. + my $mar_lang = $file->{lang} eq 'ja' ? 'ja-JP-mac' : $file->{lang}; + my $output = "$appname_mar-osx64-$file->{version}_$mar_lang.mar"; + my $step_name = "$file->{filename} -> $output"; + print "Starting $step_name\n"; + $pm->start($step_name) and next; + my $tmpdir = File::Temp->newdir(); + my (undef, $err, $success) = capture_exec('7z', 'x', "-o$tmpdir", + $file->{filename}); + exit_error "Error extracting $file->{filename}: $err" unless $success; + + # 7z does not currently extract file permissions from the dmg files + # so we also extract the old mar file to copy the permissions + # https://trac.torproject.org/projects/tor/ticket/20210 + my $tmpdir_oldmar = File::Temp->newdir(); + my $oldmar = getcwd . '/' . $output; + exit_error "Error extracting $output" + unless system('mar', '-C', $tmpdir_oldmar, '-x', $oldmar) == 0; + my $wanted = sub { + my $file = $File::Find::name; + $file =~ s{^$tmpdir/$appname.app/}{}; + if (-f "$tmpdir_oldmar/$file") { + my (undef, undef, $mode) = stat("$tmpdir_oldmar/$file"); + chmod $mode, $File::Find::name; + return; + } + chmod 0644, $File::Find::name if -f $File::Find::name; + chmod 0755, $File::Find::name if -d $File::Find::name; + }; + find($wanted, "$tmpdir/$appname.app"); + + unlink $output; + (undef, $err, $success) = capture_exec('make_full_update.sh', '-q', + $output, "$tmpdir/$appname.app"); + exit_error "Error updating $output: $err" unless $success; + $pm->finish; + } + $pm->wait_all_children; +} + +sub remove_incremental_mars { + exit_error "Missing sha256sums-unsigned-build.incrementals.txt file" + unless -f 'sha256sums-unsigned-build.incrementals.txt'; + foreach my $line (read_file('sha256sums-unsigned-build.incrementals.txt')) { + my (undef, $filename) = split ' ', $line; + chomp $filename; + next unless $filename =~ m/^$appname_mar-osx64.+.incremental.mar$/; + next unless -f $filename; + print "Removing $filename\n"; + unlink $filename; + } +} + +# Set LC_ALL=C to avoid reproducibility issues when creating mar files +$ENV{LC_ALL} = 'C'; + +extract_martools; +convert_files; +remove_incremental_mars;