commit 23bea36eb8d8d27ca00b2394757be08530cb7f19 Author: Nicolas Vigier <boklm@torproject.org> Date: Fri Sep 9 13:46:49 2016 +0200 Bug 19410: Add script to convert dmg files to mar files We are also adding the dmg2mars and dmg2mars-alpha makefile rules to run the script and regenerate the incremental mars. --- gitian/Makefile | 10 +++ tools/dmg2mar | 136 +++++++++++++++++++++++++++++ tools/update-responses/get_channel_version | 1 + tools/update-responses/update_responses | 6 ++ 4 files changed, 153 insertions(+) diff --git a/gitian/Makefile b/gitian/Makefile index 50d4328..59efd0e 100644 --- a/gitian/Makefile +++ b/gitian/Makefile @@ -45,6 +45,16 @@ signmars-alpha: signmars-nightly: ./signmars.sh versions.nightly +dmg2mars: + cd $(shell ../tools/update-responses/get_channel_version release) && ../../tools/dmg2mar + $(TORSOCKS) ../tools/update-responses/download_missing_versions release + ../tools/update-responses/gen_incrementals release + +dmg2mars-alpha: + cd $(shell ../tools/update-responses/get_channel_version alpha) && ../../tools/dmg2mar + $(TORSOCKS) ../tools/update-responses/download_missing_versions alpha + ../tools/update-responses/gen_incrementals alpha + update_responses: ../tools/update-responses/update_responses release diff --git a/tools/dmg2mar b/tools/dmg2mar new file mode 100755 index 0000000..47a543a --- /dev/null +++ b/tools/dmg2mar @@ -0,0 +1,136 @@ +#!/usr/bin/perl -w +# +# This script converts all dmg files from the current directory and +# listed in the sha256sums-unsigned-build.txt file to full update +# mar files. After code signing the dmg files, this script can be used +# to update the mar files. +# +# A recent version of p7zip is required to extract the dmg files, such +# as 15.14. The version in Debian Jessie (9.20) is not recent enough. +# It is possible to install the p7zip-full package from Debian testing, +# or build p7zip from sources: +# $ p7zipdir=/some_directory/p7zip +# $ mkdir $p7zipdir +# $ cd $p7zipdir +# $ wget http://snapshot.debian.org/archive/debian/20160417T044336Z/pool/main/p/p7zip... +# $ echo 'e9e696e2fa77b00445a4d85fa07506debeae01943fdc1bee1472152d7d1386af p7zip_15.14.1+dfsg.orig.tar.xz' | sha256sum -c +# $ wget http://snapshot.debian.org/archive/debian/20160515T161830Z/pool/main/p/p7zip... +# $ echo 'f4db6803535fc30b6ae9db5aabfd9f57a851c6773d72073847ec5e3731b7af37 p7zip_15.14.1+dfsg-2.debian.tar.xz' | sha256sum -c +# $ tar xvf p7zip_15.14.1+dfsg-2.debian.tar.xz +# $ tar xvf p7zip_15.14.1+dfsg.orig.tar.xz +# $ cd p7zip_15.14.1/ +# $ for patch in $(cat ../debian/patches/series ); do patch -p1 < ../debian/patches/$patch; done +# $ make 7z +# $ mkdir $p7zipdir/bin +# $ echo '#!/bin/sh' > $p7zipdir/bin/7z +# $ echo "export LD_LIBRARY_PATH=$PWD/bin" >> $p7zipdir/bin/7z +# $ echo "exec $PWD/bin/7z "'"$@"' >> $p7zipdir/bin/7z +# $ chmod +x $p7zipdir/bin/7z +# $ export "PATH=$p7zipdir/bin:$PATH" + +use strict; +use IO::CaptureOutput qw(capture_exec); +use File::Slurp; +use Parallel::ForkManager; +use Cwd; + +sub exit_error { + print STDERR "Error: ", $_[0], "\n"; + chdir '/'; + exit (exists $_[1] ? $_[1] : 1); +} + +sub osname { + my ($osname) = capture_exec('uname', '-s'); + my ($arch) = capture_exec('uname', '-m'); + chomp($osname, $arch); + if ($osname eq 'Linux' && $arch eq 'x86_64') { + return 'linux64'; + } + if ($osname eq 'Linux' && $arch =~ m/^i.86$/) { + return 'linux32'; + } + exit_error 'Unknown OS'; +} + +my $martools_tmpdir; +sub extract_martools { + my $osname = osname; + my $marzip = getcwd . "/mar-tools-$osname.zip"; + $martools_tmpdir = File::Temp->newdir(); + my $old_cwd = getcwd; + chdir $martools_tmpdir; + my (undef, undef, $success) = capture_exec('unzip', $marzip); + chdir $old_cwd; + exit_error "Error extracting $marzip" unless $success; + $ENV{PATH} = "$martools_tmpdir/mar-tools:$ENV{PATH}"; + if ($ENV{LD_LIBRARY_PATH}) { + $ENV{LD_LIBRARY_PATH} .= ":$martools_tmpdir/mar-tools"; + } else { + $ENV{LD_LIBRARY_PATH} = "$martools_tmpdir/mar-tools"; + } + $ENV{MAR} = "$martools_tmpdir/mar-tools/mar"; + $ENV{MSBDIFF} = "$martools_tmpdir/mar-tools/mbsdiff"; +} + +sub get_nbprocs { + return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS}; + if (-f '/proc/cpuinfo') { + return scalar grep { m/^processor\s+:\s/ } read_file '/proc/cpuinfo'; + } + return 4; +} + +sub get_dmg_files_from_sha256sums { + exit_error "Missing sha256sums-unsigned-build.txt file" + unless -f 'sha256sums-unsigned-build.txt'; + my @files; + foreach my $line (read_file('sha256sums-unsigned-build.txt')) { + my (undef, $filename) = split ' ', $line; + chomp $filename; + next unless $filename =~ m/^TorBrowser-(.+)-osx64_(.+)\.dmg$/; + push @files, { filename => $filename, version => $1, lang => $2 }; + } + return @files; +} + +sub convert_files { + my $pm = Parallel::ForkManager->new(get_nbprocs); + $pm->run_on_finish(sub { print "Finished $_[2]\n" }); + foreach my $file (get_dmg_files_from_sha256sums) { + my $output = "tor-browser-osx64-$file->{version}_$file->{lang}.mar"; + my $step_name = "$file->{filename} -> $output"; + print "Starting $step_name\n"; + $pm->start($step_name) and next; + my $tmpdir = File::Temp->newdir(); + my (undef, $err, $success) = capture_exec('7z', 'x', "-o$tmpdir", + $file->{filename}); + exit_error "Error extracting $file->{filename}: $err" unless $success; + unlink $output; + (undef, $err, $success) = capture_exec('make_full_update.sh', '-q', + $output, "$tmpdir/TorBrowser.app"); + exit_error "Error updating $output: $err" unless $success; + $pm->finish; + } + $pm->wait_all_children; +} + +sub remove_incremental_mars { + exit_error "Missing sha256sums-unsigned-build.incrementals.txt file" + unless -f 'sha256sums-unsigned-build.incrementals.txt'; + foreach my $line (read_file('sha256sums-unsigned-build.incrementals.txt')) { + my (undef, $filename) = split ' ', $line; + chomp $filename; + next unless $filename =~ m/^tor-browser-osx64.+\.incremental\.mar$/; + next unless -f $filename; + print "Removing $filename\n"; + unlink $filename; + } +} + +# Set LC_ALL=C to avoid reproducibility issues when creating mar files +$ENV{LC_ALL} = 'C'; + +extract_martools; +convert_files; +remove_incremental_mars; diff --git a/tools/update-responses/get_channel_version b/tools/update-responses/get_channel_version new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/get_channel_version @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index 498132e..b83ac29 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -596,6 +596,12 @@ my %actions = ( download_missing_versions($config, @channels); }, check_update_responses_deployement => \&check_update_responses, + get_channel_version => sub { + my ($config) = @_; + exit_error "Wrong arguments" unless @ARGV == 1; + exit_error "Unknown channel" unless $config->{channels}{$ARGV[0]}; + print $config->{channels}{$ARGV[0]}, "\n"; + }, ); my $action = fileparse($PROGRAM_NAME);