[tbb-commits] [tor-browser-bundle/master] Bug 19410: Add script to convert dmg files to mar files

gk at torproject.org gk at torproject.org
Fri Sep 9 13:07:47 UTC 2016


commit 23bea36eb8d8d27ca00b2394757be08530cb7f19
Author: Nicolas Vigier <boklm at torproject.org>
Date:   Fri Sep 9 13:46:49 2016 +0200

    Bug 19410: Add script to convert dmg files to mar files
    
    We are also adding the dmg2mars and dmg2mars-alpha makefile rules to
    run the script and regenerate the incremental mars.
---
 gitian/Makefile                            |  10 +++
 tools/dmg2mar                              | 136 +++++++++++++++++++++++++++++
 tools/update-responses/get_channel_version |   1 +
 tools/update-responses/update_responses    |   6 ++
 4 files changed, 153 insertions(+)

diff --git a/gitian/Makefile b/gitian/Makefile
index 50d4328..59efd0e 100644
--- a/gitian/Makefile
+++ b/gitian/Makefile
@@ -45,6 +45,16 @@ signmars-alpha:
 signmars-nightly:
 	./signmars.sh versions.nightly
 
+dmg2mars:
+	cd $(shell ../tools/update-responses/get_channel_version release) && ../../tools/dmg2mar
+	$(TORSOCKS) ../tools/update-responses/download_missing_versions release
+	../tools/update-responses/gen_incrementals release
+
+dmg2mars-alpha:
+	cd $(shell ../tools/update-responses/get_channel_version alpha) && ../../tools/dmg2mar
+	$(TORSOCKS) ../tools/update-responses/download_missing_versions alpha
+	../tools/update-responses/gen_incrementals alpha
+
 update_responses:
 	../tools/update-responses/update_responses release
 
diff --git a/tools/dmg2mar b/tools/dmg2mar
new file mode 100755
index 0000000..47a543a
--- /dev/null
+++ b/tools/dmg2mar
@@ -0,0 +1,136 @@
+#!/usr/bin/perl -w
+#
+# This script converts all dmg files from the current directory and
+# listed in the sha256sums-unsigned-build.txt file to full update
+# mar files. After code signing the dmg files, this script can be used
+# to update the mar files.
+#
+# A recent version of p7zip is required to extract the dmg files, such
+# as 15.14. The version in Debian Jessie (9.20) is not recent enough.
+# It is possible to install the p7zip-full package from Debian testing,
+# or build p7zip from sources:
+#  $ p7zipdir=/some_directory/p7zip
+#  $ mkdir $p7zipdir
+#  $ cd $p7zipdir
+#  $ wget http://snapshot.debian.org/archive/debian/20160417T044336Z/pool/main/p/p7zip/p7zip_15.14.1%2Bdfsg.orig.tar.xz
+#  $ echo 'e9e696e2fa77b00445a4d85fa07506debeae01943fdc1bee1472152d7d1386af p7zip_15.14.1+dfsg.orig.tar.xz' | sha256sum -c
+#  $ wget http://snapshot.debian.org/archive/debian/20160515T161830Z/pool/main/p/p7zip/p7zip_15.14.1%2Bdfsg-2.debian.tar.xz
+#  $ echo 'f4db6803535fc30b6ae9db5aabfd9f57a851c6773d72073847ec5e3731b7af37  p7zip_15.14.1+dfsg-2.debian.tar.xz' | sha256sum -c
+#  $ tar xvf p7zip_15.14.1+dfsg-2.debian.tar.xz
+#  $ tar xvf p7zip_15.14.1+dfsg.orig.tar.xz
+#  $ cd p7zip_15.14.1/
+#  $ for patch in $(cat ../debian/patches/series ); do patch -p1 < ../debian/patches/$patch; done
+#  $ make 7z
+#  $ mkdir $p7zipdir/bin
+#  $ echo '#!/bin/sh' > $p7zipdir/bin/7z
+#  $ echo "export LD_LIBRARY_PATH=$PWD/bin" >> $p7zipdir/bin/7z
+#  $ echo "exec $PWD/bin/7z "'"$@"' >> $p7zipdir/bin/7z
+#  $ chmod +x $p7zipdir/bin/7z
+#  $ export "PATH=$p7zipdir/bin:$PATH"
+
+use strict;
+use IO::CaptureOutput qw(capture_exec);
+use File::Slurp;
+use Parallel::ForkManager;
+use Cwd;
+
+sub exit_error {
+    print STDERR "Error: ", $_[0], "\n";
+    chdir '/';
+    exit (exists $_[1] ? $_[1] : 1);
+}
+
+sub osname {
+    my ($osname) = capture_exec('uname', '-s');
+    my ($arch) = capture_exec('uname', '-m');
+    chomp($osname, $arch);
+    if ($osname eq 'Linux' && $arch eq 'x86_64') {
+        return 'linux64';
+    }
+    if ($osname eq 'Linux' && $arch =~ m/^i.86$/) {
+        return 'linux32';
+    }
+    exit_error 'Unknown OS';
+}
+
+my $martools_tmpdir;
+sub extract_martools {
+    my $osname = osname;
+    my $marzip = getcwd . "/mar-tools-$osname.zip";
+    $martools_tmpdir = File::Temp->newdir();
+    my $old_cwd = getcwd;
+    chdir $martools_tmpdir;
+    my (undef, undef, $success) = capture_exec('unzip', $marzip);
+    chdir $old_cwd;
+    exit_error "Error extracting $marzip" unless $success;
+    $ENV{PATH} = "$martools_tmpdir/mar-tools:$ENV{PATH}";
+    if ($ENV{LD_LIBRARY_PATH}) {
+        $ENV{LD_LIBRARY_PATH} .= ":$martools_tmpdir/mar-tools";
+    } else {
+        $ENV{LD_LIBRARY_PATH} = "$martools_tmpdir/mar-tools";
+    }
+    $ENV{MAR} = "$martools_tmpdir/mar-tools/mar";
+    $ENV{MSBDIFF} = "$martools_tmpdir/mar-tools/mbsdiff";
+}
+
+sub get_nbprocs {
+    return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS};
+    if (-f '/proc/cpuinfo') {
+        return scalar grep { m/^processor\s+:\s/ } read_file '/proc/cpuinfo';
+    }
+    return 4;
+}
+
+sub get_dmg_files_from_sha256sums {
+    exit_error "Missing sha256sums-unsigned-build.txt file"
+        unless -f 'sha256sums-unsigned-build.txt';
+    my @files;
+    foreach my $line (read_file('sha256sums-unsigned-build.txt')) {
+        my (undef, $filename) = split '  ', $line;
+        chomp $filename;
+        next unless $filename =~ m/^TorBrowser-(.+)-osx64_(.+)\.dmg$/;
+        push @files, { filename => $filename, version => $1, lang => $2 };
+    }
+    return @files;
+}
+
+sub convert_files {
+    my $pm = Parallel::ForkManager->new(get_nbprocs);
+    $pm->run_on_finish(sub { print "Finished $_[2]\n" });
+    foreach my $file (get_dmg_files_from_sha256sums) {
+        my $output = "tor-browser-osx64-$file->{version}_$file->{lang}.mar";
+        my $step_name = "$file->{filename} -> $output";
+        print "Starting $step_name\n";
+        $pm->start($step_name) and next;
+        my $tmpdir = File::Temp->newdir();
+        my (undef, $err, $success) = capture_exec('7z', 'x', "-o$tmpdir",
+                                                        $file->{filename});
+        exit_error "Error extracting $file->{filename}: $err" unless $success;
+        unlink $output;
+        (undef, $err, $success) =  capture_exec('make_full_update.sh', '-q',
+                                        $output, "$tmpdir/TorBrowser.app");
+        exit_error "Error updating $output: $err" unless $success;
+        $pm->finish;
+    }
+    $pm->wait_all_children;
+}
+
+sub remove_incremental_mars {
+    exit_error "Missing sha256sums-unsigned-build.incrementals.txt file"
+        unless -f 'sha256sums-unsigned-build.incrementals.txt';
+    foreach my $line (read_file('sha256sums-unsigned-build.incrementals.txt')) {
+        my (undef, $filename) = split '  ', $line;
+        chomp $filename;
+        next unless $filename =~ m/^tor-browser-osx64.+\.incremental\.mar$/;
+        next unless -f $filename;
+        print "Removing $filename\n";
+        unlink $filename;
+    }
+}
+
+# Set LC_ALL=C to avoid reproducibility issues when creating mar files
+$ENV{LC_ALL} = 'C';
+
+extract_martools;
+convert_files;
+remove_incremental_mars;
diff --git a/tools/update-responses/get_channel_version b/tools/update-responses/get_channel_version
new file mode 120000
index 0000000..3766925
--- /dev/null
+++ b/tools/update-responses/get_channel_version
@@ -0,0 +1 @@
+update_responses
\ No newline at end of file
diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses
index 498132e..b83ac29 100755
--- a/tools/update-responses/update_responses
+++ b/tools/update-responses/update_responses
@@ -596,6 +596,12 @@ my %actions = (
         download_missing_versions($config, @channels);
     },
     check_update_responses_deployement => \&check_update_responses,
+    get_channel_version => sub {
+        my ($config) = @_;
+        exit_error "Wrong arguments" unless @ARGV == 1;
+        exit_error "Unknown channel" unless $config->{channels}{$ARGV[0]};
+        print $config->{channels}{$ARGV[0]}, "\n";
+    },
 );
 
 my $action = fileparse($PROGRAM_NAME);



More information about the tbb-commits mailing list