[tbb-commits] [tor-browser-bundle/master] update_responses: generate incremental MAR files

mikeperry at torproject.org mikeperry at torproject.org
Fri Oct 24 23:56:04 UTC 2014


commit 389f843dd3bb19f703a4448528e3d5b92942bf04
Author: Nicolas Vigier <boklm at mars-attacks.org>
Date:   Thu Oct 2 15:34:06 2014 +0200

    update_responses: generate incremental MAR files
    
    Generate incremental MAR file when they are not already there.
    
    The list of versions from which to generate incremental MAR is defined in
    config.yml as incremental_from for each version.
    
    The script expects the MAR tools (the mar and make_incremental_update.sh
    commands) to be available in the PATH.
---
 tools/update-responses/README.md        |    9 ++--
 tools/update-responses/config.yml       |    2 +
 tools/update-responses/update_responses |   80 ++++++++++++++++++++++++++++++-
 3 files changed, 87 insertions(+), 4 deletions(-)

diff --git a/tools/update-responses/README.md b/tools/update-responses/README.md
index ee21bac..c7ecdd4 100644
--- a/tools/update-responses/README.md
+++ b/tools/update-responses/README.md
@@ -12,19 +12,22 @@ Dependencies
 ------------
 
 The following perl modules need to be installed to run the script:
-  FindBin YAML File::Slurp Digest::SHA XML::Writer
+  FindBin YAML File::Slurp Digest::SHA XML::Writer File::Temp
+  IO::CaptureOutput File::Which
 
 On Debian / Ubuntu you can install them with:
 
 ```
   # apt-get install libfindbin-libs-perl libyaml-perl libfile-slurp-perl \
-                    libdigest-sha-perl libxml-writer-perl
+                    libdigest-sha-perl libxml-writer-perl \
+                    libio-captureoutput-perl libfile-which-perl
 ```
 
 On Red Hat / Fedora you can install them with:
 
 ```
-  # for module in FindBin YAML File::Slurp Digest::SHA XML::Writer
+  # for module in FindBin YAML File::Slurp Digest::SHA XML::Writer \
+                  File::Temp IO::CaptureOutput File::Which
     do yum install "perl($module)"; done
 ```
 
diff --git a/tools/update-responses/config.yml b/tools/update-responses/config.yml
index b49ce1a..ad1c512 100644
--- a/tools/update-responses/config.yml
+++ b/tools/update-responses/config.yml
@@ -16,6 +16,8 @@ versions:
         platformVersion: 24.8.1
         detailsURL: https://www.torproject.org/projects/torbrowser.html.en
         download_url: https://www.torproject.org/dist/torbrowser/4.0-alpha-3
+        incremental_from:
+            - 4.0-alpha-2
 #       osx32:
 #          minSupportedOSVersion: 10.7
 #          unsupported: 1
diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses
index e86979b..1946278 100755
--- a/tools/update-responses/update_responses
+++ b/tools/update-responses/update_responses
@@ -6,13 +6,20 @@ use YAML qw(LoadFile);
 use File::Slurp;
 use Digest::SHA;
 use XML::Writer;
+use Cwd;
+use File::Temp;
+use File::Find;
+use File::Which;
+use IO::CaptureOutput qw(capture_exec);
 
 my $htdocsdir = "$FindBin::Bin/htdocs";
 my $config = LoadFile("$FindBin::Bin/config.yml");
 my %htdocsfiles = ( '.' => 1, '..' => 1, 'no-update.xml' => 1 );
+my $releases_dir = "$FindBin::Bin/../../gitian";
 
 sub exit_error {
     print STDERR "Error: ", $_[0], "\n";
+    chdir '/';
     exit (exists $_[1] ? $_[1] : 1);
 }
 
@@ -45,7 +52,7 @@ sub get_version_files {
     my ($config, $version) = @_;
     return if $config->{versions}{$version}{files};
     my $files = {};
-    my $vdir = "$FindBin::Bin/../../gitian/$version";
+    my $vdir = "$releases_dir/$version";
     opendir(my $d, $vdir) or exit_error "Error opening directory $vdir";
     foreach my $file (readdir $d) {
         next unless -f "$vdir/$file";
@@ -75,6 +82,69 @@ sub get_version_files {
     $config->{versions}{$version}{files} = $files;
 }
 
+sub extract_mar {
+    my ($mar_file, $dest_dir) = @_;
+    my $old_cwd = getcwd;
+    mkdir $dest_dir;
+    chdir $dest_dir or exit_error "Cannot enter $dest_dir";
+    my $res = system('mar', '-x', $mar_file);
+    exit_error "Error extracting $mar_file" if $res;
+    my $bunzip_file = sub {
+        return unless -f $File::Find::name;
+        rename $File::Find::name, "$File::Find::name.bz2";
+        system('bunzip2', "$File::Find::name.bz2") == 0
+                || exit_error "Error decompressing $File::Find::name";
+    };
+    find($bunzip_file, $dest_dir);
+    chdir $old_cwd;
+}
+
+sub mar_filename {
+    my ($version, $os, $lang) = @_;
+    "$releases_dir/$version/tor-browser-$os-${version}_$lang.mar";
+}
+
+sub create_incremental_mar {
+    my ($config, $from_version, $new_version, $os, $lang) = @_;
+    print "Creating incremental mar file ",
+          "$from_version -> $new_version ($os $lang)\n";
+    my $tmpdir = File::Temp->newdir();
+    extract_mar(mar_filename($from_version, $os, $lang), "$tmpdir/A");
+    extract_mar(mar_filename($new_version, $os, $lang), "$tmpdir/B");
+    my $mar_file = "tor-browser-$os-${from_version}-${new_version}_$lang.mar";
+    my $mar_file_path = "$releases_dir/$new_version/$mar_file";
+    my ($out, $err, $success) = capture_exec('make_incremental_update.sh',
+                                   $mar_file_path, "$tmpdir/A", "$tmpdir/B");
+    if (!$success) {
+        unlink $mar_file_path if -f $mar_file_path;
+        exit_error "making incremental mar:\n" . $err;
+    }
+    $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = {
+        type => 'partial',
+        URL => "$config->{versions}{$new_version}{download_url}/$mar_file",
+        size => -s $mar_file_path,
+        hashFunction => 'SHA512',
+        hashValue => get_sha512_hex_of_file($mar_file_path),
+    };
+}
+
+sub create_missing_incremental_mars {
+    my ($config, $version) = @_;
+    my $v = $config->{versions}{$version};
+    foreach my $from_version (@{$v->{incremental_from}}) {
+        $config->{versions}{$from_version} //= { download_url => '' };
+        get_version_files($config, $from_version);
+        my $from_v = $config->{versions}{$from_version};
+        foreach my $os (keys %{$v->{files}}) {
+            foreach my $lang (keys %{$v->{files}{$os}}) {
+                next if defined $v->{files}{$os}{$lang}{partial}{$from_version};
+                next unless defined $from_v->{files}{$os}{$lang}{complete};
+                create_incremental_mar($config, $from_version, $version, $os, $lang);
+            }
+        }
+    }
+}
+
 sub get_config {
     my ($config, $version, $os, $name) = @_;
     return $config->{versions}{$version}{$os}{$name}
@@ -121,6 +191,7 @@ sub write_responses {
     my ($config) = @_;
     foreach my $version (values %{$config->{channels}}) {
         get_version_files($config, $version);
+        create_missing_incremental_mars($config, $version);
         my $files = $config->{versions}{$version}{files};
         foreach my $os (keys %$files) {
             foreach my $lang (keys %{$files->{$os}}) {
@@ -165,6 +236,13 @@ sub write_htaccess {
     write_htdocs('.htaccess', $htaccess);
 }
 
+sub check_deps {
+    foreach my $bin (qw(bunzip2 mar mbsdiff make_incremental_update.sh)) {
+        exit_error "Cannot find $bin in PATH" unless which($bin);
+    }
+}
+
+check_deps;
 write_responses($config);
 write_htaccess($config);
 clean_htdocs;





More information about the tbb-commits mailing list