[tor-commits] [tor-browser-bundle/master] update_responses: generate incremental MAR files
mikeperry at torproject.org
mikeperry at torproject.org
Fri Oct 24 23:56:04 UTC 2014
commit 389f843dd3bb19f703a4448528e3d5b92942bf04
Author: Nicolas Vigier <boklm at mars-attacks.org>
Date: Thu Oct 2 15:34:06 2014 +0200
update_responses: generate incremental MAR files
Generate incremental MAR file when they are not already there.
The list of versions from which to generate incremental MAR is defined in
config.yml as incremental_from for each version.
The script expects the MAR tools (the mar and make_incremental_update.sh
commands) to be available in the PATH.
---
tools/update-responses/README.md | 9 ++--
tools/update-responses/config.yml | 2 +
tools/update-responses/update_responses | 80 ++++++++++++++++++++++++++++++-
3 files changed, 87 insertions(+), 4 deletions(-)
diff --git a/tools/update-responses/README.md b/tools/update-responses/README.md
index ee21bac..c7ecdd4 100644
--- a/tools/update-responses/README.md
+++ b/tools/update-responses/README.md
@@ -12,19 +12,22 @@ Dependencies
------------
The following perl modules need to be installed to run the script:
- FindBin YAML File::Slurp Digest::SHA XML::Writer
+ FindBin YAML File::Slurp Digest::SHA XML::Writer File::Temp
+ IO::CaptureOutput File::Which
On Debian / Ubuntu you can install them with:
```
# apt-get install libfindbin-libs-perl libyaml-perl libfile-slurp-perl \
- libdigest-sha-perl libxml-writer-perl
+ libdigest-sha-perl libxml-writer-perl \
+ libio-captureoutput-perl libfile-which-perl
```
On Red Hat / Fedora you can install them with:
```
- # for module in FindBin YAML File::Slurp Digest::SHA XML::Writer
+ # for module in FindBin YAML File::Slurp Digest::SHA XML::Writer \
+ File::Temp IO::CaptureOutput File::Which
do yum install "perl($module)"; done
```
diff --git a/tools/update-responses/config.yml b/tools/update-responses/config.yml
index b49ce1a..ad1c512 100644
--- a/tools/update-responses/config.yml
+++ b/tools/update-responses/config.yml
@@ -16,6 +16,8 @@ versions:
platformVersion: 24.8.1
detailsURL: https://www.torproject.org/projects/torbrowser.html.en
download_url: https://www.torproject.org/dist/torbrowser/4.0-alpha-3
+ incremental_from:
+ - 4.0-alpha-2
# osx32:
# minSupportedOSVersion: 10.7
# unsupported: 1
diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses
index e86979b..1946278 100755
--- a/tools/update-responses/update_responses
+++ b/tools/update-responses/update_responses
@@ -6,13 +6,20 @@ use YAML qw(LoadFile);
use File::Slurp;
use Digest::SHA;
use XML::Writer;
+use Cwd;
+use File::Temp;
+use File::Find;
+use File::Which;
+use IO::CaptureOutput qw(capture_exec);
my $htdocsdir = "$FindBin::Bin/htdocs";
my $config = LoadFile("$FindBin::Bin/config.yml");
my %htdocsfiles = ( '.' => 1, '..' => 1, 'no-update.xml' => 1 );
+my $releases_dir = "$FindBin::Bin/../../gitian";
sub exit_error {
print STDERR "Error: ", $_[0], "\n";
+ chdir '/';
exit (exists $_[1] ? $_[1] : 1);
}
@@ -45,7 +52,7 @@ sub get_version_files {
my ($config, $version) = @_;
return if $config->{versions}{$version}{files};
my $files = {};
- my $vdir = "$FindBin::Bin/../../gitian/$version";
+ my $vdir = "$releases_dir/$version";
opendir(my $d, $vdir) or exit_error "Error opening directory $vdir";
foreach my $file (readdir $d) {
next unless -f "$vdir/$file";
@@ -75,6 +82,69 @@ sub get_version_files {
$config->{versions}{$version}{files} = $files;
}
+sub extract_mar {
+ my ($mar_file, $dest_dir) = @_;
+ my $old_cwd = getcwd;
+ mkdir $dest_dir;
+ chdir $dest_dir or exit_error "Cannot enter $dest_dir";
+ my $res = system('mar', '-x', $mar_file);
+ exit_error "Error extracting $mar_file" if $res;
+ my $bunzip_file = sub {
+ return unless -f $File::Find::name;
+ rename $File::Find::name, "$File::Find::name.bz2";
+ system('bunzip2', "$File::Find::name.bz2") == 0
+ || exit_error "Error decompressing $File::Find::name";
+ };
+ find($bunzip_file, $dest_dir);
+ chdir $old_cwd;
+}
+
+sub mar_filename {
+ my ($version, $os, $lang) = @_;
+ "$releases_dir/$version/tor-browser-$os-${version}_$lang.mar";
+}
+
+sub create_incremental_mar {
+ my ($config, $from_version, $new_version, $os, $lang) = @_;
+ print "Creating incremental mar file ",
+ "$from_version -> $new_version ($os $lang)\n";
+ my $tmpdir = File::Temp->newdir();
+ extract_mar(mar_filename($from_version, $os, $lang), "$tmpdir/A");
+ extract_mar(mar_filename($new_version, $os, $lang), "$tmpdir/B");
+ my $mar_file = "tor-browser-$os-${from_version}-${new_version}_$lang.mar";
+ my $mar_file_path = "$releases_dir/$new_version/$mar_file";
+ my ($out, $err, $success) = capture_exec('make_incremental_update.sh',
+ $mar_file_path, "$tmpdir/A", "$tmpdir/B");
+ if (!$success) {
+ unlink $mar_file_path if -f $mar_file_path;
+ exit_error "making incremental mar:\n" . $err;
+ }
+ $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = {
+ type => 'partial',
+ URL => "$config->{versions}{$new_version}{download_url}/$mar_file",
+ size => -s $mar_file_path,
+ hashFunction => 'SHA512',
+ hashValue => get_sha512_hex_of_file($mar_file_path),
+ };
+}
+
+sub create_missing_incremental_mars {
+ my ($config, $version) = @_;
+ my $v = $config->{versions}{$version};
+ foreach my $from_version (@{$v->{incremental_from}}) {
+ $config->{versions}{$from_version} //= { download_url => '' };
+ get_version_files($config, $from_version);
+ my $from_v = $config->{versions}{$from_version};
+ foreach my $os (keys %{$v->{files}}) {
+ foreach my $lang (keys %{$v->{files}{$os}}) {
+ next if defined $v->{files}{$os}{$lang}{partial}{$from_version};
+ next unless defined $from_v->{files}{$os}{$lang}{complete};
+ create_incremental_mar($config, $from_version, $version, $os, $lang);
+ }
+ }
+ }
+}
+
sub get_config {
my ($config, $version, $os, $name) = @_;
return $config->{versions}{$version}{$os}{$name}
@@ -121,6 +191,7 @@ sub write_responses {
my ($config) = @_;
foreach my $version (values %{$config->{channels}}) {
get_version_files($config, $version);
+ create_missing_incremental_mars($config, $version);
my $files = $config->{versions}{$version}{files};
foreach my $os (keys %$files) {
foreach my $lang (keys %{$files->{$os}}) {
@@ -165,6 +236,13 @@ sub write_htaccess {
write_htdocs('.htaccess', $htaccess);
}
+sub check_deps {
+ foreach my $bin (qw(bunzip2 mar mbsdiff make_incremental_update.sh)) {
+ exit_error "Cannot find $bin in PATH" unless which($bin);
+ }
+}
+
+check_deps;
write_responses($config);
write_htaccess($config);
clean_htdocs;
More information about the tor-commits
mailing list