commit 803049749f966e63e7c4f86dd556c49fa0fdd48d Author: Nicolas Vigier boklm@torproject.org Date: Mon Mar 16 18:49:43 2015 +0100
Bug 14959: Download missing MAR files for incrementals --- gitian/Makefile | 6 ++- gitian/gpg/torbrowser.gpg | Bin 0 -> 6015 bytes tools/update-responses/config.yml | 3 ++ tools/update-responses/download_missing_versions | 1 + tools/update-responses/update_responses | 58 +++++++++++++++++++++- 5 files changed, 65 insertions(+), 3 deletions(-)
diff --git a/gitian/Makefile b/gitian/Makefile index ea82151..8f89bca 100644 --- a/gitian/Makefile +++ b/gitian/Makefile @@ -29,10 +29,12 @@ build-alpha: ./mkbundle-mac.sh versions.alpha
incrementals: - ../tools/update-responses/gen_incrementals release || echo 'Warning: could not generate incremental MARs.' >&2 + $(TORSOCKS) ../tools/update-responses/download_missing_versions release + ../tools/update-responses/gen_incrementals release
incrementals-alpha: - ../tools/update-responses/gen_incrementals alpha || echo 'Warning: could not generate incremental MARs.' >&2 + $(TORSOCKS) ../tools/update-responses/download_missing_versions alpha + ../tools/update-responses/gen_incrementals alpha
signmars: ./signmars.sh versions diff --git a/gitian/gpg/torbrowser.gpg b/gitian/gpg/torbrowser.gpg new file mode 100644 index 0000000..7267d4d Binary files /dev/null and b/gitian/gpg/torbrowser.gpg differ diff --git a/tools/update-responses/config.yml b/tools/update-responses/config.yml index a338745..403be5c 100644 --- a/tools/update-responses/config.yml +++ b/tools/update-responses/config.yml @@ -1,4 +1,7 @@ --- +download: + archive_url: https://archive.torproject.org/tor-package-archive/torbrowser + gpg_keyring: ../../gitian/gpg/torbrowser.gpg build_targets: linux32: Linux_x86-gcc3 linux64: Linux_x86_64-gcc3 diff --git a/tools/update-responses/download_missing_versions b/tools/update-responses/download_missing_versions new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/download_missing_versions @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index 8b8c217..07efc7c 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -6,9 +6,10 @@ use English; use FindBin; use YAML qw(LoadFile); use File::Slurp; -use Digest::SHA; +use Digest::SHA qw(sha256_hex); use XML::Writer; use Cwd; +use File::Copy; use File::Temp; use File::Find; use File::Which; @@ -408,6 +409,55 @@ sub check_update_responses_channel { } }
+sub download_version { + my ($config, $version) = @_; + my $tmpdir = File::Temp->newdir(); + my $destdir = "$releases_dir/$version"; + my $urldir = "$config->{download}{archive_url}/$version"; + print "Downloading version $version\n"; + foreach my $file (qw(sha256sums.txt sha256sums.txt.asc)) { + if (getstore("$urldir/$file", "$tmpdir/$file") != 200) { + exit_error "Error downloading $urldir/$file"; + } + } + if (system('gpg', '--no-default-keyring', '--keyring', + $config->{download}{gpg_keyring}, '--verify', + "$tmpdir/sha256sums.txt.asc", "$tmpdir/sha256sums.txt")) { + exit_error "Error checking gpg signature for version $version"; + } + mkdir $destdir; + move "$tmpdir/sha256sums.txt.asc", "$destdir/sha256sums.txt.asc"; + move "$tmpdir/sha256sums.txt", "$destdir/sha256sums.txt"; + my %sums = map { chomp; reverse split ' ', $_ } read_file "$destdir/sha256sums.txt"; + foreach my $file (sort grep { $_ =~ m/.mar$/ } keys %sums) { + print "Downloading $file\n"; + exit_error "Error downloading $urldir/$file\n" + unless getstore("$urldir/$file", "$tmpdir/$file") == 200; + if ($sums{$file} ne sha256_hex(read_file("$tmpdir/$file"))) { + exit_error "Error unsigning $file" + if system('signmar', '-r', "$tmpdir/$file", "$tmpdir/$file.u"); + exit_error "Wrong checksum for $file" + unless $sums{$file} eq sha256_hex(read_file("$tmpdir/$file.u")); + move "$tmpdir/$file.u", "$tmpdir/$file"; + } + move "$tmpdir/$file", "$destdir/$file"; + } +} + +sub download_missing_versions { + my ($config, @channels) = @_; + foreach my $channel (@channels) { + exit_error "Unknown channel $channel" + unless $config->{channels}{$channel}; + my $cversion = $config->{channels}{$channel}; + next unless $config->{versions}{$cversion}{incremental_from}; + foreach my $version (@{$config->{versions}{$cversion}{incremental_from}}) { + next if -d "$releases_dir/$version"; + download_version($config, $version); + } + } +} + sub check_update_responses { my ($config) = @_; exit_error "usage: $PROGRAM_NAME <base_url> [channels...]" unless @ARGV; @@ -453,6 +503,12 @@ my %actions = ( create_incremental_mars_for_version($config, $version); } }, + download_missing_versions => sub { + my ($config) = @_; + my @channels = @ARGV ? @ARGV : keys %{$config->{channels}}; + extract_martools; + download_missing_versions($config, @channels); + }, check_update_responses_deployement => &check_update_responses, );