morgan pushed to branch main at The Tor Project / Applications / tor-browser-build
Commits: 7b2ef30f by Nicolas Vigier at 2024-10-03T10:17:59+02:00 Bug 41247: Simplify tools/update-responses/update_responses
Most commands implemented in update_responses expect the channel to be an optional argument, doing the operation on all channels if no channel is provided. However we don't really need that feature. In this commit we simplify the script by making the channel a required argument.
At the same time we remove the get_channel_version command, which was not used anywhere.
- - - - - 97935517 by Nicolas Vigier at 2024-10-03T10:18:01+02:00 Bug 41247: Add support for multiple versions in update_responses
- - - - - 0a476854 by Nicolas Vigier at 2024-10-03T16:19:53+02:00 Bug 41247: Add torbrowser_legacy_version to update-responses
- - - - - d64bc351 by Nicolas Vigier at 2024-10-03T16:19:55+02:00 Bug 41247: Drop sha512 hash from update_responses files (except for nightly)
With tor-browser#42737, the sha512 hash included in the update_responses files is not checked anymore. Since we're doing a watershed update, support for older versions checking the hash is not needed.
However since we don't do watershed updates in nightly, we'll keep the sha512 hash in nightly for a little more time.
- - - - -
8 changed files:
- .gitlab/issue_templates/Release Prep - Tor Browser Stable.md - projects/release/create_update_responses_tar - projects/release/update_responses_config.yml - rbm.conf - tools/signing/nightly/config.yml - tools/signing/nightly/sign-nightly - − tools/update-responses/get_channel_version - tools/update-responses/update_responses
Changes:
===================================== .gitlab/issue_templates/Release Prep - Tor Browser Stable.md ===================================== @@ -39,6 +39,8 @@ Tor Browser Stable lives in the various `maint-$(TOR_BROWSER_MAJOR).$(TOR_BROWSE - [ ] `var/torbrowser_version` : update to next version - [ ] `var/torbrowser_build` : update to `$(TOR_BROWSER_BUILD_N)` - [ ] `var/browser_release_date` : update to build date. For the build to be reproducible, the date should be in the past when building. + - [ ] `var/torbrowser_legacy_version` : update to next version in the legacy-13.5 branch + - [ ] `var/torbrowser_legacy_platform_version` : update to firefox platform version in the legacy-13.5 branch - [ ] ***(Desktop Only)***`var/torbrowser_incremental_from` : update to previous Desktop version - **NOTE**: We try to build incrementals for the previous 3 desktop versions except in the case of a watershed update - **IMPORTANT**: Really *actually* make sure this is the previous Desktop version or else the `make torbrowser-incrementals-*` step will fail
===================================== projects/release/create_update_responses_tar ===================================== @@ -1,8 +1,8 @@ #!/bin/bash [% c("var/set_default_env") -%] -[% shell_quote(c("basedir")) %]/tools/update-responses/update_responses -mkdir -p [% shell_quote(path(dest_dir)) %]/update-responses [% SET channel = c('var/channel') -%] +[% shell_quote(c("basedir")) %]/tools/update-responses/update_responses [% channel %] +mkdir -p [% shell_quote(path(dest_dir)) %]/update-responses mv [% shell_quote(c("basedir")) %]/tools/update-responses/htdocs/[% channel %] [% channel %] chmod 775 [% channel %] chmod 664 [% channel %]/.htaccess [% channel %]/*
===================================== projects/release/update_responses_config.yml ===================================== @@ -27,7 +27,11 @@ build_targets: - Darwin_x86_64-gcc3 - Darwin_aarch64-gcc3 channels: - [% c('var/channel') %]: [% c("var/torbrowser_version") %] + [% c('var/channel') %]: + - [% c("var/torbrowser_version") %] +[% IF c("var/torbrowser_legacy_version") -%] + - [% c("var/torbrowser_legacy_version") %] +[% END -%] versions: [% c("var/torbrowser_version") %]: [% IF c("var/create_unsigned_incrementals") -%] @@ -68,6 +72,26 @@ versions: minSupportedInstructionSet: SSE2 linux-x86_64: minSupportedInstructionSet: SSE2 +[% IF c("var/torbrowser_legacy_version") -%] + [% c("var/torbrowser_legacy_version") %]: + mar_channel_id: [% c('var/mar_channel_id') %] + platformVersion: [% c('var/torbrowser_legacy_platform_version') %] + detailsURL: https://blog.torproject.org/new%5B% IF c("var/alpha") %]-alpha[% END %]-release-tor-browser-[% c("var/torbrowser_legacy_version") FILTER remove('.') %] + incremental_from: [] + # minSupportedOsVersion on macOS corresponds to the Darwin version ( https://en.wikipedia.org/wiki/Darwin_(operating_system) ) + macos: + # macOS v10.12.0 + minSupportedOSVersion: 16.0.0 + # minSupportedOsVersion on Windows corresponds to the operating system version ( https://docs.microsoft.com/en-us/windows/win32/sysinfo/operating-system-vers... ) + windows-i686: + # Windows 7 + minSupportedOSVersion: 6.1 + minSupportedInstructionSet: SSE2 + windows-x86_64: + # Windows 7 + minSupportedOSVersion: 6.1 + minSupportedInstructionSet: SSE2 +[% END -%] mar_compression: xz [% IF c("var/tor-browser") -%] tag: 'tbb-[% c("var/torbrowser_version") %]-[% c("var/torbrowser_build") %]'
===================================== rbm.conf ===================================== @@ -87,6 +87,9 @@ var: - 14.0a4 mar_channel_id: '[% c("var/projectname") %]-torproject-[% c("var/channel") %]'
+ torbrowser_legacy_version: 13.5a11 + torbrowser_legacy_platform_version: 115.16.0 + # By default, we sort the list of installed packages. This allows sharing # containers with identical list of packages, even if they are not listed # in the same order. In the cases where the installation order is
===================================== tools/signing/nightly/config.yml ===================================== @@ -1,4 +1,5 @@ --- +hashes_in_responses: 1 martools_version: 9.0.2 martools_url: https://archive.torproject.org/tor-package-archive/torbrowser/ martools_gpg_keyring: keyring/torbrowser.gpg
===================================== tools/signing/nightly/sign-nightly ===================================== @@ -256,7 +256,7 @@ sub update_responses { my $htdocsdir = "$topdir/tools/update-responses/htdocs/nightly"; path($htdocsdir)->remove_tree({ safe => 0 }); exit_error "Error running update_responses for $publish_dir" unless - system("$topdir/tools/update-responses/update_responses") == 0; + system("$topdir/tools/update-responses/update_responses", 'nightly') == 0; path("$topdir/nightly/updates/$publish_dir/nightly")->remove_tree({ safe => 0 }); make_path("$topdir/nightly/updates/$publish_dir"); dirmove($htdocsdir, "$topdir/nightly/updates/$publish_dir/nightly")
===================================== tools/update-responses/get_channel_version deleted ===================================== @@ -1 +0,0 @@ -update_responses \ No newline at end of file
===================================== tools/update-responses/update_responses ===================================== @@ -42,6 +42,10 @@ sub exit_error { exit (exists $_[1] ? $_[1] : 1); }
+sub as_array { + ref $_[0] eq 'ARRAY' ? $_[0] : [ $_[0] ]; +} + sub get_tmpdir { my ($config) = @_; return File::Temp->newdir($config->{tmp_dir} ? @@ -72,13 +76,11 @@ sub write_htdocs { }
sub clean_htdocs { - my (@channels) = @_; - foreach my $channel (@channels) { - opendir(my $d, "$htdocsdir/$channel"); - my @files = grep { ! $htdocsfiles{$channel}->{$_} } readdir $d; - closedir $d; - unlink map { "$htdocsdir/$channel/$_" } @files; - } + my ($channel) = @_; + opendir(my $d, "$htdocsdir/$channel"); + my @files = grep { ! $htdocsfiles{$channel}->{$_} } readdir $d; + closedir $d; + unlink map { "$htdocsdir/$channel/$_" } @files; }
sub get_sha512_hex_of_file { @@ -105,8 +107,10 @@ sub get_version_files { type => 'complete', URL => "$download_url/$file", size => -s "$vdir/$file", - hashFunction => 'SHA512', - hashValue => get_sha512_hex_of_file("$vdir/$file"), + $config->{hashes_in_responses} ? ( + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file("$vdir/$file"), + ) : (), }; next; } @@ -116,9 +120,11 @@ sub get_version_files { type => 'partial', URL => "$download_url/$file", size => -s "$vdir/$file", - hashFunction => 'SHA512', - hashValue => get_sha512_hex_of_file("$vdir/$file"), - } + $config->{hashes_in_responses} ? ( + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file("$vdir/$file"), + ) : (), + }; } } closedir $d; @@ -239,8 +245,10 @@ sub create_incremental_mar { type => 'partial', URL => "$download_url/$mar_file", size => -s $mar_file_path, - hashFunction => 'SHA512', - hashValue => get_sha512_hex_of_file($mar_file_path), + $config->{hashes_in_responses} ? ( + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file($mar_file_path), + ) : (), }; }; return if $pm->start($finished_file); @@ -307,16 +315,6 @@ sub version_dir { return get_config($config, $version, 'any', 'releases_dir') . "/$version"; }
-sub channel_to_version { - my ($config, @channels) = @_; - return values %{$config->{channels}} unless @channels; - foreach my $channel (@channels) { - exit_error "Unknown channel $channel" - unless $config->{channels}{$channel}; - } - return map { $config->{channels}{$_} } @channels; -} - sub get_buildinfos { my ($config, $version) = @_; return if exists $config->{versions}{$version}{buildID}; @@ -347,49 +345,58 @@ sub get_buildinfos { }
sub get_response { - my ($config, $version, $os, @patches) = @_; + my ($config, $versions, $os, $lang, $from_version) = @_; my $res; my $writer = XML::Writer->new(OUTPUT => $res, ENCODING => 'UTF-8'); $writer->xmlDecl; $writer->startTag('updates'); - if (get_config($config, $version, $os, 'unsupported')) { + foreach my $version (@$versions) { + if (get_config($config, $version, $os, 'unsupported')) { + $writer->startTag('update', + unsupported => 'true', + detailsURL => get_config($config, $version, $os, 'detailsURL'), + ); + goto CLOSETAGS; + } + my $minversion = get_config($config, $version, $os, 'minSupportedOSVersion'); + my $mininstruc = get_config($config, $version, $os, 'minSupportedInstructionSet'); $writer->startTag('update', - unsupported => 'true', + type => 'minor', + displayVersion => $version, + appVersion => $version, + platformVersion => get_config($config, $version, $os, 'platformVersion'), + buildID => get_config($config, $version, $os, 'buildID'), detailsURL => get_config($config, $version, $os, 'detailsURL'), + actions => 'showURL', + openURL => get_config($config, $version, $os, 'detailsURL'), + defined $minversion ? ( minSupportedOSVersion => $minversion ) : (), + defined $mininstruc ? ( minSupportedInstructionSet => $mininstruc ) : (), ); - goto CLOSETAGS; - } - my $minversion = get_config($config, $version, $os, 'minSupportedOSVersion'); - my $mininstruc = get_config($config, $version, $os, 'minSupportedInstructionSet'); - $writer->startTag('update', - type => 'minor', - displayVersion => $version, - appVersion => $version, - platformVersion => get_config($config, $version, $os, 'platformVersion'), - buildID => get_config($config, $version, $os, 'buildID'), - detailsURL => get_config($config, $version, $os, 'detailsURL'), - actions => 'showURL', - openURL => get_config($config, $version, $os, 'detailsURL'), - defined $minversion ? ( minSupportedOSVersion => $minversion ) : (), - defined $mininstruc ? ( minSupportedInstructionSet => $mininstruc ) : (), - ); - foreach my $patch (@patches) { - my @sorted_patch = map { $_ => $patch->{$_} } sort keys %$patch; - $writer->startTag('patch', @sorted_patch); - $writer->endTag('patch'); + if (my $patch = $config->{versions}{$version}{files}{$os}{$lang}{complete}) { + my @sorted_patch = map { $_ => $patch->{$_} } sort keys %$patch; + $writer->startTag('patch', @sorted_patch); + $writer->endTag('patch'); + } + if ($from_version) { + if (my $patch = $config->{versions}{$version}{files}{$os}{$lang}{partial}{$from_version}) { + my @sorted_patch = map { $_ => $patch->{$_} } sort keys %$patch; + $writer->startTag('patch', @sorted_patch); + $writer->endTag('patch'); + } + } + CLOSETAGS: + $writer->endTag('update'); } - CLOSETAGS: - $writer->endTag('update'); $writer->endTag('updates'); $writer->end; return $res; }
sub write_responses { - my ($config, @channels) = @_; - @channels = keys %{$config->{channels}} unless @channels; - foreach my $channel (@channels) { - my $version = $config->{channels}{$channel}; + my ($config, $channel) = @_; + my $versions = as_array($config->{channels}{$channel}); + my (%oses, %langs, %from_versions); + foreach my $version (@$versions) { get_version_files($config, $version); get_buildinfos($config, $version); my $files = $config->{versions}{$version}{files}; @@ -398,79 +405,96 @@ sub write_responses { my $new_os = $migrate_archs->{$old_os}; foreach my $lang (keys %{$files->{$new_os}}) { $files->{$old_os}{$lang}{complete} = - $files->{$new_os}{$lang}{complete}; + $files->{$new_os}{$lang}{complete}; } } foreach my $os (keys %$files) { + $oses{$os} = 1; foreach my $lang (keys %{$files->{$os}}) { - my $resp = get_response($config, $version, $os, - $files->{$os}{$lang}{complete}); - write_htdocs($channel, "$version-$os-$lang.xml", $resp); + $langs{$lang} = 1; foreach my $from_version (keys %{$files->{$os}{$lang}{partial}}) { - $resp = get_response($config, $version, $os, - $files->{$os}{$lang}{complete}, - $files->{$os}{$lang}{partial}{$from_version}); - write_htdocs($channel, "$from_version-$version-$os-$lang.xml", $resp); + $from_versions{$from_version} = 1; } } } - write_htdocs($channel, 'no-update.xml', - '<?xml version="1.0" encoding="UTF-8"?>' - . "\n<updates></updates>\n"); } + my $versions_str = join('+', @$versions); + foreach my $os (keys %oses) { + foreach my $lang (keys %langs) { + my $resp = get_response($config, $versions, $os, $lang); + write_htdocs($channel, "$versions_str-$os-$lang.xml", $resp); + foreach my $from_version (keys %from_versions) { + $resp = get_response($config, $versions, $os, $lang, $from_version); + write_htdocs($channel, "$from_version-$versions_str-$os-$lang.xml", $resp); + } + } + } + write_htdocs($channel, 'no-update.xml', + '<?xml version="1.0" encoding="UTF-8"?>' + . "\n<updates></updates>\n"); }
sub write_htaccess { - my ($config, @channels) = @_; - @channels = keys %{$config->{channels}} unless @channels; + my ($config, $channel) = @_; my $flags = "[last]"; - foreach my $channel (@channels) { - my $htaccess = "RewriteEngine On\n"; - $htaccess .= $config->{htaccess_rewrite_rules}{$channel} // ''; - my $version = $config->{channels}{$channel}; - my $migrate_langs = $config->{versions}{$version}{migrate_langs} // {}; + my $htaccess = "RewriteEngine On\n"; + $htaccess .= $config->{htaccess_rewrite_rules}{$channel} // ''; + my $versions = as_array($config->{channels}{$channel}); + my $versions_str = join('+', @$versions); + my (%oses, %langs, %from_versions); + my $migrate_langs; + foreach my $version (@$versions) { + $migrate_langs = $config->{versions}{$version}{migrate_langs} + if $config->{versions}{$version}{migrate_langs}; my $files = $config->{versions}{$version}{files}; - $htaccess .= "RewriteRule ^[^/]+/$version/ no-update.xml $flags\n"; - foreach my $os (sort keys %$files) { - foreach my $bt (build_targets_by_os($os)) { - foreach my $lang (sort keys %{$files->{$os}}) { - foreach my $from_version (sort keys %{$files->{$os}{$lang}{partial}}) { - $htaccess .= "RewriteRule ^$bt/$from_version/$lang " - . "$from_version-$version-$os-$lang.xml $flags\n"; - } - $htaccess .= "RewriteRule ^$bt/[^/]+/$lang " - . "$version-$os-$lang.xml $flags\n"; + foreach my $os (keys %$files) { + $oses{$os} = 1; + foreach my $lang (keys %{$files->{$os}}) { + $langs{$lang} = 1; + foreach my $from_version (keys %{$files->{$os}{$lang}{partial}}) { + $from_versions{$from_version} = 1; } - foreach my $lang (sort keys %$migrate_langs) { - $htaccess .= "RewriteRule ^$bt/[^/]+/$lang " - . "$version-$os-$migrate_langs->{$lang}.xml $flags\n"; + } + } + $htaccess .= "RewriteRule ^[^/]+/$version/ no-update.xml $flags\n"; + } + foreach my $os (sort keys %oses) { + foreach my $bt (build_targets_by_os($os)) { + foreach my $lang (sort keys %langs) { + foreach my $from_version (sort keys %from_versions) { + $htaccess .= "RewriteRule ^$bt/$from_version/$lang " + . "$from_version-$versions_str-$os-$lang.xml $flags\n"; } - $htaccess .= "RewriteRule ^$bt/ $version-$os-ALL.xml $flags\n"; + $htaccess .= "RewriteRule ^$bt/[^/]+/$lang " + . "$versions_str-$os-$lang.xml $flags\n"; + } + foreach my $lang (sort keys %$migrate_langs) { + $htaccess .= "RewriteRule ^$bt/[^/]+/$lang " + . "$versions_str-$os-$migrate_langs->{$lang}.xml $flags\n"; } + $htaccess .= "RewriteRule ^$bt/ $versions_str-$os-ALL.xml $flags\n"; } - write_htdocs($channel, '.htaccess', $htaccess); } + write_htdocs($channel, '.htaccess', $htaccess); }
sub write_downloads_json { - my ($config, @channels) = @_; + my ($config, $channel) = @_; return unless $config->{create_downloads_json}; - @channels = keys %{$config->{channels}} unless @channels; - foreach my $channel (@channels) { - my $version = $config->{channels}{$channel}; - my $tag = get_config($config, $version, 'any', 'tag'); - my $data = { - version => "$version", - tag => "$tag", - downloads => get_version_downloads($config, $version), - }; - write_htdocs($channel, 'downloads.json', - JSON->new->utf8->canonical->encode($data)); - my $pp_downloads = get_perplatform_downloads($config, $version, $tag); - foreach my $os (keys %{$pp_downloads}) { - write_htdocs($channel, "download-$os.json", + my $versions = as_array($config->{channels}{$channel}); + my ($version) = @$versions; + my $tag = get_config($config, $version, 'any', 'tag'); + my $data = { + version => "$version", + tag => "$tag", + downloads => get_version_downloads($config, $version), + }; + write_htdocs($channel, 'downloads.json', + JSON->new->utf8->canonical->encode($data)); + my $pp_downloads = get_perplatform_downloads($config, $version, $tag); + foreach my $os (keys %{$pp_downloads}) { + write_htdocs($channel, "download-$os.json", JSON->new->utf8->canonical->encode($pp_downloads->{$os})); - } } }
@@ -486,8 +510,10 @@ sub marzip_path { }
my $martools_tmpdir; +my $extracted_martools; sub extract_martools { my ($config, $version) = @_; + return if $extracted_martools; my $marzip = marzip_path($config, $version); $martools_tmpdir = get_tmpdir($config); my $old_cwd = getcwd; @@ -503,6 +529,7 @@ sub extract_martools { } else { $ENV{LD_LIBRARY_PATH} = "$martools_tmpdir/mar-tools"; } + $extracted_martools = 1; }
sub log_step { @@ -559,7 +586,8 @@ sub build_targets_list {
sub check_update_responses_channel { my ($config, $base_url, $channel) = @_; - my $channel_version = $config->{channels}{$channel}; + my $channel_versions = as_array($config->{channels}{$channel}); + my ($channel_version) = @$channel_versions; foreach my $build_target (build_targets_list()) { foreach my $lang (qw(en-US de)) { my $url = "$base_url/$channel/$build_target/1.0/$lang"; @@ -621,11 +649,11 @@ sub download_version { }
sub download_missing_versions { - my ($config, @channels) = @_; - foreach my $channel (@channels) { - exit_error "Unknown channel $channel" - unless $config->{channels}{$channel}; - my $cversion = $config->{channels}{$channel}; + my ($config, $channel) = @_; + exit_error "Unknown channel $channel" + unless $config->{channels}{$channel}; + my $cversions = as_array($config->{channels}{$channel}); + foreach my $cversion (@$cversions) { next unless $config->{versions}{$cversion}{incremental_from}; foreach my $version (@{$config->{versions}{$cversion}{incremental_from}}) { next if -d version_dir($config, $version); @@ -636,11 +664,9 @@ sub download_missing_versions {
sub check_update_responses { my ($config) = @_; - exit_error "usage: $PROGRAM_NAME <base_url> [channels...]" unless @ARGV; - my ($base_url, @channels) = @ARGV; - foreach my $channel (@channels ? @channels : keys %{$config->{channels}}) { - check_update_responses_channel($config, $base_url, $channel); - } + exit_error "usage: $PROGRAM_NAME <base_url> <channel>" unless @ARGV; + my ($base_url, $channel) = @ARGV; + check_update_responses_channel($config, $base_url, $channel); if (!@check_errors) { print "\n\nNo errors\n"; return; @@ -660,38 +686,34 @@ sub check_update_responses { my %actions = ( update_responses => sub { my ($config) = @_; - my @channels = @ARGV ? @ARGV : keys %{$config->{channels}}; - foreach my $channel (@channels) { - exit_error "Unknown channel $channel" - unless $config->{channels}{$channel}; - $htdocsfiles{$channel} = { '.' => 1, '..' => 1 }; - } - write_responses($config, @channels); - write_htaccess($config, @channels); - write_downloads_json($config, @channels); - clean_htdocs(@channels); + exit_error "Wrong arguments" unless @ARGV == 1; + my $channel = $ARGV[0]; + exit_error "Unknown channel $channel" unless $config->{channels}{$channel}; + $htdocsfiles{$channel} = { '.' => 1, '..' => 1 }; + write_responses($config, $channel); + write_htaccess($config, $channel); + write_downloads_json($config, $channel); + clean_htdocs($channel); }, gen_incrementals => sub { my ($config) = @_; - foreach my $channel (@ARGV) { - my ($version) = channel_to_version($config, $channel); + exit_error "Wrong arguments" unless @ARGV == 1; + my $channel = $ARGV[0]; + exit_error "Unknown channel" unless $config->{channels}{$channel}; + my $versions = as_array($config->{channels}{$channel}); + foreach my $version (@$versions) { extract_martools($config, $version); get_version_files($config, $version); create_incremental_mars_for_version($config, $version, $channel); } }, download_missing_versions => sub { - my ($config) = @_; - my @channels = @ARGV ? @ARGV : keys %{$config->{channels}}; - download_missing_versions($config, @channels); - }, - check_update_responses_deployement => &check_update_responses, - get_channel_version => sub { my ($config) = @_; exit_error "Wrong arguments" unless @ARGV == 1; - exit_error "Unknown channel" unless $config->{channels}{$ARGV[0]}; - print $config->{channels}{$ARGV[0]}, "\n"; + my $channel = $ARGV[0]; + download_missing_versions($config, $channel); }, + check_update_responses_deployement => &check_update_responses, );
my $action = fileparse($PROGRAM_NAME);
View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/compare/b...