morgan pushed to branch maint-14.0 at The Tor Project / Applications / tor-browser-build

Commits: ae7a43c7 by Nicolas Vigier at 2025-03-13T10:36:59+01:00 Bug 41374: Remove support for migrate_archs and migrate_langs in update_responses

Those options have not been used for some time, and removing them simplifies the following changes we're doing to split files per platform.

- - - - - 5abd1cb7 by Nicolas Vigier at 2025-03-13T10:37:22+01:00 Bug 40799: Remove legacy locale iteration in update-responses

- - - - - 7c05ce80 by Nicolas Vigier at 2025-03-17T10:38:03+01:00 Bug 40799: Remove legacy locale support in tools/dmg2mar

- - - - - b1bd59c3 by Nicolas Vigier at 2025-03-17T10:38:51+01:00 Bug 41363: Split update_responses files per platform

To make reverting update for a single platform easier, we split update response files, using a separate directory for each platform.

- - - - - 7945d332 by Nicolas Vigier at 2025-03-17T10:39:15+01:00 Bug 41363: Make separate update_responses commit for each platform

- - - - - 80a959de by Nicolas Vigier at 2025-03-17T10:39:26+01:00 Bug 41363: Fix error message about $update_responses_repository_dir being undefined

- - - - - 03e261aa by Nicolas Vigier at 2025-03-17T10:40:27+01:00 Bug 41394: Don't try to add download-android-*.json files for Mullvad Browser

- - - - -

6 changed files:

- projects/release/create_update_responses_tar - projects/release/update_responses_config.yml - tools/dmg2mar - tools/signing/functions - tools/signing/upload-update_responses-to-staticiforme - tools/update-responses/update_responses

Changes:

===================================== projects/release/create_update_responses_tar ===================================== @@ -1,9 +1,10 @@ #!/bin/bash [% c("var/set_default_env") -%] [% SET channel = c('var/channel') -%] +rm -Rf [% shell_quote(c("basedir")) %]/tools/update-responses/htdocs/[% channel %] [% shell_quote(c("basedir")) %]/tools/update-responses/update_responses [% channel %] mkdir -p [% shell_quote(path(dest_dir)) %]/update-responses mv [% shell_quote(c("basedir")) %]/tools/update-responses/htdocs/[% channel %] [% channel %] -chmod 775 [% channel %] -chmod 664 [% channel %]/.htaccess [% channel %]/* +find [% channel %] -type d -exec chmod 775 {} ; +find [% channel %] -type f -exec chmod 664 {} ; tar cf [% shell_quote(path(dest_dir)) %]/update-responses/update-responses-[% channel %]-[% c("version") %].tar [% channel %]

===================================== projects/release/update_responses_config.yml ===================================== @@ -53,10 +53,6 @@ versions: - [% ver %] [% END; END -%] - # for example, osx32: osx64 - migrate_archs: - # for example, pt-PT: pt-BR - migrate_langs: # minSupportedOsVersion on macOS corresponds to the Darwin version ( https://en.wikipedia.org/wiki/Darwin_(operating_system) ) macos: # macOS v10.15.0

===================================== tools/dmg2mar ===================================== @@ -102,7 +102,7 @@ sub get_dmg_files_from_sha256sums { next unless $filename; chomp $filename; next unless $filename =~ m/^$appname_dmg-macos-(.+).dmg$/; - push @files, { filename => $filename, version => $1, lang => 'ALL' }; + push @files, { filename => $filename, version => $1 }; } return @files; } @@ -116,11 +116,7 @@ sub convert_files { print "Finished $_[2]\n"; }); foreach my $file (get_dmg_files_from_sha256sums) { - # The 'ja' locale is a special case: it is called 'ja-JP-mac' - # internally on OSX, but the dmg file still uses 'ja' to avoid - # confusing users. - my $mar_lang = $file->{lang} eq 'ja' ? 'ja-JP-mac' : $file->{lang}; - my $output = "$appname_mar-macos-$file->{version}_$mar_lang.mar"; + my $output = "$appname_mar-macos-$file->{version}_ALL.mar"; my $step_name = "$file->{filename} -> $output"; print "Starting $step_name\n"; $pm->start($step_name) and next;

===================================== tools/signing/functions ===================================== @@ -23,7 +23,7 @@ function check_update_responses_repository_dir { if test -z "$update_responses_repository_dir" || ! test -d "$update_responses_repository_dir" then cat << 'EOF' > /dev/stderr -$aus1_repository_dir is not defined, or the directory does not exist +$update_responses_repository_dir is not defined, or the directory does not exist You should clone git@gitlab.torproject.org:tpo/applications/tor-browser-update-responses.git and set $update_responses_repository_dir in set-config.update-responses EOF

===================================== tools/signing/upload-update_responses-to-staticiforme ===================================== @@ -39,8 +39,34 @@ do mv "$file" "$tbb_version_type/$fname" done

+# Keep directory from previous release if they are not in the new release. +# This happens when a release does not include some platforms. +for file in $(ls -1 "$old_ur/$tbb_version_type") +do + test -d "$old_ur/$tbb_version_type/$file" || continue + test -d "$tbb_version_type/$file" && continue + mv -f "$old_ur/$tbb_version_type/$file" "$tbb_version_type/$file" +done + +# Commit each sub-directory separately +for file in $(ls -1 "$tbb_version_type") +do + test -d "$tbb_version_type/$file" || continue + git add "$tbb_version_type/$file" + git add "$tbb_version_type/download-$file.json" + git diff --quiet --cached --exit-code || \ + git commit -m "$tbb_version_type: new version, $tbb_version ($file)" +done + +if is_project torbrowser; then + git add "$tbb_version_type"/download-android-*.json + git diff --quiet --cached --exit-code || \ + git commit -m "$tbb_version_type: new version, $tbb_version (android)" +fi + git add "$tbb_version_type" -git commit -m "$tbb_version_type: new version, $tbb_version" +git diff --quiet --cached --exit-code || \ + git commit -m "$tbb_version_type: new version, $tbb_version" git push

# we just need to push mullvadbrowser's update responses to git, not deploy to staticiforme

===================================== tools/update-responses/update_responses ===================================== @@ -29,7 +29,6 @@ setlocale(LC_ALL, "C");

my $htdocsdir = "$FindBin::Bin/htdocs"; my $config = LoadFile("$FindBin::Bin/config.yml"); -my %htdocsfiles; my $releases_dir = $config->{releases_dir}; $releases_dir = "$FindBin::Bin/$releases_dir" unless $releases_dir =~ m/^//; my @check_errors; @@ -51,12 +50,6 @@ sub get_tmpdir { : ()); }

-sub build_targets_by_os { - exit_error "Unknown build target for OS $_[0]" unless $config->{build_targets}{$_[0]}; - my $r = $config->{build_targets}{$_[0]}; - return ref $r eq 'ARRAY' ? @$r : ($r); -} - sub get_nbprocs { return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS}; if (-f '/proc/cpuinfo') { @@ -80,19 +73,11 @@ sub setup_martools { }

sub write_htdocs { - my ($channel, $file, $content) = @_; + my ($channel, $dir, $file, $content) = @_; mkdir $htdocsdir unless -d $htdocsdir; mkdir "$htdocsdir/$channel" unless -d "$htdocsdir/$channel"; - write_file("$htdocsdir/$channel/$file", $content); - $htdocsfiles{$channel}->{$file} = 1; -} - -sub clean_htdocs { - my ($channel) = @_; - opendir(my $d, "$htdocsdir/$channel"); - my @files = grep { ! $htdocsfiles{$channel}->{$_} } readdir $d; - closedir $d; - unlink map { "$htdocsdir/$channel/$_" } @files; + mkdir "$htdocsdir/$channel/$dir" unless -d "$htdocsdir/$channel/$dir"; + write_file("$htdocsdir/$channel/$dir/$file", $content); }

sub get_sha512_hex_of_file { @@ -113,9 +98,9 @@ sub get_version_files { foreach my $file (readdir $d) { next unless -f "$vdir/$file"; if ($file !~ m/incremental.mar$/ && - $file =~ m/^$appname-(.+)-${version}_(.+).mar$/) { - my ($os, $lang) = ($1, $2); - $files->{$os}{$lang}{complete} = { + $file =~ m/^$appname-(.+)-${version}_ALL.mar$/) { + my $os = $1; + $files->{$os}{complete} = { type => 'complete', URL => "$download_url/$file", size => -s "$vdir/$file", @@ -126,9 +111,9 @@ sub get_version_files { }; next; } - if ($file =~ m/^$appname-(.+)--(.+)-${version}_(.+).incremental.mar$/) { - my ($os, $from_version, $lang) = ($1, $2, $3); - $files->{$os}{$lang}{partial}{$from_version} = { + if ($file =~ m/^$appname-(.+)--(.+)-${version}_ALL.incremental.mar$/) { + my ($os, $from_version) = ($1, $2); + $files->{$os}{partial}{$from_version} = { type => 'partial', URL => "$download_url/$file", size => -s "$vdir/$file", @@ -235,14 +220,14 @@ sub extract_mar { }

sub mar_filename { - my ($config, $appname, $version, $os, $lang) = @_; - version_dir($config, $version) . "/$appname-$os-${version}_$lang.mar"; + my ($config, $appname, $version, $os) = @_; + version_dir($config, $version) . "/$appname-$os-${version}_ALL.mar"; }

sub create_incremental_mar { - my ($config, $pm, $from_version, $new_version, $os, $lang, $channel) = @_; + my ($config, $pm, $from_version, $new_version, $os, $channel) = @_; my $appname = $config->{appname_marfile}; - my $mar_file = "$appname-$os--${from_version}-${new_version}_$lang.incremental.mar"; + my $mar_file = "$appname-$os--${from_version}-${new_version}_ALL.incremental.mar"; my $mar_file_path = version_dir($config, $new_version) . '/' . $mar_file; if ($ENV{MAR_SKIP_EXISTING} && -f $mar_file_path) { print "Skipping $mar_file\n"; @@ -253,7 +238,7 @@ sub create_incremental_mar { my $finished_file = sub { exit_error "Error creating $mar_file" unless $_[1] == 0; print "Finished $mar_file\n"; - $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = { + $config->{versions}{$new_version}{files}{$os}{partial}{$from_version} = { type => 'partial', URL => "$download_url/$mar_file", size => -s $mar_file_path, @@ -267,9 +252,9 @@ sub create_incremental_mar { my $tmpdir = get_tmpdir($config); my $mar_c_from = get_config($config, $from_version, $os, 'mar_compression'); my $mar_c_new = get_config($config, $new_version, $os, 'mar_compression'); - extract_mar(mar_filename($config, $appname, $from_version, $os, $lang), + extract_mar(mar_filename($config, $appname, $from_version, $os), "$tmpdir/A", $mar_c_from); - extract_mar(mar_filename($config, $appname, $new_version, $os, $lang), + extract_mar(mar_filename($config, $appname, $new_version, $os), "$tmpdir/B", $mar_c_new); # bug 26054: make sure previous macOS version is code signed if (!$ENV{NO_CODESIGNATURE} && ($os eq 'macos') @@ -306,10 +291,8 @@ sub create_incremental_mars_for_version { get_version_files($config, $from_version); my $from_v = $config->{versions}{$from_version}; foreach my $os (keys %{$v->{files}}) { - foreach my $lang (keys %{$v->{files}{$os}}) { - next unless defined $from_v->{files}{$os}{$lang}{complete}; - create_incremental_mar($config, $pm, $from_version, $version, $os, $lang, $channel); - } + next unless defined $from_v->{files}{$os}{complete}; + create_incremental_mar($config, $pm, $from_version, $version, $os, $channel); } } $pm->wait_all_children; @@ -333,31 +316,29 @@ sub get_buildinfos { setup_martools; my $files = $config->{versions}{$version}{files}; foreach my $os (keys %$files) { - foreach my $lang (keys %{$files->{$os}}) { - next unless $files->{$os}{$lang}{complete}; - my $tmpdir = get_tmpdir($config); - my $mar_compression = get_config($config, $version, $os, 'mar_compression'); - extract_mar( - mar_filename($config, $config->{appname_marfile}, $version, $os, $lang), - "$tmpdir", - $mar_compression); - my $appfile = "$tmpdir/application.ini" if -f "$tmpdir/application.ini"; - $appfile = "$tmpdir/Contents/Resources/application.ini" - if -f "$tmpdir/Contents/Resources/application.ini"; - exit_error "Could not find application.ini" unless $appfile; - foreach my $line (read_file($appfile)) { - if ($line =~ m/^BuildID=(.*)$/) { - $config->{versions}{$version}{buildID} = $1; - return; - } + next unless $files->{$os}{complete}; + my $tmpdir = get_tmpdir($config); + my $mar_compression = get_config($config, $version, $os, 'mar_compression'); + extract_mar( + mar_filename($config, $config->{appname_marfile}, $version, $os), + "$tmpdir", + $mar_compression); + my $appfile = "$tmpdir/application.ini" if -f "$tmpdir/application.ini"; + $appfile = "$tmpdir/Contents/Resources/application.ini" + if -f "$tmpdir/Contents/Resources/application.ini"; + exit_error "Could not find application.ini" unless $appfile; + foreach my $line (read_file($appfile)) { + if ($line =~ m/^BuildID=(.*)$/) { + $config->{versions}{$version}{buildID} = $1; + return; } - exit_error "Could not extract buildID from application.ini"; } + exit_error "Could not extract buildID from application.ini"; } }

sub get_response { - my ($config, $versions, $os, $lang, $from_version) = @_; + my ($config, $versions, $os, $from_version) = @_; my $res; my $writer = XML::Writer->new(OUTPUT => $res, ENCODING => 'UTF-8'); $writer->xmlDecl; @@ -384,13 +365,13 @@ sub get_response { defined $minversion ? ( minSupportedOSVersion => $minversion ) : (), defined $mininstruc ? ( minSupportedInstructionSet => $mininstruc ) : (), ); - if (my $patch = $config->{versions}{$version}{files}{$os}{$lang}{complete}) { + if (my $patch = $config->{versions}{$version}{files}{$os}{complete}) { my @sorted_patch = map { $_ => $patch->{$_} } sort keys %$patch; $writer->startTag('patch', @sorted_patch); $writer->endTag('patch'); } if ($from_version) { - if (my $patch = $config->{versions}{$version}{files}{$os}{$lang}{partial}{$from_version}) { + if (my $patch = $config->{versions}{$version}{files}{$os}{partial}{$from_version}) { my @sorted_patch = map { $_ => $patch->{$_} } sort keys %$patch; $writer->startTag('patch', @sorted_patch); $writer->endTag('patch'); @@ -407,87 +388,69 @@ sub get_response { sub write_responses { my ($config, $channel) = @_; my $versions = as_array($config->{channels}{$channel}); - my (%oses, %langs, %from_versions); + my (%oses, %from_versions); foreach my $version (@$versions) { get_version_files($config, $version); get_buildinfos($config, $version); my $files = $config->{versions}{$version}{files}; - my $migrate_archs = $config->{versions}{$version}{migrate_archs} // {}; - foreach my $old_os (keys %$migrate_archs) { - my $new_os = $migrate_archs->{$old_os}; - foreach my $lang (keys %{$files->{$new_os}}) { - $files->{$old_os}{$lang}{complete} = - $files->{$new_os}{$lang}{complete}; - } - } foreach my $os (keys %$files) { $oses{$os} = 1; - foreach my $lang (keys %{$files->{$os}}) { - $langs{$lang} = 1; - foreach my $from_version (keys %{$files->{$os}{$lang}{partial}}) { - $from_versions{$from_version} = 1; - } + foreach my $from_version (keys %{$files->{$os}{partial}}) { + $from_versions{$from_version} = 1; } } } my $versions_str = join('+', @$versions); foreach my $os (keys %oses) { - foreach my $lang (keys %langs) { - my $resp = get_response($config, $versions, $os, $lang); - write_htdocs($channel, "$versions_str-$os-$lang.xml", $resp); - foreach my $from_version (keys %from_versions) { - $resp = get_response($config, $versions, $os, $lang, $from_version); - write_htdocs($channel, "$from_version-$versions_str-$os-$lang.xml", $resp); - } + my $resp = get_response($config, $versions, $os); + write_htdocs($channel, $os, "$versions_str-$os.xml", $resp); + foreach my $from_version (keys %from_versions) { + $resp = get_response($config, $versions, $os, $from_version); + write_htdocs($channel, $os, "$from_version-$versions_str-$os.xml", $resp); } + write_htdocs($channel, $os, 'no-update.xml', + '<?xml version="1.0" encoding="UTF-8"?>' + . "\n<updates></updates>\n"); } - write_htdocs($channel, 'no-update.xml', - '<?xml version="1.0" encoding="UTF-8"?>' - . "\n<updates></updates>\n"); }

sub write_htaccess { my ($config, $channel) = @_; - my $flags = "[last]"; - my $htaccess = "RewriteEngine On\n"; - $htaccess .= $config->{htaccess_rewrite_rules}{$channel} // ''; + + my $htaccess_main = "RewriteEngine On\n"; + $htaccess_main .= $config->{htaccess_rewrite_rules}{$channel} // ''; my $versions = as_array($config->{channels}{$channel}); my $versions_str = join('+', @$versions); - my (%oses, %langs, %from_versions); - my $migrate_langs; + foreach my $os (sort keys %{$config->{build_targets}}) { + foreach my $bt (@{ as_array($config->{build_targets}{$os}) }) { + $htaccess_main .= "RewriteRule ^$bt/(.*) $os/$1 [last]\n"; + } + } + write_htdocs($channel, '.', '.htaccess', $htaccess_main); + + my (%oses, %from_versions); foreach my $version (@$versions) { - $migrate_langs = $config->{versions}{$version}{migrate_langs} - if $config->{versions}{$version}{migrate_langs}; my $files = $config->{versions}{$version}{files}; foreach my $os (keys %$files) { $oses{$os} = 1; - foreach my $lang (keys %{$files->{$os}}) { - $langs{$lang} = 1; - foreach my $from_version (keys %{$files->{$os}{$lang}{partial}}) { - $from_versions{$from_version} = 1; - } + foreach my $from_version (keys %{$files->{$os}{partial}}) { + $from_versions{$from_version} = 1; } } - $htaccess .= "RewriteRule ^[^/]+/$version/ no-update.xml $flags\n"; } - foreach my $os (sort keys %oses) { - foreach my $bt (build_targets_by_os($os)) { - foreach my $lang (sort keys %langs) { - foreach my $from_version (sort keys %from_versions) { - $htaccess .= "RewriteRule ^$bt/$from_version/$lang " - . "$from_version-$versions_str-$os-$lang.xml $flags\n"; - } - $htaccess .= "RewriteRule ^$bt/[^/]+/$lang " - . "$versions_str-$os-$lang.xml $flags\n"; - } - foreach my $lang (sort keys %$migrate_langs) { - $htaccess .= "RewriteRule ^$bt/[^/]+/$lang " - . "$versions_str-$os-$migrate_langs->{$lang}.xml $flags\n"; + foreach my $version (@$versions) { + my $files = $config->{versions}{$version}{files}; + foreach my $os (sort keys %oses) { + my $htaccess_os = "RewriteEngine On\n"; + $htaccess_os .= "RewriteRule ^$version/ no-update.xml [last]\n"; + foreach my $from_version (sort keys %from_versions) { + $htaccess_os .= "RewriteRule ^$from_version/ " + . "$from_version-$versions_str-$os.xml [last]\n"; } - $htaccess .= "RewriteRule ^$bt/ $versions_str-$os-ALL.xml $flags\n"; + $htaccess_os .= "RewriteRule ^[^/]+/ $versions_str-$os.xml [last]\n"; + write_htdocs($channel, $os, '.htaccess', $htaccess_os); } } - write_htdocs($channel, '.htaccess', $htaccess); }

sub write_downloads_json { @@ -501,11 +464,11 @@ sub write_downloads_json { tag => "$tag", downloads => get_version_downloads($config, $version), }; - write_htdocs($channel, 'downloads.json', + write_htdocs($channel, '.', 'downloads.json', JSON->new->utf8->canonical->encode($data)); my $pp_downloads = get_perplatform_downloads($config, $version, $tag); foreach my $os (keys %{$pp_downloads}) { - write_htdocs($channel, "download-$os.json", + write_htdocs($channel, '.', "download-$os.json", JSON->new->utf8->canonical->encode($pp_downloads->{$os})); } } @@ -567,28 +530,26 @@ sub check_update_responses_channel { my $channel_versions = as_array($config->{channels}{$channel}); my ($channel_version) = @$channel_versions; foreach my $build_target (build_targets_list()) { - foreach my $lang (qw(en-US de)) { - my $url = "$base_url/$channel/$build_target/1.0/$lang"; - my $dom = get_remote_xml($url); - if ($dom) { - my $version = check_get_version($dom); - log_step($url, 'version', $version eq $channel_version, - "expected: $channel_version received: $version"); - } - $url = "$base_url/$channel/$build_target/$channel_version/$lang"; + my $url = "$base_url/$channel/$build_target/1.0/ALL"; + my $dom = get_remote_xml($url); + if ($dom) { + my $version = check_get_version($dom); + log_step($url, 'version', $version eq $channel_version, + "expected: $channel_version received: $version"); + } + $url = "$base_url/$channel/$build_target/$channel_version/ALL"; + $dom = get_remote_xml($url); + log_step($url, 'no_update', check_no_update($dom)) if $dom; + my @inc = @{$config->{versions}{$channel_version}{incremental_from}} + if $config->{versions}{$channel_version}{incremental_from}; + foreach my $inc_from (@inc) { + my $url = "$base_url/$channel/$build_target/$inc_from/ALL"; $dom = get_remote_xml($url); - log_step($url, 'no_update', check_no_update($dom)) if $dom; - my @inc = @{$config->{versions}{$channel_version}{incremental_from}} - if $config->{versions}{$channel_version}{incremental_from}; - foreach my $inc_from (@inc) { - my $url = "$base_url/$channel/$build_target/$inc_from/$lang"; - $dom = get_remote_xml($url); - next unless $dom; - my $version = check_get_version($dom); - log_step($url, 'version', $version eq $channel_version, - "expected: $channel_version received: $version"); - log_step($url, 'has_incremental', check_has_incremental($dom)); - } + next unless $dom; + my $version = check_get_version($dom); + log_step($url, 'version', $version eq $channel_version, + "expected: $channel_version received: $version"); + log_step($url, 'has_incremental', check_has_incremental($dom)); } } } @@ -667,11 +628,9 @@ my %actions = ( exit_error "Wrong arguments" unless @ARGV == 1; my $channel = $ARGV[0]; exit_error "Unknown channel $channel" unless $config->{channels}{$channel}; - $htdocsfiles{$channel} = { '.' => 1, '..' => 1 }; write_responses($config, $channel); write_htaccess($config, $channel); write_downloads_json($config, $channel); - clean_htdocs($channel); }, gen_incrementals => sub { my ($config) = @_;

View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/compare/0...