- tbb-commits - lists.torproject.org

[tor-browser-bundle-testsuite/master] Bug 25101: Remove tools/prune-old-builds
by gk＠torproject.org 18 Dec '19

18 Dec '19

commit ea0942f9da2bdbaf3f140f3baaed1bb56ee9af69 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Dec 16 21:35:49 2019 +0100 Bug 25101: Remove tools/prune-old-builds The prune-old-builds script has been moved to tor-browser-build.git, in tools/ansible/roles/tbb-nightly-build/files/prune-old-builds. --- .gitmodules | 3 --- tools/prune-old-builds | 1 - 2 files changed, 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index a6d0dd9..067f871 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "packages/rbm"] path = packages/rbm url = https://github.com/boklm/rbm -[submodule "tools/prune-old-builds"] - path = tools/prune-old-builds - url = https://github.com/boklm/prune-old-builds diff --git a/tools/prune-old-builds b/tools/prune-old-builds deleted file mode 160000 index 8cb7522..0000000 --- a/tools/prune-old-builds +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8cb752270ef0c1d82a07196c5fc5024f042d7885

1 0

[tor-browser-bundle-testsuite/master] Bug 25101: Generate incremental mars for nightly builds
by gk＠torproject.org 18 Dec '19

18 Dec '19

commit 4261c714139e0d9896d17c92af54747f34bd9f9b Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Dec 10 21:13:10 2019 +0100 Bug 25101: Generate incremental mars for nightly builds --- TBBTestSuite/TestSuite/TorBrowserBuild.pm | 58 +++++++++++++++++++++++++++++++ tmpl/details_make_incrementals.html | 1 + 2 files changed, 59 insertions(+) diff --git a/TBBTestSuite/TestSuite/TorBrowserBuild.pm b/TBBTestSuite/TestSuite/TorBrowserBuild.pm index c52d5c4..d391bac 100644 --- a/TBBTestSuite/TestSuite/TorBrowserBuild.pm +++ b/TBBTestSuite/TestSuite/TorBrowserBuild.pm @@ -3,9 +3,11 @@ package TBBTestSuite::TestSuite::TorBrowserBuild; use strict; use parent 'TBBTestSuite::TestSuite::RBMBuild'; +use TBBTestSuite::Common qw(run_to_file); use TBBTestSuite::GitRepo; use File::Copy; use IO::CaptureOutput qw(capture_exec); +use Path::Tiny; sub description { 'Tor Browser Build'; @@ -15,6 +17,13 @@ sub type { 'tor-browser_build'; }; +sub test_types { + my $self = shift; + my $res = $self->SUPER::test_types(); + $res->{make_incrementals} = \&make_incrementals; + return $res; +} + sub set_tests { my ($testsuite) = @_; $testsuite->{tests} = [ @@ -32,6 +41,12 @@ sub set_tests { publish_dir => 'nightly-linux-x86_64', }, { + name => 'incrementals-nightly-linux-x86_64', + descr => 'create incrementals for tor-browser nightly linux-x86_64', + type => 'make_incrementals', + publish_dir => 'nightly-linux-x86_64', + }, + { name => 'nightly-linux-i686', descr => 'build tor-browser nightly linux-i686', type => 'rbm_build', @@ -44,6 +59,12 @@ sub set_tests { publish_dir => 'nightly-linux-i686', }, { + name => 'incrementals-nightly-linux-i686', + descr => 'create incrementals for tor-browser nightly linux-i686', + type => 'make_incrementals', + publish_dir => 'nightly-linux-i686', + }, + { name => 'nightly-windows-i686', descr => 'build tor-browser nightly windows-i686', type => 'rbm_build', @@ -56,6 +77,12 @@ sub set_tests { publish_dir => 'nightly-windows-i686', }, { + name => 'incrementals-nightly-windows-i686', + descr => 'create incrementals for tor-browser nightly windows-i686', + type => 'make_incrementals', + publish_dir => 'nightly-windows-i686', + }, + { name => 'nightly-windows-x86_64', descr => 'build tor-browser nightly windows-x86_64', type => 'rbm_build', @@ -68,6 +95,12 @@ sub set_tests { publish_dir => 'nightly-windows-x86_64', }, { + name => 'incrementals-nightly-windows-x86_64', + descr => 'create incrementals for tor-browser nightly windows-x86_64', + type => 'make_incrementals', + publish_dir => 'nightly-windows-x86_64', + }, + { name => 'nightly-osx-x86_64', descr => 'build tor-browser nightly osx-x86_64', type => 'rbm_build', @@ -80,6 +113,12 @@ sub set_tests { publish_dir => 'nightly-osx-x86_64', }, { + name => 'incrementals-nightly-osx-x86_64', + descr => 'create incrementals for tor-browser nightly osx-x86_64', + type => 'make_incrementals', + publish_dir => 'nightly-osx-x86_64', + }, + { name => 'nightly-android-armv7', descr => 'build tor-browser nightly android-armv7', type => 'rbm_build', @@ -130,6 +169,25 @@ sub set_tests { ]; } +sub make_incrementals { + my ($testsuite, $test) = @_; + $test->{results}{success} = 0; + mkdir 'nightly' unless -d 'nightly'; + # Clean the nightly directory + foreach my $subdir (path('nightly')->children) { + unlink $subdir if -l $subdir; + } + foreach my $builddir (path($testsuite->{publish_dir} . '/..')->children) { + if (-f "$builddir/$test->{publish_dir}/sha256sums-unsigned-build.txt") { + symlink("$builddir/$test->{publish_dir}", 'nightly/' . $builddir->basename); + } + } + my @cmd = ('make', 'incrementals-nightly'); + run_to_file("$testsuite->{'results-dir'}/$test->{name}.build.txt", @cmd) + or return; + $test->{results}{success} = 1; +} + sub pre_tests { my ($tbbinfos) = @_; my $gr = TBBTestSuite::GitRepo->new({ diff --git a/tmpl/details_make_incrementals.html b/tmpl/details_make_incrementals.html new file mode 120000 index 0000000..d9bb49c --- /dev/null +++ b/tmpl/details_make_incrementals.html @@ -0,0 +1 @@ +details_rbm_build.html \ No newline at end of file

1 0

[tor-browser-bundle-testsuite/master] Bug 25101: Set TORBROWSER_NIGHTLY_VERSION env variable
by gk＠torproject.org 18 Dec '19

18 Dec '19

commit 679ee355425b0cb4bf77626a8b16755c1c43db60 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Dec 10 20:13:18 2019 +0100 Bug 25101: Set TORBROWSER_NIGHTLY_VERSION env variable --- TBBTestSuite/TestSuite/TorBrowserBuild.pm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TBBTestSuite/TestSuite/TorBrowserBuild.pm b/TBBTestSuite/TestSuite/TorBrowserBuild.pm index 1389092..c52d5c4 100644 --- a/TBBTestSuite/TestSuite/TorBrowserBuild.pm +++ b/TBBTestSuite/TestSuite/TorBrowserBuild.pm @@ -149,6 +149,9 @@ sub pre_tests { return; } } + if ($tbbinfos->{tbb_version}) { + $ENV{TORBROWSER_NIGHTLY_VERSION} = $tbbinfos->{tbb_version}; + } } 1;

1 0

[tor-browser-build/master] Bug 25101: Add license and usage information to the prune-old-builds script
by gk＠torproject.org 18 Dec '19

18 Dec '19

commit 434a7d9e9ad8785c92a6dae014de990a9595c77f Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Dec 16 21:00:31 2019 +0100 Bug 25101: Add license and usage information to the prune-old-builds script --- .../roles/tbb-nightly-build/files/prune-old-builds | 57 ++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tools/ansible/roles/tbb-nightly-build/files/prune-old-builds b/tools/ansible/roles/tbb-nightly-build/files/prune-old-builds index 5d54229..1b4e30b 100755 --- a/tools/ansible/roles/tbb-nightly-build/files/prune-old-builds +++ b/tools/ansible/roles/tbb-nightly-build/files/prune-old-builds @@ -1,4 +1,61 @@ #!/usr/bin/perl -w + +# Copyright (c) 2019, The Tor Project, Inc. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# +# * Neither the names of the copyright owners nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +# 'prune-old-builds' is a script to prune old builds. +# +# +# Usage: +# $ ./prune-old-builds [options] <directory> +# +# +# Available options: +# +# --dry-run +# Don't delete anything, but say what would be deleted. +# +# --prefix <prefix> +# Prefix of the directories to be removed. Default is 'tbb-nightly-'. +# +# --days <n> +# Number of days that we should keep. Default is 6. +# +# --weeks <n> +# Number of monday builds that we should keep. Default is 3. +# +# --months <n> +# Number of 1st day of the month builds that we should keep. +# Default is 3. + use strict; use Getopt::Long; use DateTime;

1 0

[tor-browser-build/master] Bug 25101: Generate incremental mars for nightly builds
by gk＠torproject.org 18 Dec '19

18 Dec '19

commit 608fbb6f06237c0e0786b650433ca0c4cc203f72 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Wed Dec 11 00:15:33 2019 +0100 Bug 25101: Generate incremental mars for nightly builds Update testsuite_git_commit so that incremental mars are generated as part of our nightly builds. --- tools/ansible/roles/tbb-nightly-build/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ansible/roles/tbb-nightly-build/defaults/main.yml b/tools/ansible/roles/tbb-nightly-build/defaults/main.yml index f7f63f5..0a9e94a 100644 --- a/tools/ansible/roles/tbb-nightly-build/defaults/main.yml +++ b/tools/ansible/roles/tbb-nightly-build/defaults/main.yml @@ -5,7 +5,7 @@ nightly_build_cron_minute: 20 nightly_build_keep_builds: 3 testsuite_dir: "/home/{{ nightly_build_user }}/tbb-testsuite" testsuite_git_url: https://git.torproject.org/tor-browser-bundle-testsuite.git -testsuite_git_commit: d5e23151bf4ff48146c0c33355fe70def5e4d3f2 +testsuite_git_commit: ea0942f9da2bdbaf3f140f3baaed1bb56ee9af69 nightly_build_wwwdir: "/home/{{ nightly_build_user }}/www" nightly_build_nginx_enable: true nightly_build_nginx_listen: 127.0.0.1:80

1 0

[tor-browser-build/master] Bug 25101: Keep 3 nightly builds
by gk＠torproject.org 18 Dec '19

18 Dec '19

commit 8489c017a01460d3e88e06dfc7a1edd3f02b71f8 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Dec 16 21:09:20 2019 +0100 Bug 25101: Keep 3 nightly builds In rbm.conf we currently have var/max_torbrowser_incremental_from set to 2. In order to be able to generate incrementals from the previous two versions, we need to keep 3 builds. --- tools/ansible/roles/tbb-nightly-build/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ansible/roles/tbb-nightly-build/defaults/main.yml b/tools/ansible/roles/tbb-nightly-build/defaults/main.yml index 34d3929..f7f63f5 100644 --- a/tools/ansible/roles/tbb-nightly-build/defaults/main.yml +++ b/tools/ansible/roles/tbb-nightly-build/defaults/main.yml @@ -2,7 +2,7 @@ nightly_build_user: tbb-nightly nightly_build_cron_hour: 2 nightly_build_cron_minute: 20 -nightly_build_keep_builds: 2 +nightly_build_keep_builds: 3 testsuite_dir: "/home/{{ nightly_build_user }}/tbb-testsuite" testsuite_git_url: https://git.torproject.org/tor-browser-bundle-testsuite.git testsuite_git_commit: d5e23151bf4ff48146c0c33355fe70def5e4d3f2

1 0

[tor-browser-build/master] Bug 25101: Change nightly builds directory to be the version
by gk＠torproject.org 18 Dec '19

18 Dec '19

commit 2d5508bfea59900b8d5952d895ac0e9610c70503 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Wed Dec 11 00:10:56 2019 +0100 Bug 25101: Change nightly builds directory to be the version To make generation of incremental mars easier, we use the version as the directory name (instead of just the date). We also define the tbb_version option, which is used to set the TORBROWSER_NIGHTLY_VERSION environment variable. --- .../roles/tbb-nightly-build/files/prune-old-builds | 17 +++++++++++------ .../roles/tbb-nightly-build/templates/start-tbb-nightly | 2 +- .../roles/tbb-nightly-build/templates/testsuite-config | 6 ++++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/tools/ansible/roles/tbb-nightly-build/files/prune-old-builds b/tools/ansible/roles/tbb-nightly-build/files/prune-old-builds index 1b4e30b..852a9da 100755 --- a/tools/ansible/roles/tbb-nightly-build/files/prune-old-builds +++ b/tools/ansible/roles/tbb-nightly-build/files/prune-old-builds @@ -44,7 +44,11 @@ # Don't delete anything, but say what would be deleted. # # --prefix <prefix> -# Prefix of the directories to be removed. Default is 'tbb-nightly-'. +# Prefix of the directories to be removed. Default is 'tbb-nightly.'. +# +# --separator <c> +# Separator character to separate the year, month, day in the directory +# names. Default is '.'. # # --days <n> # Number of days that we should keep. Default is 6. @@ -66,7 +70,8 @@ my %options = ( days => 6, weeks => 3, months => 3, - prefix => 'tbb-nightly-', + prefix => 'tbb-nightly.', + separator => '.', ); sub keep_builds { @@ -79,7 +84,7 @@ sub keep_builds { my $n = $options{days}; my $dt = DateTime->now; while ($n) { - $res{ $options{prefix} . $dt->ymd } = 1; + $res{ $options{prefix} . $dt->ymd($options{separator}) } = 1; $dt = $dt - $day; $n--; } @@ -89,7 +94,7 @@ sub keep_builds { $dt = $dt - $day; } while ($w) { - $res{ $options{prefix} . $dt->ymd } = 1; + $res{ $options{prefix} . $dt->ymd($options{separator}) } = 1; $dt = $dt - $week; $w--; } @@ -100,7 +105,7 @@ sub keep_builds { $dt = $dt - $day; } while ($m) { - $res{ $options{prefix} . $dt->ymd } = 1; + $res{ $options{prefix} . $dt->ymd($options{separator}) } = 1; $dt = $dt - $month; $m--; } @@ -113,7 +118,7 @@ sub clean_directory { my $k = keep_builds; chdir $directory || die "Error entering $directory"; foreach my $file (glob "$options{prefix}*") { - next unless $file =~ m/^$options{prefix}\d{4}-\d{2}-\d{2}$/; + next unless $file =~ m/^$options{prefix}\d{4}$options{separator}\d{2}$options{separator}\d{2}$/; next if $k->{$file}; if ($options{'dry-run'}) { print "Would remove $file\n"; diff --git a/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly b/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly index c93bc94..fafc1d9 100644 --- a/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly +++ b/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly @@ -2,4 +2,4 @@ cd {{ testsuite_dir }} export RBM_NO_DEBUG=1 ./tbb-testsuite --config=tbb-nightly "$@" -/home/{{ nightly_build_user }}/prune-old-builds --prefix '' --days {{ nightly_build_keep_builds }} ./tor-browser-builds +/home/{{ nightly_build_user }}/prune-old-builds --days {{ nightly_build_keep_builds }} ./tor-browser-builds diff --git a/tools/ansible/roles/tbb-nightly-build/templates/testsuite-config b/tools/ansible/roles/tbb-nightly-build/templates/testsuite-config index c07c52c..90b08ac 100644 --- a/tools/ansible/roles/tbb-nightly-build/templates/testsuite-config +++ b/tools/ansible/roles/tbb-nightly-build/templates/testsuite-config @@ -5,6 +5,7 @@ use DateTime; use TBBTestSuite::TestSuite::TorBrowserBuild; my $date = DateTime->now->ymd; +my $tbb_version = 'tbb-nightly.' . DateTime->now->ymd('.'); my $name = "tor-browser-$date"; if (-d "$options->{'reports-dir'}/r/$name") { @@ -13,8 +14,9 @@ if (-d "$options->{'reports-dir'}/r/$name") { } my $testsuite = TBBTestSuite::TestSuite::TorBrowserBuild->new({ - publish_dir => "$FindBin::Bin/tor-browser-builds/$date", - publish_url => "{{ nightly_build_url }}/tor-browser-builds/$date", + tbb_version => $tbb_version, + publish_dir => "$FindBin::Bin/tor-browser-builds/$tbb_version", + publish_url => "{{ nightly_build_url }}/tor-browser-builds/$tbb_version", rbm_local_conf => "$FindBin::Bin/rbm-config/tbb-nightly.rbm.local.conf", make_clean => 1, });

1 0

[tor-browser-build/master] Bug 25101: Add the prune-old-builds script
by gk＠torproject.org 18 Dec '19

18 Dec '19

commit 073dfa5f7402b54cfe7772d71a1c931552908463 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Dec 16 20:52:54 2019 +0100 Bug 25101: Add the prune-old-builds script Previously we were using the prune-old-builds script which was located in the testsuite repository in tools/prune-old-builds, as a git submodule. As this script is not used anywhere other than here, it doesn't make much sense to keep it as a separate git submodule. To simplify modification of this script, we move it here. --- .../roles/tbb-nightly-build/files/prune-old-builds | 74 ++++++++++++++++++++++ .../ansible/roles/tbb-nightly-build/tasks/main.yml | 7 ++ .../tbb-nightly-build/templates/start-tbb-nightly | 2 +- 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/tools/ansible/roles/tbb-nightly-build/files/prune-old-builds b/tools/ansible/roles/tbb-nightly-build/files/prune-old-builds new file mode 100755 index 0000000..5d54229 --- /dev/null +++ b/tools/ansible/roles/tbb-nightly-build/files/prune-old-builds @@ -0,0 +1,74 @@ +#!/usr/bin/perl -w +use strict; +use Getopt::Long; +use DateTime; +use DateTime::Duration; +use File::Path qw(remove_tree); + +my %options = ( + days => 6, + weeks => 3, + months => 3, + prefix => 'tbb-nightly-', +); + +sub keep_builds { + my %res; + + my $day = DateTime::Duration->new(days => 1); + my $week = DateTime::Duration->new(weeks => 1); + my $month = DateTime::Duration->new(months => 1); + + my $n = $options{days}; + my $dt = DateTime->now; + while ($n) { + $res{ $options{prefix} . $dt->ymd } = 1; + $dt = $dt - $day; + $n--; + } + + my $w = $options{weeks}; + while ($dt->day_of_week != 1) { + $dt = $dt - $day; + } + while ($w) { + $res{ $options{prefix} . $dt->ymd } = 1; + $dt = $dt - $week; + $w--; + } + + my $m = $options{months}; + $dt = DateTime->now; + while ($dt->day != 1) { + $dt = $dt - $day; + } + while ($m) { + $res{ $options{prefix} . $dt->ymd } = 1; + $dt = $dt - $month; + $m--; + } + + return \%res; +} + +sub clean_directory { + my ($directory) = @_; + my $k = keep_builds; + chdir $directory || die "Error entering $directory"; + foreach my $file (glob "$options{prefix}*") { + next unless $file =~ m/^$options{prefix}\d{4}-\d{2}-\d{2}$/; + next if $k->{$file}; + if ($options{'dry-run'}) { + print "Would remove $file\n"; + } else { + remove_tree($file); + } + } +} + +my @opts = qw(days=i weeks=i months=i prefix=s dry-run!); +Getopt::Long::GetOptions(\%options, @opts); +die "Missing argument: directory to clean" unless @ARGV; +foreach my $dir (@ARGV) { + clean_directory($dir); +} diff --git a/tools/ansible/roles/tbb-nightly-build/tasks/main.yml b/tools/ansible/roles/tbb-nightly-build/tasks/main.yml index 64d31a9..2c795b8 100644 --- a/tools/ansible/roles/tbb-nightly-build/tasks/main.yml +++ b/tools/ansible/roles/tbb-nightly-build/tasks/main.yml @@ -39,6 +39,13 @@ mode: 0644 owner: "{{ nightly_build_user }}" +- name: prune-old-builds + copy: + src: prune-old-builds + dest: "/home/{{ nightly_build_user }}/prune-old-builds" + mode: 0755 + owner: "{{ nightly_build_user }}" + - name: add start-tbb-nightly script template: src: start-tbb-nightly diff --git a/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly b/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly index 3c1847d..c93bc94 100644 --- a/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly +++ b/tools/ansible/roles/tbb-nightly-build/templates/start-tbb-nightly @@ -2,4 +2,4 @@ cd {{ testsuite_dir }} export RBM_NO_DEBUG=1 ./tbb-testsuite --config=tbb-nightly "$@" -./tools/prune-old-builds/prune-old-builds --prefix '' --days {{ nightly_build_keep_builds }} ./tor-browser-builds +/home/{{ nightly_build_user }}/prune-old-builds --prefix '' --days {{ nightly_build_keep_builds }} ./tor-browser-builds

1 0

[tor-browser-build/master] Merge remote-tracking branch 'boklm/bug_25101_v12'
by gk＠torproject.org 18 Dec '19

18 Dec '19

commit 16e629f680bbbe07a408a3022c3b819acd833bd2 Merge: b61a049 608fbb6 Author: Georg Koppen <gk(a)torproject.org> Date: Wed Dec 18 13:58:26 2019 +0000 Merge remote-tracking branch 'boklm/bug_25101_v12' .../roles/tbb-nightly-build/defaults/main.yml | 4 +- .../roles/tbb-nightly-build/files/prune-old-builds | 136 +++++++++++++++++++++ .../ansible/roles/tbb-nightly-build/tasks/main.yml | 7 ++ .../tbb-nightly-build/templates/start-tbb-nightly | 2 +- .../tbb-nightly-build/templates/testsuite-config | 6 +- 5 files changed, 150 insertions(+), 5 deletions(-)

1 0

[tor-browser-build/master] Bug 32750: Sign nightly sha256sums files with gpg
by gk＠torproject.org 17 Dec '19

17 Dec '19

commit 32ce9d2bf8bbd09017533175ffebf0b3599eb617 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Fri Dec 13 17:52:17 2019 +0100 Bug 32750: Sign nightly sha256sums files with gpg --- tools/ansible/group_vars/boklm-tbb-nightly/tbb-nightly-build.yml | 1 + tools/ansible/roles/tbb-nightly-build/templates/rbm.local.conf | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/ansible/group_vars/boklm-tbb-nightly/tbb-nightly-build.yml b/tools/ansible/group_vars/boklm-tbb-nightly/tbb-nightly-build.yml index 0d19cff..8e4bc05 100644 --- a/tools/ansible/group_vars/boklm-tbb-nightly/tbb-nightly-build.yml +++ b/tools/ansible/group_vars/boklm-tbb-nightly/tbb-nightly-build.yml @@ -3,3 +3,4 @@ nightly_build_hostname: f4amtbsowhix7rrf.onion nightly_build_url: 'http://{{ nightly_build_hostname }}' nightly_build_email_from: "'Tor Browser Nightly Builds (boklm) <boklm-tbb-nightly(a)riseup.net>'," nightly_build_email_to: "[ 'boklm(a)torproject.org', 'sysrqb(a)torproject.org', ]," +nightly_build_sign_build: 1 diff --git a/tools/ansible/roles/tbb-nightly-build/templates/rbm.local.conf b/tools/ansible/roles/tbb-nightly-build/templates/rbm.local.conf index f83d8f3..4380480 100644 --- a/tools/ansible/roles/tbb-nightly-build/templates/rbm.local.conf +++ b/tools/ansible/roles/tbb-nightly-build/templates/rbm.local.conf @@ -45,12 +45,14 @@ var: local_conf: 1 ### The var/sign_build option defines if you want to sign the - ### sha256sums-unsigned-build.txt file with gpg. - #sign_build: 1 + ### sha256sums-unsigned-build.txt and + ### sha256sums-unsigned-build.incrementals.txt files with gpg. + {% if nightly_build_sign_build is defined %}sign_build: {{ nightly_build_sign_build }}{% endif %} + ### The var/sign_build_gpg_opts option can be used to define some gpg ### options to select the key to use to sign the sha256sums-unsigned-build.txt - ### file. + ### and sha256sums-unsigned-build.incrementals.txt files. #sign_build_gpg_opts: '--local-user XXXXXXXX' ### The clean configuration is used by the cleaning script to find the

1 0

[tor-browser-build/master] Merge remote-tracking branch 'boklm/bug_32750_v4'
by gk＠torproject.org 17 Dec '19

17 Dec '19

commit b61a0497f0258ffebb2586c41267462e16a69cf0 Merge: 38624f8 32ce9d2 Author: Georg Koppen <gk(a)torproject.org> Date: Tue Dec 17 20:09:03 2019 +0000 Merge remote-tracking branch 'boklm/bug_32750_v4' tools/ansible/group_vars/boklm-tbb-nightly/tbb-nightly-build.yml | 1 + tools/ansible/roles/tbb-nightly-build/templates/rbm.local.conf | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-)

1 0

[tor-browser-build/master] Fold in stable changelog for 9.0.3
by sysrqb＠torproject.org 17 Dec '19

17 Dec '19

commit 38624f870024a93c907c15e6a16838faedda8ead Author: Matthew Finkel <sysrqb(a)torproject.org> Date: Tue Dec 17 18:53:54 2019 +0000 Fold in stable changelog for 9.0.3 --- projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt b/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt index c4866d4..fd891e0 100644 --- a/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt +++ b/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt @@ -1,3 +1,24 @@ +Tor Browser 9.0.2 -- December 3 2019 + * All Platforms + * Update Firefox to 68.3.0esr + * Bump NoScript to 11.0.9 + * Bug 32362: NoScript TRUSTED setting doesn't work + * Bug 32429: Issues with about:blank and NoScript on .onion sites + * Bump HTTPS Everywhere to 2019.11.7 + * Bug 27268: Preferences clean-up in Torbutton code + * Translations update + * Windows + OS X + Linux + * Bug 32125: Fix circuit display for bridge without a fingerprint + * Bug 32250: Backport enhanced letterboxing support (bug 1546832 and 1556017) + * Windows + * Bug 31989: Backport backout of old mingw-gcc patch + * Bug 32616: Disable GetSecureOutputDirectoryPath() functionality + * Android + * Bug 32365: Localization is broken in Tor Browser 9 on Android + * Build System + * All Platforms + * Bug 32413: Bump Go version to 1.12.13 + Tor Browser 9.5a3 -- December 3 2019 * All Platforms * Update Firefox to 68.3.0esr

1 0

[tor-browser/tor-browser-68.3.0esr-9.0-1] Bug #32405: Crash immediately after bootstrap on Android
by sysrqb＠torproject.org 17 Dec '19

17 Dec '19

commit 4eb56ec0936e03dcc5d7e83322961c986065dc5d Author: sisbell <shane.isbell(a)gmail.com> Date: Wed Nov 27 21:08:02 2019 -0800 Bug #32405: Crash immediately after bootstrap on Android --- mobile/android/base/java/org/mozilla/gecko/BrowserApp.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mobile/android/base/java/org/mozilla/gecko/BrowserApp.java b/mobile/android/base/java/org/mozilla/gecko/BrowserApp.java index 36f0b0dbb66c..38dac85ef3bc 100644 --- a/mobile/android/base/java/org/mozilla/gecko/BrowserApp.java +++ b/mobile/android/base/java/org/mozilla/gecko/BrowserApp.java @@ -3032,7 +3032,10 @@ public class BrowserApp extends GeckoApp // When the content loaded in the background (such as about:tor), // it was loaded while mBrowserChrome was GONE. We should refresh the // height now so the page is rendered correctly. - Tabs.getInstance().getSelectedTab().doReload(true); + final Tab currentTab = Tabs.getInstance().getSelectedTab(); + if (currentTab != null) { + currentTab.doReload(true); + } // If we finished, then Tor bootstrapped 100% mTorNeedsStart = false;

1 0

[tor-browser-build/master] Bug 32751: Allow gpg to find its keys in hash_incrementals
by gk＠torproject.org 17 Dec '19

17 Dec '19

commit b8ade5ac89f85dc122894a458b22f47802530a63 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Dec 16 21:29:34 2019 +0100 Bug 32751: Allow gpg to find its keys in hash_incrementals var/set_default_env is changing $HOME, preventing gpg to find the keys in its default configuration directory. We avoid that by reseting $HOME to its previous value before running gpg. --- projects/release/hash_incrementals | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/release/hash_incrementals b/projects/release/hash_incrementals index cf31bfd..4ab0f4d 100644 --- a/projects/release/hash_incrementals +++ b/projects/release/hash_incrementals @@ -1,4 +1,8 @@ #!/bin/bash +[% IF c("var/sign_build") -%] + # var/set_default_env will reset HOME. We need to restore it to use gpg. + old_HOME="$HOME" +[% END -%] [% c("var/set_default_env") -%] [% IF c("var/nightly") -%] cd [% shell_quote(path(dest_dir)) %]/[% c("version") %] @@ -7,5 +11,6 @@ [% END -%] sha256sum `ls -1 | grep '\.incremental\.mar$' | sort` > sha256sums-[% c("var/signed_status") %]-build.incrementals.txt [% IF c("var/sign_build") -%] + export HOME="$old_HOME" gpg -abs [% c("var/sign_build_gpg_opts") %] sha256sums-[% c("var/signed_status") %]-build.incrementals.txt [% END -%]

1 0

[tor-browser-build/maint-9.0] Bug 32547: Add new default bridge at UMN.
by gk＠torproject.org 17 Dec '19

17 Dec '19

commit 095dfed8762cd1756b3fc50ed4aa6e8f2d0dc849 Author: Philipp Winter <phw(a)nymity.ch> Date: Tue Dec 10 13:33:29 2019 -0800 Bug 32547: Add new default bridge at UMN. Researchers at the University of Minnesota generously agreed to running a default bridge. This fixes <https://bugs.torproject.org/32547>. --- projects/tor-browser/Bundle-Data/PTConfigs/bridge_prefs.js | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/tor-browser/Bundle-Data/PTConfigs/bridge_prefs.js b/projects/tor-browser/Bundle-Data/PTConfigs/bridge_prefs.js index 0004046..da3f2cd 100644 --- a/projects/tor-browser/Bundle-Data/PTConfigs/bridge_prefs.js +++ b/projects/tor-browser/Bundle-Data/PTConfigs/bridge_prefs.js @@ -15,6 +15,7 @@ pref("extensions.torlauncher.default_bridge.obfs4.10", "obfs4 193.11.166.194:270 pref("extensions.torlauncher.default_bridge.obfs4.11", "obfs4 193.11.166.194:27020 86AC7B8D430DAC4117E9F42C9EAED18133863AAF cert=0LDeJH4JzMDtkJJrFphJCiPqKx7loozKN7VNfuukMGfHO0Z8OGdzHVkhVAOfo1mUdv9cMg iat-mode=0"); pref("extensions.torlauncher.default_bridge.obfs4.12", "obfs4 193.11.166.194:27025 1AE2C08904527FEA90C4C4F8C1083EA59FBC6FAF cert=ItvYZzW5tn6v3G4UnQa6Qz04Npro6e81AP70YujmK/KXwDFPTs3aHXcHp4n8Vt6w/bv8cA iat-mode=0"); pref("extensions.torlauncher.default_bridge.obfs4.13", "obfs4 209.148.46.65:443 74FAD13168806246602538555B5521A0383A1875 cert=ssH+9rP8dG2NLDN2XuFw63hIO/9MNNinLmxQDpVa+7kTOa9/m+tGWT1SmSYpQ9uTBGa6Hw iat-mode=0"); +pref("extensions.torlauncher.default_bridge.obfs4.14", "obfs4 146.57.248.225:22 10A6CD36A537FCE513A322361547444B393989F0 cert=K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw iat-mode=0"); pref("extensions.torlauncher.default_bridge.meek-azure.1", "meek_lite 0.0.2.0:2 97700DFE9F483596DDA6264C4D7DF7641E1E39CE url=https://meek.azureedge.net/ front=ajax.aspnetcdn.com");

1 0

[tor-browser-build/maint-9.0] Pick up new obfs4 bridge for mobile (#32547)
by gk＠torproject.org 17 Dec '19

17 Dec '19

commit d6bd267e1747a14d2a326a36099bf8e79bbc6a91 Author: Georg Koppen <gk(a)torproject.org> Date: Tue Dec 17 08:29:49 2019 +0000 Pick up new obfs4 bridge for mobile (#32547) --- projects/tor-android-service/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/tor-android-service/config b/projects/tor-android-service/config index 669733b..b65c657 100644 --- a/projects/tor-android-service/config +++ b/projects/tor-android-service/config @@ -1,7 +1,7 @@ # vim: filetype=yaml sw=2 version: '[% c("abbrev") %]' filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %]' -git_hash: 1a2f8b9d7e4d10b2057eee0e68d5aacc5d26f8ae +git_hash: 4a81e5806f36e7d0bf9492437d824599ecf4ff43 git_url: https://git.torproject.org/tor-android-service.git git_submodule: 1

1 0

[tor-browser-build/master] Bug 32547: Add new default bridge at UMN.
by gk＠torproject.org 17 Dec '19

17 Dec '19

commit 00f4ea3a18f1cbce16f256ca2b7c1c90ad3b1daf Author: Philipp Winter <phw(a)nymity.ch> Date: Tue Dec 10 13:33:29 2019 -0800 Bug 32547: Add new default bridge at UMN. Researchers at the University of Minnesota generously agreed to running a default bridge. This fixes <https://bugs.torproject.org/32547>. --- projects/tor-browser/Bundle-Data/PTConfigs/bridge_prefs.js | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/tor-browser/Bundle-Data/PTConfigs/bridge_prefs.js b/projects/tor-browser/Bundle-Data/PTConfigs/bridge_prefs.js index 0004046..da3f2cd 100644 --- a/projects/tor-browser/Bundle-Data/PTConfigs/bridge_prefs.js +++ b/projects/tor-browser/Bundle-Data/PTConfigs/bridge_prefs.js @@ -15,6 +15,7 @@ pref("extensions.torlauncher.default_bridge.obfs4.10", "obfs4 193.11.166.194:270 pref("extensions.torlauncher.default_bridge.obfs4.11", "obfs4 193.11.166.194:27020 86AC7B8D430DAC4117E9F42C9EAED18133863AAF cert=0LDeJH4JzMDtkJJrFphJCiPqKx7loozKN7VNfuukMGfHO0Z8OGdzHVkhVAOfo1mUdv9cMg iat-mode=0"); pref("extensions.torlauncher.default_bridge.obfs4.12", "obfs4 193.11.166.194:27025 1AE2C08904527FEA90C4C4F8C1083EA59FBC6FAF cert=ItvYZzW5tn6v3G4UnQa6Qz04Npro6e81AP70YujmK/KXwDFPTs3aHXcHp4n8Vt6w/bv8cA iat-mode=0"); pref("extensions.torlauncher.default_bridge.obfs4.13", "obfs4 209.148.46.65:443 74FAD13168806246602538555B5521A0383A1875 cert=ssH+9rP8dG2NLDN2XuFw63hIO/9MNNinLmxQDpVa+7kTOa9/m+tGWT1SmSYpQ9uTBGa6Hw iat-mode=0"); +pref("extensions.torlauncher.default_bridge.obfs4.14", "obfs4 146.57.248.225:22 10A6CD36A537FCE513A322361547444B393989F0 cert=K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw iat-mode=0"); pref("extensions.torlauncher.default_bridge.meek-azure.1", "meek_lite 0.0.2.0:2 97700DFE9F483596DDA6264C4D7DF7641E1E39CE url=https://meek.azureedge.net/ front=ajax.aspnetcdn.com");

1 0

[tor-browser-build/master] Pick up new obfs4 bridge for mobile (#32547)
by gk＠torproject.org 17 Dec '19

17 Dec '19

commit 8f76d71e2a0923d3ca317f82ab5eeec124706e22 Author: Georg Koppen <gk(a)torproject.org> Date: Tue Dec 17 08:29:49 2019 +0000 Pick up new obfs4 bridge for mobile (#32547) --- projects/tor-android-service/config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/tor-android-service/config b/projects/tor-android-service/config index 9d7bc84..d7a5882 100644 --- a/projects/tor-android-service/config +++ b/projects/tor-android-service/config @@ -1,7 +1,7 @@ # vim: filetype=yaml sw=2 version: '[% c("abbrev") %]' filename: '[% project %]-[% c("version") %]-[% c("var/build_id") %]' -git_hash: 1a2f8b9d7e4d10b2057eee0e68d5aacc5d26f8ae +git_hash: 4a81e5806f36e7d0bf9492437d824599ecf4ff43 git_url: https://git.torproject.org/tor-android-service.git git_submodule: 1

1 0

[tor-android-service/master] Bug 32547: Add new default bridge at UMN.
by gk＠torproject.org 17 Dec '19

17 Dec '19

commit 4a81e5806f36e7d0bf9492437d824599ecf4ff43 Author: Philipp Winter <phw(a)nymity.ch> Date: Tue Dec 10 13:37:20 2019 -0800 Bug 32547: Add new default bridge at UMN. Researchers at the University of Minnesota generously agreed to running a default bridge. This fixes <https://bugs.torproject.org/32547>. --- service/src/main/assets/common/bridges.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/service/src/main/assets/common/bridges.txt b/service/src/main/assets/common/bridges.txt index 36ed076..adc6b8c 100644 --- a/service/src/main/assets/common/bridges.txt +++ b/service/src/main/assets/common/bridges.txt @@ -10,4 +10,5 @@ obfs4 193.11.166.194:27015 2D82C2E354D531A68469ADF7F878FA6060C6BACA cert=4TLQPJr obfs4 193.11.166.194:27020 86AC7B8D430DAC4117E9F42C9EAED18133863AAF cert=0LDeJH4JzMDtkJJrFphJCiPqKx7loozKN7VNfuukMGfHO0Z8OGdzHVkhVAOfo1mUdv9cMg iat-mode=0 obfs4 193.11.166.194:27025 1AE2C08904527FEA90C4C4F8C1083EA59FBC6FAF cert=ItvYZzW5tn6v3G4UnQa6Qz04Npro6e81AP70YujmK/KXwDFPTs3aHXcHp4n8Vt6w/bv8cA iat-mode=0 obfs4 209.148.46.65:443 74FAD13168806246602538555B5521A0383A1875 cert=ssH+9rP8dG2NLDN2XuFw63hIO/9MNNinLmxQDpVa+7kTOa9/m+tGWT1SmSYpQ9uTBGa6Hw iat-mode=0 +obfs4 146.57.248.225:22 10A6CD36A537FCE513A322361547444B393989F0 cert=K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw iat-mode=0 meek_lite 0.0.2.0:2 97700DFE9F483596DDA6264C4D7DF7641E1E39CE url=https://meek.azureedge.net/ front=ajax.aspnetcdn.com

1 0

[tor-browser-build/master] Revert "Bug 31130: Use Debian 10 for our Android container images"
by boklm＠torproject.org 16 Dec '19

16 Dec '19

commit 2d7921cd17c52fe2a16d0a3dc19235d0d48fa575 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Dec 16 14:47:58 2019 +0100 Revert "Bug 31130: Use Debian 10 for our Android container images" This reverts commit 26be3f16054b7d9b1c4c6cf0332d90f92511fc3e. This commit causes the android builds to fail. See #32761. --- projects/debootstrap-image/config | 17 ++++++++--------- projects/firefox/build | 1 - projects/firefox/config | 5 +++++ projects/https-everywhere/config | 1 - projects/tor-android-service/config | 6 ++++++ projects/tor-browser/build.android | 2 +- projects/tor-browser/config | 1 + projects/tor-onion-proxy-library/config | 6 ++++++ rbm.conf | 13 ++----------- 9 files changed, 29 insertions(+), 23 deletions(-) diff --git a/projects/debootstrap-image/config b/projects/debootstrap-image/config index f7b9e57..a50cbf1 100644 --- a/projects/debootstrap-image/config +++ b/projects/debootstrap-image/config @@ -4,7 +4,7 @@ version: 2 pkg_type: build var: - ubuntu_version: 19.10 + ubuntu_version: 18.04.1 container: use_container: 1 @@ -15,6 +15,8 @@ pre: | #!/bin/sh set -e export DEBIAN_FRONTEND=noninteractive + # Bug 29158: install fixed packages for apt vulnerability (CVE-2019-3462) + dpkg -i ./apt_1.6.6ubuntu0.1_amd64.deb ./libapt-pkg5.0_1.6.6ubuntu0.1_amd64.deb apt-get update -y -q apt-get install -y -q debian-archive-keyring ubuntu-keyring debootstrap debootstrap --arch=[% c("var/container/arch") %] [% c("var/container/debootstrap_opt") %] [% c("var/container/suite") %] base-image [% c("var/container/debootstrap_mirror") %] @@ -63,17 +65,14 @@ targets: suite: stretch arch: amd64 - buster-amd64: - var: - minimal_apt_version: 1.8.2 - container: - suite: buster - arch: amd64 - input_files: - URL: 'http://cdimage.ubuntu.com/ubuntu-base/releases/[% c("var/ubuntu_version") %]/release/ubuntu-base-[% c("var/ubuntu_version") %]-base-amd64.tar.gz' filename: 'container-image_ubuntu-base-[% c("var/ubuntu_version") %]-base-amd64.tar.gz' - sha256sum: eedcb1dc0ccc86b59eb1f89960c322a2ba3ed3e0323a20a1da8bcc0e6f100f4f + sha256sum: ed76e649f65548a80b361b68011085ec4dde7bb762d667657acbef87765e1a12 + - URL: http://security.ubuntu.com/ubuntu/pool/main/a/apt/apt_1.6.6ubuntu0.1_amd64.… + sha256sum: df210f9e30cf9deba5fbe815203af854e5e77bdbbe0b96d0d1c0da46a6a8dd0a + - URL: http://security.ubuntu.com/ubuntu/pool/main/a/apt/libapt-pkg5.0_1.6.6ubuntu… + sha256sum: 0a05a97b1e9b8d52ee8df040a14c5fabdebbb2c2235ac495db29df34f4c8cec3 - URL: '[% c("var/apt_packages_baseurl") %]/[% c("var/apt_package_filename") %]' sha256sum: '[% c("var/apt_package_sha256sum") %]' enable: '[% c("var/apt_package_filename") %]' diff --git a/projects/firefox/build b/projects/firefox/build index 067453d..5179634 100644 --- a/projects/firefox/build +++ b/projects/firefox/build @@ -57,7 +57,6 @@ mv -f $rootdir/[% c('input_files_by_name/mozconfig') %] .mozconfig [% END -%] [% IF c("var/android") %] - export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64 gradle_repo=/var/tmp/dist/gradle-dependencies export GRADLE_MAVEN_REPOSITORIES="file://$gradle_repo" export GRADLE_FLAGS="--no-daemon --offline" diff --git a/projects/firefox/config b/projects/firefox/config index 2c7c037..c6f840b 100644 --- a/projects/firefox/config +++ b/projects/firefox/config @@ -54,6 +54,11 @@ targets: var: branding_directory: '[% IF c("var/android") %]mobile/android[% ELSE %]browser[% END %]/branding/nightly' + android: + var: + arch_deps: + - openjdk-8-jdk + linux: var: post_pkginst: | diff --git a/projects/https-everywhere/config b/projects/https-everywhere/config index c66d7fe..1bc6f98 100644 --- a/projects/https-everywhere/config +++ b/projects/https-everywhere/config @@ -27,7 +27,6 @@ var: - rsync - zip - unzip - pre_pkginst: '' input_files: - project: container-image diff --git a/projects/tor-android-service/config b/projects/tor-android-service/config index 4a07c87..9d7bc84 100644 --- a/projects/tor-android-service/config +++ b/projects/tor-android-service/config @@ -13,6 +13,12 @@ var: # this should be updated when the list of gradle dependencies is changed gradle_dependencies_version: 3 +targets: + android: + var: + arch_deps: + - openjdk-8-jdk + input_files: - project: container-image - name: '[% c("var/compiler") %]' diff --git a/projects/tor-browser/build.android b/projects/tor-browser/build.android index 751db11..f8f3a8d 100644 --- a/projects/tor-browser/build.android +++ b/projects/tor-browser/build.android @@ -55,4 +55,4 @@ cd tmp }) %] # Sign a QA build. This apk is not a debug version and doesn't contain a debug flag in the manifest -java -jar /usr/lib/android-sdk/build-tools/debian/apksigner.jar sign --verbose --min-sdk-version [% c("var/android_min_api") %] --ks $rootdir/android-qa.keystore --out $qa_apk --in $apk --ks-key-alias androidqakey --key-pass pass:android --ks-pass pass:android +java -jar /usr/share/apksigner/apksigner.jar sign --verbose --min-sdk-version [% c("var/android_min_api") %] --ks $rootdir/android-qa.keystore --out $qa_apk --in $apk --ks-key-alias androidqakey --key-pass pass:android --ks-pass pass:android diff --git a/projects/tor-browser/config b/projects/tor-browser/config index 4a26049..819ebad 100644 --- a/projects/tor-browser/config +++ b/projects/tor-browser/config @@ -45,6 +45,7 @@ targets: build: '[% INCLUDE build.android %]' var: arch_deps: + - openjdk-8-jdk - apksigner input_files: diff --git a/projects/tor-onion-proxy-library/config b/projects/tor-onion-proxy-library/config index 0849623..ff3432c 100644 --- a/projects/tor-onion-proxy-library/config +++ b/projects/tor-onion-proxy-library/config @@ -15,6 +15,12 @@ var: # this should be updated when the list of gradle dependencies is changed gradle_dependencies_version: 3 +targets: + android: + var: + arch_deps: + - openjdk-8-jdk + input_files: - project: container-image - name: '[% c("var/compiler") %]' diff --git a/rbm.conf b/rbm.conf index f94cf4a..0fe7acb 100644 --- a/rbm.conf +++ b/rbm.conf @@ -236,7 +236,7 @@ targets: CC: '$ANDROID_NDK_HOME/[% c("var/toolchain_arch") %]/bin/clang' CXX: '$ANDROID_NDK_HOME/[% c("var/toolchain_arch") %]/bin/clang++' container: - suite: buster + suite: stretch arch: amd64 deps: - build-essential @@ -245,16 +245,7 @@ targets: - libtool - zip - unzip - pre_pkginst: | - SNAPSHOT_VERSION=20191201T212855Z - OPENJDK_URL=https://snapshot.debian.org/archive/debian/$SNAPSHOT_VERSION/po… - JDK_VERSION=8u232-b09-1~deb9u1_amd64 - apt-get install -y -q wget ca-certificates-java - wget $OPENJDK_URL/openjdk-8-jdk-headless_$JDK_VERSION.deb - wget $OPENJDK_URL/openjdk-8-jre-headless_$JDK_VERSION.deb - echo 92b4f8fb77d793a86e0b03b3b0750592b40a26a5d75956d10dd984a7b3aad4c9 openjdk-8-jdk-headless_$JDK_VERSION.deb | sha256sum -c - echo 84bf52b6cce20ead08b0d5b9fd9b81b4aa3da385ca951b313fe11d5cb1aa4d17 openjdk-8-jre-headless_$JDK_VERSION.deb | sha256sum -c - dpkg -i ./openjdk-8-jre-headless_$JDK_VERSION.deb ./openjdk-8-jdk-headless_$JDK_VERSION.deb + torbrowser-linux-x86_64: - linux-x86_64 - linux

1 0

[tor-browser-build/master] Changelog update
by gk＠torproject.org 14 Dec '19

14 Dec '19

commit 9bbc66a2faff49b37da7fccbc862879c0d48b3ae Author: Georg Koppen <gk(a)torproject.org> Date: Sat Dec 14 20:10:34 2019 +0000 Changelog update --- projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt b/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt index d9ecbbf..c4866d4 100644 --- a/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt +++ b/projects/tor-browser/Bundle-Data/Docs/ChangeLog.txt @@ -4,6 +4,7 @@ Tor Browser 9.5a3 -- December 3 2019 * Bump NoScript to 11.0.9 * Bug 32362: NoScript TRUSTED setting doesn't work * Bug 32429: Issues with about:blank and NoScript on .onion sites + * Bug 32549: NoScript makes requests to sync-messages.invalid * Update HTTPS Everywhere to 2019.11.7 * Bug 32618: Backport fixes from Mozilla bugs 1467970 and 1590526 * Bug 32606: Set up default bridge at Georgetown University @@ -38,6 +39,7 @@ Tor Browser 9.0.2 -- December 3 2019 * Bump NoScript to 11.0.9 * Bug 32362: NoScript TRUSTED setting doesn't work * Bug 32429: Issues with about:blank and NoScript on .onion sites + * Bug 32549: NoScript makes requests to sync-messages.invalid * Bump HTTPS Everywhere to 2019.11.7 * Bug 27268: Preferences clean-up in Torbutton code * Translations update

1 0

[tor-browser-build/master] Bug 32751: Sign incrementals sha256sums too if var/sign_build is set
by gk＠torproject.org 14 Dec '19

14 Dec '19

commit adf23abdceb488864de0639fc74affd3556eb2fc Author: Nicolas Vigier <boklm(a)torproject.org> Date: Fri Dec 13 17:12:23 2019 +0100 Bug 32751: Sign incrementals sha256sums too if var/sign_build is set --- README | 10 +++++----- projects/release/hash_incrementals | 3 +++ rbm.local.conf.example | 5 +++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/README b/README index d77a460..5dbf77e 100644 --- a/README +++ b/README @@ -155,11 +155,11 @@ Signing builds -------------- If the environment variable RBM_SIGN_BUILD is set to 1, the -sha256sums-unsigned-build.txt file will be signed with gpg. -You can use the RBM_GPG_OPTS environment variable to add some options -to the gpg command used to sign the file. You can also set the -var/sign_build and var/sign_build_gpg_opts options in the rbm.local.conf -file. +sha256sums-unsigned-build.txt and sha256sums-unsigned-build.incrementals.txt +files will be signed with gpg. You can use the RBM_GPG_OPTS environment +variable to add some options to the gpg command used to sign the file. +You can also set the var/sign_build and var/sign_build_gpg_opts options +in the rbm.local.conf file. Cleaning obsolete files and containers images diff --git a/projects/release/hash_incrementals b/projects/release/hash_incrementals index ba95ac7..cf31bfd 100644 --- a/projects/release/hash_incrementals +++ b/projects/release/hash_incrementals @@ -6,3 +6,6 @@ cd [% shell_quote(path(dest_dir)) %]/[% c("var/signed_status") %]/[% c("version") %]-[% c("var/torbrowser_build") %] [% END -%] sha256sum `ls -1 | grep '\.incremental\.mar$' | sort` > sha256sums-[% c("var/signed_status") %]-build.incrementals.txt +[% IF c("var/sign_build") -%] + gpg -abs [% c("var/sign_build_gpg_opts") %] sha256sums-[% c("var/signed_status") %]-build.incrementals.txt +[% END -%] diff --git a/rbm.local.conf.example b/rbm.local.conf.example index dc3f038..dd59034 100644 --- a/rbm.local.conf.example +++ b/rbm.local.conf.example @@ -45,12 +45,13 @@ var: local_conf: 1 ### The var/sign_build option defines if you want to sign the - ### sha256sums-unsigned-build.txt file with gpg. + ### sha256sums-unsigned-build.txt and + ### sha256sums-unsigned-build.incrementals.txt files with gpg. #sign_build: 1 ### The var/sign_build_gpg_opts option can be used to define some gpg ### options to select the key to use to sign the sha256sums-unsigned-build.txt - ### file. + ### and sha256sums-unsigned-build.incrementals.txt files. #sign_build_gpg_opts: '--local-user XXXXXXXX' ### The clean configuration is used by the cleaning script to find the

1 0

[tor-browser-build/master] Bug 32053: Fix LLVM reproducibility issues
by boklm＠torproject.org 13 Dec '19

13 Dec '19

commit 5cafc16c0e5af59f95f685025b487b306c10bd01 Author: Georg Koppen <gk(a)torproject.org> Date: Fri Dec 6 13:43:24 2019 +0000 Bug 32053: Fix LLVM reproducibility issues This patch contains two backports for reproducibility issues found and fixed in LLVM's optimization code. The patch from bug 42574 might not be necessary in our context as I only witnessed problems when compiling Stylo with an older Rust (1.32.0) and then omptimizing with LLVM 8. However, it's hard to exclude any mismatching builds we encountered so far from being affected by that issue, so I think it is safer to backport that patch as well. Bug 43909 is an issue we and Mozilla definitely hit and very likely responsible for at least the vast majority of different builds we got over the course of the last weeks and months. Many thanks to Alex Crichton for the invaluable help in tracking this issue down. --- projects/clang/42574.patch | 236 ++++++++++++++++++++++++++++++++++++++++++ projects/clang/43909.patch | 252 +++++++++++++++++++++++++++++++++++++++++++++ projects/clang/build | 4 + projects/clang/config | 2 + projects/rust/42574.patch | 236 ++++++++++++++++++++++++++++++++++++++++++ projects/rust/43909.patch | 252 +++++++++++++++++++++++++++++++++++++++++++++ projects/rust/build | 7 ++ projects/rust/config | 2 + 8 files changed, 991 insertions(+) diff --git a/projects/clang/42574.patch b/projects/clang/42574.patch new file mode 100644 index 0000000..285d254 --- /dev/null +++ b/projects/clang/42574.patch @@ -0,0 +1,236 @@ +From 3757213db371dcea53cae357cf9c56d1b0604f98 Mon Sep 17 00:00:00 2001 +From: Alina Sbirlea <asbirlea(a)google.com> +Date: Fri, 12 Jul 2019 22:30:30 +0000 +Subject: [PATCH] [MemorySSA] Use SetVector to avoid nondeterminism. + +Summary: +Use a SetVector for DeadBlockSet. +Resolves PR42574. + +Reviewers: george.burgess.iv, uabelho, dblaikie + +Subscribers: jlebar, Prazek, mgrang, llvm-commits + +Tags: #llvm + +Differential Revision: https://reviews.llvm.org/D64601 + +llvm-svn: 365970 + +diff --git a/llvm/include/llvm/Analysis/MemorySSAUpdater.h b/llvm/include/llvm/Analysis/MemorySSAUpdater.h +index 169d5bd9fa8..276620bd445 100644 +--- a/llvm/include/llvm/Analysis/MemorySSAUpdater.h ++++ b/llvm/include/llvm/Analysis/MemorySSAUpdater.h +@@ -32,6 +32,7 @@ + #ifndef LLVM_ANALYSIS_MEMORYSSAUPDATER_H + #define LLVM_ANALYSIS_MEMORYSSAUPDATER_H + ++#include "llvm/ADT/SetVector.h" + #include "llvm/ADT/SmallPtrSet.h" + #include "llvm/ADT/SmallSet.h" + #include "llvm/ADT/SmallVector.h" +@@ -239,7 +240,7 @@ public: + /// Deleted blocks still have successor info, but their predecessor edges and + /// Phi nodes may already be updated. Instructions in DeadBlocks should be + /// deleted after this call. +- void removeBlocks(const SmallPtrSetImpl<BasicBlock *> &DeadBlocks); ++ void removeBlocks(const SmallSetVector<BasicBlock *, 8> &DeadBlocks); + + /// Get handle on MemorySSA. + MemorySSA* getMemorySSA() const { return MSSA; } +diff --git a/llvm/lib/Analysis/MemorySSAUpdater.cpp b/llvm/lib/Analysis/MemorySSAUpdater.cpp +index 6c817d20368..a6c7142a697 100644 +--- a/llvm/lib/Analysis/MemorySSAUpdater.cpp ++++ b/llvm/lib/Analysis/MemorySSAUpdater.cpp +@@ -1101,7 +1101,7 @@ void MemorySSAUpdater::removeMemoryAccess(MemoryAccess *MA) { + } + + void MemorySSAUpdater::removeBlocks( +- const SmallPtrSetImpl<BasicBlock *> &DeadBlocks) { ++ const SmallSetVector<BasicBlock *, 8> &DeadBlocks) { + // First delete all uses of BB in MemoryPhis. + for (BasicBlock *BB : DeadBlocks) { + Instruction *TI = BB->getTerminator(); +diff --git a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp +index 2e5927f9a06..f464df26a02 100644 +--- a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp ++++ b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp +@@ -388,8 +388,8 @@ private: + void deleteDeadLoopBlocks() { + DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager); + if (MSSAU) { +- SmallPtrSet<BasicBlock *, 8> DeadLoopBlocksSet(DeadLoopBlocks.begin(), +- DeadLoopBlocks.end()); ++ SmallSetVector<BasicBlock *, 8> DeadLoopBlocksSet(DeadLoopBlocks.begin(), ++ DeadLoopBlocks.end()); + MSSAU->removeBlocks(DeadLoopBlocksSet); + } + for (auto *BB : DeadLoopBlocks) { +diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp +index 5a67178cef3..814cf814989 100644 +--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp ++++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp +@@ -1436,8 +1436,8 @@ deleteDeadClonedBlocks(Loop &L, ArrayRef<BasicBlock *> ExitBlocks, + + // Remove all MemorySSA in the dead blocks + if (MSSAU) { +- SmallPtrSet<BasicBlock *, 16> DeadBlockSet(DeadBlocks.begin(), +- DeadBlocks.end()); ++ SmallSetVector<BasicBlock *, 8> DeadBlockSet(DeadBlocks.begin(), ++ DeadBlocks.end()); + MSSAU->removeBlocks(DeadBlockSet); + } + +@@ -1455,7 +1455,7 @@ static void deleteDeadBlocksFromLoop(Loop &L, + MemorySSAUpdater *MSSAU) { + // Find all the dead blocks tied to this loop, and remove them from their + // successors. +- SmallPtrSet<BasicBlock *, 16> DeadBlockSet; ++ SmallSetVector<BasicBlock *, 8> DeadBlockSet; + + // Start with loop/exit blocks and get a transitive closure of reachable dead + // blocks. +diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp +index 499e611acb5..83dabcd7952 100644 +--- a/llvm/lib/Transforms/Utils/Local.cpp ++++ b/llvm/lib/Transforms/Utils/Local.cpp +@@ -2211,7 +2211,7 @@ bool llvm::removeUnreachableBlocks(Function &F, LazyValueInfo *LVI, + assert(Reachable.size() < F.size()); + NumRemoved += F.size()-Reachable.size(); + +- SmallPtrSet<BasicBlock *, 16> DeadBlockSet; ++ SmallSetVector<BasicBlock *, 8> DeadBlockSet; + for (Function::iterator I = ++F.begin(), E = F.end(); I != E; ++I) { + auto *BB = &*I; + if (Reachable.count(BB)) +diff --git a/llvm/test/Analysis/MemorySSA/nondeterminism.ll b/llvm/test/Analysis/MemorySSA/nondeterminism.ll +new file mode 100644 +index 00000000000..0bb3df30b58 +--- /dev/null ++++ b/llvm/test/Analysis/MemorySSA/nondeterminism.ll +@@ -0,0 +1,122 @@ ++; RUN: opt -simplifycfg -enable-mssa-loop-dependency -S --preserve-ll-uselistorder %s | FileCheck %s ++; REQUIRES: x86-registered-target ++; CHECK-LABEL: @n ++; CHECK: uselistorder i16 0, { 3, 2, 4, 1, 5, 0, 6 } ++ ++; Note: test was added in an effort to ensure determinism when updating memoryssa. See PR42574. ++; If the uselistorder check becomes no longer relevant, the test can be disabled or removed. ++ ++%rec9 = type { i16, i32, i32 } ++ ++@a = global [1 x [1 x %rec9]] zeroinitializer ++ ++define i16 @n() { ++ br label %..split_crit_edge ++ ++..split_crit_edge: ; preds = %0 ++ br label %.split ++ ++bb4.us4: ; preds = %bb2.split.us32, %bb6.us28 ++ %i.4.01.us5 = phi i16 [ %_tmp49.us30, %bb6.us28 ] ++ br label %g.exit4.us21 ++ ++bb1.i.us14: ; preds = %bb4.us4 ++ br label %g.exit4.us21 ++ ++g.exit4.us21: ; preds = %bb1.i.us14, %g.exit4.critedge.us9 ++ %i.4.02.us22 = phi i16 [ %i.4.01.us5, %bb4.us4 ], [ %i.4.01.us5, %bb1.i.us14 ] ++ br label %bb6.us28 ++ ++bb5.us26: ; preds = %g.exit4.us21 ++ br label %bb6.us28 ++ ++bb6.us28: ; preds = %bb5.us26, %g.exit4.us21 ++ %i.4.03.us29 = phi i16 [ %i.4.02.us22, %bb5.us26 ], [ %i.4.02.us22, %g.exit4.us21 ] ++ %_tmp49.us30 = add nuw nsw i16 %i.4.03.us29, 1 ++ br label %bb4.us4 ++ ++bb4.us.us: ; preds = %bb2.split.us.us, %bb6.us.us ++ %i.4.01.us.us = phi i16 [ %_tmp49.us.us, %bb6.us.us ] ++ br label %bb1.i.us.us ++ ++bb1.i.us.us: ; preds = %bb4.us.us ++ br label %g.exit4.us.us ++ ++g.exit4.us.us: ; preds = %bb1.i.us.us, %g.exit4.critedge.us.us ++ %i.4.02.us.us = phi i16 [ %i.4.01.us.us, %bb1.i.us.us ] ++ br label %bb5.us.us ++ ++bb5.us.us: ; preds = %g.exit4.us.us ++ br label %bb6.us.us ++ ++bb6.us.us: ; preds = %bb5.us.us, %g.exit4.us.us ++ %i.4.03.us.us = phi i16 [ %i.4.02.us.us, %bb5.us.us ] ++ %_tmp49.us.us = add nuw nsw i16 %i.4.03.us.us, 1 ++ br label %bb4.us.us ++ ++ ++.split: ; preds = %..split_crit_edge ++ br label %bb2 ++ ++bb2: ; preds = %.split, %bb7 ++ %h.3.0 = phi i16 [ undef, %.split ], [ %_tmp53, %bb7 ] ++ br label %bb2.bb2.split_crit_edge ++ ++bb2.bb2.split_crit_edge: ; preds = %bb2 ++ br label %bb2.split ++ ++bb2.split.us: ; preds = %bb2 ++ br label %bb4.us ++ ++bb4.us: ; preds = %bb6.us, %bb2.split.us ++ %i.4.01.us = phi i16 [ 0, %bb2.split.us ] ++ br label %bb1.i.us ++ ++g.exit4.critedge.us: ; preds = %bb4.us ++ br label %g.exit4.us ++ ++bb1.i.us: ; preds = %bb4.us ++ br label %g.exit4.us ++ ++g.exit4.us: ; preds = %bb1.i.us, %g.exit4.critedge.us ++ %i.4.02.us = phi i16 [ %i.4.01.us, %g.exit4.critedge.us ], [ %i.4.01.us, %bb1.i.us ] ++ br label %bb5.us ++ ++bb5.us: ; preds = %g.exit4.us ++ br label %bb7 ++ ++bb2.split: ; preds = %bb2.bb2.split_crit_edge ++ br label %bb4 ++ ++bb4: ; preds = %bb2.split, %bb6 ++ %i.4.01 = phi i16 [ 0, %bb2.split ] ++ %_tmp16 = getelementptr [1 x [1 x %rec9]], [1 x [1 x %rec9]]* @a, i16 0, i16 %h.3.0, i16 %i.4.01, i32 0 ++ %_tmp17 = load i16, i16* %_tmp16, align 1 ++ br label %g.exit4.critedge ++ ++bb1.i: ; preds = %bb4 ++ br label %g.exit4 ++ ++g.exit4.critedge: ; preds = %bb4 ++ %_tmp28.c = getelementptr [1 x [1 x %rec9]], [1 x [1 x %rec9]]* @a, i16 0, i16 %h.3.0, i16 %i.4.01, i32 1 ++ %_tmp29.c = load i32, i32* %_tmp28.c, align 1 ++ %_tmp30.c = trunc i32 %_tmp29.c to i16 ++ br label %g.exit4 ++ ++g.exit4: ; preds = %g.exit4.critedge, %bb1.i ++ %i.4.02 = phi i16 [ %i.4.01, %g.exit4.critedge ], [ %i.4.01, %bb1.i ] ++ %_tmp41 = getelementptr [1 x [1 x %rec9]], [1 x [1 x %rec9]]* @a, i16 0, i16 %h.3.0, i16 %i.4.02, i32 2 ++ br label %bb6 ++ ++bb5: ; preds = %g.exit4 ++ br label %bb6 ++ ++bb6: ; preds = %bb5, %g.exit4 ++ %i.4.03 = phi i16 [ %i.4.02, %bb5 ], [ %i.4.02, %g.exit4 ] ++ %_tmp49 = add nuw nsw i16 %i.4.03, 1 ++ br label %bb7 ++ ++bb7: ; preds = %bb7.us-lcssa.us, %bb7.us-lcssa ++ %_tmp53 = add nsw i16 %h.3.0, 1 ++ br label %bb2 ++} +-- +2.24.0 + diff --git a/projects/clang/43909.patch b/projects/clang/43909.patch new file mode 100644 index 0000000..78d2a75 --- /dev/null +++ b/projects/clang/43909.patch @@ -0,0 +1,252 @@ +From c95310f2d4fd3c88241c3b5d6dbf6251d34a3256 Mon Sep 17 00:00:00 2001 +From: Nikita Popov <nikita.ppv(a)gmail.com> +Date: Sat, 16 Nov 2019 16:22:18 +0100 +Subject: [PATCH] Restructure caching + +Variant on D70103. The caching is switched to always use a BB to +cache entry map, which then contains per-value caches. A separate +set contains value handles with a deletion callback. This allows us +to properly invalidate overdefined values. + +A possible alternative would be to always cache by value first and +have per-BB maps/sets in the each cache entry. In that case we could +use a ValueMap and would avoid the separate value handle set. I went +with the BB indexing at the top level to make it easier to integrate +D69914, but possibly that's not the right choice. + +Differential Revision: https://reviews.llvm.org/D70376 + +diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp +index 110c085d3f3..aa6862cb588 100644 +--- a/llvm/lib/Analysis/LazyValueInfo.cpp ++++ b/llvm/lib/Analysis/LazyValueInfo.cpp +@@ -133,12 +133,9 @@ namespace { + /// A callback value handle updates the cache when values are erased. + class LazyValueInfoCache; + struct LVIValueHandle final : public CallbackVH { +- // Needs to access getValPtr(), which is protected. +- friend struct DenseMapInfo<LVIValueHandle>; +- + LazyValueInfoCache *Parent; + +- LVIValueHandle(Value *V, LazyValueInfoCache *P) ++ LVIValueHandle(Value *V, LazyValueInfoCache *P = nullptr) + : CallbackVH(V), Parent(P) { } + + void deleted() override; +@@ -152,89 +149,63 @@ namespace { + /// This is the cache kept by LazyValueInfo which + /// maintains information about queries across the clients' queries. + class LazyValueInfoCache { +- /// This is all of the cached block information for exactly one Value*. +- /// The entries are sorted by the BasicBlock* of the +- /// entries, allowing us to do a lookup with a binary search. +- /// Over-defined lattice values are recorded in OverDefinedCache to reduce +- /// memory overhead. +- struct ValueCacheEntryTy { +- ValueCacheEntryTy(Value *V, LazyValueInfoCache *P) : Handle(V, P) {} +- LVIValueHandle Handle; +- SmallDenseMap<PoisoningVH<BasicBlock>, ValueLatticeElement, 4> BlockVals; ++ /// This is all of the cached information for one basic block. It contains ++ /// the per-value lattice elements, as well as a separate set for ++ /// overdefined values to reduce memory usage. ++ struct BlockCacheEntryTy { ++ SmallDenseMap<AssertingVH<Value>, ValueLatticeElement, 4> LatticeElements; ++ SmallDenseSet<AssertingVH<Value>, 4> OverDefined; + }; + +- /// This tracks, on a per-block basis, the set of values that are +- /// over-defined at the end of that block. +- typedef DenseMap<PoisoningVH<BasicBlock>, SmallPtrSet<Value *, 4>> +- OverDefinedCacheTy; +- /// Keep track of all blocks that we have ever seen, so we +- /// don't spend time removing unused blocks from our caches. +- DenseSet<PoisoningVH<BasicBlock> > SeenBlocks; +- +- /// This is all of the cached information for all values, +- /// mapped from Value* to key information. +- DenseMap<Value *, std::unique_ptr<ValueCacheEntryTy>> ValueCache; +- OverDefinedCacheTy OverDefinedCache; +- ++ /// Cached information per basic block. ++ DenseMap<PoisoningVH<BasicBlock>, BlockCacheEntryTy> BlockCache; ++ /// Set of value handles used to erase values from the cache on deletion. ++ DenseSet<LVIValueHandle, DenseMapInfo<Value *>> ValueHandles; + + public: + void insertResult(Value *Val, BasicBlock *BB, + const ValueLatticeElement &Result) { +- SeenBlocks.insert(BB); +- ++ auto &CacheEntry = BlockCache.try_emplace(BB).first->second; + // Insert over-defined values into their own cache to reduce memory + // overhead. + if (Result.isOverdefined()) +- OverDefinedCache[BB].insert(Val); +- else { +- auto It = ValueCache.find_as(Val); +- if (It == ValueCache.end()) { +- ValueCache[Val] = make_unique<ValueCacheEntryTy>(Val, this); +- It = ValueCache.find_as(Val); +- assert(It != ValueCache.end() && "Val was just added to the map!"); +- } +- It->second->BlockVals[BB] = Result; +- } +- } +- +- bool isOverdefined(Value *V, BasicBlock *BB) const { +- auto ODI = OverDefinedCache.find(BB); +- +- if (ODI == OverDefinedCache.end()) +- return false; ++ CacheEntry.OverDefined.insert(Val); ++ else ++ CacheEntry.LatticeElements.insert({ Val, Result }); + +- return ODI->second.count(V); ++ auto HandleIt = ValueHandles.find_as(Val); ++ if (HandleIt == ValueHandles.end()) ++ ValueHandles.insert({ Val, this }); + } + + bool hasCachedValueInfo(Value *V, BasicBlock *BB) const { +- if (isOverdefined(V, BB)) +- return true; +- +- auto I = ValueCache.find_as(V); +- if (I == ValueCache.end()) ++ auto It = BlockCache.find(BB); ++ if (It == BlockCache.end()) + return false; + +- return I->second->BlockVals.count(BB); ++ return It->second.OverDefined.count(V) || ++ It->second.LatticeElements.count(V); + } + + ValueLatticeElement getCachedValueInfo(Value *V, BasicBlock *BB) const { +- if (isOverdefined(V, BB)) ++ auto It = BlockCache.find(BB); ++ if (It == BlockCache.end()) ++ return ValueLatticeElement(); ++ ++ if (It->second.OverDefined.count(V)) + return ValueLatticeElement::getOverdefined(); + +- auto I = ValueCache.find_as(V); +- if (I == ValueCache.end()) ++ auto LatticeIt = It->second.LatticeElements.find(V); ++ if (LatticeIt == It->second.LatticeElements.end()) + return ValueLatticeElement(); +- auto BBI = I->second->BlockVals.find(BB); +- if (BBI == I->second->BlockVals.end()) +- return ValueLatticeElement(); +- return BBI->second; ++ ++ return LatticeIt->second; + } + + /// clear - Empty the cache. + void clear() { +- SeenBlocks.clear(); +- ValueCache.clear(); +- OverDefinedCache.clear(); ++ BlockCache.clear(); ++ ValueHandles.clear(); + } + + /// Inform the cache that a given value has been deleted. +@@ -248,23 +219,18 @@ namespace { + /// OldSucc might have (unless also overdefined in NewSucc). This just + /// flushes elements from the cache and does not add any. + void threadEdgeImpl(BasicBlock *OldSucc,BasicBlock *NewSucc); +- +- friend struct LVIValueHandle; + }; + } + + void LazyValueInfoCache::eraseValue(Value *V) { +- for (auto I = OverDefinedCache.begin(), E = OverDefinedCache.end(); I != E;) { +- // Copy and increment the iterator immediately so we can erase behind +- // ourselves. +- auto Iter = I++; +- SmallPtrSetImpl<Value *> &ValueSet = Iter->second; +- ValueSet.erase(V); +- if (ValueSet.empty()) +- OverDefinedCache.erase(Iter); ++ for (auto &Pair : BlockCache) { ++ Pair.second.LatticeElements.erase(V); ++ Pair.second.OverDefined.erase(V); + } + +- ValueCache.erase(V); ++ auto HandleIt = ValueHandles.find_as(V); ++ if (HandleIt != ValueHandles.end()) ++ ValueHandles.erase(HandleIt); + } + + void LVIValueHandle::deleted() { +@@ -274,18 +240,7 @@ void LVIValueHandle::deleted() { + } + + void LazyValueInfoCache::eraseBlock(BasicBlock *BB) { +- // Shortcut if we have never seen this block. +- DenseSet<PoisoningVH<BasicBlock> >::iterator I = SeenBlocks.find(BB); +- if (I == SeenBlocks.end()) +- return; +- SeenBlocks.erase(I); +- +- auto ODI = OverDefinedCache.find(BB); +- if (ODI != OverDefinedCache.end()) +- OverDefinedCache.erase(ODI); +- +- for (auto &I : ValueCache) +- I.second->BlockVals.erase(BB); ++ BlockCache.erase(BB); + } + + void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc, +@@ -303,10 +258,11 @@ void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc, + std::vector<BasicBlock*> worklist; + worklist.push_back(OldSucc); + +- auto I = OverDefinedCache.find(OldSucc); +- if (I == OverDefinedCache.end()) ++ auto I = BlockCache.find(OldSucc); ++ if (I == BlockCache.end() || I->second.OverDefined.empty()) + return; // Nothing to process here. +- SmallVector<Value *, 4> ValsToClear(I->second.begin(), I->second.end()); ++ SmallVector<Value *, 4> ValsToClear(I->second.OverDefined.begin(), ++ I->second.OverDefined.end()); + + // Use a worklist to perform a depth-first search of OldSucc's successors. + // NOTE: We do not need a visited list since any blocks we have already +@@ -320,10 +276,10 @@ void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc, + if (ToUpdate == NewSucc) continue; + + // If a value was marked overdefined in OldSucc, and is here too... +- auto OI = OverDefinedCache.find(ToUpdate); +- if (OI == OverDefinedCache.end()) ++ auto OI = BlockCache.find(ToUpdate); ++ if (OI == BlockCache.end() || OI->second.OverDefined.empty()) + continue; +- SmallPtrSetImpl<Value *> &ValueSet = OI->second; ++ auto &ValueSet = OI->second.OverDefined; + + bool changed = false; + for (Value *V : ValsToClear) { +@@ -333,11 +289,6 @@ void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc, + // If we removed anything, then we potentially need to update + // blocks successors too. + changed = true; +- +- if (ValueSet.empty()) { +- OverDefinedCache.erase(OI); +- break; +- } + } + + if (!changed) continue; +-- +2.24.0 + diff --git a/projects/clang/build b/projects/clang/build index c1afa6d..825b21e 100644 --- a/projects/clang/build +++ b/projects/clang/build @@ -20,6 +20,10 @@ tar -xf $rootdir/[% c('input_files_by_name/libcxxabi') %] tar -xf $rootdir/[% c('input_files_by_name/lld') %] tar -xf $rootdir/[% c('input_files_by_name/compiler-rt') %] mv llvm-* llvm +# LLVM has reproducibility issues when optimizing bitcode, which we need to +# patch. See: #32053 for more details. +patch -p1 < $rootdir/42574.patch +patch -p1 < $rootdir/43909.patch mv cfe-* llvm/tools/clang mv libcxx-* llvm/projects/libcxx mv libcxxabi-* llvm/projects/libcxxabi diff --git a/projects/clang/config b/projects/clang/config index 67798f8..cf4dcb0 100644 --- a/projects/clang/config +++ b/projects/clang/config @@ -45,3 +45,5 @@ input_files: enable: '[% c("var/windows") %]' - filename: timestamp.patch enable: '[% c("var/windows") %]' + - filename: 42574.patch + - filename: 43909.patch diff --git a/projects/rust/42574.patch b/projects/rust/42574.patch new file mode 100644 index 0000000..285d254 --- /dev/null +++ b/projects/rust/42574.patch @@ -0,0 +1,236 @@ +From 3757213db371dcea53cae357cf9c56d1b0604f98 Mon Sep 17 00:00:00 2001 +From: Alina Sbirlea <asbirlea(a)google.com> +Date: Fri, 12 Jul 2019 22:30:30 +0000 +Subject: [PATCH] [MemorySSA] Use SetVector to avoid nondeterminism. + +Summary: +Use a SetVector for DeadBlockSet. +Resolves PR42574. + +Reviewers: george.burgess.iv, uabelho, dblaikie + +Subscribers: jlebar, Prazek, mgrang, llvm-commits + +Tags: #llvm + +Differential Revision: https://reviews.llvm.org/D64601 + +llvm-svn: 365970 + +diff --git a/llvm/include/llvm/Analysis/MemorySSAUpdater.h b/llvm/include/llvm/Analysis/MemorySSAUpdater.h +index 169d5bd9fa8..276620bd445 100644 +--- a/llvm/include/llvm/Analysis/MemorySSAUpdater.h ++++ b/llvm/include/llvm/Analysis/MemorySSAUpdater.h +@@ -32,6 +32,7 @@ + #ifndef LLVM_ANALYSIS_MEMORYSSAUPDATER_H + #define LLVM_ANALYSIS_MEMORYSSAUPDATER_H + ++#include "llvm/ADT/SetVector.h" + #include "llvm/ADT/SmallPtrSet.h" + #include "llvm/ADT/SmallSet.h" + #include "llvm/ADT/SmallVector.h" +@@ -239,7 +240,7 @@ public: + /// Deleted blocks still have successor info, but their predecessor edges and + /// Phi nodes may already be updated. Instructions in DeadBlocks should be + /// deleted after this call. +- void removeBlocks(const SmallPtrSetImpl<BasicBlock *> &DeadBlocks); ++ void removeBlocks(const SmallSetVector<BasicBlock *, 8> &DeadBlocks); + + /// Get handle on MemorySSA. + MemorySSA* getMemorySSA() const { return MSSA; } +diff --git a/llvm/lib/Analysis/MemorySSAUpdater.cpp b/llvm/lib/Analysis/MemorySSAUpdater.cpp +index 6c817d20368..a6c7142a697 100644 +--- a/llvm/lib/Analysis/MemorySSAUpdater.cpp ++++ b/llvm/lib/Analysis/MemorySSAUpdater.cpp +@@ -1101,7 +1101,7 @@ void MemorySSAUpdater::removeMemoryAccess(MemoryAccess *MA) { + } + + void MemorySSAUpdater::removeBlocks( +- const SmallPtrSetImpl<BasicBlock *> &DeadBlocks) { ++ const SmallSetVector<BasicBlock *, 8> &DeadBlocks) { + // First delete all uses of BB in MemoryPhis. + for (BasicBlock *BB : DeadBlocks) { + Instruction *TI = BB->getTerminator(); +diff --git a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp +index 2e5927f9a06..f464df26a02 100644 +--- a/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp ++++ b/llvm/lib/Transforms/Scalar/LoopSimplifyCFG.cpp +@@ -388,8 +388,8 @@ private: + void deleteDeadLoopBlocks() { + DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager); + if (MSSAU) { +- SmallPtrSet<BasicBlock *, 8> DeadLoopBlocksSet(DeadLoopBlocks.begin(), +- DeadLoopBlocks.end()); ++ SmallSetVector<BasicBlock *, 8> DeadLoopBlocksSet(DeadLoopBlocks.begin(), ++ DeadLoopBlocks.end()); + MSSAU->removeBlocks(DeadLoopBlocksSet); + } + for (auto *BB : DeadLoopBlocks) { +diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp +index 5a67178cef3..814cf814989 100644 +--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp ++++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp +@@ -1436,8 +1436,8 @@ deleteDeadClonedBlocks(Loop &L, ArrayRef<BasicBlock *> ExitBlocks, + + // Remove all MemorySSA in the dead blocks + if (MSSAU) { +- SmallPtrSet<BasicBlock *, 16> DeadBlockSet(DeadBlocks.begin(), +- DeadBlocks.end()); ++ SmallSetVector<BasicBlock *, 8> DeadBlockSet(DeadBlocks.begin(), ++ DeadBlocks.end()); + MSSAU->removeBlocks(DeadBlockSet); + } + +@@ -1455,7 +1455,7 @@ static void deleteDeadBlocksFromLoop(Loop &L, + MemorySSAUpdater *MSSAU) { + // Find all the dead blocks tied to this loop, and remove them from their + // successors. +- SmallPtrSet<BasicBlock *, 16> DeadBlockSet; ++ SmallSetVector<BasicBlock *, 8> DeadBlockSet; + + // Start with loop/exit blocks and get a transitive closure of reachable dead + // blocks. +diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp +index 499e611acb5..83dabcd7952 100644 +--- a/llvm/lib/Transforms/Utils/Local.cpp ++++ b/llvm/lib/Transforms/Utils/Local.cpp +@@ -2211,7 +2211,7 @@ bool llvm::removeUnreachableBlocks(Function &F, LazyValueInfo *LVI, + assert(Reachable.size() < F.size()); + NumRemoved += F.size()-Reachable.size(); + +- SmallPtrSet<BasicBlock *, 16> DeadBlockSet; ++ SmallSetVector<BasicBlock *, 8> DeadBlockSet; + for (Function::iterator I = ++F.begin(), E = F.end(); I != E; ++I) { + auto *BB = &*I; + if (Reachable.count(BB)) +diff --git a/llvm/test/Analysis/MemorySSA/nondeterminism.ll b/llvm/test/Analysis/MemorySSA/nondeterminism.ll +new file mode 100644 +index 00000000000..0bb3df30b58 +--- /dev/null ++++ b/llvm/test/Analysis/MemorySSA/nondeterminism.ll +@@ -0,0 +1,122 @@ ++; RUN: opt -simplifycfg -enable-mssa-loop-dependency -S --preserve-ll-uselistorder %s | FileCheck %s ++; REQUIRES: x86-registered-target ++; CHECK-LABEL: @n ++; CHECK: uselistorder i16 0, { 3, 2, 4, 1, 5, 0, 6 } ++ ++; Note: test was added in an effort to ensure determinism when updating memoryssa. See PR42574. ++; If the uselistorder check becomes no longer relevant, the test can be disabled or removed. ++ ++%rec9 = type { i16, i32, i32 } ++ ++@a = global [1 x [1 x %rec9]] zeroinitializer ++ ++define i16 @n() { ++ br label %..split_crit_edge ++ ++..split_crit_edge: ; preds = %0 ++ br label %.split ++ ++bb4.us4: ; preds = %bb2.split.us32, %bb6.us28 ++ %i.4.01.us5 = phi i16 [ %_tmp49.us30, %bb6.us28 ] ++ br label %g.exit4.us21 ++ ++bb1.i.us14: ; preds = %bb4.us4 ++ br label %g.exit4.us21 ++ ++g.exit4.us21: ; preds = %bb1.i.us14, %g.exit4.critedge.us9 ++ %i.4.02.us22 = phi i16 [ %i.4.01.us5, %bb4.us4 ], [ %i.4.01.us5, %bb1.i.us14 ] ++ br label %bb6.us28 ++ ++bb5.us26: ; preds = %g.exit4.us21 ++ br label %bb6.us28 ++ ++bb6.us28: ; preds = %bb5.us26, %g.exit4.us21 ++ %i.4.03.us29 = phi i16 [ %i.4.02.us22, %bb5.us26 ], [ %i.4.02.us22, %g.exit4.us21 ] ++ %_tmp49.us30 = add nuw nsw i16 %i.4.03.us29, 1 ++ br label %bb4.us4 ++ ++bb4.us.us: ; preds = %bb2.split.us.us, %bb6.us.us ++ %i.4.01.us.us = phi i16 [ %_tmp49.us.us, %bb6.us.us ] ++ br label %bb1.i.us.us ++ ++bb1.i.us.us: ; preds = %bb4.us.us ++ br label %g.exit4.us.us ++ ++g.exit4.us.us: ; preds = %bb1.i.us.us, %g.exit4.critedge.us.us ++ %i.4.02.us.us = phi i16 [ %i.4.01.us.us, %bb1.i.us.us ] ++ br label %bb5.us.us ++ ++bb5.us.us: ; preds = %g.exit4.us.us ++ br label %bb6.us.us ++ ++bb6.us.us: ; preds = %bb5.us.us, %g.exit4.us.us ++ %i.4.03.us.us = phi i16 [ %i.4.02.us.us, %bb5.us.us ] ++ %_tmp49.us.us = add nuw nsw i16 %i.4.03.us.us, 1 ++ br label %bb4.us.us ++ ++ ++.split: ; preds = %..split_crit_edge ++ br label %bb2 ++ ++bb2: ; preds = %.split, %bb7 ++ %h.3.0 = phi i16 [ undef, %.split ], [ %_tmp53, %bb7 ] ++ br label %bb2.bb2.split_crit_edge ++ ++bb2.bb2.split_crit_edge: ; preds = %bb2 ++ br label %bb2.split ++ ++bb2.split.us: ; preds = %bb2 ++ br label %bb4.us ++ ++bb4.us: ; preds = %bb6.us, %bb2.split.us ++ %i.4.01.us = phi i16 [ 0, %bb2.split.us ] ++ br label %bb1.i.us ++ ++g.exit4.critedge.us: ; preds = %bb4.us ++ br label %g.exit4.us ++ ++bb1.i.us: ; preds = %bb4.us ++ br label %g.exit4.us ++ ++g.exit4.us: ; preds = %bb1.i.us, %g.exit4.critedge.us ++ %i.4.02.us = phi i16 [ %i.4.01.us, %g.exit4.critedge.us ], [ %i.4.01.us, %bb1.i.us ] ++ br label %bb5.us ++ ++bb5.us: ; preds = %g.exit4.us ++ br label %bb7 ++ ++bb2.split: ; preds = %bb2.bb2.split_crit_edge ++ br label %bb4 ++ ++bb4: ; preds = %bb2.split, %bb6 ++ %i.4.01 = phi i16 [ 0, %bb2.split ] ++ %_tmp16 = getelementptr [1 x [1 x %rec9]], [1 x [1 x %rec9]]* @a, i16 0, i16 %h.3.0, i16 %i.4.01, i32 0 ++ %_tmp17 = load i16, i16* %_tmp16, align 1 ++ br label %g.exit4.critedge ++ ++bb1.i: ; preds = %bb4 ++ br label %g.exit4 ++ ++g.exit4.critedge: ; preds = %bb4 ++ %_tmp28.c = getelementptr [1 x [1 x %rec9]], [1 x [1 x %rec9]]* @a, i16 0, i16 %h.3.0, i16 %i.4.01, i32 1 ++ %_tmp29.c = load i32, i32* %_tmp28.c, align 1 ++ %_tmp30.c = trunc i32 %_tmp29.c to i16 ++ br label %g.exit4 ++ ++g.exit4: ; preds = %g.exit4.critedge, %bb1.i ++ %i.4.02 = phi i16 [ %i.4.01, %g.exit4.critedge ], [ %i.4.01, %bb1.i ] ++ %_tmp41 = getelementptr [1 x [1 x %rec9]], [1 x [1 x %rec9]]* @a, i16 0, i16 %h.3.0, i16 %i.4.02, i32 2 ++ br label %bb6 ++ ++bb5: ; preds = %g.exit4 ++ br label %bb6 ++ ++bb6: ; preds = %bb5, %g.exit4 ++ %i.4.03 = phi i16 [ %i.4.02, %bb5 ], [ %i.4.02, %g.exit4 ] ++ %_tmp49 = add nuw nsw i16 %i.4.03, 1 ++ br label %bb7 ++ ++bb7: ; preds = %bb7.us-lcssa.us, %bb7.us-lcssa ++ %_tmp53 = add nsw i16 %h.3.0, 1 ++ br label %bb2 ++} +-- +2.24.0 + diff --git a/projects/rust/43909.patch b/projects/rust/43909.patch new file mode 100644 index 0000000..78d2a75 --- /dev/null +++ b/projects/rust/43909.patch @@ -0,0 +1,252 @@ +From c95310f2d4fd3c88241c3b5d6dbf6251d34a3256 Mon Sep 17 00:00:00 2001 +From: Nikita Popov <nikita.ppv(a)gmail.com> +Date: Sat, 16 Nov 2019 16:22:18 +0100 +Subject: [PATCH] Restructure caching + +Variant on D70103. The caching is switched to always use a BB to +cache entry map, which then contains per-value caches. A separate +set contains value handles with a deletion callback. This allows us +to properly invalidate overdefined values. + +A possible alternative would be to always cache by value first and +have per-BB maps/sets in the each cache entry. In that case we could +use a ValueMap and would avoid the separate value handle set. I went +with the BB indexing at the top level to make it easier to integrate +D69914, but possibly that's not the right choice. + +Differential Revision: https://reviews.llvm.org/D70376 + +diff --git a/llvm/lib/Analysis/LazyValueInfo.cpp b/llvm/lib/Analysis/LazyValueInfo.cpp +index 110c085d3f3..aa6862cb588 100644 +--- a/llvm/lib/Analysis/LazyValueInfo.cpp ++++ b/llvm/lib/Analysis/LazyValueInfo.cpp +@@ -133,12 +133,9 @@ namespace { + /// A callback value handle updates the cache when values are erased. + class LazyValueInfoCache; + struct LVIValueHandle final : public CallbackVH { +- // Needs to access getValPtr(), which is protected. +- friend struct DenseMapInfo<LVIValueHandle>; +- + LazyValueInfoCache *Parent; + +- LVIValueHandle(Value *V, LazyValueInfoCache *P) ++ LVIValueHandle(Value *V, LazyValueInfoCache *P = nullptr) + : CallbackVH(V), Parent(P) { } + + void deleted() override; +@@ -152,89 +149,63 @@ namespace { + /// This is the cache kept by LazyValueInfo which + /// maintains information about queries across the clients' queries. + class LazyValueInfoCache { +- /// This is all of the cached block information for exactly one Value*. +- /// The entries are sorted by the BasicBlock* of the +- /// entries, allowing us to do a lookup with a binary search. +- /// Over-defined lattice values are recorded in OverDefinedCache to reduce +- /// memory overhead. +- struct ValueCacheEntryTy { +- ValueCacheEntryTy(Value *V, LazyValueInfoCache *P) : Handle(V, P) {} +- LVIValueHandle Handle; +- SmallDenseMap<PoisoningVH<BasicBlock>, ValueLatticeElement, 4> BlockVals; ++ /// This is all of the cached information for one basic block. It contains ++ /// the per-value lattice elements, as well as a separate set for ++ /// overdefined values to reduce memory usage. ++ struct BlockCacheEntryTy { ++ SmallDenseMap<AssertingVH<Value>, ValueLatticeElement, 4> LatticeElements; ++ SmallDenseSet<AssertingVH<Value>, 4> OverDefined; + }; + +- /// This tracks, on a per-block basis, the set of values that are +- /// over-defined at the end of that block. +- typedef DenseMap<PoisoningVH<BasicBlock>, SmallPtrSet<Value *, 4>> +- OverDefinedCacheTy; +- /// Keep track of all blocks that we have ever seen, so we +- /// don't spend time removing unused blocks from our caches. +- DenseSet<PoisoningVH<BasicBlock> > SeenBlocks; +- +- /// This is all of the cached information for all values, +- /// mapped from Value* to key information. +- DenseMap<Value *, std::unique_ptr<ValueCacheEntryTy>> ValueCache; +- OverDefinedCacheTy OverDefinedCache; +- ++ /// Cached information per basic block. ++ DenseMap<PoisoningVH<BasicBlock>, BlockCacheEntryTy> BlockCache; ++ /// Set of value handles used to erase values from the cache on deletion. ++ DenseSet<LVIValueHandle, DenseMapInfo<Value *>> ValueHandles; + + public: + void insertResult(Value *Val, BasicBlock *BB, + const ValueLatticeElement &Result) { +- SeenBlocks.insert(BB); +- ++ auto &CacheEntry = BlockCache.try_emplace(BB).first->second; + // Insert over-defined values into their own cache to reduce memory + // overhead. + if (Result.isOverdefined()) +- OverDefinedCache[BB].insert(Val); +- else { +- auto It = ValueCache.find_as(Val); +- if (It == ValueCache.end()) { +- ValueCache[Val] = make_unique<ValueCacheEntryTy>(Val, this); +- It = ValueCache.find_as(Val); +- assert(It != ValueCache.end() && "Val was just added to the map!"); +- } +- It->second->BlockVals[BB] = Result; +- } +- } +- +- bool isOverdefined(Value *V, BasicBlock *BB) const { +- auto ODI = OverDefinedCache.find(BB); +- +- if (ODI == OverDefinedCache.end()) +- return false; ++ CacheEntry.OverDefined.insert(Val); ++ else ++ CacheEntry.LatticeElements.insert({ Val, Result }); + +- return ODI->second.count(V); ++ auto HandleIt = ValueHandles.find_as(Val); ++ if (HandleIt == ValueHandles.end()) ++ ValueHandles.insert({ Val, this }); + } + + bool hasCachedValueInfo(Value *V, BasicBlock *BB) const { +- if (isOverdefined(V, BB)) +- return true; +- +- auto I = ValueCache.find_as(V); +- if (I == ValueCache.end()) ++ auto It = BlockCache.find(BB); ++ if (It == BlockCache.end()) + return false; + +- return I->second->BlockVals.count(BB); ++ return It->second.OverDefined.count(V) || ++ It->second.LatticeElements.count(V); + } + + ValueLatticeElement getCachedValueInfo(Value *V, BasicBlock *BB) const { +- if (isOverdefined(V, BB)) ++ auto It = BlockCache.find(BB); ++ if (It == BlockCache.end()) ++ return ValueLatticeElement(); ++ ++ if (It->second.OverDefined.count(V)) + return ValueLatticeElement::getOverdefined(); + +- auto I = ValueCache.find_as(V); +- if (I == ValueCache.end()) ++ auto LatticeIt = It->second.LatticeElements.find(V); ++ if (LatticeIt == It->second.LatticeElements.end()) + return ValueLatticeElement(); +- auto BBI = I->second->BlockVals.find(BB); +- if (BBI == I->second->BlockVals.end()) +- return ValueLatticeElement(); +- return BBI->second; ++ ++ return LatticeIt->second; + } + + /// clear - Empty the cache. + void clear() { +- SeenBlocks.clear(); +- ValueCache.clear(); +- OverDefinedCache.clear(); ++ BlockCache.clear(); ++ ValueHandles.clear(); + } + + /// Inform the cache that a given value has been deleted. +@@ -248,23 +219,18 @@ namespace { + /// OldSucc might have (unless also overdefined in NewSucc). This just + /// flushes elements from the cache and does not add any. + void threadEdgeImpl(BasicBlock *OldSucc,BasicBlock *NewSucc); +- +- friend struct LVIValueHandle; + }; + } + + void LazyValueInfoCache::eraseValue(Value *V) { +- for (auto I = OverDefinedCache.begin(), E = OverDefinedCache.end(); I != E;) { +- // Copy and increment the iterator immediately so we can erase behind +- // ourselves. +- auto Iter = I++; +- SmallPtrSetImpl<Value *> &ValueSet = Iter->second; +- ValueSet.erase(V); +- if (ValueSet.empty()) +- OverDefinedCache.erase(Iter); ++ for (auto &Pair : BlockCache) { ++ Pair.second.LatticeElements.erase(V); ++ Pair.second.OverDefined.erase(V); + } + +- ValueCache.erase(V); ++ auto HandleIt = ValueHandles.find_as(V); ++ if (HandleIt != ValueHandles.end()) ++ ValueHandles.erase(HandleIt); + } + + void LVIValueHandle::deleted() { +@@ -274,18 +240,7 @@ void LVIValueHandle::deleted() { + } + + void LazyValueInfoCache::eraseBlock(BasicBlock *BB) { +- // Shortcut if we have never seen this block. +- DenseSet<PoisoningVH<BasicBlock> >::iterator I = SeenBlocks.find(BB); +- if (I == SeenBlocks.end()) +- return; +- SeenBlocks.erase(I); +- +- auto ODI = OverDefinedCache.find(BB); +- if (ODI != OverDefinedCache.end()) +- OverDefinedCache.erase(ODI); +- +- for (auto &I : ValueCache) +- I.second->BlockVals.erase(BB); ++ BlockCache.erase(BB); + } + + void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc, +@@ -303,10 +258,11 @@ void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc, + std::vector<BasicBlock*> worklist; + worklist.push_back(OldSucc); + +- auto I = OverDefinedCache.find(OldSucc); +- if (I == OverDefinedCache.end()) ++ auto I = BlockCache.find(OldSucc); ++ if (I == BlockCache.end() || I->second.OverDefined.empty()) + return; // Nothing to process here. +- SmallVector<Value *, 4> ValsToClear(I->second.begin(), I->second.end()); ++ SmallVector<Value *, 4> ValsToClear(I->second.OverDefined.begin(), ++ I->second.OverDefined.end()); + + // Use a worklist to perform a depth-first search of OldSucc's successors. + // NOTE: We do not need a visited list since any blocks we have already +@@ -320,10 +276,10 @@ void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc, + if (ToUpdate == NewSucc) continue; + + // If a value was marked overdefined in OldSucc, and is here too... +- auto OI = OverDefinedCache.find(ToUpdate); +- if (OI == OverDefinedCache.end()) ++ auto OI = BlockCache.find(ToUpdate); ++ if (OI == BlockCache.end() || OI->second.OverDefined.empty()) + continue; +- SmallPtrSetImpl<Value *> &ValueSet = OI->second; ++ auto &ValueSet = OI->second.OverDefined; + + bool changed = false; + for (Value *V : ValsToClear) { +@@ -333,11 +289,6 @@ void LazyValueInfoCache::threadEdgeImpl(BasicBlock *OldSucc, + // If we removed anything, then we potentially need to update + // blocks successors too. + changed = true; +- +- if (ValueSet.empty()) { +- OverDefinedCache.erase(OI); +- break; +- } + } + + if (!changed) continue; +-- +2.24.0 + diff --git a/projects/rust/build b/projects/rust/build index 9a6b54a..8b45e3f 100644 --- a/projects/rust/build +++ b/projects/rust/build @@ -51,6 +51,13 @@ mkdir /var/tmp/build tar -C /var/tmp/build -xf [% c('input_files_by_name/rust') %] cd /var/tmp/build/rustc-[% c('version') %]-src +# LLVM has reproducibility issues when optimizing bitcode, which we need to +# patch. See: #32053 for more details. +cd src/llvm-project +patch -p1 < $rootdir/42574.patch +patch -p1 < $rootdir/43909.patch +cd ../../ + [% IF c("var/windows-i686") %] # Cross-compiling for Windows 32bit is currently not possible without any # patches. The reason for that is libstd expecting DWARF unwinding while most diff --git a/projects/rust/config b/projects/rust/config index 03d7be0..848ccd0 100644 --- a/projects/rust/config +++ b/projects/rust/config @@ -106,3 +106,5 @@ input_files: gpg_keyring: rust.gpg - filename: unwind.patch enable: '[% c("var/windows-i686") %]' + - filename: 42574.patch + - filename: 43909.patch

1 0

[tor-browser-build/master] Merge remote-tracking branch 'JeremyRand/reproducible-go'
by boklm＠torproject.org 13 Dec '19

13 Dec '19

commit 0c7a319ca0be61832c3c1d0cec769f0feb089c8d Merge: 7e179ed 0ab4900 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Fri Dec 13 12:04:37 2019 +0100 Merge remote-tracking branch 'JeremyRand/reproducible-go' projects/go/build | 4 ++++ 1 file changed, 4 insertions(+)

1 0

[tor-browser-build/master] Bug 32520: go: Remove go/pkg/obj/go-build/ from output
by boklm＠torproject.org 13 Dec '19

13 Dec '19

commit 0ab49005efd962074e80cafb52fe4dd9cb286229 Author: JeremyRand <biolizard89(a)gmail.com> Date: Mon Dec 2 02:55:33 2019 +0000 Bug 32520: go: Remove go/pkg/obj/go-build/ from output --- projects/go/build | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/go/build b/projects/go/build index 2ebfd58..06af067 100644 --- a/projects/go/build +++ b/projects/go/build @@ -73,6 +73,10 @@ cd /var/tmp/dist/go/src CGO_ENABLED=1 CC_FOR_TARGET=[% c("var/CC") %] CGO_CFLAGS='-D__ANDROID_API__=[% c("var/android_min_api") %]' CC= CFLAGS= LDFLAGS= ./make.bash [% END -%] +# This directory is non-reproducible and doesn't exist in official Go releases, +# so remove it to preserve reproducibility of the output. +rm -rf /var/tmp/dist/go/pkg/obj/go-build/ + cd /var/tmp/dist [% c('tar', { tar_src => [ project ],

1 0