[tor-commits] [tor-browser-build/master] Bug 21655: add a script to clean old built components
boklm at torproject.org
boklm at torproject.org
Mon Mar 20 17:38:22 UTC 2017
commit 5d046ea3c4af539e36a886ee3a7b03c07f5d7b48
Author: Nicolas Vigier <boklm at torproject.org>
Date: Mon Mar 20 18:29:03 2017 +0100
Bug 21655: add a script to clean old built components
---
Makefile | 5 ++-
README | 10 ++++-
rbm | 2 +-
rbm.local.conf.example | 28 ++++++++++++
tools/clean-old | 114 +++++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 155 insertions(+), 4 deletions(-)
diff --git a/Makefile b/Makefile
index 9481537..b82309c 100644
--- a/Makefile
+++ b/Makefile
@@ -86,6 +86,9 @@ submodule-update:
fetch: submodule-update
$(rbm) fetch
-clean-old: submodule-update
+clean: submodule-update
./tools/clean-old
+clean-dry-run: submodule-update
+ ./tools/clean-old --dry-run
+
diff --git a/README b/README
index a14fb42..f4a74e0 100644
--- a/README
+++ b/README
@@ -171,8 +171,14 @@ file.
Cleaning obsolete files and containers images
---------------------------------------------
-There will be a script to clean old build files and containers that are
-no longer used, but it has not been added yet.
+You can run 'make clean' to clean old build files and containers that
+are no longer used in current builds. Before doing that, you need to
+configure the branches and build targets you are using in the
+rbm.local.conf file. The cleaning script will checkout all the configured
+branches to create a list of used build files, and delete the files
+from the 'out' directory that are not used. If you want to see the list
+of files and containers that would be removed without doing it, you can
+use 'make clean-dry-run'.
Multiple build directories on the same host
diff --git a/rbm b/rbm
index 845bb90..3f3886e 160000
--- a/rbm
+++ b/rbm
@@ -1 +1 @@
-Subproject commit 845bb904fb87b6c56ab789f35cd02bce32453f52
+Subproject commit 3f3886e1f210ad2853209c5aecd0951350a6f758
diff --git a/rbm.local.conf.example b/rbm.local.conf.example
index 2a86647..78de08f 100644
--- a/rbm.local.conf.example
+++ b/rbm.local.conf.example
@@ -39,6 +39,34 @@ var:
### file.
#sign_build_gpg_opts: '--local-user XXXXXXXX'
+ ### The clean configuration is used by the cleaning script to find the
+ ### branches and build targets you are using, to compute the list of
+ ### files that should be kept.
+ ###
+ ### If you only do alpha builds for all platforms, you can use the
+ ### following configuration:
+ #clean:
+ # HEAD:
+ # - project: release
+ # target:
+ # - alpha
+ # - torbrowser-all
+ #
+ ### If you are doing 'release' builds in the maint-7.0 branch and
+ ### 'alpha' builds in the master branch, you can use the following
+ ### configuration:
+ #clean:
+ # master:
+ # - project: release
+ # target:
+ # - alpha
+ # - torbrowser-all
+ # maint-7.0:
+ # - project: release
+ # target:
+ # - release
+ # - torbrowser-all
+
targets:
### testbuild is based on alpha by default. Uncomment this if you want it
diff --git a/tools/clean-old b/tools/clean-old
new file mode 100755
index 0000000..4d603fc
--- /dev/null
+++ b/tools/clean-old
@@ -0,0 +1,114 @@
+#!/usr/bin/perl -w
+use strict;
+use IO::CaptureOutput qw(capture_exec);
+use FindBin;
+use lib "$FindBin::Bin/../rbm/lib";
+use RBM;
+use File::Slurp qw(read_dir write_file read_file);
+use Getopt::Long;
+
+my %options;
+my @options_list = qw(dry-run! list-used-files=s project=s target=s@);
+Getopt::Long::GetOptionsFromArray(\@ARGV, \%options, @options_list) || exit 1;
+
+sub clean_file {
+ my ($file, $used_files) = @_;
+ return if $used_files->{$file};
+ if (-d $file) {
+ my @l = read_dir($file);
+ foreach my $subfile (@l) {
+ clean_file("$file/$subfile", $used_files);
+ }
+ @l = read_dir($file);
+ rmdir $file unless @l;
+ } else {
+ print "Removing file $file\n";
+ unlink $file unless $options{'dry-run'};
+ }
+}
+
+sub clean_docker_images {
+ my ($dockerdir, $used_files) = @_;
+ my $imgprefix = RBM::project_config('docker-image', 'docker_image_prefix');
+ my @imgs = read_dir($dockerdir);
+ foreach my $dockerimage (@imgs) {
+ next if $used_files->{"$dockerdir/$dockerimage"};
+ my $img = "$imgprefix:$dockerimage";
+ print "Cleaning docker image $img\n";
+ next if $options{'dry-run'};
+ my ($out, $err, $success) = capture_exec('docker', 'rmi', '-f', $img);
+ if (!$success) {
+ print STDERR "Error removing docker image $img:\n$err\n";
+ exit 1;
+ }
+ unlink "$dockerdir/$dockerimage";
+ }
+}
+
+sub get_project_input_files {
+ my ($project, @targets) = @_;
+ print "Getting input files for $project ", join(' ', @targets), "\n";
+ $RBM::config->{run}{target} = \@targets;
+ my $res = RBM::project_config($project, 'input_files_paths',
+ {error_if_undef => 1});
+ return ref $res eq 'ARRAY' ? @$res : ();
+}
+
+sub get_input_files {
+ my ($c) = @_;
+ my @files;
+ foreach my $p (@$c) {
+ my $tmp = File::Temp->new();
+ my @args = ("$FindBin::Bin/$FindBin::Script", '--list-used-files', $tmp,
+ '--project', $p->{project});
+ foreach my $target (@{$p->{target}}) {
+ push @args, ('--target', $target);
+ }
+ RBM::exit_error 'Error getting used files' unless system(@args) == 0;
+ push @files, map { chomp; $_ } read_file($tmp);
+ }
+ return @files;
+}
+
+RBM::load_config;
+RBM::load_system_config;
+RBM::load_local_config;
+RBM::set_default_env;
+
+if ($options{'list-used-files'}) {
+ if (!$options{project} || !$options{target}) {
+ RBM::exit_error 'Missing option project or target';
+ }
+ my @files = get_project_input_files($options{project}, @{$options{target}});
+ write_file($options{'list-used-files'}, join("\n", @files));
+ exit 0;
+}
+
+my $clean = RBM::project_config('clean', 'var/clean');
+if (!$clean) {
+ print STDERR "Clean configuration is missing. ",
+ "You should add it to rbm.local.conf.\n";
+ exit 1;
+}
+
+my @files = get_input_files($clean->{HEAD}) if $clean->{HEAD};
+
+foreach my $branch (keys %$clean) {
+ next if $branch eq 'HEAD';
+ print "Checking out $branch branch\n";
+ RBM::exit_error("Error checking out $branch branch")
+ unless system('git', 'checkout', $branch) == 0;
+ RBM::exit_error("Error running git submodule update --init")
+ unless system('git', 'submodule', 'update', '--init') == 0;
+ push @files, get_input_files($clean->{$branch});
+ RBM::exit_error('Error checking out @{-1}')
+ unless system('git', 'checkout', '@{-1}') == 0;
+ RBM::exit_error("Error running git submodule update --init")
+ unless system('git', 'submodule', 'update', '--init') == 0;
+}
+my %used_files = map { $_ => 1 } @files;
+my $outdir = $RBM::config->{basedir} . '/out';
+# Don't clean docker-image files yet
+$used_files{"$outdir/docker-image"} = 1;
+clean_file($outdir, \%used_files);
+clean_docker_images("$outdir/docker-image", \%used_files);
More information about the tor-commits
mailing list