commit 5d046ea3c4af539e36a886ee3a7b03c07f5d7b48 Author: Nicolas Vigier <boklm@torproject.org> Date: Mon Mar 20 18:29:03 2017 +0100 Bug 21655: add a script to clean old built components --- Makefile | 5 ++- README | 10 ++++- rbm | 2 +- rbm.local.conf.example | 28 ++++++++++++ tools/clean-old | 114 +++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 155 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 9481537..b82309c 100644 --- a/Makefile +++ b/Makefile @@ -86,6 +86,9 @@ submodule-update: fetch: submodule-update $(rbm) fetch -clean-old: submodule-update +clean: submodule-update ./tools/clean-old +clean-dry-run: submodule-update + ./tools/clean-old --dry-run + diff --git a/README b/README index a14fb42..f4a74e0 100644 --- a/README +++ b/README @@ -171,8 +171,14 @@ file. Cleaning obsolete files and containers images --------------------------------------------- -There will be a script to clean old build files and containers that are -no longer used, but it has not been added yet. +You can run 'make clean' to clean old build files and containers that +are no longer used in current builds. Before doing that, you need to +configure the branches and build targets you are using in the +rbm.local.conf file. The cleaning script will checkout all the configured +branches to create a list of used build files, and delete the files +from the 'out' directory that are not used. If you want to see the list +of files and containers that would be removed without doing it, you can +use 'make clean-dry-run'. Multiple build directories on the same host diff --git a/rbm b/rbm index 845bb90..3f3886e 160000 --- a/rbm +++ b/rbm @@ -1 +1 @@ -Subproject commit 845bb904fb87b6c56ab789f35cd02bce32453f52 +Subproject commit 3f3886e1f210ad2853209c5aecd0951350a6f758 diff --git a/rbm.local.conf.example b/rbm.local.conf.example index 2a86647..78de08f 100644 --- a/rbm.local.conf.example +++ b/rbm.local.conf.example @@ -39,6 +39,34 @@ var: ### file. #sign_build_gpg_opts: '--local-user XXXXXXXX' + ### The clean configuration is used by the cleaning script to find the + ### branches and build targets you are using, to compute the list of + ### files that should be kept. + ### + ### If you only do alpha builds for all platforms, you can use the + ### following configuration: + #clean: + # HEAD: + # - project: release + # target: + # - alpha + # - torbrowser-all + # + ### If you are doing 'release' builds in the maint-7.0 branch and + ### 'alpha' builds in the master branch, you can use the following + ### configuration: + #clean: + # master: + # - project: release + # target: + # - alpha + # - torbrowser-all + # maint-7.0: + # - project: release + # target: + # - release + # - torbrowser-all + targets: ### testbuild is based on alpha by default. Uncomment this if you want it diff --git a/tools/clean-old b/tools/clean-old new file mode 100755 index 0000000..4d603fc --- /dev/null +++ b/tools/clean-old @@ -0,0 +1,114 @@ +#!/usr/bin/perl -w +use strict; +use IO::CaptureOutput qw(capture_exec); +use FindBin; +use lib "$FindBin::Bin/../rbm/lib"; +use RBM; +use File::Slurp qw(read_dir write_file read_file); +use Getopt::Long; + +my %options; +my @options_list = qw(dry-run! list-used-files=s project=s target=s@); +Getopt::Long::GetOptionsFromArray(\@ARGV, \%options, @options_list) || exit 1; + +sub clean_file { + my ($file, $used_files) = @_; + return if $used_files->{$file}; + if (-d $file) { + my @l = read_dir($file); + foreach my $subfile (@l) { + clean_file("$file/$subfile", $used_files); + } + @l = read_dir($file); + rmdir $file unless @l; + } else { + print "Removing file $file\n"; + unlink $file unless $options{'dry-run'}; + } +} + +sub clean_docker_images { + my ($dockerdir, $used_files) = @_; + my $imgprefix = RBM::project_config('docker-image', 'docker_image_prefix'); + my @imgs = read_dir($dockerdir); + foreach my $dockerimage (@imgs) { + next if $used_files->{"$dockerdir/$dockerimage"}; + my $img = "$imgprefix:$dockerimage"; + print "Cleaning docker image $img\n"; + next if $options{'dry-run'}; + my ($out, $err, $success) = capture_exec('docker', 'rmi', '-f', $img); + if (!$success) { + print STDERR "Error removing docker image $img:\n$err\n"; + exit 1; + } + unlink "$dockerdir/$dockerimage"; + } +} + +sub get_project_input_files { + my ($project, @targets) = @_; + print "Getting input files for $project ", join(' ', @targets), "\n"; + $RBM::config->{run}{target} = \@targets; + my $res = RBM::project_config($project, 'input_files_paths', + {error_if_undef => 1}); + return ref $res eq 'ARRAY' ? @$res : (); +} + +sub get_input_files { + my ($c) = @_; + my @files; + foreach my $p (@$c) { + my $tmp = File::Temp->new(); + my @args = ("$FindBin::Bin/$FindBin::Script", '--list-used-files', $tmp, + '--project', $p->{project}); + foreach my $target (@{$p->{target}}) { + push @args, ('--target', $target); + } + RBM::exit_error 'Error getting used files' unless system(@args) == 0; + push @files, map { chomp; $_ } read_file($tmp); + } + return @files; +} + +RBM::load_config; +RBM::load_system_config; +RBM::load_local_config; +RBM::set_default_env; + +if ($options{'list-used-files'}) { + if (!$options{project} || !$options{target}) { + RBM::exit_error 'Missing option project or target'; + } + my @files = get_project_input_files($options{project}, @{$options{target}}); + write_file($options{'list-used-files'}, join("\n", @files)); + exit 0; +} + +my $clean = RBM::project_config('clean', 'var/clean'); +if (!$clean) { + print STDERR "Clean configuration is missing. ", + "You should add it to rbm.local.conf.\n"; + exit 1; +} + +my @files = get_input_files($clean->{HEAD}) if $clean->{HEAD}; + +foreach my $branch (keys %$clean) { + next if $branch eq 'HEAD'; + print "Checking out $branch branch\n"; + RBM::exit_error("Error checking out $branch branch") + unless system('git', 'checkout', $branch) == 0; + RBM::exit_error("Error running git submodule update --init") + unless system('git', 'submodule', 'update', '--init') == 0; + push @files, get_input_files($clean->{$branch}); + RBM::exit_error('Error checking out @{-1}') + unless system('git', 'checkout', '@{-1}') == 0; + RBM::exit_error("Error running git submodule update --init") + unless system('git', 'submodule', 'update', '--init') == 0; +} +my %used_files = map { $_ => 1 } @files; +my $outdir = $RBM::config->{basedir} . '/out'; +# Don't clean docker-image files yet +$used_files{"$outdir/docker-image"} = 1; +clean_file($outdir, \%used_files); +clean_docker_images("$outdir/docker-image", \%used_files);