This is an automated email from the git hooks/post-receive script.

gk pushed a commit to branch master in repository builders/rbm.

commit 278411fb12ef71b6f2973ce70669118748474124 Author: Nicolas Vigier boklm@torproject.org AuthorDate: Mon Jun 27 15:40:21 2022 +0200

Bug 40028: Load projects config from modules --- lib/RBM.pm | 31 ++++++++++++++++++++++++------ test.pl | 17 +++++++++++++++- test/modules/module_1/projects/a/config | 1 + test/modules/module_1/projects/m1_a/config | 2 ++ 4 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/lib/RBM.pm b/lib/RBM.pm index 7e29d3e..ee987c8 100644 --- a/lib/RBM.pm +++ b/lib/RBM.pm @@ -90,6 +90,13 @@ sub load_modules_config { my $cfile = "$d/$module/rbm.module.conf"; $config->{modules}{$module} = load_config_file($cfile) if -f $cfile; + next unless -d "$d/$module/projects"; + for my $project (map { $_->basename } path("$d/$module/projects")->children) { + next if $config->{projects}{$project}; + next unless -f "$d/$module/projects/$project/config"; + $config->{modules}{$module}{projects}{$project} = + load_config_file("$d/$module/projects/$project/config"); + } } } } @@ -249,8 +256,17 @@ sub project_config { $config->{opt} = { %{$config->{opt}}, %$options } if $options; my @modules = map { [ 'modules', $_ ] } sort keys %{ $config->{modules} }; + my $project_path = ['projects', $project]; + if (!$config->{projects}{$project}) { + for my $module (sort keys %{ $config->{modules} }) { + if ($config->{modules}{$module}{projects}{$project}) { + $project_path = [ 'modules', $module, 'projects', $project ]; + last; + } + } + } $res = config($project, $name, $options, ['opt', 'norec'], ['opt'], - ['run'], ['projects', $project], ['local'], [], + ['run'], $project_path, ['local'], [], @modules, ['system'], ['default']); if (!$options->{no_tmpl} && defined($res) && !ref $res && !notmpl(confkey_str($name), $project)) { @@ -381,8 +397,11 @@ sub valid_id {

sub valid_project { my ($project) = @_; - exists $config->{projects}{$project} - || exit_error "Unknown project $project"; + return 1 if $config->{projects}{$project}; + for my $module (keys %{$config->{modules}}) { + return 1 if $config->{modules}{$module}{projects}{$project}; + } + exit_error "Unknown project $project"; }

sub create_dir { @@ -395,7 +414,7 @@ sub create_dir {

sub git_need_fetch { my ($project, $options) = @_; - return 0 if $config->{projects}{$project}{fetched}; + return 0 if $config->{_rbm}{fetched_projects}{$project}; my $fetch = project_config($project, 'fetch', $options); if ($fetch eq 'if_needed') { my $git_hash = project_config($project, 'git_hash', $options) @@ -440,13 +459,13 @@ sub git_clone_fetch_chdir { system('git', 'fetch', @fetch_submod, 'origin', '+refs/tags/*:refs/tags/*') == 0 || exit_error "Error fetching git repository"; - $config->{projects}{$project}{fetched} = 1; + $config->{_rbm}{fetched_projects}{$project} = 1; } }

sub hg_need_fetch { my ($project, $options) = @_; - return 0 if $config->{projects}{$project}{fetched}; + return 0 if $config->{_rbm}{fetched_projects}{$project}; my $fetch = project_config($project, 'fetch', $options); if ($fetch eq 'if_needed') { my $hg_hash = project_config($project, 'hg_hash', $options) diff --git a/test.pl b/test.pl index 0700d9c..d71baaa 100755 --- a/test.pl +++ b/test.pl @@ -1,7 +1,7 @@ #!/usr/bin/perl -w use strict; use Path::Tiny; -use Test::More tests => 31; +use Test::More tests => 34; use lib 'lib/';

sub set_target { @@ -158,6 +158,21 @@ my @tests = ( config => [ 'b', 'module_m'], expected => '1', }, + { + name => 'Using option defined in a module project', + config => [ 'm1_a', 'm1_a' ], + expected => 'm1_a', + }, + { + name => 'Using option defined in a module project and rbm.module.conf', + config => [ 'm1_a', 'project_m' ], + expected => 'm1_a', + }, + { + name => 'Using option defined in main projects and a module project', + config => [ 'a', 'project_a' ], + expected => 'a', + }, { name => 'build + steps config - 1', target => [ 'version_1' ], diff --git a/test/modules/module_1/projects/a/config b/test/modules/module_1/projects/a/config new file mode 100644 index 0000000..2f2e097 --- /dev/null +++ b/test/modules/module_1/projects/a/config @@ -0,0 +1 @@ +project_a: from_module_1 diff --git a/test/modules/module_1/projects/m1_a/config b/test/modules/module_1/projects/m1_a/config new file mode 100644 index 0000000..9f8b6c6 --- /dev/null +++ b/test/modules/module_1/projects/m1_a/config @@ -0,0 +1,2 @@ +m1_a: m1_a +project_m: m1_a

This is an automated email from the git hooks/post-receive script.

gk pushed a commit to branch master in repository builders/rbm.

commit 03ab77d9190a1c700ca442160cae0e4d7b732858 Author: Nicolas Vigier boklm@torproject.org AuthorDate: Tue Jun 28 14:10:59 2022 +0200

Bug 40028: Add support for keyring directories in modules

We also remove the `gpg_keyring_dir` option since this would make things more complex, and changing the `keyring` directory name is not very useful. --- doc/options_tar.asc | 7 +------ doc/rbm_input_files.asc | 6 +++--- doc/rbm_layout.asc | 3 +-- lib/RBM/DefaultConfig.pm | 18 +++++++++++++++--- 4 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/doc/options_tar.asc b/doc/options_tar.asc index e617c0b..b615dec 100644 --- a/doc/options_tar.asc +++ b/doc/options_tar.asc @@ -57,12 +57,7 @@ gpg_wrapper::

gpg_keyring:: The filename of the gpg keyring to use. Path is relative to the - +gpg_keyring_dir+ directory. This can also be an absolute path. - -gpg_keyring_dir:: - The directory containing gpg keyring files. The default is - +$basedir/keyring+ (with $basedir the directory where the main - config file is located). + keyring directory. This can also be an absolute path.

gpg_bin:: The gpg command to be used. The default is +gpg+. diff --git a/doc/rbm_input_files.asc b/doc/rbm_input_files.asc index abe366b..0c450eb 100644 --- a/doc/rbm_input_files.asc +++ b/doc/rbm_input_files.asc @@ -112,9 +112,9 @@ file_gpg_id:: If this option is set to 1, the file is checked for a gpg signature. If it is set to an other non zero value, or an array of non zero values, those are expected to be the valid gpg key - IDs. The +gpg_wrapper+, +gpg_keyring+, +gpg_keyring_dir+, - +gpg_bin+, +gpg_args+ options can be used to configure the gpg - check. Those options can be overriden in the file descriptor. + IDs. The +gpg_wrapper+, +gpg_keyring+, +gpg_bin+, +gpg_args+ + options can be used to configure the gpg check. Those options + can be overriden in the file descriptor.

sig_ext:: An array of file extensions for the gpg signature file. Those diff --git a/doc/rbm_layout.asc b/doc/rbm_layout.asc index ef700cb..c85658d 100644 --- a/doc/rbm_layout.asc +++ b/doc/rbm_layout.asc @@ -96,8 +96,7 @@ hg_clones:: keyring:: This is the directory where you can store gpg keyring files. Those keyrings contains the keys that are used to check signed - commits and tags. The path of this directory can be changed with - the 'gpg_keyring_dir' option. + commits and tags.

projects:: This is the directory containing projects definitions. The path diff --git a/lib/RBM/DefaultConfig.pm b/lib/RBM/DefaultConfig.pm index 4fbe079..109ca8d 100644 --- a/lib/RBM/DefaultConfig.pm +++ b/lib/RBM/DefaultConfig.pm @@ -146,13 +146,25 @@ our %default_config = ( #### gpg_bin => 'gpg', gpg_args => '', - gpg_keyring_dir => '[% config.basedir %]/keyring', + gpg_keyring_path => sub { + my ($project, $options) = @_; + my $gpg_keyring = RBM::project_config($project, 'gpg_keyring', $options); + return undef unless $gpg_keyring; + return $gpg_keyring if $gpg_keyring =~ m|^/|; + my $rootpath = RBM::rbm_path("keyring/$gpg_keyring"); + return $rootpath if -f $rootpath; + for my $module (sort keys %{$RBM::config->{modules}}) { + my $modulepath = RBM::rbm_path("modules/$module/keyring/$gpg_keyring"); + return $modulepath if -f $modulepath; + } + RBM::exit_error("keyring file $gpg_keyring is missing") + }, gpg_wrapper => <<GPGEND, #!/bin/sh export LC_ALL=C [% - IF c('gpg_keyring'); - SET gpg_kr = '--keyring ' _ path(c('gpg_keyring'), path(c('gpg_keyring_dir'))) + IF c('gpg_keyring_path'); + SET gpg_kr = '--keyring ' _ c('gpg_keyring_path') _ ' --no-default-keyring --no-auto-check-trustdb --trust-model always'; END; -%]

This is an automated email from the git hooks/post-receive script.

gk pushed a commit to branch master in repository builders/rbm.

commit ce18f12cc5114ca82dc2964a62bce835dcc5a1f0 Author: Nicolas Vigier boklm@torproject.org AuthorDate: Wed Jun 29 16:26:41 2022 +0200

Bug 40028: Add documentation for modules --- doc/Makefile | 1 + doc/documentation.asc | 1 + doc/rbm_config.asc | 36 +++++++++++++++++++----- doc/rbm_layout.asc | 77 ++++++++++++++++++++++++++++++++++----------------- doc/rbm_modules.asc | 54 ++++++++++++++++++++++++++++++++++++ 5 files changed, 137 insertions(+), 32 deletions(-)

diff --git a/doc/Makefile b/doc/Makefile index 8564a19..7aeb0f2 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -33,6 +33,7 @@ MAN1_ASC += rbm-build.asc MAN7_ASC += rbm_tutorial.asc MAN7_ASC += rbm_layout.asc MAN7_ASC += rbm_config.asc +MAN7_ASC += rbm_modules.asc MAN7_ASC += rbm_templates.asc MAN7_ASC += rbm_remote.asc MAN7_ASC += rbm_targets.asc diff --git a/doc/documentation.asc b/doc/documentation.asc index 4c5c9bc..eabf738 100644 --- a/doc/documentation.asc +++ b/doc/documentation.asc @@ -1,6 +1,7 @@ - link:rbm_tutorial.html[rbm_tutorial(7)] : A tutorial introduction to rbm - link:rbm_layout.html[rbm_layout(7)] : The rbm layout - link:rbm_config.html[rbm_config(7)] : The rbm configuration +- link:rbm_modules.html[rbm_modules(7)] : The rbm modules - link:rbm_cli.html[rbm_cli(7)] : The rbm command line interface - link:rbm_templates.html[rbm_templates(7)] : The rbm templates - link:rbm_remote.html[rbm_remote(7)] : The rbm remote builds diff --git a/doc/rbm_config.asc b/doc/rbm_config.asc index 0bfed04..46c1caf 100644 --- a/doc/rbm_config.asc +++ b/doc/rbm_config.asc @@ -8,10 +8,13 @@ rbm_config - The rbm configuration DESCRIPTION -----------

-All configuration options can be defined in 5 different places : +All configuration options can be defined in different places :

- in the main configuration in your working directory (+rbm.conf+)

+- in a module configuration at the root of a module directory + (+rbm.module.conf+) + - in the local configuration in your working directory (+rbm.local.conf+)

- in the global system configuration (+/etc/rbm.conf+) @@ -48,6 +51,14 @@ The option values are used with the following priority order :

- workspace config

+- modules config for matching step and target + +- modules config for matching step + +- modules config for matching target + +- modules config + - system config for matching step and target

- system config for matching step @@ -79,23 +90,33 @@ file is used to override some options from +rbm.conf+ relevent to your local setup, whithout modifying the +rbm.conf+ file which is usually maintained in a VCS.

+The module configuration files are +rbm.module.conf+ located at the +root of each module directory: a module named +A+ will have its +configuration in +modules/A/rbm.module.conf+. Module configuration +is used in alphabetical order: configuration from a module named ++A+ will have priority over a module named +B+. + An example +rbm.conf+ file will look like this :

---- -projects_dir: projects compress_tar: xz +output_dir: "out/[% project %]" ----

-The +projects_dir+ option define the path to the directory containing -the projects definitions. - Adding a new project is done by creating a directory with the name of -the project inside the +projects_dir+ directory, and adding a +config+ +the project inside the +projects+ directory, and adding a +config+ file in this new directory. The +config+ file contains the configuration for the project. At the minimum it should contain the +git_url+ configuration, and any other configuration option you want to set for this project.

+Projects can also be defined inside a module. In this case the module +directory will include a +projects+ directory working in a similar way +to the main +projects+ directory. If a project exists with the same +name both in a module and the main +projects+ directory, the module one +will be ignored. If a project is defined in multiple modules, only the +project from the first module in alphabetic order will be used. + OPTIONS -------

@@ -337,4 +358,5 @@ SEE ALSO -------- link:rbm.html[rbm(1)], link:rbm_targets.html[rbm_targets(7)], -link:rbm_templates.html[rbm_templates(7)] +link:rbm_templates.html[rbm_templates(7)], +link:rbm_modules.html[rbm_modules(7)] diff --git a/doc/rbm_layout.asc b/doc/rbm_layout.asc index c85658d..1261cf2 100644 --- a/doc/rbm_layout.asc +++ b/doc/rbm_layout.asc @@ -26,15 +26,17 @@ and 'project_2' will look like this : | | | |-- project_1 | | | - | | |-- project_1-0.1.tar.gz + | | |-- project_1-0.1-src.tar.gz | | | | | +-- project_1-0.1-1.x86_64.rpm + | | | + | | +-- project_1-0.1-1.x86_64.tar.gz | | | +-- project_2 | | - | |-- project_2-7.3.tar.bz2 + | |-- project_2-7.3.tar-src.bz2 | | - | +-- project_2-7.1-1.x86_64.rpm + | +-- project_2-7.1-1.x86_64.tar.gz | |-- git_clones | | @@ -46,29 +48,43 @@ and 'project_2' will look like this : | |-- keyring | | - | |-- keyring-1 + | |-- project_1.gpg | | - | +-- keyring-2 + | +-- project_2.gpg | +-- projects + | | + | |-- project_1 + | | | + | | |-- config + | | | + | | |-- project_1.spec + | | | + | | |-- bugfix.patch + | | + | +-- project_2 + | | + | |-- config + | | + | +-- build + | + +-- modules | - |-- project_1 - | | - | |-- config - | | - | |-- project_1.spec - | | - | |-- bugfix.patch - | | - | +-- debian_rules - | - +-- project_2 + +-- container | - |-- config + |-- rbm.module.conf | - |-- project_2.spec + |-- projects + | | + | +-- container-image + | | + | |-- config + | | + | +-- build | - +-- debian_rules + +-- keyring + | + +-- container-image.gpg ----

You will file the following files and directories : @@ -115,13 +131,24 @@ projects/project_1/project_1.spec:: projects/project_1/bugfix.patch:: This is an example patch file, used in the rpm package.

-projects/project_1/debian_rules:: - This is an example path for a debian/rules file to be used in - the debian packaging for the project_1 project. All the debian - packaging can be defined entirely inside the 'config' file, but - it is usually convenient to split some of it in separate files. +projects/project_1/build:: + This is a build script for project_1, containing instructions + for building the project and generating a tar.gz file containing + the build output. + +modules/container/rbm.local.conf:: + In this example we use an external rbm module named 'container' + (usually included as a git-submodule). The 'rbm.local.conf' file + located at the root of the module contains the options related + to the projects included in the module. + +modules/container/projects:: + The projects included in the projects directory of a module + work in the same way as those in the main projects directory. +

SEE ALSO -------- link:rbm.html[rbm(1)], -link:rbm_config.html[rbm_config(1)] +link:rbm_config.html[rbm_config(7)], +link:rbm_modules.html[rbm_modules(7)] diff --git a/doc/rbm_modules.asc b/doc/rbm_modules.asc new file mode 100644 index 0000000..9d0517a --- /dev/null +++ b/doc/rbm_modules.asc @@ -0,0 +1,54 @@ +rbm_modules(7) +============= + +NAME +---- +rbm_modules - rbm modules + +DESCRIPTION +----------- + +An rbm workspace can contain a number of different projects. In some +cases it can be useful to share some of those projects between multiple +workspaces. This can be done using modules. + +A module is a subdirectory in the +modules+ directory. If you are using +git, it is possible to store each module directory using a separate ++git-submodule+. + +A module directory structure is similar to the main workspace structure: +each module includes a +projects+ directory containing the projects, +a +keyring+ directory containing the gpg keyrings used by the projects, +and a +rbm.module.conf+ containing rbm configuration related to the +projects included in the module. + + +RBM CONFIGURATION +----------------- + +The rbm configuration from a module (the +rbm.module.conf+ and the +project +config+ files) can define any configuration option. +However, for clarity and to avoid name collisions with other modules or +with core rbm options, it is recommended to prefix all option names +with +var_${module_name}+ (where +${module_name}+ is the module name). + +The workspace configuration (+rbm.conf+) has priority over the modules +configuration (+rbm.module.conf+). Module configuration is used in +alphabetical order: configuration from a module named +A+ will have +priority over a module named +B+. See +link:rbm_config.html[rbm_config(1)] for more details. + +KEYRING +------- + +The keyring directory in a module contains gpg keyring files (see the ++gpg_keyring+ option). If a keyring file is available in multiple +modules, the one from the first module in alphabetical order will be +used. If a keyring file is also available in the main workspace's +keyring directory, then it will be used. + + +SEE ALSO +-------- +link:rbm.html[rbm(1)], +link:rbm_config.html[rbm_config(1)]