GitLab

at 2026-01-28T09:51:39+01:00

Bug 40093: Add option to get the checksum of an input file

input_files_ids_by_name/$name can be used to get the id (sha256sum or
sha512sum) of an input file.

At the same time we remove the notmpl option for simplification, which
we have not used. We can add it back later in case it is useful.

@@ -246,8 +246,17 @@ input_files_by_name::
          with their +name+ as index. The input files without a +name+
          are not in this hash.
 +input_files_ids_by_name::
 +        This option contains the ids (hashes) of all the +input_files+,
 +        with their +name+ as index. The id of an input file will be the
 +        value of the option +input_file_id+ if it is defined, or
 +        +filename:hash+, where +filename+ is the filename of the file,
 +        and +hash+ is the sha512sum if the +sha512sum+ option is defined,
 +        the sha256sum otherwise. This option does not work for input
 +        files of type +project+, +exec+ or +content+.
++
  input_files_id::
 -        The value of this option is an identifier of the input_files.
 +        The value of this option is a single identifier of all the input_files.
          When any of the input files is changed, the identifier changes.
          This identifier is something that can be used in a project's
          filename to trigger a rebuild when any of its input files is
@@ -280,11 +289,6 @@ timestamp::
          commit time of the commit used. If set to 0 it will use the
          current time.
 -notmpl::
 -        An array containing a list of options that should not be
 -        processed as template (see the +template+ section below for
 -        details).
+-
  step::
          The value of this option is the name of the build script we are
          going to be running (by default 'build', but you could have an

@@ -53,7 +53,7 @@ filename::
  name::
          Optionaly the input_file can have a name. This name is used as
 -        key in the +input_files_by_name+ option.
 +        key in the +input_files_by_name+ and +input_files_ids_by_name+ options.
  content::
          The content of the file.

@@ -15,10 +15,7 @@ however, for the options that are needed to process templates, so they
  can't be templated themself. The following options are not templated :
   - projects_dir
+-
 -If you want to make other options not templated, add them to the
 -+notmpl+ config option, which is an array. All the other options are
 -automatically processed as template.
 + - modules_dir
  The template are made using perl Template Toolkit. You can read more
  about the syntax on the http://www.template-toolkit.org/[Template

@@ -237,11 +237,8 @@ sub config {
+ }
  sub notmpl {
 -    my ($name, $project) = @_;
 -    return 1 if $name eq 'notmpl';
 -    my @n = (@{$config->{default}{notmpl}},
 -        @{project_config($project, 'notmpl')});
 -    return grep { $name eq $_ } @n;
 +    my ($name) = @_;
 +    return grep { $name eq $_ } @{$config->{default}{notmpl}};
+ }
  sub confkey_str {
@@ -282,9 +279,13 @@ sub project_config {
                          ['run'], $project_path, ['local'], [],
                          @modules, ['system'], ['default']);
      if (!$options->{no_tmpl} && defined($res) && !ref $res
 -        && !notmpl(confkey_str($name), $project)) {
 -        $res = process_template($project, $res,
 -            confkey_str($name) eq 'output_dir' ? '.' : undef);
 +        && !notmpl(confkey_str($name))) {
 +        my $output_dir = undef;
 +        if (confkey_str($name) eq 'output_dir' ||
 +            confkey_str($name) eq 'getting_id_value/output_dir') {
 +            $output_dir = '.';
 +        }
 +        $res = process_template($project, $res, $output_dir);
+     }
      $config_cache{$project}{$step}{$name_str}{$cache_id} = $res;
      $config->{opt} = $opt_save;
@@ -733,7 +734,7 @@ sub process_template {
+         }
          return $res;
+     }
 -    $dest_dir //= rbm_path(project_config($project, 'output_dir'));
 +    $dest_dir //= rbm_path(project_config($project, 'getting_id_value/output_dir'));
      my $project_dir = modules_project_dir($project);
      my $common_dirs = join(':', modules_common_dirs($project));
      my $template = Template->new(
@@ -823,7 +824,7 @@ sub input_file_need_dl {
              $fname = undef;
+         }
+     }
 -    if ($action eq 'input_files_id') {
 +    if ($action eq 'input_files_id' || $action eq 'getfids') {
          return undef if $input_file->{input_file_id};
          for my $checksum (qw/sha512sum sha256sum/) {
              if ( ($input_file->{$checksum} || $input_file->{norec}{$checksum})
@@ -861,7 +862,7 @@ sub input_file_id {
              return $filename . ':' . $t->($checksum);
+         }
+     }
 -    my $opts = { norec => { output_dir => '/out', getting_id => 1, }};
 +    my $opts = { norec => { getting_id => 1, }};
      return $filename . ':' . sha256_hex($t->('exec', $opts))
                  if $input_file->{exec};
      shafile('sha256sum', $fname, { remove_cache => 1 }) if $t->('refresh_input');
@@ -923,6 +924,7 @@ sub input_files {
      my @res_copy;
      my %res_getfnames;
      my @res_getfpaths;
 +    my %res_getfids;
      my $getfnames_noname = 0;
      my $input_files_id = '';
      $options = {$options ? %$options : ()};
@@ -1016,6 +1018,11 @@ sub input_files {
              };
              next;
+         }
 +        if ($action eq 'getfids') {
 +            next if $input_file->{project};
 +            next if $input_file->{exec};
 +            next if $input_file->{content};
 +        }
          my $proj_out_dir;
          if ($input_file->{project}) {
              $proj_out_dir = rbm_path(project_step_config($t->('project'), 'output_dir',
@@ -1038,38 +1045,51 @@ sub input_files {
          my ($fname) = file_in_dir($name, $src_dir, $proj_out_dir,
                                          @modules_common_dirs);
          my $file_gpg_id = gpg_id($t->('file_gpg_id'));
 +        my $dl_file = sub {};
          if (input_file_need_dl($input_file, $t, $fname, $action)) {
 -            if ($t->('content')) {
 -                my $dname = dirname("$proj_out_dir/$name");
 -                make_path($dname) unless -d $dname;
 -                path("$proj_out_dir/$name")->spew_utf8($t->('content'));
 -            } elsif ($t->('URL')) {
 -                urlget($project, {%$options, %$input_file, filename => $name}, 1);
 -            } elsif ($t->('exec')) {
 -                my $exec_script = project_config($project, 'exec',
 -                    { $options ? %$options : (), %$input_file });
 -                if (run_script($project, $exec_script,
 -                        sub { system(@_) }) != 0) {
 -                    exit_error "Error creating $name";
 +            $dl_file = sub {
 +                if ($t->('content')) {
 +                    my $dname = dirname("$proj_out_dir/$name");
 +                    make_path($dname) unless -d $dname;
 +                    path("$proj_out_dir/$name")->spew_utf8($t->('content'));
 +                } elsif ($t->('URL')) {
 +                    urlget($project, {%$options, %$input_file, filename => $name}, 1);
 +                } elsif ($t->('exec')) {
 +                    my $exec_script = project_config($project, 'exec',
 +                        { $options ? %$options : (), %$input_file });
 +                    if (run_script($project, $exec_script,
 +                            sub { system(@_) }) != 0) {
 +                        exit_error "Error creating $name";
 +                    }
 +                } elsif ($input_file->{project} && $t->('project')) {
 +                    my $p = $t->('project');
 +                    print "Building project $p - $name\n";
 +                    my $run_save = $config->{run};
 +                    $config->{run} = { target => $input_file->{target} };
 +                    $config->{run}{target} //= $run_save->{target};
 +                    build_pkg($p, {%$options, origin_project => $project,
 +                            %$input_file});
 +                    $config->{run} = $run_save;
 +                    print "Finished build of project $p - $name\n";
 +                } else {
 +                    dd $input_file;
 +                    exit_error "Missing file $name";
+                 }
 -            } elsif ($input_file->{project} && $t->('project')) {
 -                my $p = $t->('project');
 -                print "Building project $p - $name\n";
 -                my $run_save = $config->{run};
 -                $config->{run} = { target => $input_file->{target} };
 -                $config->{run}{target} //= $run_save->{target};
 -                build_pkg($p, {%$options, origin_project => $project,
 -                               %$input_file});
 -                $config->{run} = $run_save;
 -                print "Finished build of project $p - $name\n";
 -            } else {
 -                dd $input_file;
 -                exit_error "Missing file $name";
 +                ($fname) = file_in_dir($name, $src_dir, $proj_out_dir,
 +                    @modules_common_dirs);
 +                exit_error "Error getting file $name" unless $fname;
+             }
 -            ($fname) = file_in_dir($name, $src_dir, $proj_out_dir,
 -                                        @modules_common_dirs);
 -            exit_error "Error getting file $name" unless $fname;
+         }
 +        if ($action eq 'getfids') {
 +            my $if_name = $t->('name') if $input_file->{name};
 +            next unless $if_name;
 +            $res_getfids{$if_name} = sub {
 +                $dl_file->();
 +                return input_file_id($input_file, $t, $fname, $name);
 +            };
 +            next;
 +        }
 +        $dl_file->();
          if ($action eq 'input_files_id') {
              $input_files_id .= input_file_id($input_file, $t, $fname, $name);
              $input_files_id .= "\n";
@@ -1145,6 +1165,7 @@ sub input_files {
      return @res_copy if ($action eq 'copy' || $action eq 'link');
      return \%res_getfnames if $action eq 'getfnames';
      return \@res_getfpaths if $action eq 'getfpaths';
 +    return \%res_getfids if $action eq 'getfids';
+ }
  sub system_log {

@@ -114,6 +114,12 @@ our %default_config = (
      projects_dir  => 'projects',
      modules_dir   => 'modules',
      output_dir    => 'out',
 +    # `getting_id_value/output_dir` is used as `output_dir` when processing
 +    # template. To avoid having ids being affected by changing `output_dir`,
 +    # a fixed value is returned when `getting_id` is set.
 +    getting_id_value   => {
 +        output_dir => '[% IF c("getting_id"); GET "/out"; ELSE; GET c("output_dir"); END %]',
 +    },
      git_clone_dir => 'git_clones',
      hg_clone_dir  => 'hg_clones',
      hg_clone_subdir  => '.',
@@ -122,7 +128,7 @@ our %default_config = (
      build         => '[% INCLUDE build -%]',
      build_log     => '-',
      build_log_append => '1',
 -    notmpl        => [ qw(projects_dir modules_dir) ],
 +    notmpl        => [ qw(projects_dir modules_dir getting_id) ],
      abbrev_length => '12',
      abbrev        => '[%
                           IF c("git_url");
@@ -626,6 +632,7 @@ ZIP_END
      touch => "[% USE date %]touch -m -t [% date.format(c('timestamp'), format = '%Y%m%d%H%M') %]",
      arch   => \&get_arch,
      input_files_by_name => sub { RBM::input_files('getfnames', @_); },
 +    input_files_ids_by_name => sub { RBM::input_files('getfids', @_); },
      input_files_id => sub { RBM::input_files('input_files_id', @_); },
      input_files_paths => sub { RBM::input_files('getfpaths', @_); },
      link_input_files => '[% IF c("remote_exec") %]1[% END %]',

boklm pushed to branch main at The Tor Project / Applications / RBM

Commits:

5 changed files:

Changes: