richard pushed to branch main at The Tor Project / Applications / RBM

Commits:

6 changed files:

Changes:

  • doc/rbm_input_files.asc
    ... ... @@ -26,7 +26,7 @@ The following input sources are available:
    26 26
    27 27 
     The file that has been retrieved can be verified with:
    28 28
    29 
    -- matching a specific sha256 checksum
    29 
    +- matching a specific sha256 or sha512 checksum
    30 30
    31 31 
     - a gpg signature file from a specific key or keyring
    32 32
    ... ... @@ -104,17 +104,22 @@ enable::
    104 104
    105 105 
     refresh_input::
    106 106 
             By default, if the file is already present, it is not downloaded
    107 
    -        or created again, except when an sha256sum is defined and the
    108 
    -        file present is not matching. If this option is set to a true
    109 
    -        value, the file will be removed and created again on each run,
    110 
    -        except when an sha256sum is defined and the file present is
    111 
    -        matching.
    107 
    +        or created again, except when an sha256sum or sha512sum is
    108 
    +        defined and the file present is not matching. If this option is
    109 
    +        set to a true value, the file will be removed and created again
    110 
    +        on each run, except when an sha256sum or sha512sum is defined
    111 
    +        and the file present is matching.
    112 112
    113 113 
     sha256sum::
    114 114 
             The sha256 checksum of the file. The build will fail with an
    115 115 
             error if the file does not match the expected sha256 checksum.
    116 116 
             If the value is empty, the checksum is not checked.
    117 117
    118 
    +sha512sum::
    119 
    +        The sha512 checksum of the file. The build will fail with an
    120 
    +        error if the file does not match the expected sha512 checksum.
    121 
    +        If the value is empty, the checksum is not checked.
    122 
    +
    118 123 
     file_gpg_id::
    119 124 
             If this option is set to 1, the file is checked for a gpg
    120 125 
             signature. If it is set to an other non zero value, or an array

  • doc/rbm_templates.asc
    ... ... @@ -138,10 +138,18 @@ sha256::
    138 138 
             A function returning the sha256 digest of its argument as an
    139 139 
             hexadecimal string.
    140 140
    141 
    +sha512::
    142 
    +        A function returning the sha512 digest of its argument as an
    143 
    +        hexadecimal string.
    144 
    +
    141 145 
     sha256file::
    142 146 
             A function returning the sha256 digest of a file as an hexadecimal
    143 147 
             string. If the file does not exist, an empty string is returned.
    144 148
    149 
    +sha512file::
    150 
    +        A function returning the sha512 digest of a file as an hexadecimal
    151 
    +        string. If the file does not exist, an empty string is returned.
    152 
    +
    145 153 
     fileparse::
    146 154 
             A function to parse a path. Returns an array containing the
    147 155 
             filename, and the directory path. This is the fileparse routine

  • lib/RBM.pm
    ... ... @@ -19,7 +19,7 @@ use String::ShellQuote;
    19 19 
     use Sort::Versions;
    20 20 
     use RBM::CaptureExec qw(capture_exec);
    21 21 
     use RBM::DefaultConfig;
    22 
    -use Digest::SHA qw(sha256_hex);
    22 
    +use Digest::SHA qw(sha256_hex sha512_hex);
    23 23 
     use Data::UUID;
    24 24 
     use Data::Dump qw(dd pp);
    25 25 
     use FindBin;
    ... ... @@ -673,16 +673,22 @@ sub maketar {
    673 673 
         return $tar_file;
    674 674 
     }
    675 675
    676 
    -sub sha256file {
    676 
    +sub shafile {
    677 677 
         CORE::state %res;
    678 
    +    my $type = shift;
    678 679 
         my $f = rbm_path(shift);
    679 680 
         my $opt = shift;
    681 
    +    my %sha_hex = (
    682 
    +        sha256sum => \&sha256_hex,
    683 
    +        sha512sum => \&sha512_hex,
    684 
    +    );
    685 
    +    exit_error "Unknown sha type $type" unless $sha_hex{$type};
    680 686 
         if (ref $opt eq 'HASH' && $opt->{remove_cache}) {
    681 
    -        delete $res{$f};
    687 
    +        delete $res{$type}{$f};
    682 688 
             return;
    683 689 
         }
    684 
    -    return $res{$f} if exists $res{$f};
    685 
    -    return $res{$f} = -f $f ? sha256_hex(path($f)->slurp_raw) : '';
    690 
    +    return $res{$type}{$f} if exists $res{$type}{$f};
    691 
    +    return $res{$type}{$f} = -f $f ? $sha_hex{$type}->(path($f)->slurp_raw) : '';
    686 692 
     }
    687 693
    688 694 
     sub process_template_opt {
    ... ... @@ -731,7 +737,11 @@ sub process_template {
    731 737 
             sha256      => sub {
    732 738 
                 return sha256_hex(encode("utf8", $_[0]));
    733 739 
             },
    734 
    -        sha256file  => \&sha256file,
    740 
    +        sha512      => sub {
    741 
    +            return sha512_hex(encode("utf8", $_[0]));
    742 
    +        },
    743 
    +        sha256file  => sub { return shafile('sha256sum', @_) },
    744 
    +        sha512file  => sub { return shafile('sha512sum', @_) },
    735 745 
             fileparse   => \&fileparse,
    736 746 
             ENV         => \%ENV,
    737 747 
         };
    ... ... @@ -782,18 +792,22 @@ sub file_in_dir {
    782 792 
     sub input_file_need_dl {
    783 793 
         my ($input_file, $t, $fname, $action) = @_;
    784 794 
         return undef if $action eq 'getfpaths';
    785 
    -    if ($fname
    786 
    -        && ($input_file->{sha256sum} || $input_file->{norec}{sha256sum})
    787 
    -        && $t->('sha256sum')
    788 
    -        && $t->('sha256sum') ne sha256file($fname)) {
    789 
    -        sha256file($fname, { remove_cache => 1 });
    790 
    -        $fname = undef;
    795 
    +    for my $checksum (qw/sha512sum sha256sum/) {
    796 
    +        if ($fname
    797 
    +            && ($input_file->{$checksum} || $input_file->{norec}{$checksum})
    798 
    +            && $t->($checksum)
    799 
    +            && $t->($checksum) ne shafile($checksum, $fname)) {
    800 
    +            shafile($checksum, $fname, { remove_cache => 1 });
    801 
    +            $fname = undef;
    802 
    +        }
    791 803 
         }
    792 804 
         if ($action eq 'input_files_id') {
    793 805 
             return undef if $input_file->{input_file_id};
    794 
    -        if ( ($input_file->{sha256sum} || $input_file->{norec}{sha256sum})
    795 
    -             && $t->('sha256sum') ) {
    796 
    -            return undef;
    806 
    +        for my $checksum (qw/sha512sum sha256sum/) {
    807 
    +            if ( ($input_file->{$checksum} || $input_file->{norec}{$checksum})
    808 
    +                && $t->($checksum) ) {
    809 
    +                return undef;
    810 
    +            }
    797 811 
             }
    798 812 
             return undef if $input_file->{exec};
    799 813 
             return undef if ($fname && !$t->('refresh_input'));
    ... ... @@ -808,8 +822,8 @@ sub input_file_need_dl {
    808 822 
     sub input_file_id_hash {
    809 823 
         my ($fname, $filename) = @_;
    810 824 
         exit_error "input_file_id: file $filename is missing" unless $fname;
    811 
    -    return $filename . ':' . sha256file($fname) if -f $fname;
    812 
    -    return $filename . ':' . sha256file(readlink $fname) if -l $fname;
    825 
    +    return $filename . ':' . shafile('sha256sum', $fname) if -f $fname;
    826 
    +    return $filename . ':' . shafile('sha256sum', readlink $fname) if -l $fname;
    813 827 
         my @subdirs = sort(map { $_->basename } path($fname)->children);
    814 828 
         my @hashes = map { input_file_id_hash("$fname/$_", "$filename/$_") } @subdirs;
    815 829 
         return join("\n", @hashes);
    ... ... @@ -819,9 +833,11 @@ sub input_file_id {
    819 833 
         my ($input_file, $t, $fname, $filename) = @_;
    820 834 
         return $t->('input_file_id') if $input_file->{input_file_id};
    821 835 
         return $input_file->{project} . ':' . $filename if $input_file->{project};
    822 
    -    if ( ($input_file->{sha256sum} || $input_file->{norec}{sha256sum})
    823 
    -         && $t->('sha256sum') ) {
    824 
    -        return $filename . ':' . $t->('sha256sum');
    836 
    +    for my $checksum (qw/sha512sum sha256sum/) {
    837 
    +        if ( ($input_file->{$checksum} || $input_file->{norec}{$checksum})
    838 
    +            && $t->($checksum) ) {
    839 
    +            return $filename . ':' . $t->($checksum);
    840 
    +        }
    825 841 
         }
    826 842 
         my $opts = { norec => { output_dir => '/out', getting_id => 1, }};
    827 843 
         return $filename . ':' . sha256_hex($t->('exec', $opts))
    ... ... @@ -1057,11 +1073,13 @@ sub input_files {
    1057 1073 
                 next;
    1058 1074 
             }
    1059 1075 
             exit_error "Missing file $name" unless $fname;
    1060 
    -        if ($t->('sha256sum')
    1061 
    -            && $t->('sha256sum') ne sha256file($fname)) {
    1062 
    -            exit_error "Can't have sha256sum on directory: $fname" if -d $fname;
    1063 
    -            exit_error "Wrong sha256sum for $fname.\n" .
    1064 
    -                       "Expected sha256sum: " . $t->('sha256sum');
    1076 
    +        for my $checksum (qw/sha512sum sha256sum/) {
    1077 
    +            if ($t->($checksum)
    1078 
    +                && $t->($checksum) ne shafile($checksum, $fname)) {
    1079 
    +                exit_error "Can't have $checksum on directory: $fname" if -d $fname;
    1080 
    +                exit_error "Wrong $checksum for $fname.\n" .
    1081 
    +                           "Expected $checksum: " . $t->($checksum);
    1082 
    +            }
    1065 1083 
             }
    1066 1084 
             if ($file_gpg_id) {
    1067 1085 
                 exit_error "Can't have gpg sig on directory: $fname" if -d $fname;

  • test.pl
    1 1 
     #!/usr/bin/perl -w
    2 2 
     use strict;
    3 3 
     use Path::Tiny;
    4 
    -use Test::More tests => 41;
    4 
    +use Test::More tests => 45;
    5 5 
     use lib 'lib/';
    6 6
    7 7 
     sub set_target {
    ... ... @@ -248,8 +248,7 @@ my @tests = (
    248 248 
             build => [ 'mozmill-automation', 'build' ],
    249 249 
             files => {
    250 250 
                 'out/mozmill-automation-bbad7215c713_sha256sum.txt' =>
    251 
    -            '13660d3f3ebbc363056ccbd3794f8f78a940dd394a464093bee5fc0575ee4090  '
    252 
    -            . "mozmill-automation-bbad7215c713.tar\n",
    251 
    +            "ceeda3cd3285b6ed53233dc65e3beac82f2b284402a80ef6c1fcdf5b9861f068  s.txt\n",
    253 252 
             },
    254 253 
         },
    255 254 
         {
    ... ... @@ -261,6 +260,28 @@ my @tests = (
    261 260 
                 "1\n2\n3\n4\n1\n2\n",
    262 261 
             },
    263 262 
         },
    263 
    +    {
    264 
    +        name => 'sha256sum input_files',
    265 
    +        target => [ 'sha256sum' ],
    266 
    +        build  => [ 'shasum', 'build' ],
    267 
    +        files  => {},
    268 
    +    },
    269 
    +    {
    270 
    +        name => 'sha512sum input_files',
    271 
    +        target => [ 'sha512sum' ],
    272 
    +        build  => [ 'shasum', 'build' ],
    273 
    +        files  => {},
    274 
    +    },
    275 
    +    {
    276 
    +        name => 'wrong sha256sum input_files',
    277 
    +        target => [ 'wrong_sha256sum' ],
    278 
    +        fail_build  => [ 'shasum', 'build' ],
    279 
    +    },
    280 
    +    {
    281 
    +        name => 'wrong sha512sum input_files',
    282 
    +        target => [ 'wrong_sha512sum' ],
    283 
    +        fail_build  => [ 'shasum', 'build' ],
    284 
    +    },
    264 285 
     );
    265 286
    266 287 
     foreach my $test (@tests) {
    ... ... @@ -279,4 +300,16 @@ foreach my $test (@tests) {
    279 300 
             my $res = grep { path($_)->slurp_utf8 ne $test->{files}{$_} } keys %{$test->{files}};
    280 301 
             ok(!$res, $test->{name});
    281 302 
         }
    303 
    +    if ($test->{fail_build}) {
    304 
    +        my $pid = fork;
    305 
    +        if (!$pid) {
    306 
    +            close STDOUT;
    307 
    +            close STDERR;
    308 
    +            RBM::build_run(@{$test->{fail_build}});
    309 
    +            exit 0;
    310 
    +        }
    311 
    +        wait;
    312 
    +        my $exit_code = $?;
    313 
    +        ok($exit_code, $test->{name});
    314 
    +    }
    282 315 
     }

  • test/projects/mozmill-automation/config
    ... ... @@ -5,4 +5,11 @@ compress_tar: ''
    5 5 
     t: '[% sha256(exec("cat testrun_remote.py")) %]'
    6 6 
     build: |
    7 7 
       #!/bin/sh
    8 
    -  sha256sum [% project %]-[% c("version") %].tar > [% dest_dir %]/[% project %]-[% c("version") %]_sha256sum.txt
    8 
    +  mkdir t
    9 
    +  tar -C t -xf mozmill-automation-bbad7215c713.tar
    10 
    +  files=$(find t | sort)
    11 
    +  for file in $files
    12 
    +  do
    13 
    +    sha256sum $file >> s.txt
    14 
    +  done
    15 
    +  sha256sum s.txt > [% dest_dir %]/[% project %]-[% c("version") %]_sha256sum.txt

  • test/projects/shasum/config
    1 
    +# vim: filetype=yaml sw=2
    2 
    +debug: 1
    3 
    +filename: 'shasum_project-[% c("input_files_id") %]'
    4 
    +build: |
    5 
    +  #!/bin/sh
    6 
    +  echo ok > [% dest_dir _ '/' _ c("filename") %]
    7 
    +
    8 
    +targets:
    9 
    +  sha256sum:
    10 
    +    input_files:
    11 
    +      - filename: sha256sums-signed-build.txt
    12 
    +        URL: https://archive.torproject.org/tor-package-archive/torbrowser/13.0.15/sha256sums-signed-build.txt
    13 
    +        sha256sum: 380c611762cf02a89a5885e7182ce17fc653f6b910c00ce50295c03c488b13ac
    14 
    +  sha512sum:
    15 
    +    input_files:
    16 
    +      - filename: sha256sums-signed-build.txt
    17 
    +        URL: https://archive.torproject.org/tor-package-archive/torbrowser/13.0.15/sha256sums-signed-build.txt
    18 
    +        sha512sum: 5a1a5199f2135dd75bfeddafc25a62ce473083d371b13f90582b5faf3a3e7c415c4b4990d4927d8a468dca88bc8376fb55143020e7dadcc69b316f6212a7f825
    19 
    +  wrong_sha256sum:
    20 
    +    input_files:
    21 
    +      - filename: sha256sums-signed-build.txt
    22 
    +        URL: https://archive.torproject.org/tor-package-archive/torbrowser/13.0.15/sha256sums-signed-build.txt
    23 
    +        sha256sum: aaa
    24 
    +  wrong_sha512sum:
    25 
    +    input_files:
    26 
    +      - filename: sha256sums-signed-build.txt
    27 
    +        URL: https://archive.torproject.org/tor-package-archive/torbrowser/13.0.15/sha256sums-signed-build.txt
    28 
    +        sha512sum: aaa

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.torproject.org. Manage all notifications · Help