[Git][tpo/applications/tor-browser-build][main] 4 commits: Bug 41314: Add tools/setup-martools and tools/mar

10 Feb 2025


      morgan pushed to branch main at The Tor Project / Applications / tor-browser-build


Commits:
bbe7bf81 by Nicolas Vigier at 2025-02-10T14:12:04+00:00
Bug 41314: Add tools/setup-martools and tools/mar

tools/setup-martools will download and extract mar-tools to
tools/local/mar-tools (if not already present).

tools/mar will run mar after running tools/setup-martools.

- - - - -
7f90f5cd by Nicolas Vigier at 2025-02-10T14:12:04+00:00
Bug 41314: Use setup-martools in update_responses

- - - - -
75179413 by Nicolas Vigier at 2025-02-10T14:12:04+00:00
Bug 41314: Update mar-tools version in projects/mar-tools/config

- - - - -
dbc66b84 by Nicolas Vigier at 2025-02-10T14:12:04+00:00
Bug 41314: Use setup-martools in dmg2mar

- - - - -


8 changed files:

- projects/mar-tools/config
- tools/dmg2mar
- + tools/mar
- + tools/setup-martools
- tools/signing/machines-setup/setup-signing-machine
- tools/signing/machines-setup/upload-tbb-to-signing-machine
- + tools/signmar
- tools/update-responses/update_responses


Changes:

=====================================
projects/mar-tools/config
=====================================
@@ -1,20 +1,22 @@
 # vim: filetype=yaml sw=2
 #
 # Used by tools/signing/machines-setup/upload-tbb-to-signing-machine
-# to fetch mar-tools for signing machine setup
+# to fetch mar-tools for signing machine setup, and by tools/setup-martools
+# to allow running `mar` as `tools/mar`.
 #
-version: 12.0.4
-filename: 'mar-tools-linux64.zip'
+version: 14.0.4
+filename: 'mar-tools.zip'
 container:
   use_container: 0
 gpg_keyring: torbrowser.gpg
 tag_gpg_id: 1
 input_files:
-  - URL: 'https://archive.torproject.org/tor-package-archive/torbrowser/[% c("version") %]/mar-tools-linux64.zip'
-    sha256sum: 726ec4192de61a9342b3262c7ac722cbd59eaba07879be9589c65599d2d69584
+  - URL: 'https://archive.torproject.org/tor-package-archive/torbrowser/[% c("version") %]/mar-tools-linux-x86_64-[% c("version") %].zip'
+    sha256sum: 414254d2471c7f0bcc25de955dbbe716c416d0b80e7688dc2dd48ac6e7fdbf0a
 
 steps:
   fetch_martools:
     fetch_martools: |
       #!/bin/bash
-      echo ok
+      mkdir -p '[% dest_dir %]'
+      cp -f mar-tools-linux-x86_64-[% c("version") %].zip '[% dest_dir %]/mar-tools.zip'


=====================================
tools/dmg2mar
=====================================
@@ -34,6 +34,7 @@ use File::Slurp;
 use File::Find;
 use Parallel::ForkManager;
 use Cwd;
+use FindBin;
 
 # If the application is not TorBrowser (for instance, TorMessenger)
 # set the application name in the TOR_APPNAME_BUNDLE_OSX,
@@ -70,25 +71,18 @@ sub osname {
     exit_error 'Unknown OS';
 }
 
-my $martools_tmpdir;
-sub extract_martools {
-    my $osname = osname;
-    my $marzip = glob(getcwd . "/mar-tools-$osname-*.zip");
-    exit_error "Could not find mar-tools zip" unless $marzip;
-    $martools_tmpdir = File::Temp->newdir();
-    my $old_cwd = getcwd;
-    chdir $martools_tmpdir;
-    my (undef, undef, $success) = capture_exec('unzip', $marzip);
-    chdir $old_cwd;
-    exit_error "Error extracting $marzip" unless $success;
-    $ENV{PATH} = "$martools_tmpdir/mar-tools:$ENV{PATH}";
-    if ($ENV{LD_LIBRARY_PATH}) {
-        $ENV{LD_LIBRARY_PATH} .= ":$martools_tmpdir/mar-tools";
-    } else {
-        $ENV{LD_LIBRARY_PATH} = "$martools_tmpdir/mar-tools";
-    }
-    $ENV{MAR} = "$martools_tmpdir/mar-tools/mar";
-    $ENV{MSBDIFF} = "$martools_tmpdir/mar-tools/mbsdiff";
+sub setup_martools {
+  my ($out, $err, $exit) = capture {
+    system("$FindBin::Bin/setup-martools");
+  };
+  exit_error "Error setting up mar-tools: $err" if $exit;
+  my $martoolsdir = "$FindBin::Bin/local/mar-tools";
+  if ($ENV{LD_LIBRARY_PATH}) {
+    $ENV{LD_LIBRARY_PATH} = "$ENV{LD_LIBRARY_PATH}:$martoolsdir";
+  } else {
+    $ENV{LD_LIBRARY_PATH} = "$martoolsdir/mar-tools";
+  }
+  $ENV{PATH} = "$martoolsdir:$ENV{PATH}";
 }
 
 sub get_nbprocs {
@@ -192,6 +186,6 @@ $ENV{LC_ALL} = 'C';
 exit_error "Please specify the mar channel id" unless @ARGV == 1;
 my $mar_channel_id = $ARGV[0];
 
-extract_martools;
+setup_martools;
 convert_files $mar_channel_id;
 remove_incremental_mars;


=====================================
tools/mar
=====================================
@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+
+script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+cmdname=$(basename -- "${BASH_SOURCE[0]}")
+setupmt=$("$script_dir/setup-martools")
+eval "$setupmt"
+exec "$MARTOOLSDIR/$cmdname" "$@"


=====================================
tools/setup-martools
=====================================
@@ -0,0 +1,49 @@
+#!/bin/bash
+set -e
+
+script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+localdir="$script_dir/local"
+test $# -eq 1 && localdir="$1"
+
+cd "$script_dir/.."
+
+function download_extract_martools {
+  make submodule-update > /dev/null
+  martools_version=$(./rbm/rbm showconf --step fetch_martools mar-tools version)
+  if test -h "$localdir/mar-tools"; then
+    test "$(readlink $localdir/mar-tools)" = "mar-tools-$martools_version" && return 0
+    rm -f "$localdir/mar-tools"
+  fi
+  if test -d "$localdir/mar-tools"; then
+    echo "Error: $localdir/mar-tools already exists and is not a symlink"
+    exit 1
+  fi
+  echo "Setting up mar-tools"
+  mkdir -p "$localdir"
+  cd "$localdir"
+  if ! test -d "mar-tools-$martools_version"; then
+    cd "$script_dir/.."
+    ./rbm/rbm build --step fetch_martools mar-tools
+    martools_zip="$(pwd)/out/mar-tools/mar-tools.zip"
+    cd "$localdir"
+    test -f "$martools_zip"
+    unzip -jd "mar-tools-$martools_version" "$martools_zip"
+  fi
+  ln -sf "mar-tools-$martools_version" mar-tools
+  echo "mar-tools have been set up in $localdir/mar-tools"
+}
+
+function print_martools_vars {
+  mardir="$localdir/mar-tools"
+  if test -z "$LD_LIBRARY_PATH"; then
+    echo "export LD_LIBRARY_PATH=$mardir"
+  else
+    echo "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$mardir"
+  fi
+  echo "export MARTOOLSDIR=$mardir"
+  echo "export MAR=$mardir/mar"
+}
+
+download_extract_martools >&2
+print_martools_vars


=====================================
tools/signing/machines-setup/setup-signing-machine
=====================================
@@ -144,7 +144,7 @@ fi
 # install mar-tools
 if ! test -d /home/signing-mar/mar-tools; then
   tmpdir=$(mktemp -d)
-  unzip -d "$tmpdir" /signing/mar-tools-linux64.zip
+  unzip -d "$tmpdir" /signing/mar-tools.zip
   chown -R signing-mar:signing-mar "$tmpdir/mar-tools"
   chmod go+rX "$tmpdir/mar-tools"/*
   mv "$tmpdir/mar-tools" /home/signing-mar/mar-tools


=====================================
tools/signing/machines-setup/upload-tbb-to-signing-machine
=====================================
@@ -25,7 +25,7 @@ git archive --prefix=rbm/ --output="$tmpdir/rbm.tar" HEAD .
 echo "Created rbm.tar"
 cd ..
 
-martools_filename=mar-tools-linux64.zip
+martools_filename=mar-tools.zip
 if ! test -f "./out/mar-tools/$martools_filename"; then
   ./rbm/rbm build --step fetch_martools mar-tools
   echo "Downloaded $martools_filename"


=====================================
tools/signmar
=====================================
@@ -0,0 +1 @@
+mar
\ No newline at end of file


=====================================
tools/update-responses/update_responses
=====================================
@@ -33,8 +33,6 @@ my %htdocsfiles;
 my $releases_dir = $config->{releases_dir};
 $releases_dir = "$FindBin::Bin/$releases_dir" unless $releases_dir =~ m/^\//;
 my @check_errors;
-my $initPATH = $ENV{PATH};
-my $initLD_LIBRARY_PATH = $ENV{LD_LIBRARY_PATH};
 
 sub exit_error {
     print STDERR "Error: ", $_[0], "\n";
@@ -67,6 +65,20 @@ sub get_nbprocs {
     return 4;
 }
 
+sub setup_martools {
+  my ($out, $err, $exit) = capture {
+    system("$FindBin::Bin/../setup-martools");
+  };
+  exit_error "Error setting up mar-tools: $err" if $exit;
+  my $martoolsdir = "$FindBin::Bin/../local/mar-tools";
+  if ($ENV{LD_LIBRARY_PATH}) {
+    $ENV{LD_LIBRARY_PATH} = "$ENV{LD_LIBRARY_PATH}:$martoolsdir";
+  } else {
+    $ENV{LD_LIBRARY_PATH} = "$martoolsdir/mar-tools";
+  }
+  $ENV{PATH} = "$martoolsdir:$ENV{PATH}";
+}
+
 sub write_htdocs {
     my ($channel, $file, $content) = @_;
     mkdir $htdocsdir unless -d $htdocsdir;
@@ -318,7 +330,7 @@ sub version_dir {
 sub get_buildinfos {
     my ($config, $version) = @_;
     return if exists $config->{versions}{$version}{buildID};
-    extract_martools($config, $version);
+    setup_martools;
     my $files = $config->{versions}{$version}{files};
     foreach my $os (keys %$files) {
         foreach my $lang (keys %{$files->{$os}}) {
@@ -498,40 +510,6 @@ sub write_downloads_json {
     }
 }
 
-sub marzip_path {
-    my ($config, $version) = @_;
-    for my $osname (qw/linux-x86_64 linux-i686 linux-aarch64 macos-x86_64 windows-x86_64 windows-i686/) {
-        my $marzip = glob(version_dir($config, $version) . "/mar-tools-$osname-*.zip");
-        if ($marzip && -f $marzip) {
-          return $marzip;
-        }
-    }
-    exit_error 'Could not find mar-tools';
-}
-
-my $martools_tmpdir;
-my $extracted_martools;
-sub extract_martools {
-    my ($config, $version) = @_;
-    return if $extracted_martools;
-    my $marzip = marzip_path($config, $version);
-    $martools_tmpdir = get_tmpdir($config);
-    my $old_cwd = getcwd;
-    chdir $martools_tmpdir;
-    my (undef, undef, $exit) = capture {
-      system('unzip', $marzip);
-    };
-    chdir $old_cwd;
-    exit_error "Error extracting $marzip" if $exit;
-    $ENV{PATH} = "$martools_tmpdir/mar-tools:$initPATH";
-    if ($initLD_LIBRARY_PATH) {
-        $ENV{LD_LIBRARY_PATH} = "$initLD_LIBRARY_PATH:$martools_tmpdir/mar-tools";
-    } else {
-        $ENV{LD_LIBRARY_PATH} = "$martools_tmpdir/mar-tools";
-    }
-    $extracted_martools = 1;
-}
-
 sub log_step {
     my ($url, $step, $status, $details) = @_;
     state $u;
@@ -701,9 +679,9 @@ my %actions = (
         my $channel = $ARGV[0];
         exit_error "Unknown channel" unless $config->{channels}{$channel};
         my $versions = as_array($config->{channels}{$channel});
+        setup_martools;
         foreach my $version (@$versions) {
             next unless $config->{versions}{$version}{incremental_from};
-            extract_martools($config, $version);
             get_version_files($config, $version);
             create_incremental_mars_for_version($config, $version, $channel);
         }



View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/compare/8...

-- 
View it on GitLab: https://gitlab.torproject.org/tpo/applications/tor-browser-build/-/compare/8...
You're receiving this email because of your account on gitlab.torproject.org.

    

morgan (＠morgan)

tags

participants (1)