[tor-commits] [tor-browser-bundle/master] update_responses: generate multiple MAR files in parallel

[mikeperry at torproject.org]

commit 49c5f2e28f172e85c66f0ba3cd63645a7373858e
Author: Nicolas Vigier <boklm at mars-attacks.org>
Date:   Thu Oct 2 18:35:08 2014 +0200

    update_responses: generate multiple MAR files in parallel
    
    Generating incremental MAR files takes a lot of time but only uses one
    thread. To make that faster, we will be creating multiple MAR files in
    parallel.
    
    The number of files we generate in parallel will be equal to the number
    of CPU cores, unless the NUM_PROCS environment variable is set.
---
 tools/update-responses/README.md        |    8 +++---
 tools/update-responses/update_responses |   43 +++++++++++++++++++++----------
 2 files changed, 35 insertions(+), 16 deletions(-)

diff --git a/tools/update-responses/README.md b/tools/update-responses/README.md
index c7ecdd4..206bd7a 100644
--- a/tools/update-responses/README.md
+++ b/tools/update-responses/README.md
@@ -13,21 +13,23 @@ Dependencies
 
 The following perl modules need to be installed to run the script:
   FindBin YAML File::Slurp Digest::SHA XML::Writer File::Temp
-  IO::CaptureOutput File::Which
+  IO::CaptureOutput File::Which Parallel::ForkManager
 
 On Debian / Ubuntu you can install them with:
 
 ```
   # apt-get install libfindbin-libs-perl libyaml-perl libfile-slurp-perl \
                     libdigest-sha-perl libxml-writer-perl \
-                    libio-captureoutput-perl libfile-which-perl
+                    libio-captureoutput-perl libfile-which-perl \
+                    libparallel-forkmanager-perl
 ```
 
 On Red Hat / Fedora you can install them with:
 
 ```
   # for module in FindBin YAML File::Slurp Digest::SHA XML::Writer \
-                  File::Temp IO::CaptureOutput File::Which
+                  File::Temp IO::CaptureOutput File::Which \
+                  Parallel::ForkManager
     do yum install "perl($module)"; done
 ```
 
diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses
index 1946278..3590227 100755
--- a/tools/update-responses/update_responses
+++ b/tools/update-responses/update_responses
@@ -11,6 +11,7 @@ use File::Temp;
 use File::Find;
 use File::Which;
 use IO::CaptureOutput qw(capture_exec);
+use Parallel::ForkManager;
 
 my $htdocsdir = "$FindBin::Bin/htdocs";
 my $config = LoadFile("$FindBin::Bin/config.yml");
@@ -27,6 +28,14 @@ sub build_target_by_os {
     $config->{build_targets}{$_[0]} ? $config->{build_targets}{$_[0]} : $_[0];
 }
 
+sub get_nbprocs {
+    return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS};
+    if (-f '/proc/cpuinfo') {
+        return scalar grep { m/^processor\s+:\s/ } read_file '/proc/cpuinfo';
+    }
+    return 4;
+}
+
 sub write_htdocs {
     my ($file, $content) = @_;
     mkdir $htdocsdir unless -d $htdocsdir;
@@ -105,31 +114,38 @@ sub mar_filename {
 }
 
 sub create_incremental_mar {
-    my ($config, $from_version, $new_version, $os, $lang) = @_;
-    print "Creating incremental mar file ",
-          "$from_version -> $new_version ($os $lang)\n";
+    my ($config, $pm, $from_version, $new_version, $os, $lang) = @_;
+    my $mar_file = "tor-browser-$os-${from_version}-${new_version}_$lang.mar";
+    print "Starting $mar_file\n";
+    my $mar_file_path = "$releases_dir/$new_version/$mar_file";
+    my $finished_file = sub {
+        exit_error "Error creating $mar_file" unless $_[1] == 0;
+        print "Finished $mar_file\n";
+        $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = {
+            type => 'partial',
+            URL => "$config->{versions}{$new_version}{download_url}/$mar_file",
+            size => -s $mar_file_path,
+            hashFunction => 'SHA512',
+            hashValue => get_sha512_hex_of_file($mar_file_path),
+        };
+    };
+    return if $pm->start($finished_file);
     my $tmpdir = File::Temp->newdir();
     extract_mar(mar_filename($from_version, $os, $lang), "$tmpdir/A");
     extract_mar(mar_filename($new_version, $os, $lang), "$tmpdir/B");
-    my $mar_file = "tor-browser-$os-${from_version}-${new_version}_$lang.mar";
-    my $mar_file_path = "$releases_dir/$new_version/$mar_file";
     my ($out, $err, $success) = capture_exec('make_incremental_update.sh',
                                    $mar_file_path, "$tmpdir/A", "$tmpdir/B");
     if (!$success) {
         unlink $mar_file_path if -f $mar_file_path;
         exit_error "making incremental mar:\n" . $err;
     }
-    $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = {
-        type => 'partial',
-        URL => "$config->{versions}{$new_version}{download_url}/$mar_file",
-        size => -s $mar_file_path,
-        hashFunction => 'SHA512',
-        hashValue => get_sha512_hex_of_file($mar_file_path),
-    };
+    $pm->finish;
 }
 
 sub create_missing_incremental_mars {
     my ($config, $version) = @_;
+    my $pm = Parallel::ForkManager->new(get_nbprocs);
+    $pm->run_on_finish(sub { $_[2]->(@_) });
     my $v = $config->{versions}{$version};
     foreach my $from_version (@{$v->{incremental_from}}) {
         $config->{versions}{$from_version} //= { download_url => '' };
@@ -139,10 +155,11 @@ sub create_missing_incremental_mars {
             foreach my $lang (keys %{$v->{files}{$os}}) {
                 next if defined $v->{files}{$os}{$lang}{partial}{$from_version};
                 next unless defined $from_v->{files}{$os}{$lang}{complete};
-                create_incremental_mar($config, $from_version, $version, $os, $lang);
+                create_incremental_mar($config, $pm, $from_version, $version, $os, $lang);
             }
         }
     }
+    $pm->wait_all_children;
 }
 
 sub get_config {