August 2017 - tor-commits - lists.torproject.org

[tor-browser-build/master] Bug 17381: add and adapt the update_responses script
by boklm＠torproject.org 21 Aug '17

21 Aug '17

commit de3f97f325df5465fa49b7c8ca2ff229b670b207 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Thu Aug 17 13:32:46 2017 +0200 Bug 17381: add and adapt the update_responses script Add the update_responses and incrementals makefile rules. The update_responses script is identical to what we have in tor-browser-bundle.git, however the config.yml file it is using is now generated using the infos from rbm. --- Makefile | 20 + README.MAKEFILE | 10 + keyring/torbrowser.gpg | Bin 0 -> 49267 bytes projects/firefox/config | 3 +- projects/release/config | 21 +- projects/release/hash_incrementals | 4 + projects/release/update_responses_config | 11 + projects/release/update_responses_config.yml | 40 ++ rbm.conf | 3 + tools/update-responses/.gitignore | 2 + tools/update-responses/README.md | 50 ++ .../check_update_responses_deployement | 1 + tools/update-responses/download_missing_versions | 1 + tools/update-responses/gen_incrementals | 1 + tools/update-responses/get_channel_version | 1 + tools/update-responses/update_responses | 643 +++++++++++++++++++++ 16 files changed, 807 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index c45b0a1..d49d670 100644 --- a/Makefile +++ b/Makefile @@ -83,6 +83,26 @@ signtag-release: submodule-update signtag-alpha: submodule-update $(rbm) build release --step signtag --target alpha +incrementals-release: submodule-update + $(rbm) build release --step update_responses_config --target release + tools/update-responses/download_missing_versions release + tools/update-responses/gen_incrementals release + $(rbm) build release --step hash_incrementals --target release + +incrementals-alpha: submodule-update + $(rbm) build release --step update_responses_config --target alpha + tools/update-responses/download_missing_versions alpha + tools/update-responses/gen_incrementals alpha + $(rbm) build release --step hash_incrementals --target alpha + +update_responses-release: submodule-update + $(rbm) build release --step update_responses_config --target release --target signed + tools/update-responses/update_responses release + +update_responses-alpha: submodule-update + $(rbm) build release --step update_responses_config --target alpha --target signed + tools/update-responses/update_responses alpha + submodule-update: git submodule update --init diff --git a/README.MAKEFILE b/README.MAKEFILE index ef33904..167f5d5 100644 --- a/README.MAKEFILE +++ b/README.MAKEFILE @@ -77,3 +77,13 @@ signtag-{release,alpha} Create a git signed tag for the selected channel, using the version and build number defined as var/torbrowser_version and var/torbrowser_build. +incrementals-{release,alpha} +---------------------------- +Create incremental mar files for an unsigned build in the release or +alpha channel. + +update_responses-{release,alpha} +-------------------------------- +Create update responses xml files for a signed build in the release or +alpha channel. + diff --git a/keyring/torbrowser.gpg b/keyring/torbrowser.gpg new file mode 100644 index 0000000..dfdee06 Binary files /dev/null and b/keyring/torbrowser.gpg differ diff --git a/projects/firefox/config b/projects/firefox/config index 1360772..034d40f 100644 --- a/projects/firefox/config +++ b/projects/firefox/config @@ -7,7 +7,8 @@ git_url: https://git.torproject.org/tor-browser.git gpg_keyring: torbutton.gpg var: - firefox_version: 52.3.0esr + firefox_platform_version: 52.3.0 + firefox_version: '[% c("var/firefox_platform_version") %]esr' torbrowser_branch: 7.5 torbrowser_update_channel: alpha copyright_year: '[% exec("git show -s --format=%ci").remove("-.*") %]' diff --git a/projects/release/config b/projects/release/config index b785878..3c771bf 100644 --- a/projects/release/config +++ b/projects/release/config @@ -1,10 +1,11 @@ # vim: filetype=yaml sw=2 version: '[% c("var/torbrowser_version") %]' -output_dir: 'release/unsigned' +output_dir: release var: + signed_status: unsigned today: '[% USE date; date.format(format = "%Y-%m-%d") %]' - publish_dir: '[% c("version") %]-[% c("var/torbrowser_build") %]' + publish_dir: '[% c("var/signed_status") %]/[% c("version") %]-[% c("var/torbrowser_build") %]' targets: torbrowser-all: @@ -50,7 +51,7 @@ targets: publish_dir: '[% c("var/today") %]' alpha: - output_dir: 'alpha/unsigned' + output_dir: alpha var: build_target: alpha @@ -60,6 +61,10 @@ targets: build_target: torbrowser-testbuild publish_dir: '' + signed: + var: + signed_status: signed + input_files: # Release @@ -112,3 +117,13 @@ steps: debug: 0 input_files: [] signtag: '[% INCLUDE signtag %]' + update_responses_config: + build_log: '-' + debug: 0 + input_files: [] + update_responses_config: '[% INCLUDE update_responses_config %]' + hash_incrementals: + build_log: '-' + debug: 0 + input_files: [] + hash_incrementals: '[% INCLUDE hash_incrementals %]' diff --git a/projects/release/hash_incrementals b/projects/release/hash_incrementals new file mode 100644 index 0000000..9cf1b27 --- /dev/null +++ b/projects/release/hash_incrementals @@ -0,0 +1,4 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +cd [% shell_quote(path(dest_dir)) %]/[% c("var/signed_status") %]/[% c("version") %]-[% c("var/torbrowser_build") %] +sha256sum `ls -1 | grep '\.incremental\.mar$' | sort` > sha256sums-[% c("var/signed_status") %]-build.incrementals.txt diff --git a/projects/release/update_responses_config b/projects/release/update_responses_config new file mode 100644 index 0000000..4ff470d --- /dev/null +++ b/projects/release/update_responses_config @@ -0,0 +1,11 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +cat > [% shell_quote(c("basedir")) %]/tools/update-responses/config.yml << 'EOF' +[% INCLUDE update_responses_config.yml -%] +EOF + +# Update / create symlink $torbrowser_version -> $torbrowser_version-buildN +versiondir=[% shell_quote(path(dest_dir)) _ '/' _ c("var/signed_status") _ '/' _ c("version") %] +test -L "$versiondir" && rm -f "$versiondir" +test -d "$versiondir" || ln -s [% shell_quote(c("version")) %]-[% shell_quote(c("var/torbrowser_build")) %] "$versiondir" + diff --git a/projects/release/update_responses_config.yml b/projects/release/update_responses_config.yml new file mode 100644 index 0000000..f0d9dbf --- /dev/null +++ b/projects/release/update_responses_config.yml @@ -0,0 +1,40 @@ +--- +appname_marfile: tor-browser +appname_bundle_osx: TorBrowser +appname_bundle_linux: tor-browser +appname_bundle_win: torbrowser-install +releases_dir: [% path(c('output_dir')) %]/[% c("var/signed_status") %] +download: + archive_url: https://archive.torproject.org/tor-package-archive/torbrowser + gpg_keyring: ../../keyring/torbrowser.gpg + bundles_url: https://dist.torproject.org/torbrowser + mars_url: https://cdn.torproject.org/aus1/torbrowser +build_targets: + linux32: Linux_x86-gcc3 + linux64: Linux_x86_64-gcc3 + win32: + - WINNT_x86-gcc3 + - WINNT_x86-gcc3-x86 + - WINNT_x86-gcc3-x64 + osx32: Darwin_x86-gcc3 + osx64: Darwin_x86_64-gcc3 +channels: + [% pc('firefox', 'var/torbrowser_update_channel') %]: [% c("var/torbrowser_version") %] +versions: + [% c("var/torbrowser_version") %]: + platformVersion: [% pc('firefox', 'var/firefox_platform_version') %] + detailsURL: https://blog.torproject.org/blog/tor-browser-[% c("var/torbrowser_version") FILTER remove('\.') %]-released + incremental_from: +[% FOREACH v IN c("var/torbrowser_incremental_from") -%] + - [% v %] +[% END -%] + migrate_archs: + osx32: osx64 + migrate_langs: + pt-PT: pt-BR + win32: + minSupportedInstructionSet: SSE2 + osx32: + minSupportedOSVersion: 13.0.0 + osx64: + minSupportedOSVersion: 13.0.0 diff --git a/rbm.conf b/rbm.conf index 5e1e66d..44fc9f5 100644 --- a/rbm.conf +++ b/rbm.conf @@ -16,6 +16,9 @@ buildconf: var: torbrowser_version: '7.5a4' torbrowser_build: 'build2' + torbrowser_incremental_from: + - 7.5a2 + - 7.5a3 project_name: tor-browser multi_lingual: 0 build_mar: 1 diff --git a/tools/update-responses/.gitignore b/tools/update-responses/.gitignore new file mode 100644 index 0000000..5947fca --- /dev/null +++ b/tools/update-responses/.gitignore @@ -0,0 +1,2 @@ +/htdocs +/config.yml diff --git a/tools/update-responses/README.md b/tools/update-responses/README.md new file mode 100644 index 0000000..5209ed5 --- /dev/null +++ b/tools/update-responses/README.md @@ -0,0 +1,50 @@ +Tor Browser Update Responses script +=================================== + +This repository contains a script to generate responses for Tor Browser +updater. + +See ticket [#12622](https://trac.torproject.org/projects/tor/ticket/12622) +for details. + + +Dependencies +------------ + +The following perl modules need to be installed to run the script: + FindBin YAML File::Slurp Digest::SHA XML::Writer File::Temp + IO::CaptureOutput Parallel::ForkManager XML::LibXML LWP JSON + +On Debian / Ubuntu you can install them with: + +``` + # apt-get install libfindbin-libs-perl libyaml-perl libfile-slurp-perl \ + libdigest-sha-perl libxml-writer-perl \ + libio-captureoutput-perl libparallel-forkmanager-perl \ + libxml-libxml-perl libwww-perl libjson-perl +``` + +On Red Hat / Fedora you can install them with: + +``` + # for module in FindBin YAML File::Slurp Digest::SHA XML::Writer \ + File::Temp IO::CaptureOutput Parallel::ForkManager \ + XML::LibXML LWP JSON + do yum install "perl($module)"; done +``` + + +URL Format +---------- + +The URL format is: + https://something/$channel/$build_target/$tb_version/$lang?force=1 + +'build_target' is the OS for which the browser was built. The correspo +ndance between the build target and the OS name that we use in archive +files is defined in the config.yml file. + +'tb_version' is the Tor Browser version. + +'lang' is the locale. + diff --git a/tools/update-responses/check_update_responses_deployement b/tools/update-responses/check_update_responses_deployement new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/check_update_responses_deployement @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/download_missing_versions b/tools/update-responses/download_missing_versions new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/download_missing_versions @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/gen_incrementals b/tools/update-responses/gen_incrementals new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/gen_incrementals @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/get_channel_version b/tools/update-responses/get_channel_version new file mode 120000 index 0000000..3766925 --- /dev/null +++ b/tools/update-responses/get_channel_version @@ -0,0 +1 @@ +update_responses \ No newline at end of file diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses new file mode 100755 index 0000000..0ed19c1 --- /dev/null +++ b/tools/update-responses/update_responses @@ -0,0 +1,643 @@ +#!/usr/bin/perl -w + +use strict; +use feature "state"; +use English; +use FindBin; +use YAML qw(LoadFile); +use File::Slurp; +use Digest::SHA qw(sha256_hex); +use XML::Writer; +use Cwd; +use File::Copy; +use File::Temp; +use File::Find; +use POSIX qw(setlocale LC_ALL); +use IO::CaptureOutput qw(capture_exec); +use Parallel::ForkManager; +use File::Basename; +use XML::LibXML '1.70'; +use LWP::Simple; +use JSON; + +# Set umask and locale to provide a consistent environment for MAR file +# generation, etc. +umask(0022); +$ENV{"LC_ALL"} = "C"; +setlocale(LC_ALL, "C"); + +my $htdocsdir = "$FindBin::Bin/htdocs"; +my $config = LoadFile("$FindBin::Bin/config.yml"); +my %htdocsfiles; +my $releases_dir = $config->{releases_dir}; +$releases_dir = "$FindBin::Bin/$releases_dir" unless $releases_dir =~ m/^\//; +my @check_errors; +my $initPATH = $ENV{PATH}; +my $initLD_LIBRARY_PATH = $ENV{LD_LIBRARY_PATH}; + +sub exit_error { + print STDERR "Error: ", $_[0], "\n"; + chdir '/'; + exit (exists $_[1] ? $_[1] : 1); +} + +sub build_targets_by_os { + return ($_[0]) unless $config->{build_targets}{$_[0]}; + my $r = $config->{build_targets}{$_[0]}; + return ref $r eq 'ARRAY' ? @$r : ($r); +} + +sub get_nbprocs { + return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS}; + if (-f '/proc/cpuinfo') { + return scalar grep { m/^processor\s+:\s/ } read_file '/proc/cpuinfo'; + } + return 4; +} + +sub write_htdocs { + my ($channel, $file, $content) = @_; + mkdir $htdocsdir unless -d $htdocsdir; + mkdir "$htdocsdir/$channel" unless -d "$htdocsdir/$channel"; + write_file("$htdocsdir/$channel/$file", $content); + $htdocsfiles{$channel}->{$file} = 1; +} + +sub clean_htdocs { + my (@channels) = @_; + foreach my $channel (@channels) { + opendir(my $d, "$htdocsdir/$channel"); + my @files = grep { ! $htdocsfiles{$channel}->{$_} } readdir $d; + closedir $d; + unlink map { "$htdocsdir/$channel/$_" } @files; + } +} + +sub get_sha512_hex_of_file { + my ($file) = @_; + my $sha = Digest::SHA->new("512"); + $sha->addfile($file); + return $sha->hexdigest; +} + +sub get_version_files { + my ($config, $version) = @_; + return if $config->{versions}{$version}{files}; + my $appname = $config->{appname_marfile}; + my $files = {}; + my $vdir = "$releases_dir/$version"; + my $download_url = "$config->{download}{mars_url}/$version"; + opendir(my $d, $vdir) or exit_error "Error opening directory $vdir"; + foreach my $file (readdir $d) { + next unless -f "$vdir/$file"; + if ($file =~ m/^$appname-([^-]+)-${version}_(.+)\.mar$/) { + my ($os, $lang) = ($1, $2); + $files->{$os}{$lang}{complete} = { + type => 'complete', + URL => "$download_url/$file", + size => -s "$vdir/$file", + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file("$vdir/$file"), + }; + next; + } + if ($file =~ m/^$appname-([^-]+)-(.+)-${version}_(.+)\.incremental\.mar$/) { + my ($os, $from_version, $lang) = ($1, $2, $3); + $files->{$os}{$lang}{partial}{$from_version} = { + type => 'partial', + URL => "$download_url/$file", + size => -s "$vdir/$file", + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file("$vdir/$file"), + } + } + } + closedir $d; + $config->{versions}{$version}{files} = $files; +} + +sub get_version_downloads { + my ($config, $version) = @_; + my $downloads = {}; + my $vdir = "$releases_dir/$version"; + my $download_url = "$config->{download}{bundles_url}/$version"; + opendir(my $d, $vdir) or exit_error "Error opening directory $vdir"; + foreach my $file (readdir $d) { + next unless -f "$vdir/$file"; + my ($os, $lang); + if ($file =~ m/^$config->{appname_bundle_osx}-$version-osx64_(.+).dmg$/) { + ($os, $lang) = ('osx64', $1); + } elsif ($file =~ m/^$config->{appname_bundle_linux}-(linux32|linux64)-${version}_(.+).tar.xz$/) { + ($os, $lang) = ($1, $2); + } elsif ($file =~ m/^$config->{appname_bundle_win}-${version}_(.+).exe$/) { + ($os, $lang) = ('win32', $1); + } else { + next; + } + $downloads->{$os}{$lang} = { + binary => "$download_url/$file", + sig => "$download_url/$file.asc", + }; + } + closedir $d; + $config->{versions}{$version}{downloads} = $downloads; +} + +sub extract_mar { + my ($mar_file, $dest_dir) = @_; + my $old_cwd = getcwd; + mkdir $dest_dir; + chdir $dest_dir or exit_error "Cannot enter $dest_dir"; + my $res = system('mar', '-x', $mar_file); + exit_error "Error extracting $mar_file" if $res; + my $bunzip_file = sub { + return unless -f $File::Find::name; + rename $File::Find::name, "$File::Find::name.bz2"; + system('bunzip2', "$File::Find::name.bz2") == 0 + || exit_error "Error decompressing $File::Find::name"; + }; + find($bunzip_file, $dest_dir); + my $manifest = -f 'updatev3.manifest' ? 'updatev3.manifest' + : 'updatev2.manifest'; + my @lines = read_file($manifest) if -f $manifest; + foreach my $line (@lines) { + if ($line =~ m/^addsymlink "(.+)" "(.+)"$/) { + exit_error "$mar_file: Could not create symlink $1 -> $2" + unless symlink $2, $1; + } + } + chdir $old_cwd; +} + +sub mar_filename { + my ($appname, $version, $os, $lang) = @_; + "$releases_dir/$version/$appname-$os-${version}_$lang.mar"; +} + +sub create_incremental_mar { + my ($config, $pm, $from_version, $new_version, $os, $lang) = @_; + my $appname = $config->{appname_marfile}; + my $mar_file = "$appname-$os-${from_version}-${new_version}_$lang.incremental.mar"; + my $mar_file_path = "$releases_dir/$new_version/$mar_file"; + if ($ENV{MAR_SKIP_EXISTING} && -f $mar_file_path) { + print "Skipping $mar_file\n"; + return; + } + print "Starting $mar_file\n"; + my $download_url = "$config->{download}{mars_url}/$new_version"; + my $finished_file = sub { + exit_error "Error creating $mar_file" unless $_[1] == 0; + print "Finished $mar_file\n"; + $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = { + type => 'partial', + URL => "$download_url/$mar_file", + size => -s $mar_file_path, + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file($mar_file_path), + }; + }; + return if $pm->start($finished_file); + my $tmpdir = File::Temp->newdir(); + extract_mar(mar_filename($appname, $from_version, $os, $lang), "$tmpdir/A"); + extract_mar(mar_filename($appname, $new_version, $os, $lang), "$tmpdir/B"); + if ($ENV{CHECK_CODESIGNATURE_EXISTS}) { + unless (-f "$tmpdir/A/Contents/_CodeSignature/CodeResources" + && -f "$tmpdir/B/Contents/_CodeSignature/CodeResources") { + exit_error "Missing code signature while creating $mar_file"; + } + } + my ($out, $err, $success) = capture_exec('make_incremental_update.sh', + $mar_file_path, "$tmpdir/A", "$tmpdir/B"); + if (!$success) { + unlink $mar_file_path if -f $mar_file_path; + exit_error "making incremental mar:\n" . $err; + } + $pm->finish; +} + +sub create_incremental_mars_for_version { + my ($config, $version) = @_; + my $pm = Parallel::ForkManager->new(get_nbprocs); + $pm->run_on_finish(sub { $_[2]->(@_) }); + my $v = $config->{versions}{$version}; + foreach my $from_version (@{$v->{incremental_from}}) { + $config->{versions}{$from_version} //= {}; + get_version_files($config, $from_version); + my $from_v = $config->{versions}{$from_version}; + foreach my $os (keys %{$v->{files}}) { + foreach my $lang (keys %{$v->{files}{$os}}) { + next unless defined $from_v->{files}{$os}{$lang}{complete}; + create_incremental_mar($config, $pm, $from_version, $version, $os, $lang); + } + } + } + $pm->wait_all_children; +} + +sub get_config { + my ($config, $version, $os, $name) = @_; + return $config->{versions}{$version}{$os}{$name} + // $config->{versions}{$version}{$name} + // $config->{$name}; +} + +sub channel_to_version { + my ($config, @channels) = @_; + return values %{$config->{channels}} unless @channels; + foreach my $channel (@channels) { + exit_error "Unknown channel $channel" + unless $config->{channels}{$channel}; + } + return map { $config->{channels}{$_} } @channels; +} + +sub get_buildinfos { + my ($config, $version) = @_; + return if exists $config->{versions}{$version}{buildID}; + extract_martools($version); + my $files = $config->{versions}{$version}{files}; + foreach my $os (keys %$files) { + foreach my $lang (keys %{$files->{$os}}) { + next unless $files->{$os}{$lang}{complete}; + my $tmpdir = File::Temp->newdir(); + extract_mar( + mar_filename($config->{appname_marfile}, $version, $os, $lang), + "$tmpdir"); + my $appfile = "$tmpdir/application.ini" if -f "$tmpdir/application.ini"; + $appfile = "$tmpdir/Contents/Resources/application.ini" + if -f "$tmpdir/Contents/Resources/application.ini"; + exit_error "Could not find application.ini" unless $appfile; + foreach my $line (read_file($appfile)) { + if ($line =~ m/^BuildID=(.*)$/) { + $config->{versions}{$version}{buildID} = $1; + return; + } + } + exit_error "Could not extract buildID from application.ini"; + } + } +} + +sub get_response { + my ($config, $version, $os, @patches) = @_; + my $res; + my $writer = XML::Writer->new(OUTPUT => \$res, ENCODING => 'UTF-8'); + $writer->xmlDecl; + $writer->startTag('updates'); + if (get_config($config, $version, $os, 'unsupported')) { + $writer->startTag('update', + unsupported => 'true', + detailsURL => get_config($config, $version, $os, 'detailsURL'), + ); + goto CLOSETAGS; + } + my $minversion = get_config($config, $version, $os, 'minSupportedOSVersion'); + my $mininstruc = get_config($config, $version, $os, 'minSupportedInstructionSet'); + $writer->startTag('update', + type => 'minor', + displayVersion => $version, + appVersion => $version, + platformVersion => get_config($config, $version, $os, 'platformVersion'), + buildID => get_config($config, $version, $os, 'buildID'), + detailsURL => get_config($config, $version, $os, 'detailsURL'), + actions => 'showURL', + openURL => get_config($config, $version, $os, 'detailsURL'), + defined $minversion ? ( minSupportedOSVersion => $minversion ) : (), + defined $mininstruc ? ( minSupportedInstructionSet => $mininstruc ) : (), + ); + foreach my $patch (@patches) { + my @sorted_patch = map { $_ => $patch->{$_} } sort keys %$patch; + $writer->startTag('patch', @sorted_patch); + $writer->endTag('patch'); + } + CLOSETAGS: + $writer->endTag('update'); + $writer->endTag('updates'); + $writer->end; + return $res; +} + +sub write_responses { + my ($config, @channels) = @_; + @channels = keys %{$config->{channels}} unless @channels; + foreach my $channel (@channels) { + my $version = $config->{channels}{$channel}; + get_version_files($config, $version); + get_buildinfos($config, $version); + my $files = $config->{versions}{$version}{files}; + my $migrate_archs = $config->{versions}{$version}{migrate_archs} // {}; + foreach my $old_os (keys %$migrate_archs) { + my $new_os = $migrate_archs->{$old_os}; + foreach my $lang (keys %{$files->{$new_os}}) { + $files->{$old_os}{$lang}{complete} = + $files->{$new_os}{$lang}{complete}; + } + } + foreach my $os (keys %$files) { + foreach my $lang (keys %{$files->{$os}}) { + my $resp = get_response($config, $version, $os, + $files->{$os}{$lang}{complete}); + write_htdocs($channel, "$version-$os-$lang.xml", $resp); + foreach my $from_version (keys %{$files->{$os}{$lang}{partial}}) { + $resp = get_response($config, $version, $os, + $files->{$os}{$lang}{complete}, + $files->{$os}{$lang}{partial}{$from_version}); + write_htdocs($channel, "$from_version-$version-$os-$lang.xml", $resp); + } + } + } + write_htdocs($channel, 'no-update.xml', + '<?xml version="1.0" encoding="UTF-8"?>' + . "\n<updates></updates>\n"); + } +} + +sub write_htaccess { + my ($config, @channels) = @_; + @channels = keys %{$config->{channels}} unless @channels; + my $flags = "[last]"; + foreach my $channel (@channels) { + my $htaccess = "RewriteEngine On\n"; + my $version = $config->{channels}{$channel}; + my $migrate_langs = $config->{versions}{$version}{migrate_langs} // {}; + my $files = $config->{versions}{$version}{files}; + $htaccess .= "RewriteRule ^[^\/]+/$version/ no-update.xml $flags\n"; + foreach my $os (sort keys %$files) { + foreach my $bt (build_targets_by_os($os)) { + foreach my $lang (sort keys %{$files->{$os}}) { + foreach my $from_version (sort keys %{$files->{$os}{$lang}{partial}}) { + $htaccess .= "RewriteRule ^$bt/$from_version/$lang " + . "$from_version-$version-$os-$lang.xml $flags\n"; + } + $htaccess .= "RewriteRule ^$bt/[^\/]+/$lang " + . "$version-$os-$lang.xml $flags\n"; + } + foreach my $lang (sort keys %$migrate_langs) { + $htaccess .= "RewriteRule ^$bt/[^\/]+/$lang " + . "$version-$os-$migrate_langs->{$lang}.xml $flags\n"; + } + $htaccess .= "RewriteRule ^$bt/ $version-$os-en-US.xml $flags\n"; + } + } + write_htdocs($channel, '.htaccess', $htaccess); + } +} + +sub write_downloads_json { + my ($config, @channels) = @_; + @channels = keys %{$config->{channels}} unless @channels; + foreach my $channel (@channels) { + my $version = $config->{channels}{$channel}; + my $data = { + version => $version, + downloads => get_version_downloads($config, $version), + }; + write_htdocs($channel, 'downloads.json', + JSON->new->utf8->canonical->encode($data)); + } +} + +sub osname { + my ($osname) = capture_exec('uname', '-s'); + my ($arch) = capture_exec('uname', '-m'); + chomp($osname, $arch); + if ($osname eq 'Linux' && $arch eq 'x86_64') { + return 'linux64'; + } + if ($osname eq 'Linux' && $arch =~ m/^i.86$/) { + return 'linux32'; + } + exit_error 'Unknown OS'; +} + +my $martools_tmpdir; +sub extract_martools { + my ($version) = @_; + my $osname = osname; + my $marzip = "$releases_dir/$version/mar-tools-$osname.zip"; + $martools_tmpdir = File::Temp->newdir(); + my $old_cwd = getcwd; + chdir $martools_tmpdir; + my (undef, undef, $success) = capture_exec('unzip', $marzip); + chdir $old_cwd; + exit_error "Error extracting $marzip" unless $success; + $ENV{PATH} = "$martools_tmpdir/mar-tools:$initPATH"; + if ($initLD_LIBRARY_PATH) { + $ENV{LD_LIBRARY_PATH} = "$initLD_LIBRARY_PATH:$martools_tmpdir/mar-tools"; + } else { + $ENV{LD_LIBRARY_PATH} = "$martools_tmpdir/mar-tools"; + } +} + +sub log_step { + my ($url, $step, $status, $details) = @_; + state $u; + if (!defined $u || $url ne $u) { + print "\n" if $u; + print "$url\n"; + $u = $url; + } + print ' ', $step, $status ? ': OK' : ': ERROR', + $details ? " - $details\n" : "\n"; + return if $status; + push @check_errors, { url => $url, step => $step, details => $details }; +} + +sub get_remote_xml { + my ($url) = @_; + my $content = get $url; + log_step($url, 'get', defined $content); + return undef unless defined $content; + my $dom = eval { XML::LibXML->load_xml(string => $content) }; + log_step($url, 'parse_xml', defined $dom, $@); + return $dom; +} + +sub check_get_version { + my ($dom) = @_; + my @updates = $dom->documentElement()->getChildrenByLocalName('update'); + return undef unless @updates; + return $updates[0]->getAttribute('appVersion'); +} + +sub check_no_update { + my ($dom) = @_; + my @updates = $dom->documentElement()->getChildrenByLocalName('update'); + return @updates == 0; +} + +sub check_has_incremental { + my ($dom) = @_; + my @updates = $dom->documentElement()->getChildrenByLocalName('update'); + return undef unless @updates; + my @patches = $updates[0]->getChildrenByLocalName('patch'); + foreach my $patch (@patches) { + return 1 if $patch->getAttribute('type') eq 'partial'; + } + return undef; +} + +sub build_targets_list { + map { ref $_ eq 'ARRAY' ? @$_ : $_ } values %{$config->{build_targets}}; +} + +sub check_update_responses_channel { + my ($config, $base_url, $channel) = @_; + my $channel_version = $config->{channels}{$channel}; + foreach my $build_target (build_targets_list()) { + foreach my $lang (qw(en-US de)) { + my $url = "$base_url/$channel/$build_target/1.0/$lang"; + my $dom = get_remote_xml($url); + if ($dom) { + my $version = check_get_version($dom); + log_step($url, 'version', $version eq $channel_version, + "expected: $channel_version received: $version"); + } + $url = "$base_url/$channel/$build_target/$channel_version/$lang"; + $dom = get_remote_xml($url); + log_step($url, 'no_update', check_no_update($dom)) if $dom; + my @inc = @{$config->{versions}{$channel_version}{incremental_from}} + if $config->{versions}{$channel_version}{incremental_from}; + foreach my $inc_from (@inc) { + my $url = "$base_url/$channel/$build_target/$inc_from/$lang"; + $dom = get_remote_xml($url); + next unless $dom; + my $version = check_get_version($dom); + log_step($url, 'version', $version eq $channel_version, + "expected: $channel_version received: $version"); + log_step($url, 'has_incremental', check_has_incremental($dom)); + } + } + } +} + +sub download_version { + my ($config, $version) = @_; + my $tmpdir = File::Temp->newdir(); + my $destdir = "$releases_dir/$version"; + my $urldir = "$config->{download}{archive_url}/$version"; + print "Downloading version $version\n"; + foreach my $file (qw(sha256sums-unsigned-build.txt sha256sums-unsigned-build.txt.asc)) { + if (getstore("$urldir/$file", "$tmpdir/$file") != 200) { + exit_error "Error downloading $urldir/$file"; + } + } + if (system('gpg', '--no-default-keyring', '--keyring', + "$FindBin::Bin/$config->{download}{gpg_keyring}", '--verify', + "$tmpdir/sha256sums-unsigned-build.txt.asc", + "$tmpdir/sha256sums-unsigned-build.txt")) { + exit_error "Error checking gpg signature for version $version"; + } + mkdir $destdir; + move "$tmpdir/sha256sums-unsigned-build.txt.asc", "$destdir/sha256sums-unsigned-build.txt.asc"; + move "$tmpdir/sha256sums-unsigned-build.txt", "$destdir/sha256sums-unsigned-build.txt"; + my %sums = map { chomp; reverse split ' ', $_ } + read_file "$destdir/sha256sums-unsigned-build.txt"; + + my $martools = 'mar-tools-' . osname . '.zip'; + exit_error "Error downloading $urldir/$martools\n" + unless getstore("$urldir/$martools", "$tmpdir/$martools") == 200; + exit_error "Error downloading $urldir/$martools.asc\n" + unless getstore("$urldir/$martools.asc", "$tmpdir/$martools.asc") == 200; + if (system('gpg', '--no-default-keyring', '--keyring', + "$FindBin::Bin/$config->{download}{gpg_keyring}", '--verify', + "$tmpdir/$martools.asc", "$tmpdir/$martools")) { + exit_error "Error checking gpg signature for $version/$martools"; + } + exit_error "Wrong checksum for $version/$martools" + unless $sums{$martools} eq sha256_hex(read_file("$tmpdir/$martools")); + move "$tmpdir/$martools", "$destdir/$martools"; + move "$tmpdir/$martools.asc", "$destdir/$martools.asc"; + extract_martools($version); + + foreach my $file (sort grep { $_ =~ m/\.mar$/ } keys %sums) { + print "Downloading $file\n"; + exit_error "Error downloading $urldir/$file\n" + unless getstore("$urldir/$file", "$tmpdir/$file") == 200; + if ($sums{$file} ne sha256_hex(read_file("$tmpdir/$file"))) { + exit_error "Error unsigning $file" + if system('signmar', '-r', "$tmpdir/$file", "$tmpdir/$file.u"); + exit_error "Wrong checksum for $file" + unless $sums{$file} eq sha256_hex(read_file("$tmpdir/$file.u")); + move "$tmpdir/$file.u", "$tmpdir/$file"; + } + move "$tmpdir/$file", "$destdir/$file"; + } +} + +sub download_missing_versions { + my ($config, @channels) = @_; + foreach my $channel (@channels) { + exit_error "Unknown channel $channel" + unless $config->{channels}{$channel}; + my $cversion = $config->{channels}{$channel}; + next unless $config->{versions}{$cversion}{incremental_from}; + foreach my $version (@{$config->{versions}{$cversion}{incremental_from}}) { + next if -d "$releases_dir/$version"; + download_version($config, $version); + } + } +} + +sub check_update_responses { + my ($config) = @_; + exit_error "usage: $PROGRAM_NAME <base_url> [channels...]" unless @ARGV; + my ($base_url, @channels) = @ARGV; + foreach my $channel (@channels ? @channels : keys %{$config->{channels}}) { + check_update_responses_channel($config, $base_url, $channel); + } + if (!@check_errors) { + print "\n\nNo errors\n"; + return; + } + print "\n\nErrors list:\n"; + my $url = ''; + foreach my $error (@check_errors) { + if ($url ne $error->{url}) { + $url = $error->{url}; + print "$url\n"; + } + print " $error->{step}", + $error->{details} ? " - $error->{details}\n" : "\n"; + } +} + +my %actions = ( + update_responses => sub { + my ($config) = @_; + my @channels = @ARGV ? @ARGV : keys %{$config->{channels}}; + foreach my $channel (@channels) { + exit_error "Unknown channel $channel" + unless $config->{channels}{$channel}; + $htdocsfiles{$channel} = { '.' => 1, '..' => 1 }; + } + write_responses($config, @channels); + write_htaccess($config, @channels); + write_downloads_json($config, @channels); + clean_htdocs(@channels); + }, + gen_incrementals => sub { + my ($config) = @_; + foreach my $version (channel_to_version($config, @ARGV)) { + extract_martools($version); + get_version_files($config, $version); + create_incremental_mars_for_version($config, $version); + } + }, + download_missing_versions => sub { + my ($config) = @_; + my @channels = @ARGV ? @ARGV : keys %{$config->{channels}}; + download_missing_versions($config, @channels); + }, + check_update_responses_deployement => \&check_update_responses, + get_channel_version => sub { + my ($config) = @_; + exit_error "Wrong arguments" unless @ARGV == 1; + exit_error "Unknown channel" unless $config->{channels}{$ARGV[0]}; + print $config->{channels}{$ARGV[0]}, "\n"; + }, +); + +my $action = fileparse($PROGRAM_NAME); +exit_error "Unknown action $action" unless $actions{$action}; +$actions{$action}->($config);

1 0

[tor-browser-build/master] Bug 17381: create tar containing update responses xml files
by boklm＠torproject.org 21 Aug '17

21 Aug '17

commit 93bb526421830f185c431d9064043e54c38d6bab Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Aug 21 19:01:16 2017 +0200 Bug 17381: create tar containing update responses xml files Create a tar file in directory {alpha,release}/update-responses/, to make it a little easier to copy them to the host where they will be deployed. --- Makefile | 4 ++-- README.MAKEFILE | 3 ++- projects/release/config | 5 +++++ projects/release/create_update_responses_tar | 9 +++++++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1229cfa..ce63e54 100644 --- a/Makefile +++ b/Makefile @@ -97,11 +97,11 @@ incrementals-alpha: submodule-update update_responses-release: submodule-update $(rbm) build release --step update_responses_config --target release --target signed - tools/update-responses/update_responses release + $(rbm) build release --step create_update_responses_tar --target release --target signed update_responses-alpha: submodule-update $(rbm) build release --step update_responses_config --target alpha --target signed - tools/update-responses/update_responses alpha + $(rbm) build release --step create_update_responses_tar --target alpha --target signed dmg2mar-release: submodule-update $(rbm) build release --step update_responses_config --target release --target signed diff --git a/README.MAKEFILE b/README.MAKEFILE index 9e28864..cd4bf2f 100644 --- a/README.MAKEFILE +++ b/README.MAKEFILE @@ -91,5 +91,6 @@ signing the OSX dmg files. update_responses-{release,alpha} -------------------------------- Create update responses xml files for a signed build in the release or -alpha channel. +alpha channel. The files can be found in a tar in the directory +{release,alpha}/update-responses. diff --git a/projects/release/config b/projects/release/config index 0a4276a..4ecbc0a 100644 --- a/projects/release/config +++ b/projects/release/config @@ -122,6 +122,11 @@ steps: debug: 0 input_files: [] update_responses_config: '[% INCLUDE update_responses_config %]' + create_update_responses_tar: + build_log: '-' + debug: 0 + input_files: [] + create_update_responses_tar: '[% INCLUDE create_update_responses_tar %]' hash_incrementals: build_log: '-' debug: 0 diff --git a/projects/release/create_update_responses_tar b/projects/release/create_update_responses_tar new file mode 100644 index 0000000..ee1180f --- /dev/null +++ b/projects/release/create_update_responses_tar @@ -0,0 +1,9 @@ +#!/bin/bash +[% c("var/set_default_env") -%] +[% shell_quote(c("basedir")) %]/tools/update-responses/update_responses +mkdir -p [% shell_quote(path(dest_dir)) %]/update-responses +[% SET channel = pc('firefox', 'var/torbrowser_update_channel') -%] +mv [% shell_quote(c("basedir")) %]/tools/update-responses/htdocs/[% channel %] [% channel %] +chmod 775 [% channel %] +chmod 664 [% channel %]/.htaccess [% channel %]/* +tar cf [% shell_quote(path(dest_dir)) %]/update-responses/update-responses-[% channel %]-[% c("version") %].tar [% channel %]

1 0

[metrics-web/master] Remove deprecated torperf.csv file.
by karsten＠torproject.org 21 Aug '17

21 Aug '17

commit 9a29d4400142542ab43fdb31ced4f1c57a389f0b Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 21 09:53:14 2017 +0200 Remove deprecated torperf.csv file. --- website/src/main/resources/web/WEB-INF/stats.jsp | 54 ------------------------ 1 file changed, 54 deletions(-) diff --git a/website/src/main/resources/web/WEB-INF/stats.jsp b/website/src/main/resources/web/WEB-INF/stats.jsp index 7912834..234d0f4 100644 --- a/website/src/main/resources/web/WEB-INF/stats.jsp +++ b/website/src/main/resources/web/WEB-INF/stats.jsp @@ -306,60 +306,6 @@ there are users by either of the two numbers.</li> </div> <div class="container"> -<h2>Performance of downloading static files over Tor (deprecated) <a href="#torperf" name="torperf" class="anchor">#</a></h2> - -<div class="bs-callout bs-callout-warning"> -<h3>Deprecated</h3> -<p>As of May 12, 2017, the linked data file has been replaced by <a href="#torperf-1.1">this data file</a>. As of June 30, 2017, the linked data file is not updated anymore. This section and the linked data file will be removed in the future.</p> -</div> - -<p>The following data file contains aggregate statistics on performance when -downloading static files of different sizes over Tor. These statistics are -generated by the <a href="https://gitweb.torproject.org/torperf.git" target="_blank">Torperf</a> -tool, which periodically fetches static files over Tor and records several -timestamps in the process. The data file contains daily medians and quartiles -as well as total numbers of requests, timeouts, and failures. Raw Torperf -measurement data are available on the <a -href="collector.html#torperf">CollecTor</a> website.</p> - -<p><b>Download as <a href="stats/torperf.csv">CSV file</a>.</b></p> - -<p>The statistics file contains the following columns:</p> -<ul> - -<li><b>date:</b> UTC date (YYYY-MM-DD) when download performance was -measured.</li> - -<li><b>size:</b> Size of the downloaded file in bytes.</li> - -<li><b>source:</b> Name of the Torperf service performing measurements. If this -column contains the empty string, all measurements are included, regardless of -which Torperf service performed them. Examples are <b>"moria"</b>, -<b>"siv"</b>, and <b>"torperf"</b>.</li> - -<li><b>q1:</b> First quartile of time until receiving the last byte in -milliseconds.</li> - -<li><b>md:</b> Median of time until receiving the last byte in -milliseconds.</li> - -<li><b>q3:</b> Third quartile of time until receiving the last byte in -milliseconds.</li> - -<li><b>timeouts:</b> Number of timeouts that occurred when attempting to -download the static file over Tor.</li> - -<li><b>failures:</b> Number of failures that occurred when attempting to -download the static file over Tor.</li> - -<li><b>requests:</b> Total number of requests made to download the static file -over Tor.</li> - -</ul> - -</div> - -<div class="container"> <h2>Performance of downloading static files over Tor <a href="#torperf-1.1" name="torperf-1.1" class="anchor">#</a></h2> <p>The following data file contains aggregate statistics on performance when

1 0

[metrics-web/master] Fix broken CollecTor link.
by karsten＠torproject.org 21 Aug '17

21 Aug '17

commit 6fbb55ab215ecf4c38b69924e0a52296cb8605c8 Author: Karsten Loesing <karsten.loesing(a)gmx.net> Date: Mon Aug 21 09:55:12 2017 +0200 Fix broken CollecTor link. --- website/src/main/resources/web/WEB-INF/stats.jsp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/src/main/resources/web/WEB-INF/stats.jsp b/website/src/main/resources/web/WEB-INF/stats.jsp index 234d0f4..629130d 100644 --- a/website/src/main/resources/web/WEB-INF/stats.jsp +++ b/website/src/main/resources/web/WEB-INF/stats.jsp @@ -315,8 +315,8 @@ its predecessor <a href="https://gitweb.torproject.org/torperf.git">Torperf</a>, which both periodically fetch static files over Tor and record several timestamps in the process. The data file contains daily medians and quartiles as well as total numbers of requests, timeouts, and failures. Raw Onionperf and -Torperf measurement data is available on the <a -href="https://collector.torproject.org/#torperf">CollecTor</a> website.</p> +Torperf measurement data is specified and available for download on the <a +href="/collector.html#torperf">CollecTor</a> page.</p> <p><b>Download as <a href="stats/torperf-1.1.csv">CSV file</a>.</b></p>

1 0

[stem/master] DaemonTask stacktraced if joined before done
by atagar＠torproject.org 20 Aug '17

20 Aug '17

commit a20438efc598cdb5c01d567fe472bd595ce57eb5 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 20 12:36:31 2017 -0700 DaemonTask stacktraced if joined before done Pox on my really fast laptop. :P Now that I'm on a slower system realizing I buggered up the order of joining... Traceback (most recent call last): File "./run_tests.py", line 384, in <module> main() File "./run_tests.py", line 266, in main task.join() # no-op if these haven't been run File "/home/atagar/Desktop/stem/test/task.py", line 209, in join self.result = self._background_process.join() File "/home/atagar/Desktop/stem/stem/util/system.py", line 265, in join response = self._pipe.recv() IOError: [Errno 4] Interrupted system call --- stem/util/system.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stem/util/system.py b/stem/util/system.py index e4b2d874..b3b57da9 100644 --- a/stem/util/system.py +++ b/stem/util/system.py @@ -262,8 +262,8 @@ class DaemonTask(object): self.run() if self.status == State.RUNNING: - response = self._pipe.recv() self._process.join() + response = self._pipe.recv() self.status = response[0] self.runtime = response[1]

1 0

[stem/master] When tests logged we printed extra parentheses
by atagar＠torproject.org 20 Aug '17

20 Aug '17

commit 5e20fa1fe4a4a9b5afbaee491dec089fc9658565 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 20 12:38:32 2017 -0700 When tests logged we printed extra parentheses Print always requires an argument. Lacking this causes python 2.7 to print '()'. Probably breaks python 3.x outright but didn't check. --- test/output.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/output.py b/test/output.py index 82eb358d..c5a9efe9 100644 --- a/test/output.py +++ b/test/output.py @@ -89,7 +89,7 @@ def print_logging(logging_buffer): for entry in logging_buffer: println(entry.replace('\n', '\n '), term.Color.MAGENTA) - print() + print('') def apply_filters(testing_output, *filters):

1 0

[stem/master] Ensure CLOCK_TICK is what we expect for unit tests
by atagar＠torproject.org 20 Aug '17

20 Aug '17

commit 3fb2faa5a295d22f08bb7aeb1d41aab46d87e564 Author: Damian Johnson <atagar(a)torproject.org> Date: Sun Aug 20 10:54:21 2017 -0700 Ensure CLOCK_TICK is what we expect for unit tests yurivict271 points out that several tests of ours are failing on FreeBSD. First one is a unit test. The values are based on a CLOCK_TICK constant so that's probably why our results differ. ====================================================================== FAIL: test_stats ---------------------------------------------------------------------- Traceback (most recent call last): File "/usr/local/lib/python2.7/site-packages/mock/mock.py", line 1305, in patched return func(*args, **keywargs) File "/usr/ports/security/py-stem/work/stem-1.5.4/test/unit/util/proc.py", line 138, in test_stats self.assertEqual(response, proc.stats(24062, *args)) AssertionError: Tuples differ: ('0.13',) != ('0.1015625',) First differing element 0: '0.13' '0.1015625' - ('0.13',) + ('0.1015625',) ---------------------------------------------------------------------- Ran 10 tests in 0.021s --- test/unit/util/proc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/util/proc.py b/test/unit/util/proc.py index b4e3aab5..a172fcb4 100644 --- a/test/unit/util/proc.py +++ b/test/unit/util/proc.py @@ -100,6 +100,7 @@ class TestProc(unittest.TestCase): @patch('stem.util.proc._get_line') @patch('stem.util.proc.system_start_time', Mock(return_value = 10)) + @patch('stem.util.proc.CLOCK_TICKS', 100) def test_stats(self, get_line_mock): """ Tests stats() with all combinations of stat_type arguments.

1 0

[stem/master] Move testing to DaemonTask
by atagar＠torproject.org 19 Aug '17

19 Aug '17

commit 75f883864d9b68b11f8f2d1611a449c5ca9db063 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Aug 18 08:05:04 2017 -0700 Move testing to DaemonTask Now that we vend running daemon tasks lets use it ourselves. :P --- stem/util/system.py | 3 +++ test/task.py | 17 ++--------------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/stem/util/system.py b/stem/util/system.py index 6301c67b..e4b2d874 100644 --- a/stem/util/system.py +++ b/stem/util/system.py @@ -258,6 +258,9 @@ class DaemonTask(object): :raises: exception raised by the function if it failed with one """ + if self.status == State.PENDING: + self.run() + if self.status == State.RUNNING: response = self._pipe.recv() self._process.join() diff --git a/test/task.py b/test/task.py index f764fe99..53606a19 100644 --- a/test/task.py +++ b/test/task.py @@ -23,7 +23,6 @@ """ import importlib -import multiprocessing import os import re import sys @@ -169,7 +168,6 @@ class Task(object): self._is_background_task = background self._background_process = None - self._background_pipe = None def run(self): start_time = time.time() @@ -180,14 +178,7 @@ class Task(object): try: if self._is_background_task: - def _run_wrapper(conn, runner, args): - os.nice(15) - conn.send(runner(*args) if args else runner()) - conn.close() - - self._background_pipe, child_pipe = multiprocessing.Pipe() - self._background_process = multiprocessing.Process(target = _run_wrapper, args = (child_pipe, self.runner, self.args)) - self._background_process.start() + self._background_process = stem.util.system.DaemonTask(self.runner, self.args, start = True) else: self.result = self.runner(*self.args) if self.args else self.runner() @@ -215,11 +206,7 @@ class Task(object): def join(self): if self._background_process: - try: - self.result = self._background_pipe.recv() - self._background_process.join() - except IOError: - pass + self.result = self._background_process.join() class ModuleVersion(Task):

1 0

[stem/master] Add DaemonTask class
by atagar＠torproject.org 19 Aug '17

19 Aug '17

commit a7f6990c4ba3cb6348c32084abbd16e4cbadd0c8 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Aug 18 05:13:53 2017 -0700 Add DaemonTask class To improve our test runtime we started using the multithreading module to parallelize. It's clunky, but with this little helper it's easy to split off long-running tasks. This can improve more than just our tests. First candidate that comes to mind is nyx's resource usage fetches so it doesn't block the draw loop. Either way, having a tested library class is better than keeping this hidden in our test helpers. --- stem/util/system.py | 111 ++++++++++++++++++++++++++++++++++++++++++++++ test/integ/util/system.py | 35 +++++++++++++++ 2 files changed, 146 insertions(+) diff --git a/stem/util/system.py b/stem/util/system.py index 6d966723..6301c67b 100644 --- a/stem/util/system.py +++ b/stem/util/system.py @@ -45,12 +45,28 @@ best-effort, providing **None** if the lookup fails. get_process_name - provides our process' name set_process_name - changes our process' name + +.. data:: Status (enum) + + State of a subprocess. + + .. versionadded:: 1.6.0 + + ==================== =========== + Status Description + ==================== =========== + PENDING not yet started + RUNNING currently being performed + DONE completed successfully + FAILED failed with an exception + ==================== =========== """ import ctypes import ctypes.util import distutils.spawn import mimetypes +import multiprocessing import os import platform import re @@ -59,12 +75,20 @@ import tarfile import threading import time +import stem.util.enum import stem.util.proc import stem.util.str_tools from stem import UNDEFINED from stem.util import str_type, log +State = stem.util.enum.UppercaseEnum( + 'PENDING', + 'RUNNING', + 'DONE', + 'FAILED', +) + # Mapping of commands to if they're available or not. CMD_AVAILABLE_CACHE = {} @@ -180,6 +204,93 @@ class CallTimeoutError(CallError): self.timeout = timeout +class DaemonTask(object): + """ + Invokes the given function in a subprocess, returning the value. + + .. versionadded:: 1.6.0 + + :var function runner: function to be invoked by the subprocess + :var tuple args: arguments to provide to the subprocess + :var int priority: subprocess nice priority + + :var stem.util.system.State status: state of the subprocess + :var float runtime: seconds subprocess took to complete + :var object result: return value of subprocess if successful + :var exception error: exception raised by subprocess if it failed + """ + + def __init__(self, runner, args = None, priority = 15, start = False): + self.runner = runner + self.args = args + self.priority = priority + + self.status = State.PENDING + self.runtime = None + self.result = None + self.error = None + + self._process = None + self._pipe = None + + if start: + self.run() + + def run(self): + """ + Invokes the task if it hasn't already been started. If it has this is a + no-op. + """ + + if self.status == State.PENDING: + self._pipe, child_pipe = multiprocessing.Pipe() + self._process = multiprocessing.Process(target = DaemonTask._run_wrapper, args = (child_pipe, self.priority, self.runner, self.args)) + self._process.start() + self.status = State.RUNNING + + def join(self): + """ + Provides the result of the daemon task. If still running this blocks until + the task is completed. + + :returns: response of the function we ran + + :raises: exception raised by the function if it failed with one + """ + + if self.status == State.RUNNING: + response = self._pipe.recv() + self._process.join() + + self.status = response[0] + self.runtime = response[1] + + if self.status == State.DONE: + self.result = response[2] + elif self.status == State.FAILED: + self.error = response[2] + + if self.status == State.DONE: + return self.result + elif self.status == State.FAILED: + raise self.error + else: + raise RuntimeError('BUG: unexpected status from daemon task, %s' % self.status) + + @staticmethod + def _run_wrapper(conn, priority, runner, args): + start_time = time.time() + os.nice(priority) + + try: + result = runner(*args) if args else runner() + conn.send((State.DONE, time.time() - start_time, result)) + except Exception as exc: + conn.send((State.FAILED, time.time() - start_time, exc)) + finally: + conn.close() + + def is_windows(): """ Checks if we are running on Windows. diff --git a/test/integ/util/system.py b/test/integ/util/system.py index 818b6392..6a310faa 100644 --- a/test/integ/util/system.py +++ b/test/integ/util/system.py @@ -13,6 +13,8 @@ import stem.util.system import test.require import test.runner +from stem.util.system import State, DaemonTask + try: # added in python 3.3 from unittest.mock import Mock, patch @@ -70,6 +72,39 @@ require_path = test.require.needs(lambda: 'PATH' in os.environ, 'requires PATH') class TestSystem(unittest.TestCase): + def test_daemon_task_when_successful(self): + """ + Checks a simple, successfully DaemonTask that simply echos a value. + """ + + task = DaemonTask(lambda arg: arg, ('hello world',)) + + self.assertEqual(None, task.result) + self.assertEqual(State.PENDING, task.status) + + task.run() + self.assertEqual('hello world', task.join()) + self.assertEqual(State.DONE, task.status) + self.assertTrue(0 < task.runtime < 1.0) + + def test_daemon_task_on_failure(self): + """ + Checks DaemonTask when an exception is raised. + """ + + def _test_task(arg): + raise RuntimeError(arg) + + task = DaemonTask(_test_task, ('hello world',)) + + self.assertEqual(None, task.result) + self.assertEqual(State.PENDING, task.status) + + task.run() + self.assertRaisesRegexp(RuntimeError, 'hello world', task.join) + self.assertEqual(State.FAILED, task.status) + self.assertTrue(0 < task.runtime < 1.0) + @require_path def test_is_available(self): """

1 0

[nyx/master] Run static checks in the background
by atagar＠torproject.org 19 Aug '17

19 Aug '17

commit c03d05abdec29eebba2d3ed0f15c7e5f46aa658b Author: Damian Johnson <atagar(a)torproject.org> Date: Wed Aug 16 07:46:03 2017 -0700 Run static checks in the background Stem got quite a substantial speedup in its tests by running long-running tasks in the background. Nyx is a much smaller test suite so not much to gain, but might as well improve things a tad here too. Running pyflakes and pycodestyle in the background. On my laptop this drops the test runtime from 3.8s to 2.4s (37% faster). --- run_tests.py | 41 +++++++++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/run_tests.py b/run_tests.py index 2b76561..e115552 100755 --- a/run_tests.py +++ b/run_tests.py @@ -9,6 +9,7 @@ render to the screen. """ import os +import multiprocessing import unittest import stem.util.conf @@ -27,6 +28,12 @@ SRC_PATHS = [os.path.join(NYX_BASE, path) for path in ( )] +def _run_wrapper(conn, runner, args): + os.nice(15) + conn.send(runner(*args) if args else runner()) + conn.close() + + @nyx.uses_settings def main(): nyx.TESTING = True @@ -38,7 +45,20 @@ def main(): for path in orphaned_pyc: print('Deleted orphaned pyc file: %s' % path) - tests = unittest.defaultTestLoader.discover('test', pattern='*.py') + pyflakes_task, pyflakes_pipe = None, None + pycodestyle_task, pycodestyle_pipe = None, None + + if stem.util.test_tools.is_pyflakes_available(): + pyflakes_pipe, child_pipe = multiprocessing.Pipe() + pyflakes_task = multiprocessing.Process(target = _run_wrapper, args = (child_pipe, stem.util.test_tools.pyflakes_issues, (SRC_PATHS,))) + pyflakes_task.start() + + if stem.util.test_tools.is_pep8_available(): + pycodestyle_pipe, child_pipe = multiprocessing.Pipe() + pycodestyle_task = multiprocessing.Process(target = _run_wrapper, args = (child_pipe, stem.util.test_tools.stylistic_issues, (SRC_PATHS, True, True, True))) + pycodestyle_task.start() + + tests = unittest.defaultTestLoader.discover('test', pattern = '*.py') test_runner = unittest.TextTestRunner() test_runner.run(tests) @@ -46,22 +66,19 @@ def main(): static_check_issues = {} - if stem.util.test_tools.is_pyflakes_available(): - pyflakes_issues = stem.util.test_tools.pyflakes_issues(SRC_PATHS) + if pyflakes_task: + pyflakes_issues = pyflakes_pipe.recv() + pyflakes_task.join() for path, issues in pyflakes_issues.items(): for issue in issues: static_check_issues.setdefault(path, []).append(issue) - if stem.util.test_tools.is_pep8_available(): - pep8_issues = stem.util.test_tools.stylistic_issues( - SRC_PATHS, - check_newlines = True, - check_exception_keyword = True, - prefer_single_quotes = True, - ) - - for path, issues in pep8_issues.items(): + if pycodestyle_task: + pycodestyle_issues = pycodestyle_pipe.recv() + pycodestyle_task.join() + + for path, issues in pycodestyle_issues.items(): for issue in issues: static_check_issues.setdefault(path, []).append(issue)

1 0