- tbb-commits - lists.torproject.org

[tor-browser-bundle/master] update_responses: rename incremental files to *.incremental.mar
by mikeperry＠torproject.org 24 Oct '14

24 Oct '14

commit f11faec46052d5996f8316fce515fbfefecf5491 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Oct 13 19:21:02 2014 +0200 update_responses: rename incremental files to *.incremental.mar This should make them easier to grep. --- tools/update-responses/update_responses | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index 7f1df2d..a42e700 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -76,7 +76,7 @@ sub get_version_files { }; next; } - if ($file =~ m/^tor-browser-([^-]+)-(.+)-${version}_(.+)\.mar$/) { + if ($file =~ m/^tor-browser-([^-]+)-(.+)-${version}_(.+)\.incremental\.mar$/) { my ($os, $from_version, $lang) = ($1, $2, $3); $files->{$os}{$lang}{partial}{$from_version} = { type => 'partial', @@ -124,7 +124,7 @@ sub mar_filename { sub create_incremental_mar { my ($config, $pm, $from_version, $new_version, $os, $lang) = @_; - my $mar_file = "tor-browser-$os-${from_version}-${new_version}_$lang.mar"; + my $mar_file = "tor-browser-$os-${from_version}-${new_version}_$lang.incremental.mar"; print "Starting $mar_file\n"; my $mar_file_path = "$releases_dir/$new_version/$mar_file"; my $finished_file = sub {

1 0

[tor-browser-bundle/master] Merge remote-tracking branch 'boklm/update-responses-3'
by mikeperry＠torproject.org 24 Oct '14

24 Oct '14

commit 6f6c2ab62c3d24201df55d25346d52933a72abb1 Merge: adcf052 f1472e5 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Fri Oct 24 16:48:41 2014 -0700 Merge remote-tracking branch 'boklm/update-responses-3' gitian/Makefile | 15 ++++ gitian/check-match.sh | 44 +++++++++- gitian/hash-bundles.sh | 8 +- gitian/upload-signature.sh | 9 +- tools/update-responses/README.md | 11 ++- tools/update-responses/config.yml | 2 + tools/update-responses/update_responses | 146 +++++++++++++++++++++++++++++-- 7 files changed, 220 insertions(+), 15 deletions(-)

1 0

[tor-browser-bundle/master] update_responses: create symlinks when extracting mars
by mikeperry＠torproject.org 24 Oct '14

24 Oct '14

commit cf83610e79951c0676212dca49d7144bf5df0f21 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Sun Oct 12 15:03:43 2014 +0200 update_responses: create symlinks when extracting mars Find the addsymlink instructions in the updatev3.manifest file and create corresponding symbolic links. If no updatev3.manifest file is present, we do it with the updatev2.manifest file if it is present. --- tools/update-responses/update_responses | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index 3590227..55ebfb9 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -105,6 +105,15 @@ sub extract_mar { || exit_error "Error decompressing $File::Find::name"; }; find($bunzip_file, $dest_dir); + my $manifest = -f 'updatev3.manifest' ? 'updatev3.manifest' + : 'updatev2.manifest'; + my @lines = read_file($manifest) if -f $manifest; + foreach my $line (@lines) { + if ($line =~ m/^addsymlink "(.+)" "(.+)"$/) { + exit_error "$mar_file: Could not create symlink $1 -> $2" + unless symlink $2, $1; + } + } chdir $old_cwd; }

1 0

[tor-browser-bundle/master] update_responses: extract mar-tools in a temporary directory
by mikeperry＠torproject.org 24 Oct '14

24 Oct '14

commit d5459fbac32c7be88e70093b18aad138dd620b7c Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Oct 13 12:31:33 2014 +0200 update_responses: extract mar-tools in a temporary directory Extract the gitian-builder/inputs/mar-tools-linux*.zip file that was created during the last firefox build. --- tools/update-responses/update_responses | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index 55ebfb9..7f1df2d 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -268,6 +268,33 @@ sub check_deps { } } +sub osname { + my ($osname) = capture_exec('uname', '-s'); + my ($arch) = capture_exec('uname', '-m'); + chomp($osname, $arch); + if ($osname eq 'Linux' && $arch eq 'x86_64') { + return 'linux64'; + } + if ($osname eq 'Linux' && $arch =~ m/^i.86$/) { + return 'linux32'; + } + exit_error 'Unknown OS'; +} + +my $martools_tmpdir; +sub extract_martools { + my $osname = osname; + my $marzip = "$FindBin::Bin/../../../gitian-builder/inputs/mar-tools-$osname.zip"; + $martools_tmpdir = File::Temp->newdir(); + my $old_cwd = getcwd; + chdir $martools_tmpdir; + my (undef, undef, $success) = capture_exec('unzip', $marzip); + chdir $old_cwd; + exit_error "Error extracting $marzip" unless $success; + $ENV{PATH} .= ":$martools_tmpdir/mar-tools"; +} + +extract_martools; check_deps; write_responses($config); write_htaccess($config);

1 0

[tor-browser-bundle/master] Add changelog for 4.0.1.
by mikeperry＠torproject.org 24 Oct '14

24 Oct '14

commit adcf0528cb53e2054d46871b36289b28f7e52498 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Fri Oct 24 16:39:31 2014 -0700 Add changelog for 4.0.1. --- Bundle-Data/Docs/ChangeLog.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Bundle-Data/Docs/ChangeLog.txt b/Bundle-Data/Docs/ChangeLog.txt index 7d8470b..92a060b 100644 --- a/Bundle-Data/Docs/ChangeLog.txt +++ b/Bundle-Data/Docs/ChangeLog.txt @@ -1,3 +1,10 @@ +Tor Browser 4.0.1 -- Oct 30 2014 + * All Platforms + * Bug 13301: Prevent extensions incompatibility error after upgrades + * Bug 13460: Fix MSVC compilation issue + * Windows + * Bug 13443: Disable DirectShow to prevent crashes on many sites + Tor Browser 4.0 -- Oct 15 2014 * All Platforms * Update Firefox to 31.2.0esr @@ -26,7 +33,6 @@ Tor Browser 4.0 -- Oct 15 2014 * Bug 13186: Disable DOM Performance timers * Bug 13028: Defense-in-depth checks for OCSP/Cert validation proxy usage - Tor Browser 4.0-alpha-3 -- Sep 24 2014 * All Platforms * Update Tor to 0.2.5.8-rc

1 0

[tor-browser-bundle/master] Revert "The poodle hotfix."
by mikeperry＠torproject.org 24 Oct '14

24 Oct '14

commit 77e6fb571a69e4b146909dfda8f7552b34ab34d8 Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Fri Oct 24 16:29:43 2014 -0700 Revert "The poodle hotfix." This reverts commit fd7e8eef09b813c445e31239bcb674de7a03d191. We set this pref in our browser preferences now. --- gitian/descriptors/linux/gitian-bundle.yml | 4 ---- gitian/descriptors/mac/gitian-bundle.yml | 4 ---- gitian/descriptors/windows/gitian-bundle.yml | 4 ---- 3 files changed, 12 deletions(-) diff --git a/gitian/descriptors/linux/gitian-bundle.yml b/gitian/descriptors/linux/gitian-bundle.yml index 3de0d49..19233b6 100644 --- a/gitian/descriptors/linux/gitian-bundle.yml +++ b/gitian/descriptors/linux/gitian-bundle.yml @@ -146,8 +146,6 @@ script: | cp defaults/preferences/000-tor-browser.js ~/build/ # Set the locale of the bundle. echo "pref(\"general.useragent.locale\", \"en-US\");" >> defaults/preferences/000-tor-browser.js - # The poodle hotfix - echo "pref(\"security.tls.version.min\", 1);" >> defaults/preferences/000-tor-browser.js zip -Xm omni.ja defaults/preferences/000-tor-browser.js rm -rf defaults popd @@ -178,8 +176,6 @@ script: | cp ~/build/000-tor-browser.js defaults/preferences/ # Set the locale of the bundle. echo "pref(\"general.useragent.locale\", \"$LANG\");" >> defaults/preferences/000-tor-browser.js - # The poodle hotfix - echo "pref(\"security.tls.version.min\", 1);" >> defaults/preferences/000-tor-browser.js zip -Xm omni.ja defaults/preferences/000-tor-browser.js rm -rf defaults popd diff --git a/gitian/descriptors/mac/gitian-bundle.yml b/gitian/descriptors/mac/gitian-bundle.yml index 9e78429..82b6595 100644 --- a/gitian/descriptors/mac/gitian-bundle.yml +++ b/gitian/descriptors/mac/gitian-bundle.yml @@ -166,8 +166,6 @@ script: | cp defaults/preferences/000-tor-browser.js ~/build/ # Set the locale of the bundle. echo "pref(\"general.useragent.locale\", \"en-US\");" >> defaults/preferences/000-tor-browser.js - # The poodle hotfix - echo "pref(\"security.tls.version.min\", 1);" >> defaults/preferences/000-tor-browser.js zip -Xm omni.ja defaults/preferences/000-tor-browser.js rm -rf defaults popd @@ -223,8 +221,6 @@ script: | cp ~/build/000-tor-browser.js defaults/preferences/ # Set the locale of the bundle. echo "pref(\"general.useragent.locale\", \"$LANG\");" >> defaults/preferences/000-tor-browser.js - # The poodle hotfix - echo "pref(\"security.tls.version.min\", 1);" >> defaults/preferences/000-tor-browser.js zip -Xm omni.ja defaults/preferences/000-tor-browser.js rm -rf defaults popd diff --git a/gitian/descriptors/windows/gitian-bundle.yml b/gitian/descriptors/windows/gitian-bundle.yml index a3c4b8a..76f5c67 100644 --- a/gitian/descriptors/windows/gitian-bundle.yml +++ b/gitian/descriptors/windows/gitian-bundle.yml @@ -126,8 +126,6 @@ script: | cp defaults/preferences/000-tor-browser.js ~/build/ # Set the locale bundle. echo "pref(\"general.useragent.locale\", \"en-US\");" >> defaults/preferences/000-tor-browser.js - # The poodle hotfix - echo "pref(\"security.tls.version.min\", 1);" >> defaults/preferences/000-tor-browser.js zip -Xm omni.ja defaults/preferences/000-tor-browser.js rm -rf defaults popd @@ -158,8 +156,6 @@ script: | mkdir -p defaults/preferences cp ~/build/000-tor-browser.js defaults/preferences/ echo "pref(\"general.useragent.locale\", \"$LANG\");" >> defaults/preferences/000-tor-browser.js - # The poodle hotfix - echo "pref(\"security.tls.version.min\", 1);" >> defaults/preferences/000-tor-browser.js zip -Xm omni.ja defaults/preferences/000-tor-browser.js rm -rf defaults popd

1 0

[tor-browser-bundle/master] Bug #13324: Address comments from mcs's review
by mikeperry＠torproject.org 24 Oct '14

24 Oct '14

commit 8ad38fa9eed101c4e44ce6c7951579cf13fd708b Author: Mike Perry <mikeperry-git(a)torproject.org> Date: Fri Oct 24 16:49:41 2014 -0700 Bug #13324: Address comments from mcs's review --- gitian/check-match.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitian/check-match.sh b/gitian/check-match.sh index 43d1ef1..0622512 100755 --- a/gitian/check-match.sh +++ b/gitian/check-match.sh @@ -52,6 +52,7 @@ do done cd ../.. +# XXX: We should refactor this code into a shared function if [ -f $TORBROWSER_VERSION/sha256sums.incrementals.txt ] then for u in $USERS @@ -94,6 +95,7 @@ then if [ -z "$VALID_incrementals" ] then echo "No incremental mars hashes or sigs published for $TORBROWSER_VERSION." + exit_val=1 else echo "Matching incremental mars exist from the following users: $VALID_incrementals" fi

1 0

[tor-browser-bundle/master] update_responses: write things in sorted order
by mikeperry＠torproject.org 24 Oct '14

24 Oct '14

commit 3cfcdc6a9b8261186063932277fffe112ff98ff1 Author: Nicolas Vigier <boklm(a)mars-attacks.org> Date: Sat Sep 27 14:37:11 2014 +0200 update_responses: write things in sorted order So that the same input files always produce the same output files. --- tools/update-responses/update_responses | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index e082592..6e9365a 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -106,7 +106,8 @@ sub get_response { defined $minversion ? ( minSupportedOSVersion => $minversion ) : (), ); foreach my $patch (@patches) { - $writer->startTag('patch', %$patch); + my @sorted_patch = map { $_ => $patch->{$_} } sort keys %$patch; + $writer->startTag('patch', @sorted_patch); $writer->endTag('patch'); } CLOSETAGS: @@ -141,16 +142,16 @@ sub write_htaccess { my ($config) = @_; my $htaccess = "RewriteEngine On\n"; my $flags = "[last]"; - foreach my $channel (keys %{$config->{channels}}) { + foreach my $channel (sort keys %{$config->{channels}}) { my $version = $config->{channels}{$channel}; my $files = $config->{versions}{$version}{files}; $htaccess .= "RewriteRule " . "^$channel/[^\/]+/$version/ " . "no-update.xml $flags\n"; - foreach my $os (keys %$files) { + foreach my $os (sort keys %$files) { my $bt = build_target_by_os($os); - foreach my $lang (keys %{$files->{$os}}) { - foreach my $from_version (keys %{$files->{$os}{$lang}{partial}}) { + foreach my $lang (sort keys %{$files->{$os}}) { + foreach my $from_version (sort keys %{$files->{$os}{$lang}{partial}}) { $htaccess .= "RewriteRule ^$channel/$bt/$from_version/$lang " . "$from_version-$version-$os-$lang.xml $flags\n"; }

1 0

[tor-browser-bundle/master] update_responses: accept '-' in version numbers
by mikeperry＠torproject.org 24 Oct '14

24 Oct '14

commit 9e26ef8acfaefd7d7a84338e1bdb75efc0087de8 Author: Nicolas Vigier <boklm(a)mars-attacks.org> Date: Thu Oct 2 14:13:55 2014 +0200 update_responses: accept '-' in version numbers Version numbers like '4.0-alpha-2' contain '-'. --- tools/update-responses/update_responses | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index e94c15f..e86979b 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -60,7 +60,7 @@ sub get_version_files { }; next; } - if ($file =~ m/^tor-browser-([^-]+)-([^-]+)-${version}_(.+)\.mar$/) { + if ($file =~ m/^tor-browser-([^-]+)-(.+)-${version}_(.+)\.mar$/) { my ($os, $from_version, $lang) = ($1, $2, $3); $files->{$os}{$lang}{partial}{$from_version} = { type => 'partial',

1 0

[tor-browser-bundle/master] update_responses: generate incremental MAR files
by mikeperry＠torproject.org 24 Oct '14

24 Oct '14

commit 389f843dd3bb19f703a4448528e3d5b92942bf04 Author: Nicolas Vigier <boklm(a)mars-attacks.org> Date: Thu Oct 2 15:34:06 2014 +0200 update_responses: generate incremental MAR files Generate incremental MAR file when they are not already there. The list of versions from which to generate incremental MAR is defined in config.yml as incremental_from for each version. The script expects the MAR tools (the mar and make_incremental_update.sh commands) to be available in the PATH. --- tools/update-responses/README.md | 9 ++-- tools/update-responses/config.yml | 2 + tools/update-responses/update_responses | 80 ++++++++++++++++++++++++++++++- 3 files changed, 87 insertions(+), 4 deletions(-) diff --git a/tools/update-responses/README.md b/tools/update-responses/README.md index ee21bac..c7ecdd4 100644 --- a/tools/update-responses/README.md +++ b/tools/update-responses/README.md @@ -12,19 +12,22 @@ Dependencies ------------ The following perl modules need to be installed to run the script: - FindBin YAML File::Slurp Digest::SHA XML::Writer + FindBin YAML File::Slurp Digest::SHA XML::Writer File::Temp + IO::CaptureOutput File::Which On Debian / Ubuntu you can install them with: ``` # apt-get install libfindbin-libs-perl libyaml-perl libfile-slurp-perl \ - libdigest-sha-perl libxml-writer-perl + libdigest-sha-perl libxml-writer-perl \ + libio-captureoutput-perl libfile-which-perl ``` On Red Hat / Fedora you can install them with: ``` - # for module in FindBin YAML File::Slurp Digest::SHA XML::Writer + # for module in FindBin YAML File::Slurp Digest::SHA XML::Writer \ + File::Temp IO::CaptureOutput File::Which do yum install "perl($module)"; done ``` diff --git a/tools/update-responses/config.yml b/tools/update-responses/config.yml index b49ce1a..ad1c512 100644 --- a/tools/update-responses/config.yml +++ b/tools/update-responses/config.yml @@ -16,6 +16,8 @@ versions: platformVersion: 24.8.1 detailsURL: https://www.torproject.org/projects/torbrowser.html.en download_url: https://www.torproject.org/dist/torbrowser/4.0-alpha-3 + incremental_from: + - 4.0-alpha-2 # osx32: # minSupportedOSVersion: 10.7 # unsupported: 1 diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index e86979b..1946278 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -6,13 +6,20 @@ use YAML qw(LoadFile); use File::Slurp; use Digest::SHA; use XML::Writer; +use Cwd; +use File::Temp; +use File::Find; +use File::Which; +use IO::CaptureOutput qw(capture_exec); my $htdocsdir = "$FindBin::Bin/htdocs"; my $config = LoadFile("$FindBin::Bin/config.yml"); my %htdocsfiles = ( '.' => 1, '..' => 1, 'no-update.xml' => 1 ); +my $releases_dir = "$FindBin::Bin/../../gitian"; sub exit_error { print STDERR "Error: ", $_[0], "\n"; + chdir '/'; exit (exists $_[1] ? $_[1] : 1); } @@ -45,7 +52,7 @@ sub get_version_files { my ($config, $version) = @_; return if $config->{versions}{$version}{files}; my $files = {}; - my $vdir = "$FindBin::Bin/../../gitian/$version"; + my $vdir = "$releases_dir/$version"; opendir(my $d, $vdir) or exit_error "Error opening directory $vdir"; foreach my $file (readdir $d) { next unless -f "$vdir/$file"; @@ -75,6 +82,69 @@ sub get_version_files { $config->{versions}{$version}{files} = $files; } +sub extract_mar { + my ($mar_file, $dest_dir) = @_; + my $old_cwd = getcwd; + mkdir $dest_dir; + chdir $dest_dir or exit_error "Cannot enter $dest_dir"; + my $res = system('mar', '-x', $mar_file); + exit_error "Error extracting $mar_file" if $res; + my $bunzip_file = sub { + return unless -f $File::Find::name; + rename $File::Find::name, "$File::Find::name.bz2"; + system('bunzip2', "$File::Find::name.bz2") == 0 + || exit_error "Error decompressing $File::Find::name"; + }; + find($bunzip_file, $dest_dir); + chdir $old_cwd; +} + +sub mar_filename { + my ($version, $os, $lang) = @_; + "$releases_dir/$version/tor-browser-$os-${version}_$lang.mar"; +} + +sub create_incremental_mar { + my ($config, $from_version, $new_version, $os, $lang) = @_; + print "Creating incremental mar file ", + "$from_version -> $new_version ($os $lang)\n"; + my $tmpdir = File::Temp->newdir(); + extract_mar(mar_filename($from_version, $os, $lang), "$tmpdir/A"); + extract_mar(mar_filename($new_version, $os, $lang), "$tmpdir/B"); + my $mar_file = "tor-browser-$os-${from_version}-${new_version}_$lang.mar"; + my $mar_file_path = "$releases_dir/$new_version/$mar_file"; + my ($out, $err, $success) = capture_exec('make_incremental_update.sh', + $mar_file_path, "$tmpdir/A", "$tmpdir/B"); + if (!$success) { + unlink $mar_file_path if -f $mar_file_path; + exit_error "making incremental mar:\n" . $err; + } + $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = { + type => 'partial', + URL => "$config->{versions}{$new_version}{download_url}/$mar_file", + size => -s $mar_file_path, + hashFunction => 'SHA512', + hashValue => get_sha512_hex_of_file($mar_file_path), + }; +} + +sub create_missing_incremental_mars { + my ($config, $version) = @_; + my $v = $config->{versions}{$version}; + foreach my $from_version (@{$v->{incremental_from}}) { + $config->{versions}{$from_version} //= { download_url => '' }; + get_version_files($config, $from_version); + my $from_v = $config->{versions}{$from_version}; + foreach my $os (keys %{$v->{files}}) { + foreach my $lang (keys %{$v->{files}{$os}}) { + next if defined $v->{files}{$os}{$lang}{partial}{$from_version}; + next unless defined $from_v->{files}{$os}{$lang}{complete}; + create_incremental_mar($config, $from_version, $version, $os, $lang); + } + } + } +} + sub get_config { my ($config, $version, $os, $name) = @_; return $config->{versions}{$version}{$os}{$name} @@ -121,6 +191,7 @@ sub write_responses { my ($config) = @_; foreach my $version (values %{$config->{channels}}) { get_version_files($config, $version); + create_missing_incremental_mars($config, $version); my $files = $config->{versions}{$version}{files}; foreach my $os (keys %$files) { foreach my $lang (keys %{$files->{$os}}) { @@ -165,6 +236,13 @@ sub write_htaccess { write_htdocs('.htaccess', $htaccess); } +sub check_deps { + foreach my $bin (qw(bunzip2 mar mbsdiff make_incremental_update.sh)) { + exit_error "Cannot find $bin in PATH" unless which($bin); + } +} + +check_deps; write_responses($config); write_htaccess($config); clean_htdocs;

1 0