December 2016 - tor-commits - lists.torproject.org

[stem/master] Merge MANIFEST.in into setup.py
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit 4581199ffd9bd65bb25d89b42663e2192607d7f2 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Dec 5 14:10:52 2016 -0800 Merge MANIFEST.in into setup.py The manifest solely exists to make our source distribution contain the files it should. Silly we can't simply specify them in our setup() call, but oh well. Lets include this in our setup.py to avoid clutter and keep this where it should be. --- MANIFEST.in | 17 ------------- setup.py | 85 ++++++++++++++++++++++++++++++++++++++++--------------------- 2 files changed, 56 insertions(+), 46 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index d35f06a..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,17 +0,0 @@ -include cache_fallback_directories.py -include cache_manual.py -include LICENSE -include MANIFEST.in -include requirements.txt -include run_tests.py -include tox.ini -graft docs -graft test -global-exclude __pycache__ -global-exclude *.orig -global-exclude *.pyc -global-exclude *.swp -global-exclude *.swo -global-exclude .tox -recursive-exclude test/data * -recursive-exclude docs/_build * diff --git a/setup.py b/setup.py index 5770898..ac780d6 100644 --- a/setup.py +++ b/setup.py @@ -31,37 +31,64 @@ To install you can either use... After that, give some `tutorials <https://stem.torproject.org/tutorials.html>`_ a try! For questions or to discuss project ideas we're available on `irc <https://www.torproject.org/about/contact.html.en#irc>`_ and the `tor-dev@ email list <https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-dev>`_. """.strip() +MANIFEST = """ +include cache_fallback_directories.py +include cache_manual.py +include LICENSE +include MANIFEST.in +include requirements.txt +include run_tests.py +include tox.ini +graft docs +graft test +global-exclude __pycache__ +global-exclude *.orig +global-exclude *.pyc +global-exclude *.swp +global-exclude *.swo +global-exclude .tox +recursive-exclude test/data * +recursive-exclude docs/_build * +""".strip() + # Ensure this is our cwd, otherwise distutils fails with 'standard file # 'setup.py' not found'. os.chdir(os.path.dirname(os.path.abspath(__file__))) -distutils.core.setup( - name = 'stem-dry-run' if DRY_RUN else 'stem', - version = stem.__version__, - description = DRY_RUN_SUMMARY if DRY_RUN else SUMMARY, - long_description = DESCRIPTION, - license = stem.__license__, - author = stem.__author__, - author_email = stem.__contact__, - url = stem.__url__, - packages = ['stem', 'stem.descriptor', 'stem.interpreter', 'stem.response', 'stem.util'], - keywords = 'tor onion controller', - scripts = ['tor-prompt'], - provides = ['stem'], - package_data = { - 'stem': ['cached_tor_manual.cfg', 'settings.cfg'], - 'stem.descriptor': ['fallback_directories.cfg'], - 'stem.interpreter': ['settings.cfg'], - 'stem.util': ['ports.cfg'], - }, classifiers = [ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)', - 'Topic :: Security', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], -) - -if os.path.exists('MANIFEST'): - os.remove('MANIFEST') +with open('MANIFEST.in', 'w') as manifest_file: + manifest_file.write(MANIFEST) + +try: + distutils.core.setup( + name = 'stem-dry-run' if DRY_RUN else 'stem', + version = stem.__version__, + description = DRY_RUN_SUMMARY if DRY_RUN else SUMMARY, + long_description = DESCRIPTION, + license = stem.__license__, + author = stem.__author__, + author_email = stem.__contact__, + url = stem.__url__, + packages = ['stem', 'stem.descriptor', 'stem.interpreter', 'stem.response', 'stem.util'], + keywords = 'tor onion controller', + scripts = ['tor-prompt'], + provides = ['stem'], + package_data = { + 'stem': ['cached_tor_manual.cfg', 'settings.cfg'], + 'stem.descriptor': ['fallback_directories.cfg'], + 'stem.interpreter': ['settings.cfg'], + 'stem.util': ['ports.cfg'], + }, classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)', + 'Topic :: Security', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + ) +finally: + if os.path.exists('MANIFEST.in'): + os.remove('MANIFEST.in') + + if os.path.exists('MANIFEST'): + os.remove('MANIFEST')

1 0

[stem/master] Exclude gitignore resources
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit cb0bdac2b82f7b5cb41442a25a37e4ce3fb69353 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Nov 25 15:43:36 2016 -0800 Exclude gitignore resources Oops! Dererk noticed our docs/_build/* is present in the tarballs we upload. Turns out test/data/* is too. Excluding things specified by our gitignore. We'll need to be careful to double check that there's no untracked files before uploads... --- MANIFEST.in | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 086a5db..d35f06a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,4 +8,10 @@ include tox.ini graft docs graft test global-exclude __pycache__ -global-exclude *.py[co] +global-exclude *.orig +global-exclude *.pyc +global-exclude *.swp +global-exclude *.swo +global-exclude .tox +recursive-exclude test/data * +recursive-exclude docs/_build *

1 0

[stem/master] Clean up auto-generated MANIFEST
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit 484b00544f6a97cfadf67068c9af1afc1280c6d8 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Dec 5 14:07:10 2016 -0800 Clean up auto-generated MANIFEST Now that we have a MANIFEST.in our setup.py generates a MANIFEST. Hate leaving turds around so cleaning it up. --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 3d79742..5770898 100644 --- a/setup.py +++ b/setup.py @@ -62,3 +62,6 @@ distutils.core.setup( 'Topic :: Software Development :: Libraries :: Python Modules', ], ) + +if os.path.exists('MANIFEST'): + os.remove('MANIFEST')

1 0

[stem/master] Allow setup.py to work with different cwd
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit 7b2af7d6810234d3847873df823edee9bae86403 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Dec 5 14:04:52 2016 -0800 Allow setup.py to work with different cwd Running our setup.py when this wasn't your cwd failed. Quick fix to account for this. --- setup.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 65c790e..3d79742 100644 --- a/setup.py +++ b/setup.py @@ -3,11 +3,12 @@ # See LICENSE for licensing information import distutils.core +import os import stem DRY_RUN = True SUMMARY = 'Stem is a Python controller library that allows applications to interact with Tor (https://www.torproject.org/)' -DRY_RUN_SUMMARY = "Ignore this package. This is dry-run release creation to work around PyPI limitations (https://github.com/pypa/packaging-problems/issues/74#issuecomment-260716129)" +DRY_RUN_SUMMARY = 'Ignore this package. This is dry-run release creation to work around PyPI limitations (https://github.com/pypa/packaging-problems/issues/74#issuecomment-260716129)' DESCRIPTION = """ For tutorials and API documentation see `stem's homepage <https://stem.torproject.org/>`_. @@ -20,7 +21,7 @@ To install you can either use... :: pip install stem - + ... or install from the source tarball. Stem supports both the python 2.x and 3.x series. To use its python3 counterpart you simply need to install using that version of python. :: @@ -30,6 +31,11 @@ To install you can either use... After that, give some `tutorials <https://stem.torproject.org/tutorials.html>`_ a try! For questions or to discuss project ideas we're available on `irc <https://www.torproject.org/about/contact.html.en#irc>`_ and the `tor-dev@ email list <https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-dev>`_. """.strip() +# Ensure this is our cwd, otherwise distutils fails with 'standard file +# 'setup.py' not found'. + +os.chdir(os.path.dirname(os.path.abspath(__file__))) + distutils.core.setup( name = 'stem-dry-run' if DRY_RUN else 'stem', version = stem.__version__,

1 0

[stem/master] Stem release 1.5.3
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit f2522efef52cfda0d3f289e082cbd4effb17fedd Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Nov 25 10:49:34 2016 -0800 Stem release 1.5.3 Juan pointed out that packagers use resources that aren't available by default as part of our source distribution. Including everything in the tarball we upload. --- docs/change_log.rst | 3 +++ docs/index.rst | 2 +- stem/__init__.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index ba496a7..a3b3d9a 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -111,6 +111,9 @@ The following are only available within Stem's `git repository * Example for `custom path selection for circuits <tutorials/to_russia_with_love.html#custom-path-selection>`_ (:trac:`8728`) * Example for `persisting ephemeral hidden service keys <tutorials/over_the_river.html#ephemeral-hidden-services>`_ + * **Version 1.5.3** (December 5th, 2016) - including tests and site in the + release tarball + .. _version_1.4: Version 1.4 (May 13th, 2015) diff --git a/docs/index.rst b/docs/index.rst index da9362b..f265f4e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ Welcome to Stem! ================ -Stem is a Python controller library for `Tor <https://www.torproject.org/>`_. With it you can use Tor's `control protocol <https://gitweb.torproject.org/torspec.git/tree/control-spec.txt>`_ to script against the Tor process, or build things such as `arm <https://www.atagar.com/arm/>`_. Stem's latest version is **1.5.2** (released November 20th, 2016). +Stem is a Python controller library for `Tor <https://www.torproject.org/>`_. With it you can use Tor's `control protocol <https://gitweb.torproject.org/torspec.git/tree/control-spec.txt>`_ to script against the Tor process, or build things such as `arm <https://www.atagar.com/arm/>`_. Stem's latest version is **1.5.3** (released December 5th, 2016). .. Main Stem Logo Source: http://www.wpclipart.com/plants/assorted/P/plant_stem.png.html diff --git a/stem/__init__.py b/stem/__init__.py index 7cf39f2..66502c8 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -470,7 +470,7 @@ Library for working with the tor process. import stem.util.enum -__version__ = '1.5.2' +__version__ = '1.5.3' __author__ = 'Damian Johnson' __contact__ = 'atagar(a)torproject.org' __url__ = 'https://stem.torproject.org/'

1 0

[stem/master] Fix unit test for data file installation
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit d2a0be85d83dd6167865aa52fe12baae75562f47 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Dec 5 19:12:51 2016 -0800 Fix unit test for data file installation Reformatting broke test_installs_all_data_files. It was a bit better so I might re-introduce the change later - just don't want to hassle with rewriting the test right now. --- setup.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index ac780d6..6b4922d 100644 --- a/setup.py +++ b/setup.py @@ -73,12 +73,8 @@ try: keywords = 'tor onion controller', scripts = ['tor-prompt'], provides = ['stem'], - package_data = { - 'stem': ['cached_tor_manual.cfg', 'settings.cfg'], - 'stem.descriptor': ['fallback_directories.cfg'], - 'stem.interpreter': ['settings.cfg'], - 'stem.util': ['ports.cfg'], - }, classifiers = [ + package_data = {'stem': ['cached_tor_manual.cfg', 'settings.cfg'], 'stem.descriptor': ['fallback_directories.cfg'], 'stem.interpreter': ['settings.cfg'], 'stem.util': ['ports.cfg']}, + classifiers = [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)',

1 0

[stem/master] Test our source distribution matches the git contents
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit 34be2ece71819ae005914317d9a492b755037d26 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Dec 5 15:41:24 2016 -0800 Test our source distribution matches the git contents Having a hardcoded Manifest.in is error prone. Unfortunately we need to do it, so to avoid shooting ourselves in the foot we need a test. This checks that source distributions we generate match the contents of the git repository. --- test/integ/installation.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/test/integ/installation.py b/test/integ/installation.py index 189c26e..9e11c4d 100644 --- a/test/integ/installation.py +++ b/test/integ/installation.py @@ -2,6 +2,7 @@ import glob import os import shutil import sys +import tarfile import unittest import stem @@ -10,6 +11,8 @@ import stem.util.system import test.runner import test.util +INSTALL_MISMATCH_MSG = "Running 'python setup.py sdist' doesn't match our git contents in the following way. The manifest in our setup.py may need to be updated...\n\n" + class TestInstallation(unittest.TestCase): # TODO: remove when dropping support for python 2.6 @@ -56,6 +59,51 @@ class TestInstallation(unittest.TestCase): return False @test.runner.only_run_once + def test_sdist_matches_git(self): + """ + Check the source distribution tarball we make for releases matches the + contents of 'git archive'. This primarily is meant to test that our + MANIFEST.in is up to date. + """ + + if self.requires_installation(): + return + elif not stem.util.system.is_available('git'): + test.runner.skip(self, '(git unavailable)') + return + + original_cwd = os.getcwd() + dist_path = os.path.join(test.util.STEM_BASE, 'dist') + git_contents = [line.split()[-1] for line in stem.util.system.call('git ls-tree --full-tree -r HEAD')] + + try: + os.chdir(test.util.STEM_BASE) + stem.util.system.call(sys.executable + ' setup.py sdist') + + # tarball has a prefix 'stem-[verion]' directory so stipping that out + + dist_tar = tarfile.open(os.path.join(dist_path, 'stem-dry-run-%s.tar.gz' % stem.__version__)) + tar_contents = ['/'.join(info.name.split('/')[1:]) for info in dist_tar.getmembers() if info.isfile()] + finally: + if os.path.exists(dist_path): + shutil.rmtree(dist_path) + + os.chdir(original_cwd) + + issues = [] + + for path in git_contents: + if path not in tar_contents and path not in ['.gitignore']: + issues.append(' * %s is missing from our release tarball' % path) + + for path in tar_contents: + if path not in git_contents and path not in ['MANIFEST.in', 'PKG-INFO']: + issues.append(" * %s isn't expected in our release tarball" % path) + + if issues: + self.fail(INSTALL_MISMATCH_MSG + '\n'.join(issues)) + + @test.runner.only_run_once def test_installing_stem(self): """ Attempt to use the package we install.

1 0

[stem/master] Merge version 1.5.3 hotfix
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit 84d8a5035b4986f40e5f3fed841b4e7cd734d1e0 Merge: c857786 f2522ef Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Dec 6 09:34:21 2016 -0800 Merge version 1.5.3 hotfix docs/change_log.rst | 3 ++ docs/index.rst | 2 +- setup.py | 83 +++++++++++++++++++++++++++++++++------------- stem/__init__.py | 2 +- test/integ/installation.py | 48 +++++++++++++++++++++++++++ 5 files changed, 113 insertions(+), 25 deletions(-) diff --cc stem/__init__.py index b02dd9c,66502c8..1acf0f4 --- a/stem/__init__.py +++ b/stem/__init__.py @@@ -470,7 -470,7 +470,7 @@@ Library for working with the tor proces import stem.util.enum - __version__ = '1.5.2-dev' -__version__ = '1.5.3' ++__version__ = '1.5.3-dev' __author__ = 'Damian Johnson' __contact__ = 'atagar(a)torproject.org' __url__ = 'https://stem.torproject.org/'

1 0

[stem/master] Empty hidden services are invalid
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit c0aae485b23698d087c1df86b6c40dbfc76bdb02 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Dec 6 09:50:33 2016 -0800 Empty hidden services are invalid Tor added some more validation for the hidden service configuration... https://trac.torproject.org/projects/tor/ticket/20860 This broke our integ tests... ====================================================================== ERROR: test_hidden_services_conf ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/atagar/Desktop/stem/test/runner.py", line 127, in wrapped return func(self, *args, **kwargs) File "/home/atagar/Desktop/stem/test/integ/control/controller.py", line 549, in test_hidden_services_conf controller.set_hidden_service_conf(initialconf) File "/home/atagar/Desktop/stem/stem/control.py", line 2582, in set_hidden_service_conf self.set_options(hidden_service_options) File "/home/atagar/Desktop/stem/stem/control.py", line 2419, in set_options raise stem.InvalidRequest(response.code, response.message) InvalidRequest: Unacceptable option value: Failed to configure rendezvous options. See logs for details. ---------------------------------------------------------------------- Reproing this with the tor-prompt the log says... WARN Hidden service ("/tmp/hs_dir") with no ports configured. That's fine. This isn't what we were testing anyway. --- test/integ/control/controller.py | 14 +++++--------- test/unit/installation.py | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/test/integ/control/controller.py b/test/integ/control/controller.py index 149dc03..c249c08 100644 --- a/test/integ/control/controller.py +++ b/test/integ/control/controller.py @@ -510,7 +510,6 @@ class TestController(unittest.TestCase): service2_path = os.path.join(test_dir, 'test_hidden_service2') service3_path = os.path.join(test_dir, 'test_hidden_service3') service4_path = os.path.join(test_dir, 'test_hidden_service4') - empty_service_path = os.path.join(test_dir, 'test_hidden_service_empty') with runner.get_tor_controller() as controller: try: @@ -541,9 +540,6 @@ class TestController(unittest.TestCase): (8032, '127.0.0.1', 8032), ] }, - empty_service_path: { - 'HiddenServicePort': [] - } } controller.set_hidden_service_conf(initialconf) @@ -565,18 +561,18 @@ class TestController(unittest.TestCase): self.assertTrue(hs_address1.endswith('.onion')) conf = controller.get_hidden_service_conf() - self.assertEqual(4, len(conf)) + self.assertEqual(3, len(conf)) self.assertEqual(2, len(conf[hs_path]['HiddenServicePort'])) # remove a hidden service, the service dir should still be there controller.remove_hidden_service(hs_path, 8888) - self.assertEqual(4, len(controller.get_hidden_service_conf())) + self.assertEqual(3, len(controller.get_hidden_service_conf())) # remove a service completely, it should now be gone controller.remove_hidden_service(hs_path, 8989) - self.assertEqual(3, len(controller.get_hidden_service_conf())) + self.assertEqual(2, len(controller.get_hidden_service_conf())) # add a new service, this time with client authentication @@ -589,13 +585,13 @@ class TestController(unittest.TestCase): self.assertTrue(hs_attributes.hostname_for_client['c2'].endswith('.onion')) conf = controller.get_hidden_service_conf() - self.assertEqual(4, len(conf)) + self.assertEqual(3, len(conf)) self.assertEqual(1, len(conf[hs_path]['HiddenServicePort'])) # remove a hidden service controller.remove_hidden_service(hs_path, 8888) - self.assertEqual(3, len(controller.get_hidden_service_conf())) + self.assertEqual(2, len(controller.get_hidden_service_conf())) finally: controller.set_hidden_service_conf({}) # drop hidden services created during the test diff --git a/test/unit/installation.py b/test/unit/installation.py index 719abfe..53d9c41 100644 --- a/test/unit/installation.py +++ b/test/unit/installation.py @@ -77,7 +77,7 @@ class TestInstallation(unittest.TestCase): path = os.path.join(directory, filename) file_type = path.split('.')[-1] - if not file_type in ['.py', '.pyc', '.swp', '.swo', 'orig']: + if file_type not in ['.py', '.pyc', '.swp', '.swo', 'orig']: continue elif path not in data_files: self.fail("setup.py doesn't install %s" % path)

1 0

[tor-browser-bundle/hardened-builds] Bug 20660: use mar-tools from the release's directory
by gk＠torproject.org 06 Dec '16

06 Dec '16

commit 1a8db0805c7532a3be2b8d524fe50956f7709d64 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Tue Dec 6 16:00:10 2016 +0100 Bug 20660: use mar-tools from the release's directory --- gitian/check-prerequisites.sh | 2 +- tools/update-responses/README.md | 12 ++++------ tools/update-responses/update_responses | 41 +++++++++++++++++++++------------ 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/gitian/check-prerequisites.sh b/gitian/check-prerequisites.sh index 6b88a90..858fd4d 100755 --- a/gitian/check-prerequisites.sh +++ b/gitian/check-prerequisites.sh @@ -68,7 +68,7 @@ then exit 1 fi -update_responses_pkg="libyaml-perl libfile-slurp-perl libxml-writer-perl libio-captureoutput-perl libfile-which-perl libparallel-forkmanager-perl libxml-libxml-perl libwww-perl libjson-perl" +update_responses_pkg="libyaml-perl libfile-slurp-perl libxml-writer-perl libio-captureoutput-perl libparallel-forkmanager-perl libxml-libxml-perl libwww-perl libjson-perl" missing_pkg='' for pkg in $update_responses_pkg do diff --git a/tools/update-responses/README.md b/tools/update-responses/README.md index 5440b0d..5209ed5 100644 --- a/tools/update-responses/README.md +++ b/tools/update-responses/README.md @@ -13,25 +13,23 @@ Dependencies The following perl modules need to be installed to run the script: FindBin YAML File::Slurp Digest::SHA XML::Writer File::Temp - IO::CaptureOutput File::Which Parallel::ForkManager XML::LibXML - LWP JSON + IO::CaptureOutput Parallel::ForkManager XML::LibXML LWP JSON On Debian / Ubuntu you can install them with: ``` # apt-get install libfindbin-libs-perl libyaml-perl libfile-slurp-perl \ libdigest-sha-perl libxml-writer-perl \ - libio-captureoutput-perl libfile-which-perl \ - libparallel-forkmanager-perl libxml-libxml-perl \ - libwww-perl libjson-perl + libio-captureoutput-perl libparallel-forkmanager-perl \ + libxml-libxml-perl libwww-perl libjson-perl ``` On Red Hat / Fedora you can install them with: ``` # for module in FindBin YAML File::Slurp Digest::SHA XML::Writer \ - File::Temp IO::CaptureOutput File::Which \ - Parallel::ForkManager XML::LibXML LWP JSON + File::Temp IO::CaptureOutput Parallel::ForkManager \ + XML::LibXML LWP JSON do yum install "perl($module)"; done ``` diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index 1064a01..c776c7f 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -12,7 +12,6 @@ use Cwd; use File::Copy; use File::Temp; use File::Find; -use File::Which; use POSIX qw(setlocale LC_ALL); use IO::CaptureOutput qw(capture_exec); use Parallel::ForkManager; @@ -33,6 +32,8 @@ my %htdocsfiles; my $releases_dir = $config->{releases_dir}; $releases_dir = "$FindBin::Bin/$releases_dir" unless $releases_dir =~ m/^\//; my @check_errors; +my $initPATH = $ENV{PATH}; +my $initLD_LIBRARY_PATH = $ENV{LD_LIBRARY_PATH}; sub exit_error { print STDERR "Error: ", $_[0], "\n"; @@ -243,6 +244,7 @@ sub channel_to_version { sub get_buildinfos { my ($config, $version) = @_; return if exists $config->{versions}{$version}{buildID}; + extract_martools($version); my $files = $config->{versions}{$version}{files}; foreach my $os (keys %$files) { foreach my $lang (keys %{$files->{$os}}) { @@ -378,12 +380,6 @@ sub write_downloads_json { } } -sub check_deps { - foreach my $bin (qw(bunzip2 mar mbsdiff make_incremental_update.sh)) { - exit_error "Cannot find $bin in PATH" unless which($bin); - } -} - sub osname { my ($osname) = capture_exec('uname', '-s'); my ($arch) = capture_exec('uname', '-m'); @@ -399,17 +395,18 @@ sub osname { my $martools_tmpdir; sub extract_martools { + my ($version) = @_; my $osname = osname; - my $marzip = "$FindBin::Bin/../../../gitian-builder/inputs/mar-tools-$osname.zip"; + my $marzip = "$releases_dir/$version/mar-tools-$osname.zip"; $martools_tmpdir = File::Temp->newdir(); my $old_cwd = getcwd; chdir $martools_tmpdir; my (undef, undef, $success) = capture_exec('unzip', $marzip); chdir $old_cwd; exit_error "Error extracting $marzip" unless $success; - $ENV{PATH} .= ":$martools_tmpdir/mar-tools"; - if ($ENV{LD_LIBRARY_PATH}) { - $ENV{LD_LIBRARY_PATH} .= ":$martools_tmpdir/mar-tools"; + $ENV{PATH} = "$martools_tmpdir/mar-tools:$initPATH"; + if ($initLD_LIBRARY_PATH) { + $ENV{LD_LIBRARY_PATH} = "$initLD_LIBRARY_PATH:$martools_tmpdir/mar-tools"; } else { $ENV{LD_LIBRARY_PATH} = "$martools_tmpdir/mar-tools"; } @@ -519,6 +516,23 @@ sub download_version { move "$tmpdir/sha256sums-unsigned-build.txt", "$destdir/sha256sums-unsigned-build.txt"; my %sums = map { chomp; reverse split ' ', $_ } read_file "$destdir/sha256sums-unsigned-build.txt"; + + my $martools = 'mar-tools-' . osname . '.zip'; + exit_error "Error downloading $urldir/$martools\n" + unless getstore("$urldir/$martools", "$tmpdir/$martools") == 200; + exit_error "Error downloading $urldir/$martools.asc\n" + unless getstore("$urldir/$martools.asc", "$tmpdir/$martools.asc") == 200; + if (system('gpg', '--no-default-keyring', '--keyring', + "$FindBin::Bin/$config->{download}{gpg_keyring}", '--verify', + "$tmpdir/$martools.asc", "$tmpdir/$martools")) { + exit_error "Error checking gpg signature for $version/$martools"; + } + exit_error "Wrong checksum for $version/$martools" + unless $sums{$martools} eq sha256_hex(read_file("$tmpdir/$martools")); + move "$tmpdir/$martools", "$destdir/$martools"; + move "$tmpdir/$martools.asc", "$destdir/$martools.asc"; + extract_martools($version); + foreach my $file (sort grep { $_ =~ m/\.mar$/ } keys %sums) { print "Downloading $file\n"; exit_error "Error downloading $urldir/$file\n" @@ -580,7 +594,6 @@ my %actions = ( unless $config->{channels}{$channel}; $htdocsfiles{$channel} = { '.' => 1, '..' => 1 }; } - extract_martools; write_responses($config, @channels); write_htaccess($config, @channels); write_downloads_json($config, @channels); @@ -588,9 +601,8 @@ my %actions = ( }, gen_incrementals => sub { my ($config) = @_; - extract_martools; - check_deps; foreach my $version (channel_to_version($config, @ARGV)) { + extract_martools($version); get_version_files($config, $version); create_incremental_mars_for_version($config, $version); } @@ -598,7 +610,6 @@ my %actions = ( download_missing_versions => sub { my ($config) = @_; my @channels = @ARGV ? @ARGV : keys %{$config->{channels}}; - extract_martools; download_missing_versions($config, @channels); }, check_update_responses_deployement => \&check_update_responses,

1 0