December 2016 - tor-commits - lists.torproject.org

[stem/master] Include docs and tests in source distribution
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit 3b79cf94c21aeeb65cb07d22c160fffb54dada22 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Nov 25 10:48:03 2016 -0800 Include docs and tests in source distribution Juan pointed out the source distribution we upload does have docs, tests, or our LICENSE file. This is because we moved from uploading a git archive to 'python setup.py sdist', which is the more proper thing to upload nowadays. --- MANIFEST.in | 11 +++++++++++ setup.py | 8 ++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..086a5db --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,11 @@ +include cache_fallback_directories.py +include cache_manual.py +include LICENSE +include MANIFEST.in +include requirements.txt +include run_tests.py +include tox.ini +graft docs +graft test +global-exclude __pycache__ +global-exclude *.py[co] diff --git a/setup.py b/setup.py index 7d7e5e9..65c790e 100644 --- a/setup.py +++ b/setup.py @@ -42,9 +42,13 @@ distutils.core.setup( packages = ['stem', 'stem.descriptor', 'stem.interpreter', 'stem.response', 'stem.util'], keywords = 'tor onion controller', scripts = ['tor-prompt'], - package_data = {'stem': ['cached_tor_manual.cfg', 'settings.cfg'], 'stem.descriptor': ['fallback_directories.cfg'], 'stem.interpreter': ['settings.cfg'], 'stem.util': ['ports.cfg']}, provides = ['stem'], - classifiers = [ + package_data = { + 'stem': ['cached_tor_manual.cfg', 'settings.cfg'], + 'stem.descriptor': ['fallback_directories.cfg'], + 'stem.interpreter': ['settings.cfg'], + 'stem.util': ['ports.cfg'], + }, classifiers = [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)',

1 0

[stem/master] Merge MANIFEST.in into setup.py
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit 4581199ffd9bd65bb25d89b42663e2192607d7f2 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Dec 5 14:10:52 2016 -0800 Merge MANIFEST.in into setup.py The manifest solely exists to make our source distribution contain the files it should. Silly we can't simply specify them in our setup() call, but oh well. Lets include this in our setup.py to avoid clutter and keep this where it should be. --- MANIFEST.in | 17 ------------- setup.py | 85 ++++++++++++++++++++++++++++++++++++++++--------------------- 2 files changed, 56 insertions(+), 46 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index d35f06a..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,17 +0,0 @@ -include cache_fallback_directories.py -include cache_manual.py -include LICENSE -include MANIFEST.in -include requirements.txt -include run_tests.py -include tox.ini -graft docs -graft test -global-exclude __pycache__ -global-exclude *.orig -global-exclude *.pyc -global-exclude *.swp -global-exclude *.swo -global-exclude .tox -recursive-exclude test/data * -recursive-exclude docs/_build * diff --git a/setup.py b/setup.py index 5770898..ac780d6 100644 --- a/setup.py +++ b/setup.py @@ -31,37 +31,64 @@ To install you can either use... After that, give some `tutorials <https://stem.torproject.org/tutorials.html>`_ a try! For questions or to discuss project ideas we're available on `irc <https://www.torproject.org/about/contact.html.en#irc>`_ and the `tor-dev@ email list <https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-dev>`_. """.strip() +MANIFEST = """ +include cache_fallback_directories.py +include cache_manual.py +include LICENSE +include MANIFEST.in +include requirements.txt +include run_tests.py +include tox.ini +graft docs +graft test +global-exclude __pycache__ +global-exclude *.orig +global-exclude *.pyc +global-exclude *.swp +global-exclude *.swo +global-exclude .tox +recursive-exclude test/data * +recursive-exclude docs/_build * +""".strip() + # Ensure this is our cwd, otherwise distutils fails with 'standard file # 'setup.py' not found'. os.chdir(os.path.dirname(os.path.abspath(__file__))) -distutils.core.setup( - name = 'stem-dry-run' if DRY_RUN else 'stem', - version = stem.__version__, - description = DRY_RUN_SUMMARY if DRY_RUN else SUMMARY, - long_description = DESCRIPTION, - license = stem.__license__, - author = stem.__author__, - author_email = stem.__contact__, - url = stem.__url__, - packages = ['stem', 'stem.descriptor', 'stem.interpreter', 'stem.response', 'stem.util'], - keywords = 'tor onion controller', - scripts = ['tor-prompt'], - provides = ['stem'], - package_data = { - 'stem': ['cached_tor_manual.cfg', 'settings.cfg'], - 'stem.descriptor': ['fallback_directories.cfg'], - 'stem.interpreter': ['settings.cfg'], - 'stem.util': ['ports.cfg'], - }, classifiers = [ - 'Development Status :: 5 - Production/Stable', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)', - 'Topic :: Security', - 'Topic :: Software Development :: Libraries :: Python Modules', - ], -) - -if os.path.exists('MANIFEST'): - os.remove('MANIFEST') +with open('MANIFEST.in', 'w') as manifest_file: + manifest_file.write(MANIFEST) + +try: + distutils.core.setup( + name = 'stem-dry-run' if DRY_RUN else 'stem', + version = stem.__version__, + description = DRY_RUN_SUMMARY if DRY_RUN else SUMMARY, + long_description = DESCRIPTION, + license = stem.__license__, + author = stem.__author__, + author_email = stem.__contact__, + url = stem.__url__, + packages = ['stem', 'stem.descriptor', 'stem.interpreter', 'stem.response', 'stem.util'], + keywords = 'tor onion controller', + scripts = ['tor-prompt'], + provides = ['stem'], + package_data = { + 'stem': ['cached_tor_manual.cfg', 'settings.cfg'], + 'stem.descriptor': ['fallback_directories.cfg'], + 'stem.interpreter': ['settings.cfg'], + 'stem.util': ['ports.cfg'], + }, classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)', + 'Topic :: Security', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + ) +finally: + if os.path.exists('MANIFEST.in'): + os.remove('MANIFEST.in') + + if os.path.exists('MANIFEST'): + os.remove('MANIFEST')

1 0

[stem/master] Exclude gitignore resources
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit cb0bdac2b82f7b5cb41442a25a37e4ce3fb69353 Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Nov 25 15:43:36 2016 -0800 Exclude gitignore resources Oops! Dererk noticed our docs/_build/* is present in the tarballs we upload. Turns out test/data/* is too. Excluding things specified by our gitignore. We'll need to be careful to double check that there's no untracked files before uploads... --- MANIFEST.in | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 086a5db..d35f06a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,4 +8,10 @@ include tox.ini graft docs graft test global-exclude __pycache__ -global-exclude *.py[co] +global-exclude *.orig +global-exclude *.pyc +global-exclude *.swp +global-exclude *.swo +global-exclude .tox +recursive-exclude test/data * +recursive-exclude docs/_build *

1 0

[stem/master] Clean up auto-generated MANIFEST
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit 484b00544f6a97cfadf67068c9af1afc1280c6d8 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Dec 5 14:07:10 2016 -0800 Clean up auto-generated MANIFEST Now that we have a MANIFEST.in our setup.py generates a MANIFEST. Hate leaving turds around so cleaning it up. --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 3d79742..5770898 100644 --- a/setup.py +++ b/setup.py @@ -62,3 +62,6 @@ distutils.core.setup( 'Topic :: Software Development :: Libraries :: Python Modules', ], ) + +if os.path.exists('MANIFEST'): + os.remove('MANIFEST')

1 0

[stem/master] Allow setup.py to work with different cwd
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit 7b2af7d6810234d3847873df823edee9bae86403 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Dec 5 14:04:52 2016 -0800 Allow setup.py to work with different cwd Running our setup.py when this wasn't your cwd failed. Quick fix to account for this. --- setup.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 65c790e..3d79742 100644 --- a/setup.py +++ b/setup.py @@ -3,11 +3,12 @@ # See LICENSE for licensing information import distutils.core +import os import stem DRY_RUN = True SUMMARY = 'Stem is a Python controller library that allows applications to interact with Tor (https://www.torproject.org/)' -DRY_RUN_SUMMARY = "Ignore this package. This is dry-run release creation to work around PyPI limitations (https://github.com/pypa/packaging-problems/issues/74#issuecomment-260716129)" +DRY_RUN_SUMMARY = 'Ignore this package. This is dry-run release creation to work around PyPI limitations (https://github.com/pypa/packaging-problems/issues/74#issuecomment-260716129)' DESCRIPTION = """ For tutorials and API documentation see `stem's homepage <https://stem.torproject.org/>`_. @@ -20,7 +21,7 @@ To install you can either use... :: pip install stem - + ... or install from the source tarball. Stem supports both the python 2.x and 3.x series. To use its python3 counterpart you simply need to install using that version of python. :: @@ -30,6 +31,11 @@ To install you can either use... After that, give some `tutorials <https://stem.torproject.org/tutorials.html>`_ a try! For questions or to discuss project ideas we're available on `irc <https://www.torproject.org/about/contact.html.en#irc>`_ and the `tor-dev@ email list <https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-dev>`_. """.strip() +# Ensure this is our cwd, otherwise distutils fails with 'standard file +# 'setup.py' not found'. + +os.chdir(os.path.dirname(os.path.abspath(__file__))) + distutils.core.setup( name = 'stem-dry-run' if DRY_RUN else 'stem', version = stem.__version__,

1 0

[stem/master] Stem release 1.5.3
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit f2522efef52cfda0d3f289e082cbd4effb17fedd Author: Damian Johnson <atagar(a)torproject.org> Date: Fri Nov 25 10:49:34 2016 -0800 Stem release 1.5.3 Juan pointed out that packagers use resources that aren't available by default as part of our source distribution. Including everything in the tarball we upload. --- docs/change_log.rst | 3 +++ docs/index.rst | 2 +- stem/__init__.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/change_log.rst b/docs/change_log.rst index ba496a7..a3b3d9a 100644 --- a/docs/change_log.rst +++ b/docs/change_log.rst @@ -111,6 +111,9 @@ The following are only available within Stem's `git repository * Example for `custom path selection for circuits <tutorials/to_russia_with_love.html#custom-path-selection>`_ (:trac:`8728`) * Example for `persisting ephemeral hidden service keys <tutorials/over_the_river.html#ephemeral-hidden-services>`_ + * **Version 1.5.3** (December 5th, 2016) - including tests and site in the + release tarball + .. _version_1.4: Version 1.4 (May 13th, 2015) diff --git a/docs/index.rst b/docs/index.rst index da9362b..f265f4e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ Welcome to Stem! ================ -Stem is a Python controller library for `Tor <https://www.torproject.org/>`_. With it you can use Tor's `control protocol <https://gitweb.torproject.org/torspec.git/tree/control-spec.txt>`_ to script against the Tor process, or build things such as `arm <https://www.atagar.com/arm/>`_. Stem's latest version is **1.5.2** (released November 20th, 2016). +Stem is a Python controller library for `Tor <https://www.torproject.org/>`_. With it you can use Tor's `control protocol <https://gitweb.torproject.org/torspec.git/tree/control-spec.txt>`_ to script against the Tor process, or build things such as `arm <https://www.atagar.com/arm/>`_. Stem's latest version is **1.5.3** (released December 5th, 2016). .. Main Stem Logo Source: http://www.wpclipart.com/plants/assorted/P/plant_stem.png.html diff --git a/stem/__init__.py b/stem/__init__.py index 7cf39f2..66502c8 100644 --- a/stem/__init__.py +++ b/stem/__init__.py @@ -470,7 +470,7 @@ Library for working with the tor process. import stem.util.enum -__version__ = '1.5.2' +__version__ = '1.5.3' __author__ = 'Damian Johnson' __contact__ = 'atagar(a)torproject.org' __url__ = 'https://stem.torproject.org/'

1 0

[stem/master] Fix unit test for data file installation
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit d2a0be85d83dd6167865aa52fe12baae75562f47 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Dec 5 19:12:51 2016 -0800 Fix unit test for data file installation Reformatting broke test_installs_all_data_files. It was a bit better so I might re-introduce the change later - just don't want to hassle with rewriting the test right now. --- setup.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index ac780d6..6b4922d 100644 --- a/setup.py +++ b/setup.py @@ -73,12 +73,8 @@ try: keywords = 'tor onion controller', scripts = ['tor-prompt'], provides = ['stem'], - package_data = { - 'stem': ['cached_tor_manual.cfg', 'settings.cfg'], - 'stem.descriptor': ['fallback_directories.cfg'], - 'stem.interpreter': ['settings.cfg'], - 'stem.util': ['ports.cfg'], - }, classifiers = [ + package_data = {'stem': ['cached_tor_manual.cfg', 'settings.cfg'], 'stem.descriptor': ['fallback_directories.cfg'], 'stem.interpreter': ['settings.cfg'], 'stem.util': ['ports.cfg']}, + classifiers = [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)',

1 0

[stem/master] Test our source distribution matches the git contents
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit 34be2ece71819ae005914317d9a492b755037d26 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon Dec 5 15:41:24 2016 -0800 Test our source distribution matches the git contents Having a hardcoded Manifest.in is error prone. Unfortunately we need to do it, so to avoid shooting ourselves in the foot we need a test. This checks that source distributions we generate match the contents of the git repository. --- test/integ/installation.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/test/integ/installation.py b/test/integ/installation.py index 189c26e..9e11c4d 100644 --- a/test/integ/installation.py +++ b/test/integ/installation.py @@ -2,6 +2,7 @@ import glob import os import shutil import sys +import tarfile import unittest import stem @@ -10,6 +11,8 @@ import stem.util.system import test.runner import test.util +INSTALL_MISMATCH_MSG = "Running 'python setup.py sdist' doesn't match our git contents in the following way. The manifest in our setup.py may need to be updated...\n\n" + class TestInstallation(unittest.TestCase): # TODO: remove when dropping support for python 2.6 @@ -56,6 +59,51 @@ class TestInstallation(unittest.TestCase): return False @test.runner.only_run_once + def test_sdist_matches_git(self): + """ + Check the source distribution tarball we make for releases matches the + contents of 'git archive'. This primarily is meant to test that our + MANIFEST.in is up to date. + """ + + if self.requires_installation(): + return + elif not stem.util.system.is_available('git'): + test.runner.skip(self, '(git unavailable)') + return + + original_cwd = os.getcwd() + dist_path = os.path.join(test.util.STEM_BASE, 'dist') + git_contents = [line.split()[-1] for line in stem.util.system.call('git ls-tree --full-tree -r HEAD')] + + try: + os.chdir(test.util.STEM_BASE) + stem.util.system.call(sys.executable + ' setup.py sdist') + + # tarball has a prefix 'stem-[verion]' directory so stipping that out + + dist_tar = tarfile.open(os.path.join(dist_path, 'stem-dry-run-%s.tar.gz' % stem.__version__)) + tar_contents = ['/'.join(info.name.split('/')[1:]) for info in dist_tar.getmembers() if info.isfile()] + finally: + if os.path.exists(dist_path): + shutil.rmtree(dist_path) + + os.chdir(original_cwd) + + issues = [] + + for path in git_contents: + if path not in tar_contents and path not in ['.gitignore']: + issues.append(' * %s is missing from our release tarball' % path) + + for path in tar_contents: + if path not in git_contents and path not in ['MANIFEST.in', 'PKG-INFO']: + issues.append(" * %s isn't expected in our release tarball" % path) + + if issues: + self.fail(INSTALL_MISMATCH_MSG + '\n'.join(issues)) + + @test.runner.only_run_once def test_installing_stem(self): """ Attempt to use the package we install.

1 0

[stem/master] Merge version 1.5.3 hotfix
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit 84d8a5035b4986f40e5f3fed841b4e7cd734d1e0 Merge: c857786 f2522ef Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Dec 6 09:34:21 2016 -0800 Merge version 1.5.3 hotfix docs/change_log.rst | 3 ++ docs/index.rst | 2 +- setup.py | 83 +++++++++++++++++++++++++++++++++------------- stem/__init__.py | 2 +- test/integ/installation.py | 48 +++++++++++++++++++++++++++ 5 files changed, 113 insertions(+), 25 deletions(-) diff --cc stem/__init__.py index b02dd9c,66502c8..1acf0f4 --- a/stem/__init__.py +++ b/stem/__init__.py @@@ -470,7 -470,7 +470,7 @@@ Library for working with the tor proces import stem.util.enum - __version__ = '1.5.2-dev' -__version__ = '1.5.3' ++__version__ = '1.5.3-dev' __author__ = 'Damian Johnson' __contact__ = 'atagar(a)torproject.org' __url__ = 'https://stem.torproject.org/'

1 0

[stem/master] Empty hidden services are invalid
by atagar＠torproject.org 06 Dec '16

06 Dec '16

commit c0aae485b23698d087c1df86b6c40dbfc76bdb02 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Dec 6 09:50:33 2016 -0800 Empty hidden services are invalid Tor added some more validation for the hidden service configuration... https://trac.torproject.org/projects/tor/ticket/20860 This broke our integ tests... ====================================================================== ERROR: test_hidden_services_conf ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/atagar/Desktop/stem/test/runner.py", line 127, in wrapped return func(self, *args, **kwargs) File "/home/atagar/Desktop/stem/test/integ/control/controller.py", line 549, in test_hidden_services_conf controller.set_hidden_service_conf(initialconf) File "/home/atagar/Desktop/stem/stem/control.py", line 2582, in set_hidden_service_conf self.set_options(hidden_service_options) File "/home/atagar/Desktop/stem/stem/control.py", line 2419, in set_options raise stem.InvalidRequest(response.code, response.message) InvalidRequest: Unacceptable option value: Failed to configure rendezvous options. See logs for details. ---------------------------------------------------------------------- Reproing this with the tor-prompt the log says... WARN Hidden service ("/tmp/hs_dir") with no ports configured. That's fine. This isn't what we were testing anyway. --- test/integ/control/controller.py | 14 +++++--------- test/unit/installation.py | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/test/integ/control/controller.py b/test/integ/control/controller.py index 149dc03..c249c08 100644 --- a/test/integ/control/controller.py +++ b/test/integ/control/controller.py @@ -510,7 +510,6 @@ class TestController(unittest.TestCase): service2_path = os.path.join(test_dir, 'test_hidden_service2') service3_path = os.path.join(test_dir, 'test_hidden_service3') service4_path = os.path.join(test_dir, 'test_hidden_service4') - empty_service_path = os.path.join(test_dir, 'test_hidden_service_empty') with runner.get_tor_controller() as controller: try: @@ -541,9 +540,6 @@ class TestController(unittest.TestCase): (8032, '127.0.0.1', 8032), ] }, - empty_service_path: { - 'HiddenServicePort': [] - } } controller.set_hidden_service_conf(initialconf) @@ -565,18 +561,18 @@ class TestController(unittest.TestCase): self.assertTrue(hs_address1.endswith('.onion')) conf = controller.get_hidden_service_conf() - self.assertEqual(4, len(conf)) + self.assertEqual(3, len(conf)) self.assertEqual(2, len(conf[hs_path]['HiddenServicePort'])) # remove a hidden service, the service dir should still be there controller.remove_hidden_service(hs_path, 8888) - self.assertEqual(4, len(controller.get_hidden_service_conf())) + self.assertEqual(3, len(controller.get_hidden_service_conf())) # remove a service completely, it should now be gone controller.remove_hidden_service(hs_path, 8989) - self.assertEqual(3, len(controller.get_hidden_service_conf())) + self.assertEqual(2, len(controller.get_hidden_service_conf())) # add a new service, this time with client authentication @@ -589,13 +585,13 @@ class TestController(unittest.TestCase): self.assertTrue(hs_attributes.hostname_for_client['c2'].endswith('.onion')) conf = controller.get_hidden_service_conf() - self.assertEqual(4, len(conf)) + self.assertEqual(3, len(conf)) self.assertEqual(1, len(conf[hs_path]['HiddenServicePort'])) # remove a hidden service controller.remove_hidden_service(hs_path, 8888) - self.assertEqual(3, len(controller.get_hidden_service_conf())) + self.assertEqual(2, len(controller.get_hidden_service_conf())) finally: controller.set_hidden_service_conf({}) # drop hidden services created during the test diff --git a/test/unit/installation.py b/test/unit/installation.py index 719abfe..53d9c41 100644 --- a/test/unit/installation.py +++ b/test/unit/installation.py @@ -77,7 +77,7 @@ class TestInstallation(unittest.TestCase): path = os.path.join(directory, filename) file_type = path.split('.')[-1] - if not file_type in ['.py', '.pyc', '.swp', '.swo', 'orig']: + if file_type not in ['.py', '.pyc', '.swp', '.swo', 'orig']: continue elif path not in data_files: self.fail("setup.py doesn't install %s" % path)

1 0