May 2019 - tor-commits - lists.torproject.org

[stem/master] Support multiple '--test' arguments
by atagar＠torproject.org 28 May '19

28 May '19

commit 99e5ac8220f6d50018191a4a76b5f30522519f9b Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 27 17:28:53 2019 -0700 Support multiple '--test' arguments When supplied our '--test' argument limits us to only running that specific test. I added this to help with single-test troubleshooting, but on reflection it could be useful to the network team as well to limit their CI to only run the tests they're concerned with. Only gotcha was that we didn't support multiple '--test' arguments. Resolving that. --- run_tests.py | 56 +++++++++++++++++++++++++++++++++---------------------- test/arguments.py | 4 ++-- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/run_tests.py b/run_tests.py index 54fbbde1..088d3f62 100755 --- a/run_tests.py +++ b/run_tests.py @@ -114,49 +114,61 @@ def log_traceback(sig, frame): os._exit(-1) -def get_unit_tests(module_prefix = None): +def get_unit_tests(module_prefixes = None): """ Provides the classes for our unit tests. - :param str module_prefix: only provide the test if the module starts with - this substring + :param list module_prefixes: only provide the test if the module starts with + any of these substrings :returns: an **iterator** for our unit tests """ - if module_prefix and not module_prefix.startswith('test.unit.'): - module_prefix = 'test.unit.' + module_prefix + return _get_tests(CONFIG['test.unit_tests'].splitlines(), module_prefixes) - return _get_tests(CONFIG['test.unit_tests'].splitlines(), module_prefix) - -def get_integ_tests(module_prefix = None): +def get_integ_tests(module_prefixes = None): """ Provides the classes for our integration tests. - :param str module_prefix: only provide the test if the module starts with - this substring + :param list module_prefixes: only provide the test if the module starts with + any of these substrings :returns: an **iterator** for our integration tests """ - if module_prefix and not module_prefix.startswith('test.integ.'): - module_prefix = 'test.integ.' + module_prefix - - return _get_tests(CONFIG['test.integ_tests'].splitlines(), module_prefix) + return _get_tests(CONFIG['test.integ_tests'].splitlines(), module_prefixes) -def _get_tests(modules, module_prefix): +def _get_tests(modules, module_prefixes): for import_name in modules: - module, module_name = import_name.rsplit('.', 1) # example: util.conf.TestConf - - if not module_prefix or module.startswith(module_prefix): + if not module_prefixes: yield import_name - elif module_prefix.startswith(module): - # single test for this module + else: + # Example import_name: test.unit.util.conf.TestConf + # + # Our '--test' argument doesn't include the prefix, so excluding it from + # the names we look for. + + if import_name.startswith('test.unit.'): + cropped_name = import_name[10:] + elif import_name.startswith('test.integ.'): + cropped_name = import_name[11:] + else: + cropped_name = import_name - test_name = module_prefix.rsplit('.', 1)[1] - yield '%s.%s' % (import_name, test_name) + cropped_name = cropped_name.rsplit('.', 1)[0] # exclude the class name + + for prefix in module_prefixes: + if cropped_name.startswith(prefix): + yield import_name + break + elif prefix.startswith(cropped_name): + # single test for this module + + test_name = prefix.rsplit('.', 1)[1] + yield '%s.%s' % (import_name, test_name) + break def main(): diff --git a/test/arguments.py b/test/arguments.py index 87735182..1ddf9240 100644 --- a/test/arguments.py +++ b/test/arguments.py @@ -26,7 +26,7 @@ CONFIG = stem.util.conf.config_dict('test', { DEFAULT_ARGS = { 'run_unit': False, 'run_integ': False, - 'specific_test': None, + 'specific_test': [], 'logging_runlevel': None, 'tor_path': 'tor', 'run_targets': [test.Target.RUN_OPEN], @@ -103,7 +103,7 @@ def parse(argv): args['attribute_targets'] = attribute_targets elif opt == '--test': - args['specific_test'] = arg + args['specific_test'].append(arg) elif opt in ('-l', '--log'): arg = arg.upper()

1 0

[stem/master] Drop python 2.6 integ test serialization
by atagar＠torproject.org 28 May '19

28 May '19

commit c2dabe1746164db61efab357cd6d5ce5d61e99f7 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 27 16:57:30 2019 -0700 Drop python 2.6 integ test serialization When running our integ tests using python 2.6 we dropped test parallelization. Turns out this no longer works, failing all asynchronous tests with... ====================================================================== FAIL: test_cached_descriptor ---------------------------------------------------------------------- Traceback (most recent call last): File "/home/atagar/Desktop/stem/stem/util/test_tools.py", line 152, in <lambda> self.method = lambda test: self.result(test) # method that can be mixed into TestCases File "/home/atagar/Desktop/stem/stem/util/test_tools.py", line 227, in result test.fail(self._result.msg) AssertionError: Traceback (most recent call last): File "/home/atagar/Desktop/stem/stem/util/test_tools.py", line 169, in _wrapper runner(*args) if args else runner() TypeError: test_cached_descriptor() takes exactly 1 argument (0 given) Python 2.x is getting deprecated at the end of this year so rather than puzzle over this lets simply drop the pretense at a python 2.6 hack. If we really need python 2.6 testing I'll dig into what's up. --- run_tests.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/run_tests.py b/run_tests.py index a2f58f83..54fbbde1 100755 --- a/run_tests.py +++ b/run_tests.py @@ -7,6 +7,7 @@ Runs unit and integration tests. For usage information run this with '--help'. """ import errno +import importlib import multiprocessing import os import signal @@ -17,14 +18,6 @@ import traceback import unittest try: - # TODO: added in python 2.7, drop check when removing 2.6 support - - import importlib - RUN_ASYNC_TESTS = True -except ImportError: - RUN_ASYNC_TESTS = False - -try: from StringIO import StringIO except ImportError: from io import StringIO @@ -256,15 +249,14 @@ def main(): async_args = test.AsyncTestArgs(default_test_dir, args.tor_path) for module_str in stem.util.test_tools.ASYNC_TESTS: - if RUN_ASYNC_TESTS and (not args.specific_test or module_str.startswith(args.specific_test)): - module = importlib.import_module(module_str.rsplit('.', 1)[0]) - test_classes = [v for k, v in module.__dict__.items() if k.startswith('Test')] + module = importlib.import_module(module_str.rsplit('.', 1)[0]) + test_classes = [v for k, v in module.__dict__.items() if k.startswith('Test')] - if len(test_classes) != 1: - print('BUG: Detected multiple tests for %s: %s' % (module_str, ', '.join(test_classes))) - sys.exit(1) + if len(test_classes) != 1: + print('BUG: Detected multiple tests for %s: %s' % (module_str, ', '.join(test_classes))) + sys.exit(1) - test_classes[0].run_tests(async_args) + test_classes[0].run_tests(async_args) if args.run_unit: test.output.print_divider('UNIT TESTS', True)

1 0

[stem/master] Drop website republication script
by atagar＠torproject.org 27 May '19

27 May '19

commit 8f1f69b752ee1e198d0790cc985434cf89921967 Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 27 16:25:20 2019 -0700 Drop website republication script While authoring stem's website I used this script to automatically republish changes. Years ago I wised up and swapped to cron... stem@staticiforme:~$ crontab -l */5 * * * * /home/stem/build_site stem@staticiforme:~$ cat /home/stem/build_site #!/bin/sh export PATH=/home/stem/bin:$PATH export PYTHONPATH=/home/stem/lib/python cd /home/stem/stem git pull cd docs make clean make html sudo -u mirroradm static-master-update-component stem.torproject.org echo "$(date)" > /home/stem/site_last_built This republication script was specifically for our own site and hasn't been used in years. Dropping the script in response to... https://trac.torproject.org/projects/tor/ticket/30593 --- docs/republish | 7 ----- docs/republish.py | 89 ------------------------------------------------------- 2 files changed, 96 deletions(-) diff --git a/docs/republish b/docs/republish deleted file mode 100755 index 052c03a3..00000000 --- a/docs/republish +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -export PATH=/home/stem/bin:$PATH -export PYTHONPATH=/home/stem/lib/python - -python /home/stem/stem/docs/republish.py "$@" & - diff --git a/docs/republish.py b/docs/republish.py deleted file mode 100644 index f2241e52..00000000 --- a/docs/republish.py +++ /dev/null @@ -1,89 +0,0 @@ -import getopt -import logging -import subprocess -import sys -import time - -LOGGER = logging.getLogger('republish') -LOGGER.setLevel(logging.INFO) - -handler = logging.FileHandler('/home/stem/republish.log') -handler.setFormatter(logging.Formatter( - fmt = '%(asctime)s [%(levelname)s] %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', -)) -LOGGER.addHandler(handler) - -OPT = 'r:h' -OPT_EXPANDED = ['repeat=', 'help'] - -HELP_MSG = """\ -Republishes stem's website. This can either be done or on a reoccurring basis. -If stem's repository is unchanged then this is a no-op. - - -r, --repeat RATE tries to republish the site at a set rate, in minutes -""" - - -def run(command, cwd = None): - # Runs the given command. This returns the stdout if successful, and raises - # an OSError if it fails. - - cmd = subprocess.Popen(command.split(' '), stdout = subprocess.PIPE, stderr = subprocess.PIPE, cwd = cwd) - - if cmd.wait() == 0: - return cmd.communicate()[0] - else: - stdout, stderr = cmd.communicate() - raise OSError("'%s' failed\n stdout: %s\n stderr: %s" % (command, stdout.strip(), stderr.strip())) - - -def republish_site(): - # Checks if stem's repository has changed, rebuilding the site if so. Ideally - # we'd use plumbing commands to check this but... meh. Patches welcome. - - if 'Already up-to-date.' not in run('git pull', cwd = '/home/stem/stem'): - start_time = time.time() - LOGGER.log(logging.INFO, "Stem's repository has changed. Republishing...") - run('make html', cwd = '/home/stem/stem/docs') - run('sudo -u mirroradm static-master-update-component stem.torproject.org') - - runtime = int(time.time() - start_time) - LOGGER.log(logging.INFO, ' site republished (took %s seconds)' % runtime) - - -if __name__ == '__main__': - try: - opts = getopt.getopt(sys.argv[1:], OPT, OPT_EXPANDED)[0] - except getopt.GetoptError as exc: - print('%s (for usage provide --help)' % exc) - sys.exit(1) - - repeat_rate = None - - for opt, arg in opts: - if opt in ('-r', '--repeat'): - if arg.isdigit(): - repeat_rate = int(arg) - else: - print("The --repeat argument must be an integer, got '%s'" % arg) - sys.exit(1) - elif opt in ('-h', '--help'): - print(HELP_MSG) - sys.exit() - - if repeat_rate: - LOGGER.log(logging.INFO, 'Starting stem site republisher') - latest_run = 0 # unix timestamp for when we last ran - - while True: - while time.time() < (latest_run + repeat_rate * 60): - time.sleep(15) - - try: - latest_run = time.time() - republish_site() - except OSError as exc: - LOGGER.log(logging.WARN, str(exc)) - else: - republish_site()

1 0

[stem/master] Empty tor data directory between integ tests
by atagar＠torproject.org 27 May '19

27 May '19

commit 8b07bca61ef2d6f88bf00662a5e493c40f8be9fc Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 27 15:41:47 2019 -0700 Empty tor data directory between integ tests When authoring our integ tests years ago I had a decision to make: retain tor's data directory between runs or start fresh. From the standpoint of testing best practices this should be a no brainer: start fresh. However, I decided against this because at the time we ran our 'ONLINE' target by default, and losing our cache added a frustrating amount of runtime. We no longer run tests that require network activity by default, and keeping our data directory around adds up over time... % du -h test/data/tor_log 5.3M test/data/tor_log % ls test/data/torrc.orig.* | wc -l 98 I'm about to change our logging runlevel which will raise our log size by an order of magnitude. This is negligible, but if we don't first change our cleaning behavior it'll add up. --- run_tests.py | 1 + test/task.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/run_tests.py b/run_tests.py index 89c0b3de..a2f58f83 100755 --- a/run_tests.py +++ b/run_tests.py @@ -223,6 +223,7 @@ def main(): test.task.CLEAN_PYC, test.task.UNUSED_TESTS, test.task.IMPORT_TESTS, + test.task.REMOVE_TOR_DATA_DIR if args.run_integ else None, test.task.PYFLAKES_TASK if not args.specific_test else None, test.task.PYCODESTYLE_TASK if not args.specific_test else None, ) diff --git a/test/task.py b/test/task.py index a7dc15a6..be518ca7 100644 --- a/test/task.py +++ b/test/task.py @@ -18,6 +18,7 @@ |- PYFLAKES_VERSION - checks our version of pyflakes |- PYCODESTYLE_VERSION - checks our version of pycodestyle |- CLEAN_PYC - removes any *.pyc without a corresponding *.py + |- REMOVE_TOR_DATA_DIR - removes our tor data directory |- IMPORT_TESTS - ensure all test modules have been imported |- UNUSED_TESTS - checks to see if any tests are missing from our settings |- PYFLAKES_TASK - static checks @@ -27,6 +28,7 @@ import os import platform import re +import shutil import sys import time import traceback @@ -136,6 +138,20 @@ def _clean_orphaned_pyc(paths): return ['removed %s' % path for path in stem.util.test_tools.clean_orphaned_pyc(paths)] +def _remove_tor_data_dir(): + """ + Empties tor's data directory. + """ + + config_test_dir = CONFIG['integ.test_directory'] + + if config_test_dir and os.path.exists(config_test_dir): + shutil.rmtree(config_test_dir, ignore_errors = True) + return 'done' + else: + return 'skipped' + + def _import_tests(): """ Ensure all tests have been imported. This is important so tests can @@ -316,6 +332,7 @@ MOCK_VERSION = ModuleVersion('mock version', ['unittest.mock', 'mock'], stem.pre PYFLAKES_VERSION = ModuleVersion('pyflakes version', 'pyflakes') PYCODESTYLE_VERSION = ModuleVersion('pycodestyle version', ['pycodestyle', 'pep8']) CLEAN_PYC = Task('checking for orphaned .pyc files', _clean_orphaned_pyc, (SRC_PATHS,), print_runtime = True) +REMOVE_TOR_DATA_DIR = Task('emptying our tor data directory', _remove_tor_data_dir) IMPORT_TESTS = Task('importing test modules', _import_tests, print_runtime = True) UNUSED_TESTS = Task('checking for unused tests', _check_for_unused_tests, [(

1 0

[stem/master] Log at debug runlevel when integ testing
by atagar＠torproject.org 27 May '19

27 May '19

commit 3670fe061278c934ff57c3ba3c9ba457f276ad9a Author: Damian Johnson <atagar(a)torproject.org> Date: Mon May 27 15:52:51 2019 -0700 Log at debug runlevel when integ testing Teor is troubleshooting integ testing issues and requested that we bump our tor logging from notice to info runlevel... https://trac.torproject.org/projects/tor/ticket/30592 If anyone looks at this log file it's because something went wrong, so lets just go to the debug runlevel instead. Folks can always grep out the info messages if it's too overwhelming. --- test/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runner.py b/test/runner.py index b92b5a01..b7da06a4 100644 --- a/test/runner.py +++ b/test/runner.py @@ -69,7 +69,7 @@ PublishServerDescriptor 0 AssumeReachable 1 DownloadExtraInfo 1 Log notice stdout -Log notice file %%s/tor_log +Log debug file %%s/tor_log """ % (SOCKS_PORT, ORPORT) # singleton Runner instance

1 0

[translation/torbutton-securitylevelproperties_completed] Update translations for torbutton-securitylevelproperties_completed
by translation＠torproject.org 27 May '19

27 May '19

commit 04df1475dd0a75b7b4d830937f765a8b6e579a64 Author: Translation commit bot <translation(a)torproject.org> Date: Mon May 27 22:49:53 2019 +0000 Update translations for torbutton-securitylevelproperties_completed --- fr/securitylevel.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fr/securitylevel.properties b/fr/securitylevel.properties index 3dd5edf73..c548747a0 100644 --- a/fr/securitylevel.properties +++ b/fr/securitylevel.properties @@ -19,4 +19,4 @@ securityLevel.safest.description3 = Le son et la vidéo (médias HTML5) sont « securityLevel.custom.summary = Les préférences personnalisées de votre navigateur ont entraîné des paramètres de sécurité inhabituels. Pour des raisons de sécurité et de protection des données personnelles, nous vous recommandons de choisir un des niveaux de sécurité par défaut. securityLevel.learnMore = En apprendre davantage securityLevel.restoreDefaults = Rétablir les paramètres par défaut -securityLevel.advancedSecuritySettings = Paramètres de sécurité avancés... +securityLevel.advancedSecuritySettings = Paramètres de sécurité avancés…

1 0

[translation/torbutton-securitylevelproperties] Update translations for torbutton-securitylevelproperties
by translation＠torproject.org 27 May '19

27 May '19

commit d0fdb87e2f1fbf73c73c9328ce3dcccf61196946 Author: Translation commit bot <translation(a)torproject.org> Date: Mon May 27 22:49:47 2019 +0000 Update translations for torbutton-securitylevelproperties --- fr/securitylevel.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fr/securitylevel.properties b/fr/securitylevel.properties index 3dd5edf73..c548747a0 100644 --- a/fr/securitylevel.properties +++ b/fr/securitylevel.properties @@ -19,4 +19,4 @@ securityLevel.safest.description3 = Le son et la vidéo (médias HTML5) sont « securityLevel.custom.summary = Les préférences personnalisées de votre navigateur ont entraîné des paramètres de sécurité inhabituels. Pour des raisons de sécurité et de protection des données personnelles, nous vous recommandons de choisir un des niveaux de sécurité par défaut. securityLevel.learnMore = En apprendre davantage securityLevel.restoreDefaults = Rétablir les paramètres par défaut -securityLevel.advancedSecuritySettings = Paramètres de sécurité avancés... +securityLevel.advancedSecuritySettings = Paramètres de sécurité avancés…

1 0

[translation/tbmanual-contentspot_completed] Update translations for tbmanual-contentspot_completed
by translation＠torproject.org 27 May '19

27 May '19

commit c49ec8d15bee985590a024f1ef43278e17956b01 Author: Translation commit bot <translation(a)torproject.org> Date: Mon May 27 22:48:03 2019 +0000 Update translations for tbmanual-contentspot_completed --- contents+fr.po | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contents+fr.po b/contents+fr.po index d4a37b516..3866feea1 100644 --- a/contents+fr.po +++ b/contents+fr.po @@ -1687,6 +1687,9 @@ msgid "" "functioning properly, so you should weigh your security needs against the " "degree of usability you require." msgstr "" +"Augmenter le niveau de sécurité du Navigateur Tor empêchera le bon " +"fonctionnement de certaines pages Web, et vous devriez évaluer vos besoins " +"de sécurité par rapport au degré de facilité d’emploi dont vous avez besoin." #: https//tb-manual.torproject.org/security-settings/ #: (content/security-settings/contents+en.lrtopic.body) @@ -1699,6 +1702,8 @@ msgid "" "The Security Settings can be accessed by clicking the Shield icon next to " "the Tor Browser URL bar." msgstr "" +"Vous pouvez accéder aux paramètres de sécurité en cliquant sur l’icône de " +"bouclier située à côté de la barre d’URL du Navigateur Tor." #: https//tb-manual.torproject.org/security-settings/ #: (content/security-settings/contents+en.lrtopic.body)

1 0

[translation/tbmanual-contentspot] Update translations for tbmanual-contentspot
by translation＠torproject.org 27 May '19

27 May '19

commit fe832566b8efd43d47be1d358e00e03836b29854 Author: Translation commit bot <translation(a)torproject.org> Date: Mon May 27 22:47:56 2019 +0000 Update translations for tbmanual-contentspot --- contents+fr.po | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contents+fr.po b/contents+fr.po index d4a37b516..3866feea1 100644 --- a/contents+fr.po +++ b/contents+fr.po @@ -1687,6 +1687,9 @@ msgid "" "functioning properly, so you should weigh your security needs against the " "degree of usability you require." msgstr "" +"Augmenter le niveau de sécurité du Navigateur Tor empêchera le bon " +"fonctionnement de certaines pages Web, et vous devriez évaluer vos besoins " +"de sécurité par rapport au degré de facilité d’emploi dont vous avez besoin." #: https//tb-manual.torproject.org/security-settings/ #: (content/security-settings/contents+en.lrtopic.body) @@ -1699,6 +1702,8 @@ msgid "" "The Security Settings can be accessed by clicking the Shield icon next to " "the Tor Browser URL bar." msgstr "" +"Vous pouvez accéder aux paramètres de sécurité en cliquant sur l’icône de " +"bouclier située à côté de la barre d’URL du Navigateur Tor." #: https//tb-manual.torproject.org/security-settings/ #: (content/security-settings/contents+en.lrtopic.body)

1 0

[translation/tails-misc_completed] Update translations for tails-misc_completed
by translation＠torproject.org 27 May '19

27 May '19

commit d5d2329ad65f378ad9943171400b7161d3d5d976 Author: Translation commit bot <translation(a)torproject.org> Date: Mon May 27 22:46:44 2019 +0000 Update translations for tails-misc_completed --- fr.po | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fr.po b/fr.po index 2cf374777..0357ff694 100644 --- a/fr.po +++ b/fr.po @@ -31,7 +31,7 @@ msgstr "" "Project-Id-Version: Tor Project\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2019-05-18 19:31+0200\n" -"PO-Revision-Date: 2019-05-22 15:35+0000\n" +"PO-Revision-Date: 2019-05-27 22:18+0000\n" "Last-Translator: AO <ao(a)localizationlab.org>\n" "Language-Team: French (http://www.transifex.com/otf/torproject/language/fr/)\n" "MIME-Version: 1.0\n" @@ -396,7 +396,7 @@ msgstr "Synchronisation de l’horloge système" msgid "" "Tor needs an accurate clock to work properly, especially for Hidden " "Services. Please wait..." -msgstr "Tor exige une horloge précise et bien réglée pour fonctionner correctement, en particulier pour les services cachés. Veuillez patienter..." +msgstr "Tor exige une horloge précise et bien réglée pour fonctionner correctement, en particulier pour les services cachés. Veuillez patienter…" #: config/chroot_local-includes/usr/local/lib/tails-htp-notify-user:87 msgid "Failed to synchronize the clock!"

1 0