January 2015 - tor-commits - lists.torproject.org

[stem/master] Standardizing run_tests.py quote type
by atagar＠torproject.org 08 Jan '15

08 Jan '15

commit 88ab59e02eb09ec30c9afe544e47b64bb9110ef1 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 6 09:40:52 2015 -0800 Standardizing run_tests.py quote type I standardized all the rest of our codebase to use single quotes a while back but evidently missed run_tests.py, probably because it's not in src or test. --- run_tests.py | 104 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/run_tests.py b/run_tests.py index dc846e7..89eb7cd 100755 --- a/run_tests.py +++ b/run_tests.py @@ -92,7 +92,7 @@ https://pypi.python.org/pypi/mock/ """ PYFLAKES_TASK = Task( - "running pyflakes", + 'running pyflakes', stem.util.test_tools.pyflakes_issues, args = (SRC_PATHS,), is_required = False, @@ -100,7 +100,7 @@ PYFLAKES_TASK = Task( ) PEP8_TASK = Task( - "running pep8", + 'running pep8', stem.util.test_tools.stylistic_issues, args = (SRC_PATHS, True, True, True, True), is_required = False, @@ -114,16 +114,16 @@ def main(): try: stem.prereq.check_requirements() except ImportError as exc: - println("%s\n" % exc) + println('%s\n' % exc) sys.exit(1) - test_config = stem.util.conf.get_config("test") - test_config.load(os.path.join(STEM_BASE, "test", "settings.cfg")) + test_config = stem.util.conf.get_config('test') + test_config.load(os.path.join(STEM_BASE, 'test', 'settings.cfg')) try: args = _get_args(sys.argv[1:]) except getopt.GetoptError as exc: - println("%s (for usage provide --help)" % exc) + println('%s (for usage provide --help)' % exc) sys.exit(1) except ValueError as exc: println(str(exc)) @@ -133,7 +133,7 @@ def main(): println(test.util.get_help_message()) sys.exit() elif not args.run_unit and not args.run_integ: - println("Nothing to run (for usage provide --help)\n") + println('Nothing to run (for usage provide --help)\n') sys.exit() if not stem.prereq.is_mock_available(): @@ -164,15 +164,15 @@ def main(): pep8_task = PEP8_TASK test.util.run_tasks( - "INITIALISING", - Task("checking stem version", test.util.check_stem_version), - Task("checking python version", test.util.check_python_version), - Task("checking pycrypto version", test.util.check_pycrypto_version), - Task("checking mock version", test.util.check_mock_version), - Task("checking pyflakes version", test.util.check_pyflakes_version), - Task("checking pep8 version", test.util.check_pep8_version), - Task("checking for orphaned .pyc files", test.util.clean_orphaned_pyc, (SRC_PATHS,)), - Task("checking for unused tests", test.util.check_for_unused_tests, ((os.path.join(STEM_BASE, 'test'),),)), + 'INITIALISING', + Task('checking stem version', test.util.check_stem_version), + Task('checking python version', test.util.check_python_version), + Task('checking pycrypto version', test.util.check_pycrypto_version), + Task('checking mock version', test.util.check_mock_version), + Task('checking pyflakes version', test.util.check_pyflakes_version), + Task('checking pep8 version', test.util.check_pep8_version), + Task('checking for orphaned .pyc files', test.util.clean_orphaned_pyc, (SRC_PATHS,)), + Task('checking for unused tests', test.util.check_for_unused_tests, ((os.path.join(STEM_BASE, 'test'),),)), pyflakes_task, pep8_task, ) @@ -200,8 +200,8 @@ def main(): skipped_tests = 0 if args.run_unit: - test.output.print_divider("UNIT TESTS", True) - error_tracker.set_category("UNIT TEST") + test.output.print_divider('UNIT TESTS', True) + error_tracker.set_category('UNIT TEST') for test_class in test.util.get_unit_tests(args.specific_test): run_result = _run_test(args, test_class, output_filters, logging_buffer) @@ -210,7 +210,7 @@ def main(): println() if args.run_integ: - test.output.print_divider("INTEGRATION TESTS", True) + test.output.print_divider('INTEGRATION TESTS', True) integ_runner = test.runner.get_runner() # Determine targets we don't meet the prereqs for. Warnings are given about @@ -233,7 +233,7 @@ def main(): try: integ_runner.start(target, args.attribute_targets, args.tor_path, extra_torrc_opts = test.util.get_torrc_entries(target)) - println("Running tests...\n", STATUS) + println('Running tests...\n', STATUS) owner = None if integ_runner.is_accessible(): @@ -252,14 +252,14 @@ def main(): active_threads = threading.enumerate() if len(active_threads) > 1: - println("Threads lingering after test run:", ERROR) + println('Threads lingering after test run:', ERROR) for lingering_thread in active_threads: - println(" %s" % lingering_thread, ERROR) + println(' %s' % lingering_thread, ERROR) break except KeyboardInterrupt: - println(" aborted starting tor: keyboard interrupt\n", ERROR) + println(' aborted starting tor: keyboard interrupt\n', ERROR) break except ValueError as exc: # can arise if get_torrc_entries() runs into a bad settings.cfg data @@ -278,7 +278,7 @@ def main(): for target in skipped_targets: req_version = test.util.get_prereq(target) - println("Unable to run target %s, this requires tor version %s" % (target, req_version), ERROR) + println('Unable to run target %s, this requires tor version %s' % (target, req_version), ERROR) println() @@ -289,29 +289,29 @@ def main(): for issue in issues: static_check_issues.setdefault(path, []).append(issue) elif not stem.util.test_tools.is_pyflakes_available(): - println("Static error checking requires pyflakes version 0.7.3 or later. Please install it from ...\n http://pypi.python.org/pypi/pyflakes\n", ERROR) + println('Static error checking requires pyflakes version 0.7.3 or later. Please install it from ...\n http://pypi.python.org/pypi/pyflakes\n', ERROR) if pep8_task and pep8_task.is_successful: for path, issues in pep8_task.result.items(): for issue in issues: static_check_issues.setdefault(path, []).append(issue) elif not stem.util.test_tools.is_pep8_available(): - println("Style checks require pep8 version 1.4.2 or later. Please install it from...\n http://pypi.python.org/pypi/pep8\n", ERROR) + println('Style checks require pep8 version 1.4.2 or later. Please install it from...\n http://pypi.python.org/pypi/pep8\n', ERROR) _print_static_issues(static_check_issues) - runtime_label = "(%i seconds)" % (time.time() - start_time) + runtime_label = '(%i seconds)' % (time.time() - start_time) if error_tracker.has_errors_occured(): - println("TESTING FAILED %s" % runtime_label, ERROR) + println('TESTING FAILED %s' % runtime_label, ERROR) for line in error_tracker: - println(" %s" % line, ERROR) + println(' %s' % line, ERROR) else: if skipped_tests > 0: - println("%i TESTS WERE SKIPPED" % skipped_tests, STATUS) + println('%i TESTS WERE SKIPPED' % skipped_tests, STATUS) - println("TESTING PASSED %s\n" % runtime_label, SUCCESS) + println('TESTING PASSED %s\n' % runtime_label, SUCCESS) sys.exit(1 if error_tracker.has_errors_occured() else 0) @@ -332,27 +332,27 @@ def _get_args(argv): args = dict(ARGS) for opt, arg in getopt.getopt(argv, OPT, OPT_EXPANDED)[0]: - if opt in ("-a", "--all"): + if opt in ('-a', '--all'): args['run_unit'] = True args['run_integ'] = True - elif opt in ("-u", "--unit"): + elif opt in ('-u', '--unit'): args['run_unit'] = True - elif opt in ("-i", "--integ"): + elif opt in ('-i', '--integ'): args['run_integ'] = True - elif opt in ("-t", "--targets"): + elif opt in ('-t', '--targets'): run_targets, attribute_targets = [], [] - integ_targets = arg.split(",") - all_run_targets = [t for t in Target if CONFIG["target.torrc"].get(t) is not None] + integ_targets = arg.split(',') + all_run_targets = [t for t in Target if CONFIG['target.torrc'].get(t) is not None] # validates the targets and split them into run and attribute targets if not integ_targets: - raise ValueError("No targets provided") + raise ValueError('No targets provided') for target in integ_targets: if target not in Target: - raise ValueError("Invalid integration target: %s" % target) + raise ValueError('Invalid integration target: %s' % target) elif target in all_run_targets: run_targets.append(target) else: @@ -366,20 +366,20 @@ def _get_args(argv): args['run_targets'] = run_targets args['attribute_targets'] = attribute_targets - elif opt in ("-l", "--test"): + elif opt in ('-l', '--test'): args['specific_test'] = arg - elif opt in ("-l", "--log"): + elif opt in ('-l', '--log'): arg = arg.upper() if arg not in stem.util.log.LOG_VALUES: raise ValueError(LOG_TYPE_ERROR % arg) args['logging_runlevel'] = arg - elif opt in ("--tor"): + elif opt in ('--tor'): args['tor_path'] = arg - elif opt in ("-v", "--verbose"): + elif opt in ('-v', '--verbose'): args['verbose'] = True - elif opt in ("-h", "--help"): + elif opt in ('-h', '--help'): args['print_help'] = True # translates our args dict into a named tuple @@ -390,10 +390,10 @@ def _get_args(argv): def _print_static_issues(static_check_issues): if static_check_issues: - println("STATIC CHECKS", STATUS) + println('STATIC CHECKS', STATUS) for file_path in static_check_issues: - println("* %s" % file_path, STATUS) + println('* %s' % file_path, STATUS) # Make a dict of line numbers to its issues. This is so we can both sort # by the line number and clear any duplicate messages. @@ -405,8 +405,8 @@ def _print_static_issues(static_check_issues): for line_number in sorted(line_to_issues.keys()): for msg in line_to_issues[line_number]: - line_count = "%-4s" % line_number - println(" line %s - %s" % (line_count, msg)) + line_count = '%-4s' % line_number + println(' line %s - %s' % (line_count, msg)) println() @@ -424,8 +424,8 @@ def _run_test(args, test_class, output_filters, logging_buffer): elif label.startswith('test.integ.'): label = label[11:] - label = " %s..." % label - label = "%-54s" % label + label = ' %s...' % label + label = '%-54s' % label println(label, STATUS, NO_NL) @@ -438,9 +438,9 @@ def _run_test(args, test_class, output_filters, logging_buffer): sys.stdout.write(test.output.apply_filters(test_results.getvalue(), *output_filters)) println() elif not run_result.failures and not run_result.errors: - println(" success (%0.2fs)" % (time.time() - start_time), SUCCESS) + println(' success (%0.2fs)' % (time.time() - start_time), SUCCESS) else: - println(" failed (%0.2fs)" % (time.time() - start_time), ERROR) + println(' failed (%0.2fs)' % (time.time() - start_time), ERROR) sys.stdout.write(test.output.apply_filters(test_results.getvalue(), *output_filters)) test.output.print_logging(logging_buffer)

1 0

[stem/master] Testing improvements suggested by Nick
by atagar＠torproject.org 08 Jan '15

08 Jan '15

commit 5a4fa6c70c8b4d9f25d68e3201f58a17a19a7966 Merge: 5312483 6c15247 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jan 8 09:47:23 2015 -0800 Testing improvements suggested by Nick While adding Stem's tests as a make target in tor Nick suggested several improvements... https://trac.torproject.org/projects/tor/ticket/14110 https://trac.torproject.org/projects/tor/ticket/14111 https://trac.torproject.org/projects/tor/ticket/14112 https://trac.torproject.org/projects/tor/ticket/14113 Most notably our run_tests.py now has a '--quiet' argument to just display failures, and gives the commands to re-run individual tests. run_tests.py | 134 +++++++++++++++++++++++++-------------------- test/integ/util/proc.py | 3 +- test/integ/util/system.py | 23 +++++--- test/output.py | 34 ++++++++++-- test/runner.py | 7 ++- test/settings.cfg | 2 +- 6 files changed, 127 insertions(+), 76 deletions(-)

1 0

[stem/master] 'run-tests.py --help' mistakenly mentioned runTests.py
by atagar＠torproject.org 08 Jan '15

08 Jan '15

commit 6f7187fb3f50ae28c82eba36bb3c6b42a02bb896 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 6 09:12:32 2015 -0800 'run-tests.py --help' mistakenly mentioned runTests.py Legacy from old naming years ago. Caught thanks to Nick... https://trac.torproject.org/projects/tor/ticket/14112 --- test/settings.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/settings.cfg b/test/settings.cfg index 96fa40d..44cb90a 100644 --- a/test/settings.cfg +++ b/test/settings.cfg @@ -30,7 +30,7 @@ integ.log ./test/data/log # ran when the user runs with '--target'. msg.help -|Usage runTests.py [OPTION] +|Usage run-tests.py [OPTION] |Runs tests for the stem library. | | -a, --all runs unit, integ, and style checks (same as '-uis')

1 0

[stem/master] Note command to just re-run test failures
by atagar＠torproject.org 08 Jan '15

08 Jan '15

commit 6c15247fbfd991090f91d0c4f28f36d0e31b4b43 Author: Damian Johnson <atagar(a)torproject.org> Date: Thu Jan 8 09:45:46 2015 -0800 Note command to just re-run test failures When our tests fail include the commands to just re-run those tests. Neat idea from Nick on... https://trac.torproject.org/projects/tor/ticket/14113 --- run_tests.py | 15 ++++++++++----- test/output.py | 9 +++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/run_tests.py b/run_tests.py index d157276..c0fa4be 100755 --- a/run_tests.py +++ b/run_tests.py @@ -307,13 +307,18 @@ def main(): runtime_label = '(%i seconds)' % (time.time() - start_time) if error_tracker.has_errors_occured(): - if args.quiet: - println('', STDERR) # extra newline - println('TESTING FAILED %s' % runtime_label, ERROR, STDERR) for line in error_tracker: println(' %s' % line, ERROR, STDERR) + + error_modules = error_tracker.get_modules() + + if len(error_modules) < 10 and not args.specific_test: + println('\nYou can re-run just these tests with:\n', ERROR, STDERR) + + for module in error_modules: + println(' %s --test %s' % (' '.join(sys.argv), module), ERROR, STDERR) else: if skipped_tests > 0: println('%i TESTS WERE SKIPPED' % skipped_tests, STATUS) @@ -373,7 +378,7 @@ def _get_args(argv): args['run_targets'] = run_targets args['attribute_targets'] = attribute_targets - elif opt in ('-l', '--test'): + elif opt in ('-t', '--test'): args['specific_test'] = arg elif opt in ('-l', '--log'): arg = arg.upper() @@ -451,7 +456,7 @@ def _run_test(args, test_class, output_filters, logging_buffer): if args.quiet: println(label, STATUS, NO_NL, STDERR) println(' failed (%0.2fs)' % (time.time() - start_time), ERROR, STDERR) - println(test.output.apply_filters(test_results.getvalue(), *output_filters), NO_NL, STDERR) + println(test.output.apply_filters(test_results.getvalue(), *output_filters), STDERR) else: println(' failed (%0.2fs)' % (time.time() - start_time), ERROR) println(test.output.apply_filters(test_results.getvalue(), *output_filters), NO_NL) diff --git a/test/output.py b/test/output.py index f338753..e88d3de 100644 --- a/test/output.py +++ b/test/output.py @@ -196,6 +196,7 @@ class ErrorTracker(object): def __init__(self): self._errors = [] + self._error_modules = set() self._category = None self._error_noted = False @@ -232,10 +233,18 @@ class ErrorTracker(object): else: self._errors.append(line_content) + module_match = re.match('.*$(test\.\S+)\.\S+$.*', line_content) + + if module_match: + self._error_modules.add(module_match.group(1)) + return line_content return _error_tracker + def get_modules(self): + return self._error_modules + def __iter__(self): for error_line in self._errors: yield error_line

1 0

[stem/master] Integ tests failed when the tor binary was named something else
by atagar＠torproject.org 08 Jan '15

08 Jan '15

commit ee538767ffcc352e75cece3a9b380c8b04ecb2d0 Author: Damian Johnson <atagar(a)torproject.org> Date: Tue Jan 6 09:31:23 2015 -0800 Integ tests failed when the tor binary was named something else Some of our tests respected non-standardly named tor binaries but not all. Fixing the system tests... https://trac.torproject.org/projects/tor/ticket/14111 --- test/integ/util/proc.py | 3 ++- test/integ/util/system.py | 23 +++++++++++++++-------- test/runner.py | 7 +++++-- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/test/integ/util/proc.py b/test/integ/util/proc.py index 66203dd..79da938 100644 --- a/test/integ/util/proc.py +++ b/test/integ/util/proc.py @@ -65,10 +65,11 @@ class TestProc(unittest.TestCase): test.runner.skip(self, '(proc unavailable)') return + tor_cmd = test.runner.get_runner().get_tor_command(True) tor_pid = test.runner.get_runner().get_pid() command, utime, stime, start_time = proc.stats(tor_pid, 'command', 'utime', 'stime', 'start time') - self.assertEqual('tor', command) + self.assertEqual(tor_cmd, command) self.assertTrue(float(utime) > 0) self.assertTrue(float(stime) >= 0) self.assertTrue(float(start_time) > proc.system_start_time()) diff --git a/test/integ/util/system.py b/test/integ/util/system.py index d754ddb..beea501 100644 --- a/test/integ/util/system.py +++ b/test/integ/util/system.py @@ -73,7 +73,7 @@ class TestSystem(unittest.TestCase): test.runner.skip(self, '(ps unavailable)') return - self.assertTrue(stem.util.system.is_running('tor')) + self.assertTrue(stem.util.system.is_running(test.runner.get_runner().get_tor_command(True))) self.assertFalse(stem.util.system.is_running('blarg_and_stuff')) def test_pid_by_name(self): @@ -90,7 +90,8 @@ class TestSystem(unittest.TestCase): return tor_pid = test.runner.get_runner().get_pid() - self.assertEqual(tor_pid, stem.util.system.pid_by_name('tor')) + tor_cmd = test.runner.get_runner().get_tor_command(True) + self.assertEqual(tor_pid, stem.util.system.pid_by_name(tor_cmd)) self.assertEqual(None, stem.util.system.pid_by_name('blarg_and_stuff')) def test_pid_by_name_pgrep(self): @@ -113,7 +114,8 @@ class TestSystem(unittest.TestCase): call_mock.side_effect = call_replacement tor_pid = test.runner.get_runner().get_pid() - self.assertEqual(tor_pid, stem.util.system.pid_by_name('tor')) + tor_cmd = test.runner.get_runner().get_tor_command(True) + self.assertEqual(tor_pid, stem.util.system.pid_by_name(tor_cmd)) def test_pid_by_name_pidof(self): """ @@ -135,7 +137,8 @@ class TestSystem(unittest.TestCase): call_mock.side_effect = call_replacement tor_pid = test.runner.get_runner().get_pid() - self.assertEqual(tor_pid, stem.util.system.pid_by_name('tor')) + tor_cmd = test.runner.get_runner().get_tor_command(True) + self.assertEqual(tor_pid, stem.util.system.pid_by_name(tor_cmd)) def test_pid_by_name_ps_linux(self): """ @@ -160,7 +163,8 @@ class TestSystem(unittest.TestCase): call_mock.side_effect = call_replacement tor_pid = test.runner.get_runner().get_pid() - self.assertEqual(tor_pid, stem.util.system.pid_by_name('tor')) + tor_cmd = test.runner.get_runner().get_tor_command(True) + self.assertEqual(tor_pid, stem.util.system.pid_by_name(tor_cmd)) def test_pid_by_name_ps_bsd(self): """ @@ -185,7 +189,8 @@ class TestSystem(unittest.TestCase): call_mock.side_effect = call_replacement tor_pid = test.runner.get_runner().get_pid() - self.assertEqual(tor_pid, stem.util.system.pid_by_name('tor')) + tor_cmd = test.runner.get_runner().get_tor_command(True) + self.assertEqual(tor_pid, stem.util.system.pid_by_name(tor_cmd)) def test_pid_by_name_lsof(self): """ @@ -210,8 +215,9 @@ class TestSystem(unittest.TestCase): with patch('stem.util.system.call') as call_mock: call_mock.side_effect = call_replacement + tor_cmd = test.runner.get_runner().get_tor_command(True) our_tor_pid = test.runner.get_runner().get_pid() - all_tor_pids = stem.util.system.pid_by_name('tor', multiple = True) + all_tor_pids = stem.util.system.pid_by_name(tor_cmd, multiple = True) if len(all_tor_pids) == 1: self.assertEqual(our_tor_pid, all_tor_pids[0]) @@ -561,7 +567,8 @@ class TestSystem(unittest.TestCase): # TODO: not sure how to check for this on windows IS_EXTRA_TOR_RUNNING = False elif not stem.util.system.is_bsd(): - pgrep_results = stem.util.system.call(stem.util.system.GET_PID_BY_NAME_PGREP % 'tor') + tor_cmd = test.runner.get_runner().get_tor_command(True) + pgrep_results = stem.util.system.call(stem.util.system.GET_PID_BY_NAME_PGREP % tor_cmd) IS_EXTRA_TOR_RUNNING = len(pgrep_results) > 1 else: ps_results = stem.util.system.call(stem.util.system.GET_PID_BY_NAME_PS_BSD) diff --git a/test/runner.py b/test/runner.py index ee261b7..9019eae 100644 --- a/test/runner.py +++ b/test/runner.py @@ -584,12 +584,15 @@ class Runner(object): except TorInaccessable: return stem.version.get_system_tor_version(self.get_tor_command()) - def get_tor_command(self): + def get_tor_command(self, base_cmd = False): """ Provides the command used to run our tor instance. + + :param bool base_cmd: provides just the command name if true rather than + the full '--tor path' argument """ - return self._get('_tor_cmd') + return os.path.basename(self._get('_tor_cmd')) if base_cmd else self._get('_tor_cmd') def _get(self, attr): """

1 0

[bridgedb/develop] Remove two spaces before start of bridgelines on HTTPS distributor.
by isis＠torproject.org 08 Jan '15

08 Jan '15

commit 7e79bc05e77aa87a06ea0e1ad5e329a92e7a8dc0 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Jan 8 14:02:18 2015 +0000 Remove two spaces before start of bridgelines on HTTPS distributor. * CHANGE bridge lines created by the HTTPS distributor to not begin with two space characters. * FIXES #13123: https://bugs.torproject.org/13123 --- lib/bridgedb/HTTPServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/bridgedb/HTTPServer.py b/lib/bridgedb/HTTPServer.py index 464ae0d..6aebba1 100644 --- a/lib/bridgedb/HTTPServer.py +++ b/lib/bridgedb/HTTPServer.py @@ -750,7 +750,7 @@ class WebResourceBridges(resource.Resource): self.nBridgesToGive, countryCode, bridgeFilterRules=rules) - bridgeLines = "".join(" %s\n" % b.getConfigLine( + bridgeLines = "".join("%s\n" % b.getConfigLine( includeFingerprint=self.includeFingerprints, addressClass=addressClass, transport=transport,

1 0

[bridgedb/develop] Merge branch 'fix/13123-bridgeline-spaces' into develop
by isis＠torproject.org 08 Jan '15

08 Jan '15

commit b2ef9342e6734d4e3d0a2c8e943c68cf0da419d8 Merge: c6d5278 7e79bc0 Author: Isis Lovecruft <isis(a)torproject.org> Date: Thu Jan 8 14:08:29 2015 +0000 Merge branch 'fix/13123-bridgeline-spaces' into develop lib/bridgedb/HTTPServer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)

1 0

[tor-browser-bundle/master] Bug 13857: allow making incrementals MARs for a specific channel
by gk＠torproject.org 08 Jan '15

08 Jan '15

commit 0f37c120647be48614d3af9512bcc06ba295fa34 Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Dec 8 14:43:12 2014 +0100 Bug 13857: allow making incrementals MARs for a specific channel The gen_incrementals can now receive a channel name as argument. In that case, incremental MARs will be created for this channel only. If there is no argument, incremental MARs for all channels are created. Makefile rules have been added for the different channels. --- gitian/Makefile | 8 +++++++- gitian/README.build | 4 ++++ tools/update-responses/update_responses | 24 +++++++++++++++--------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/gitian/Makefile b/gitian/Makefile index 9613db6..1fce05e 100644 --- a/gitian/Makefile +++ b/gitian/Makefile @@ -33,7 +33,13 @@ build-beta: ./hash-bundles.sh versions.beta incrementals: - ../tools/update-responses/gen_incrementals + ../tools/update-responses/gen_incrementals release + +incrementals-alpha: + ../tools/update-responses/gen_incrementals alpha + +incrementals-beta: + ../tools/update-responses/gen_incrementals beta signmars: ./signmars.sh versions diff --git a/gitian/README.build b/gitian/README.build index 670e84c..dd7af91 100644 --- a/gitian/README.build +++ b/gitian/README.build @@ -78,6 +78,10 @@ Detailed Explanation of Scripts: - hash-nightly: The equivalent to the 'hash' rule for nightly packages - hash-alpha: The equivalent to the 'hash' rule for alpha packages - hash-beta: The equivalent to the 'hash' rule for beta packages + - incrementals-alpha: The equivalent to the 'incrementals' rule for + alpha packages + - incrementals-beta: The equivalent to the 'incrementals' rule for + beta packages - signmars-nightly: The equivalent to the 'signmars' rule for nightly packages - signmars-alpha: The equivalent to the 'signmars' rule for alpha packages - signmars-beta: The equivalent to the 'signmars' rule for beta packages diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index 5b85037..e66b27d 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -180,14 +180,6 @@ sub create_missing_incremental_mars_for_version { $pm->wait_all_children; } -sub create_all_missing_incremental_mars { - my ($config) = @_; - foreach my $version (values %{$config->{channels}}) { - get_version_files($config, $version); - create_missing_incremental_mars_for_version($config, $version); - } -} - sub get_config { my ($config, $version, $os, $name) = @_; return $config->{versions}{$version}{$os}{$name} @@ -195,6 +187,16 @@ sub get_config { // $config->{$name}; } +sub channel_to_version { + my ($config, @channels) = @_; + return values %{$config->{channels}} unless @channels; + foreach my $channel (@channels) { + exit_error "Unknown channel $channel" + unless $config->{channels}{$channel}; + } + return map { $config->{channels}{$_} } @channels; +} + sub get_response { my ($config, $version, $os, @patches) = @_; my $res; @@ -325,9 +327,13 @@ my %actions = ( clean_htdocs; }, gen_incrementals => sub { + my ($config) = @_; extract_martools; check_deps; - create_all_missing_incremental_mars(@_); + foreach my $version (channel_to_version($config, @ARGV)) { + get_version_files($config, $version); + create_missing_incremental_mars_for_version($config, $version); + } }, );

1 0

[tor-browser-bundle/master] Bug 13857: allow updating responses for a specific channel
by gk＠torproject.org 08 Jan '15

08 Jan '15

commit 808746222a49af4e6294dc82397932e331d2a4fd Author: Nicolas Vigier <boklm(a)torproject.org> Date: Mon Dec 8 19:19:19 2014 +0100 Bug 13857: allow updating responses for a specific channel The update_responses command can now receive a channel name as argument. In that case, only the xml responses for the selected channel(s) are updated. The files are now located in directory htdocfiles/$channel and each channel has its own .htaccess file in that directory. Makefile rules have been added to update the different channels. --- gitian/Makefile | 8 +++- gitian/README.build | 4 ++ tools/update-responses/htdocs/no-update.xml | 2 - tools/update-responses/update_responses | 66 ++++++++++++++++----------- 4 files changed, 51 insertions(+), 29 deletions(-) diff --git a/gitian/Makefile b/gitian/Makefile index 1fce05e..eaa22c2 100644 --- a/gitian/Makefile +++ b/gitian/Makefile @@ -54,7 +54,13 @@ signmars-nightly: ./signmars.sh versions.nightly update_responses: - ../tools/update-responses/update_responses + ../tools/update-responses/update_responses release + +update_responses-alpha: + ../tools/update-responses/update_responses alpha + +update_responses-beta: + ../tools/update-responses/update_responses beta hash: ./hash-bundles.sh versions diff --git a/gitian/README.build b/gitian/README.build index dd7af91..b51fd96 100644 --- a/gitian/README.build +++ b/gitian/README.build @@ -82,6 +82,10 @@ Detailed Explanation of Scripts: alpha packages - incrementals-beta: The equivalent to the 'incrementals' rule for beta packages + - update_responses-alpha: The equivalent to the 'update_responses' rule + for alpha packages + - update_responses-beta: The equivalent to the 'update_responses' rule + for beta packages - signmars-nightly: The equivalent to the 'signmars' rule for nightly packages - signmars-alpha: The equivalent to the 'signmars' rule for alpha packages - signmars-beta: The equivalent to the 'signmars' rule for beta packages diff --git a/tools/update-responses/htdocs/no-update.xml b/tools/update-responses/htdocs/no-update.xml deleted file mode 100644 index 910e99d..0000000 --- a/tools/update-responses/htdocs/no-update.xml +++ /dev/null @@ -1,2 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<updates></updates> diff --git a/tools/update-responses/update_responses b/tools/update-responses/update_responses index e66b27d..0c487e3 100755 --- a/tools/update-responses/update_responses +++ b/tools/update-responses/update_responses @@ -24,7 +24,7 @@ setlocale(LC_ALL, "C"); my $htdocsdir = "$FindBin::Bin/htdocs"; my $config = LoadFile("$FindBin::Bin/config.yml"); -my %htdocsfiles = ( '.' => 1, '..' => 1, 'no-update.xml' => 1 ); +my %htdocsfiles; my $releases_dir = "$FindBin::Bin/../../gitian"; sub exit_error { @@ -46,17 +46,21 @@ sub get_nbprocs { } sub write_htdocs { - my ($file, $content) = @_; + my ($channel, $file, $content) = @_; mkdir $htdocsdir unless -d $htdocsdir; - write_file("$htdocsdir/$file", $content); - $htdocsfiles{$file} = 1; + mkdir "$htdocsdir/$channel" unless -d "$htdocsdir/$channel"; + write_file("$htdocsdir/$channel/$file", $content); + $htdocsfiles{$channel}->{$file} = 1; } sub clean_htdocs { - opendir(my $d, $htdocsdir); - my @files = grep { ! $htdocsfiles{$_} } readdir $d; - closedir $d; - unlink map { "$htdocsdir/$_" } @files; + my (@channels) = @_; + foreach my $channel (@channels) { + opendir(my $d, "$htdocsdir/$channel"); + my @files = grep { ! $htdocsfiles{$channel}->{$_} } readdir $d; + closedir $d; + unlink map { "$htdocsdir/$channel/$_" } @files; + } } sub get_sha512_hex_of_file { @@ -233,8 +237,10 @@ sub get_response { } sub write_responses { - my ($config) = @_; - foreach my $version (values %{$config->{channels}}) { + my ($config, @channels) = @_; + @channels = keys %{$config->{channels}} unless @channels; + foreach my $channel (@channels) { + my $version = $config->{channels}{$channel}; get_version_files($config, $version); my $files = $config->{versions}{$version}{files}; my $migrate_archs = $config->{versions}{$version}{migrate_archs} // {}; @@ -249,43 +255,44 @@ sub write_responses { foreach my $lang (keys %{$files->{$os}}) { my $resp = get_response($config, $version, $os, $files->{$os}{$lang}{complete}); - write_htdocs("$version-$os-$lang.xml", $resp); + write_htdocs($channel, "$version-$os-$lang.xml", $resp); foreach my $from_version (keys %{$files->{$os}{$lang}{partial}}) { $resp = get_response($config, $version, $os, $files->{$os}{$lang}{complete}, $files->{$os}{$lang}{partial}{$from_version}); - write_htdocs("$from_version-$version-$os-$lang.xml", $resp); + write_htdocs($channel, "$from_version-$version-$os-$lang.xml", $resp); } } } + write_htdocs($channel, 'no-update.xml', + '<?xml version="1.0" encoding="UTF-8"?>' + . "\n<updates></updates>\n"); } } sub write_htaccess { - my ($config) = @_; - my $htaccess = "RewriteEngine On\n"; + my ($config, @channels) = @_; + @channels = keys %{$config->{channels}} unless @channels; my $flags = "[last]"; - foreach my $channel (sort keys %{$config->{channels}}) { + foreach my $channel (@channels) { + my $htaccess = "RewriteEngine On\n"; my $version = $config->{channels}{$channel}; my $files = $config->{versions}{$version}{files}; - $htaccess .= "RewriteRule " - . "^$channel/[^\/]+/$version/ " - . "no-update.xml $flags\n"; + $htaccess .= "RewriteRule ^[^\/]+/$version/ no-update.xml $flags\n"; foreach my $os (sort keys %$files) { my $bt = build_target_by_os($os); foreach my $lang (sort keys %{$files->{$os}}) { foreach my $from_version (sort keys %{$files->{$os}{$lang}{partial}}) { - $htaccess .= "RewriteRule ^$channel/$bt/$from_version/$lang " + $htaccess .= "RewriteRule ^$bt/$from_version/$lang " . "$from_version-$version-$os-$lang.xml $flags\n"; } - $htaccess .= "RewriteRule ^$channel/$bt/[^\/]+/$lang " + $htaccess .= "RewriteRule ^$bt/[^\/]+/$lang " . "$version-$os-$lang.xml $flags\n"; } - $htaccess .= "RewriteRule ^$channel/$bt " - . "$version-$os-en-US.xml $flags\n"; + $htaccess .= "RewriteRule ^$bt $version-$os-en-US.xml $flags\n"; } + write_htdocs($channel, '.htaccess', $htaccess); } - write_htdocs('.htaccess', $htaccess); } sub check_deps { @@ -322,9 +329,16 @@ sub extract_martools { my %actions = ( update_responses => sub { - write_responses(@_); - write_htaccess(@_); - clean_htdocs; + my ($config) = @_; + my @channels = @ARGV ? @ARGV : keys %{$config->{channels}}; + foreach my $channel (@channels) { + exit_error "Unknown channel $channel" + unless $config->{channels}{$channel}; + $htdocsfiles{$channel} = { '.' => 1, '..' => 1 }; + } + write_responses($config, @channels); + write_htaccess($config, @channels); + clean_htdocs(@channels); }, gen_incrementals => sub { my ($config) = @_;

1 0

[tor-browser-bundle/master] Merge remote-tracking branch 'boklm/bug13857-v2'
by gk＠torproject.org 08 Jan '15

08 Jan '15

commit 5401e06508eae0aaa8f6357303b006dd833ba8e5 Merge: bf363a7 ca6ca42 Author: Georg Koppen <gk(a)torproject.org> Date: Thu Jan 8 14:28:00 2015 +0000 Merge remote-tracking branch 'boklm/bug13857-v2' gitian/Makefile | 16 +- gitian/README.build | 8 + tools/update-responses/README.md | 8 +- .../check_update_responses_deployement | 1 + tools/update-responses/htdocs/no-update.xml | 2 - tools/update-responses/update_responses | 196 ++++++++++++++++---- 6 files changed, 189 insertions(+), 42 deletions(-)

1 0