[tor-commits] [arm/master] Use pyflakes and pep8 apis rather than shelling out

commit be09186f744d9fd335fb354d957e5e766e847709 Author: Damian Johnson atagar@torproject.org Date: Thu Jan 2 08:29:15 2014 -0800

Use pyflakes and pep8 apis rather than shelling out

Bringing over stem's recent improvements to use pyflakes and pep8 directly rather than shelling out. This results in a 62.7% reduction in our test runtime (from 2.63 to 0.98 seconds on my system). --- arm/headerPanel.py | 4 +- run_tests.py | 138 ++++++++++++++++++++++++++++++++-------------------- 2 files changed, 88 insertions(+), 54 deletions(-)

diff --git a/arm/headerPanel.py b/arm/headerPanel.py index addde9d..00796a1 100644 --- a/arm/headerPanel.py +++ b/arm/headerPanel.py @@ -21,8 +21,6 @@ import threading

import arm.util.tracker

-import stem - from stem.control import State from stem.util import conf, log, str_tools

@@ -173,6 +171,8 @@ class HeaderPanel(panel.Panel, threading.Thread): # torTools.getConn().init(controller) # log.notice("Reconnected to Tor's control port") # arm.popups.showMsg("Tor reconnected", 1) + + pass else: isKeystrokeConsumed = False

return isKeystrokeConsumed diff --git a/run_tests.py b/run_tests.py index 3ed5e25..13570bc 100755 --- a/run_tests.py +++ b/run_tests.py @@ -79,6 +79,34 @@ def clean_orphaned_pyc(): os.remove(pyc_path)

+def is_pyflakes_available(): + """ + Checks if pyflakes is availalbe. + + :returns: **True** if we can use pyflakes and **False** otherwise + """ + + try: + import pyflakes + return True + except ImportError: + return False + + +def is_pep8_available(): + """ + Checks if pep8 is availalbe. + + :returns: **True** if we can use pep8 and **False** otherwise + """ + + try: + import pep8 + return True + except ImportError: + return False + + def get_stylistic_issues(paths): """ Checks for stylistic issues that are an issue according to the parts of PEP8 @@ -94,26 +122,25 @@ def get_stylistic_issues(paths): :returns: **dict** of the form ``path => [(line_number, message)...]`` """

- ignored_issues = ','.join(CONFIG["pep8.ignore"]) issues = {}

- for path in paths: - pep8_output = stem.util.system.call( - "pep8 --ignore %s %s" % (ignored_issues, path), - ignore_exit_status = True, - ) + if is_pep8_available(): + import pep8

- for line in pep8_output: - line_match = re.match("^(.*):(\d+):(\d+): (.*)$", line) + class StyleReport(pep8.BaseReport): + def __init__(self, options): + super(StyleReport, self).__init__(options)

- if line_match: - path, line, _, issue = line_match.groups() + def error(self, line_number, offset, text, check): + code = super(StyleReport, self).error(line_number, offset, text, check)

- if path in CONFIG["pep8.blacklist"]: - continue + if code: + issues.setdefault(self.filename, []).append((offset + line_number, "%s %s" % (code, text)))

- issues.setdefault(path, []).append((int(line), issue)) + style_checker = pep8.StyleGuide(ignore = CONFIG["pep8.ignore"], reporter = StyleReport) + style_checker.check_files(filter(lambda path: path not in CONFIG["pep8.blacklist"], _python_files(paths)))

+ for path in paths: for file_path in _get_files_with_suffix(path): if file_path in CONFIG["pep8.blacklist"]: continue @@ -121,21 +148,25 @@ def get_stylistic_issues(paths): with open(file_path) as f: file_contents = f.read()

- lines, file_issues, prev_indent = file_contents.split("\n"), [], 0 + lines, prev_indent = file_contents.split("\n"), 0 is_block_comment = False

for index, line in enumerate(lines): whitespace, content = re.match("^(\s*)(.*)$", line).groups()

+ # TODO: This does not check that block indentations are two spaces + # because differentiating source from string blocks ("""foo""") is more + # of a pita than I want to deal with right now. + if '"""' in content: is_block_comment = not is_block_comment

if "\t" in whitespace: - file_issues.append((index + 1, "indentation has a tab")) + issues.setdefault(file_path, []).append((index + 1, "indentation has a tab")) elif "\r" in content: - file_issues.append((index + 1, "contains a windows newline")) + issues.setdefault(file_path, []).append((index + 1, "contains a windows newline")) elif content != content.rstrip(): - file_issues.append((index + 1, "line has trailing whitespace")) + issues.setdefault(file_path, []).append((index + 1, "line has trailing whitespace")) elif content.lstrip().startswith("except") and content.endswith(", exc:"): # Python 2.6 - 2.7 supports two forms for exceptions... # @@ -145,10 +176,7 @@ def get_stylistic_issues(paths): # The former is the old method and no longer supported in python 3 # going forward.

- file_issues.append((index + 1, "except clause should use 'as', not comma")) - - if file_issues: - issues[file_path] = file_issues + issues.setdefault(file_path, []).append((index + 1, "except clause should use 'as', not comma"))

return issues

@@ -162,47 +190,47 @@ def get_pyflakes_issues(paths): :returns: dict of the form ``path => [(line_number, message)...]`` """

- pyflakes_ignore = {} + issues = {}

- for line in CONFIG["pyflakes.ignore"]: - path, issue = line.split("=>") - pyflakes_ignore.setdefault(path.strip(), []).append(issue.strip()) + if is_pyflakes_available(): + import pyflakes.api + import pyflakes.reporter

- def is_ignored(path, issue): - # Paths in pyflakes_ignore are relative, so we need to check to see if our - # path ends with any of them. + class Reporter(pyflakes.reporter.Reporter): + def __init__(self): + self._ignored_issues = {}

- for ignore_path in pyflakes_ignore: - if path.endswith(ignore_path) and issue in pyflakes_ignore[ignore_path]: - return True + for line in CONFIG["pyflakes.ignore"]: + path, issue = line.split("=>") + self._ignored_issues.setdefault(path.strip(), []).append(issue.strip())

- return False + def unexpectedError(self, filename, msg): + self._register_issue(filename, None, msg)

- # Pyflakes issues are of the form... - # - # FILE:LINE: ISSUE - # - # ... for instance... - # - # stem/prereq.py:73: 'long_to_bytes' imported but unused - # stem/control.py:957: undefined name 'entry' + def syntaxError(self, filename, msg, lineno, offset, text): + self._register_issue(filename, lineno, msg)

- issues = {} + def flake(self, msg): + self._register_issue(msg.filename, msg.lineno, msg.message % msg.message_args)

- for path in paths: - pyflakes_output = stem.util.system.call( - "pyflakes %s" % path, - ignore_exit_status = True, - ) + def _is_ignored(self, path, issue): + # Paths in pyflakes_ignore are relative, so we need to check to see if our + # path ends with any of them. + + for ignored_path, ignored_issues in self._ignored_issues.items(): + if path.endswith(ignored_path) and issue in ignored_issues: + return True

- for line in pyflakes_output: - line_match = re.match("^(.*):(\d+): (.*)$", line) + return False

- if line_match: - path, line, issue = line_match.groups() + def _register_issue(self, path, line_number, issue): + if not self._is_ignored(path, issue): + issues.setdefault(path, []).append((line_number, issue))

- if not is_ignored(path, issue): - issues.setdefault(path, []).append((int(line), issue)) + reporter = Reporter() + + for path in _python_files(paths): + pyflakes.api.checkPath(path, reporter)

return issues

@@ -228,5 +256,11 @@ def _get_files_with_suffix(base_path, suffix = ".py"): yield os.path.join(root, filename)

+def _python_files(paths): + for path in paths: + for file_path in _get_files_with_suffix(path): + yield file_path + + if __name__ == '__main__': main()