[tor-commits] [arm/master] Use pyflakes and pep8 apis rather than shelling out
atagar at torproject.org
atagar at torproject.org
Thu Jan 2 17:12:04 UTC 2014
commit be09186f744d9fd335fb354d957e5e766e847709
Author: Damian Johnson <atagar at torproject.org>
Date: Thu Jan 2 08:29:15 2014 -0800
Use pyflakes and pep8 apis rather than shelling out
Bringing over stem's recent improvements to use pyflakes and pep8 directly
rather than shelling out. This results in a 62.7% reduction in our test runtime
(from 2.63 to 0.98 seconds on my system).
---
arm/headerPanel.py | 4 +-
run_tests.py | 138 ++++++++++++++++++++++++++++++++--------------------
2 files changed, 88 insertions(+), 54 deletions(-)
diff --git a/arm/headerPanel.py b/arm/headerPanel.py
index addde9d..00796a1 100644
--- a/arm/headerPanel.py
+++ b/arm/headerPanel.py
@@ -21,8 +21,6 @@ import threading
import arm.util.tracker
-import stem
-
from stem.control import State
from stem.util import conf, log, str_tools
@@ -173,6 +171,8 @@ class HeaderPanel(panel.Panel, threading.Thread):
# torTools.getConn().init(controller)
# log.notice("Reconnected to Tor's control port")
# arm.popups.showMsg("Tor reconnected", 1)
+
+ pass
else: isKeystrokeConsumed = False
return isKeystrokeConsumed
diff --git a/run_tests.py b/run_tests.py
index 3ed5e25..13570bc 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -79,6 +79,34 @@ def clean_orphaned_pyc():
os.remove(pyc_path)
+def is_pyflakes_available():
+ """
+ Checks if pyflakes is availalbe.
+
+ :returns: **True** if we can use pyflakes and **False** otherwise
+ """
+
+ try:
+ import pyflakes
+ return True
+ except ImportError:
+ return False
+
+
+def is_pep8_available():
+ """
+ Checks if pep8 is availalbe.
+
+ :returns: **True** if we can use pep8 and **False** otherwise
+ """
+
+ try:
+ import pep8
+ return True
+ except ImportError:
+ return False
+
+
def get_stylistic_issues(paths):
"""
Checks for stylistic issues that are an issue according to the parts of PEP8
@@ -94,26 +122,25 @@ def get_stylistic_issues(paths):
:returns: **dict** of the form ``path => [(line_number, message)...]``
"""
- ignored_issues = ','.join(CONFIG["pep8.ignore"])
issues = {}
- for path in paths:
- pep8_output = stem.util.system.call(
- "pep8 --ignore %s %s" % (ignored_issues, path),
- ignore_exit_status = True,
- )
+ if is_pep8_available():
+ import pep8
- for line in pep8_output:
- line_match = re.match("^(.*):(\d+):(\d+): (.*)$", line)
+ class StyleReport(pep8.BaseReport):
+ def __init__(self, options):
+ super(StyleReport, self).__init__(options)
- if line_match:
- path, line, _, issue = line_match.groups()
+ def error(self, line_number, offset, text, check):
+ code = super(StyleReport, self).error(line_number, offset, text, check)
- if path in CONFIG["pep8.blacklist"]:
- continue
+ if code:
+ issues.setdefault(self.filename, []).append((offset + line_number, "%s %s" % (code, text)))
- issues.setdefault(path, []).append((int(line), issue))
+ style_checker = pep8.StyleGuide(ignore = CONFIG["pep8.ignore"], reporter = StyleReport)
+ style_checker.check_files(filter(lambda path: path not in CONFIG["pep8.blacklist"], _python_files(paths)))
+ for path in paths:
for file_path in _get_files_with_suffix(path):
if file_path in CONFIG["pep8.blacklist"]:
continue
@@ -121,21 +148,25 @@ def get_stylistic_issues(paths):
with open(file_path) as f:
file_contents = f.read()
- lines, file_issues, prev_indent = file_contents.split("\n"), [], 0
+ lines, prev_indent = file_contents.split("\n"), 0
is_block_comment = False
for index, line in enumerate(lines):
whitespace, content = re.match("^(\s*)(.*)$", line).groups()
+ # TODO: This does not check that block indentations are two spaces
+ # because differentiating source from string blocks ("""foo""") is more
+ # of a pita than I want to deal with right now.
+
if '"""' in content:
is_block_comment = not is_block_comment
if "\t" in whitespace:
- file_issues.append((index + 1, "indentation has a tab"))
+ issues.setdefault(file_path, []).append((index + 1, "indentation has a tab"))
elif "\r" in content:
- file_issues.append((index + 1, "contains a windows newline"))
+ issues.setdefault(file_path, []).append((index + 1, "contains a windows newline"))
elif content != content.rstrip():
- file_issues.append((index + 1, "line has trailing whitespace"))
+ issues.setdefault(file_path, []).append((index + 1, "line has trailing whitespace"))
elif content.lstrip().startswith("except") and content.endswith(", exc:"):
# Python 2.6 - 2.7 supports two forms for exceptions...
#
@@ -145,10 +176,7 @@ def get_stylistic_issues(paths):
# The former is the old method and no longer supported in python 3
# going forward.
- file_issues.append((index + 1, "except clause should use 'as', not comma"))
-
- if file_issues:
- issues[file_path] = file_issues
+ issues.setdefault(file_path, []).append((index + 1, "except clause should use 'as', not comma"))
return issues
@@ -162,47 +190,47 @@ def get_pyflakes_issues(paths):
:returns: dict of the form ``path => [(line_number, message)...]``
"""
- pyflakes_ignore = {}
+ issues = {}
- for line in CONFIG["pyflakes.ignore"]:
- path, issue = line.split("=>")
- pyflakes_ignore.setdefault(path.strip(), []).append(issue.strip())
+ if is_pyflakes_available():
+ import pyflakes.api
+ import pyflakes.reporter
- def is_ignored(path, issue):
- # Paths in pyflakes_ignore are relative, so we need to check to see if our
- # path ends with any of them.
+ class Reporter(pyflakes.reporter.Reporter):
+ def __init__(self):
+ self._ignored_issues = {}
- for ignore_path in pyflakes_ignore:
- if path.endswith(ignore_path) and issue in pyflakes_ignore[ignore_path]:
- return True
+ for line in CONFIG["pyflakes.ignore"]:
+ path, issue = line.split("=>")
+ self._ignored_issues.setdefault(path.strip(), []).append(issue.strip())
- return False
+ def unexpectedError(self, filename, msg):
+ self._register_issue(filename, None, msg)
- # Pyflakes issues are of the form...
- #
- # FILE:LINE: ISSUE
- #
- # ... for instance...
- #
- # stem/prereq.py:73: 'long_to_bytes' imported but unused
- # stem/control.py:957: undefined name 'entry'
+ def syntaxError(self, filename, msg, lineno, offset, text):
+ self._register_issue(filename, lineno, msg)
- issues = {}
+ def flake(self, msg):
+ self._register_issue(msg.filename, msg.lineno, msg.message % msg.message_args)
- for path in paths:
- pyflakes_output = stem.util.system.call(
- "pyflakes %s" % path,
- ignore_exit_status = True,
- )
+ def _is_ignored(self, path, issue):
+ # Paths in pyflakes_ignore are relative, so we need to check to see if our
+ # path ends with any of them.
+
+ for ignored_path, ignored_issues in self._ignored_issues.items():
+ if path.endswith(ignored_path) and issue in ignored_issues:
+ return True
- for line in pyflakes_output:
- line_match = re.match("^(.*):(\d+): (.*)$", line)
+ return False
- if line_match:
- path, line, issue = line_match.groups()
+ def _register_issue(self, path, line_number, issue):
+ if not self._is_ignored(path, issue):
+ issues.setdefault(path, []).append((line_number, issue))
- if not is_ignored(path, issue):
- issues.setdefault(path, []).append((int(line), issue))
+ reporter = Reporter()
+
+ for path in _python_files(paths):
+ pyflakes.api.checkPath(path, reporter)
return issues
@@ -228,5 +256,11 @@ def _get_files_with_suffix(base_path, suffix = ".py"):
yield os.path.join(root, filename)
+def _python_files(paths):
+ for path in paths:
+ for file_path in _get_files_with_suffix(path):
+ yield file_path
+
+
if __name__ == '__main__':
main()
More information about the tor-commits
mailing list