[tor-commits] [stem/master] Mypy static checks
atagar at torproject.org
atagar at torproject.org
Mon May 11 22:52:13 UTC 2020
commit d0ed7ecd7485bd5cfa587bcc5b35e1783bb03961
Author: Damian Johnson <atagar at torproject.org>
Date: Mon Apr 13 18:10:08 2020 -0700
Mypy static checks
Integrating Mypy (http://mypy-lang.org/) for static type checks. Presently it's
reporting hundreds of issues so this will require some more work.
---
run_tests.py | 4 +-
stem/util/test_tools.py | 112 +++++++++++++++++++++++++++++++++++++++---------
test/task.py | 15 ++++++-
3 files changed, 107 insertions(+), 24 deletions(-)
diff --git a/run_tests.py b/run_tests.py
index c9196e18..2ea07dab 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -217,12 +217,14 @@ def main():
test.task.CRYPTO_VERSION,
test.task.PYFLAKES_VERSION,
test.task.PYCODESTYLE_VERSION,
+ test.task.MYPY_VERSION,
test.task.CLEAN_PYC,
test.task.UNUSED_TESTS,
test.task.IMPORT_TESTS,
test.task.REMOVE_TOR_DATA_DIR if args.run_integ else None,
test.task.PYFLAKES_TASK if not args.specific_test else None,
test.task.PYCODESTYLE_TASK if not args.specific_test else None,
+ test.task.MYPY_TASK if not args.specific_test else None,
)
# Test logging. If '--log-file' is provided we log to that location,
@@ -334,7 +336,7 @@ def main():
static_check_issues = {}
- for task in (test.task.PYFLAKES_TASK, test.task.PYCODESTYLE_TASK):
+ for task in (test.task.PYFLAKES_TASK, test.task.PYCODESTYLE_TASK, test.task.MYPY_TASK):
if not task.is_available and task.unavailable_msg:
println(task.unavailable_msg, ERROR)
else:
diff --git a/stem/util/test_tools.py b/stem/util/test_tools.py
index d5d0f842..80de447e 100644
--- a/stem/util/test_tools.py
+++ b/stem/util/test_tools.py
@@ -23,9 +23,11 @@ to match just against the prefix or suffix. For instance...
is_pyflakes_available - checks if pyflakes is available
is_pycodestyle_available - checks if pycodestyle is available
+ is_mypy_available - checks if mypy is available
pyflakes_issues - static checks for problems via pyflakes
stylistic_issues - checks for PEP8 and other stylistic issues
+ type_issues - checks for type problems
"""
import collections
@@ -47,6 +49,7 @@ from typing import Any, Callable, Iterator, Mapping, Optional, Sequence, Tuple,
CONFIG = stem.util.conf.config_dict('test', {
'pycodestyle.ignore': [],
'pyflakes.ignore': [],
+ 'mypy.ignore': [],
'exclude_paths': [],
})
@@ -353,6 +356,16 @@ def is_pycodestyle_available() -> bool:
return hasattr(pycodestyle, 'BaseReport')
+def is_mypy_available() -> bool:
+ """
+ Checks if mypy is available.
+
+ :returns: **True** if we can use mypy and **False** otherwise
+ """
+
+ return _module_exists('mypy.api')
+
+
def stylistic_issues(paths: Sequence[str], check_newlines: bool = False, check_exception_keyword: bool = False, prefer_single_quotes: bool = False) -> Mapping[str, 'stem.util.test_tools.Issue']:
"""
Checks for stylistic issues that are an issue according to the parts of PEP8
@@ -541,27 +554,8 @@ def pyflakes_issues(paths: Sequence[str]) -> Mapping[str, 'stem.util.test_tools.
def flake(self, msg: str) -> None:
self._register_issue(msg.filename, msg.lineno, msg.message % msg.message_args, None)
- def _is_ignored(self, path: str, issue: str) -> bool:
- # Paths in pyflakes_ignore are relative, so we need to check to see if our
- # path ends with any of them.
-
- for ignored_path, ignored_issues in self._ignored_issues.items():
- if path.endswith(ignored_path):
- if issue in ignored_issues:
- return True
-
- for prefix in [i[:1] for i in ignored_issues if i.endswith('*')]:
- if issue.startswith(prefix):
- return True
-
- for suffix in [i[1:] for i in ignored_issues if i.startswith('*')]:
- if issue.endswith(suffix):
- return True
-
- return False
-
- def _register_issue(self, path: str, line_number: int, issue: str, line: int) -> None:
- if not self._is_ignored(path, issue):
+ def _register_issue(self, path: str, line_number: int, issue: str, line: str) -> None:
+ if not _is_ignored(self._ignored_issues, path, issue):
if path and line_number and not line:
line = linecache.getline(path, line_number).strip()
@@ -575,6 +569,65 @@ def pyflakes_issues(paths: Sequence[str]) -> Mapping[str, 'stem.util.test_tools.
return issues
+def type_issues(paths: Sequence[str]) -> Mapping[str, 'stem.util.test_tools.Issue']:
+ """
+ Performs type checks via mypy. False positives can be ignored via
+ 'mypy.ignore' entries in our 'test' config. For instance...
+
+ ::
+
+ mypy.ignore stem/util/system.py => Incompatible types in assignment*
+
+ :param list paths: paths to search for problems
+
+ :returns: dict of paths list of :class:`stem.util.test_tools.Issue` instances
+ """
+
+ issues = {}
+
+ if is_mypy_available():
+ import mypy.api
+
+ ignored_issues = {}
+
+ for line in CONFIG['mypy.ignore']:
+ path, issue = line.split('=>')
+ ignored_issues.setdefault(path.strip(), []).append(issue.strip())
+
+ lines = mypy.api.run(paths)[0].splitlines() # mypy returns (report, errors, exit_status)
+
+ for line in lines:
+ # example:
+ # stem/util/__init__.py:89: error: Incompatible return value type (got "Union[bytes, str]", expected "bytes")
+
+ if line.startswith('Found ') and line.endswith(' source files)'):
+ continue # ex. "Found 1786 errors in 45 files (checked 49 source files)"
+ elif line.count(':') < 3:
+ raise ValueError('Failed to parse mypy line: %s' % line)
+
+ path, line_number, _, issue = line.split(':', 3)
+ issue = issue.strip()
+
+ if line_number.isdigit():
+ line_number = int(line_number)
+ else:
+ raise ValueError('Malformed line number on: %s' % line)
+
+ if _is_ignored(ignored_issues, path, issue):
+ continue
+
+ # skip getting code if there's too many reported issues
+
+ if len(lines) < 25:
+ line = linecache.getline(path, line_number).strip()
+ else:
+ line = ''
+
+ issues.setdefault(path, []).append(Issue(line_number, issue, line))
+
+ return issues
+
+
def _module_exists(module_name: str) -> bool:
"""
Checks if a module exists.
@@ -603,3 +656,20 @@ def _python_files(paths: Sequence[str]) -> Iterator[str]:
if not skip:
yield file_path
+
+
+def _is_ignored(config: Mapping[str, Sequence[str]], path: str, issue: str) -> bool:
+ for ignored_path, ignored_issues in config.items():
+ if path.endswith(ignored_path):
+ if issue in ignored_issues:
+ return True
+
+ for prefix in [i[:1] for i in ignored_issues if i.endswith('*')]:
+ if issue.startswith(prefix):
+ return True
+
+ for suffix in [i[1:] for i in ignored_issues if i.startswith('*')]:
+ if issue.endswith(suffix):
+ return True
+
+ return False
diff --git a/test/task.py b/test/task.py
index 939e263c..2366564c 100644
--- a/test/task.py
+++ b/test/task.py
@@ -16,12 +16,14 @@
|- CRYPTO_VERSION - checks our version of cryptography
|- PYFLAKES_VERSION - checks our version of pyflakes
|- PYCODESTYLE_VERSION - checks our version of pycodestyle
+ |- MYPY_VERSION - checks our version of mypy
|- CLEAN_PYC - removes any *.pyc without a corresponding *.py
|- REMOVE_TOR_DATA_DIR - removes our tor data directory
|- IMPORT_TESTS - ensure all test modules have been imported
|- UNUSED_TESTS - checks to see if any tests are missing from our settings
|- PYFLAKES_TASK - static checks
- +- PYCODESTYLE_TASK - style checks
+ |- PYCODESTYLE_TASK - style checks
+ +- MYPY_TASK - type checks
"""
import importlib
@@ -60,12 +62,12 @@ SRC_PATHS = [os.path.join(test.STEM_BASE, path) for path in (
'cache_fallback_directories.py',
'setup.py',
'tor-prompt',
- os.path.join('docs', 'republish.py'),
os.path.join('docs', 'roles.py'),
)]
PYFLAKES_UNAVAILABLE = 'Static error checking requires pyflakes version 0.7.3 or later. Please install it from ...\n https://pypi.org/project/pyflakes/\n'
PYCODESTYLE_UNAVAILABLE = 'Style checks require pycodestyle version 1.4.2 or later. Please install it from...\n https://pypi.org/project/pycodestyle/\n'
+MYPY_UNAVAILABLE = 'Type checks require mypy. Please install it from...\n http://mypy-lang.org/\n'
def _check_stem_version():
@@ -324,6 +326,7 @@ PLATFORM_VERSION = Task('operating system', _check_platform_version)
CRYPTO_VERSION = ModuleVersion('cryptography version', 'cryptography', lambda: test.require.CRYPTOGRAPHY_AVAILABLE)
PYFLAKES_VERSION = ModuleVersion('pyflakes version', 'pyflakes')
PYCODESTYLE_VERSION = ModuleVersion('pycodestyle version', ['pycodestyle', 'pep8'])
+MYPY_VERSION = ModuleVersion('mypy version', 'mypy.version')
CLEAN_PYC = Task('checking for orphaned .pyc files', _clean_orphaned_pyc, (SRC_PATHS,), print_runtime = True)
REMOVE_TOR_DATA_DIR = Task('emptying our tor data directory', _remove_tor_data_dir)
IMPORT_TESTS = Task('importing test modules', _import_tests, print_runtime = True)
@@ -348,3 +351,11 @@ PYCODESTYLE_TASK = StaticCheckTask(
is_available = stem.util.test_tools.is_pycodestyle_available(),
unavailable_msg = PYCODESTYLE_UNAVAILABLE,
)
+
+MYPY_TASK = StaticCheckTask(
+ 'running mypy',
+ stem.util.test_tools.type_issues,
+ args = ([os.path.join(test.STEM_BASE, 'stem')],),
+ is_available = stem.util.test_tools.is_mypy_available(),
+ unavailable_msg = MYPY_UNAVAILABLE,
+)
More information about the tor-commits
mailing list