commit d0ed7ecd7485bd5cfa587bcc5b35e1783bb03961 Author: Damian Johnson atagar@torproject.org Date: Mon Apr 13 18:10:08 2020 -0700

Mypy static checks

Integrating Mypy (http://mypy-lang.org/) for static type checks. Presently it's reporting hundreds of issues so this will require some more work. --- run_tests.py | 4 +- stem/util/test_tools.py | 112 +++++++++++++++++++++++++++++++++++++++--------- test/task.py | 15 ++++++- 3 files changed, 107 insertions(+), 24 deletions(-)

diff --git a/run_tests.py b/run_tests.py index c9196e18..2ea07dab 100755 --- a/run_tests.py +++ b/run_tests.py @@ -217,12 +217,14 @@ def main(): test.task.CRYPTO_VERSION, test.task.PYFLAKES_VERSION, test.task.PYCODESTYLE_VERSION, + test.task.MYPY_VERSION, test.task.CLEAN_PYC, test.task.UNUSED_TESTS, test.task.IMPORT_TESTS, test.task.REMOVE_TOR_DATA_DIR if args.run_integ else None, test.task.PYFLAKES_TASK if not args.specific_test else None, test.task.PYCODESTYLE_TASK if not args.specific_test else None, + test.task.MYPY_TASK if not args.specific_test else None, )

# Test logging. If '--log-file' is provided we log to that location, @@ -334,7 +336,7 @@ def main():

static_check_issues = {}

- for task in (test.task.PYFLAKES_TASK, test.task.PYCODESTYLE_TASK): + for task in (test.task.PYFLAKES_TASK, test.task.PYCODESTYLE_TASK, test.task.MYPY_TASK): if not task.is_available and task.unavailable_msg: println(task.unavailable_msg, ERROR) else: diff --git a/stem/util/test_tools.py b/stem/util/test_tools.py index d5d0f842..80de447e 100644 --- a/stem/util/test_tools.py +++ b/stem/util/test_tools.py @@ -23,9 +23,11 @@ to match just against the prefix or suffix. For instance...

is_pyflakes_available - checks if pyflakes is available is_pycodestyle_available - checks if pycodestyle is available + is_mypy_available - checks if mypy is available

pyflakes_issues - static checks for problems via pyflakes stylistic_issues - checks for PEP8 and other stylistic issues + type_issues - checks for type problems """

import collections @@ -47,6 +49,7 @@ from typing import Any, Callable, Iterator, Mapping, Optional, Sequence, Tuple, CONFIG = stem.util.conf.config_dict('test', { 'pycodestyle.ignore': [], 'pyflakes.ignore': [], + 'mypy.ignore': [], 'exclude_paths': [], })

@@ -353,6 +356,16 @@ def is_pycodestyle_available() -> bool: return hasattr(pycodestyle, 'BaseReport')

+def is_mypy_available() -> bool: + """ + Checks if mypy is available. + + :returns: **True** if we can use mypy and **False** otherwise + """ + + return _module_exists('mypy.api') + + def stylistic_issues(paths: Sequence[str], check_newlines: bool = False, check_exception_keyword: bool = False, prefer_single_quotes: bool = False) -> Mapping[str, 'stem.util.test_tools.Issue']: """ Checks for stylistic issues that are an issue according to the parts of PEP8 @@ -541,27 +554,8 @@ def pyflakes_issues(paths: Sequence[str]) -> Mapping[str, 'stem.util.test_tools. def flake(self, msg: str) -> None: self._register_issue(msg.filename, msg.lineno, msg.message % msg.message_args, None)

- def _is_ignored(self, path: str, issue: str) -> bool: - # Paths in pyflakes_ignore are relative, so we need to check to see if our - # path ends with any of them. - - for ignored_path, ignored_issues in self._ignored_issues.items(): - if path.endswith(ignored_path): - if issue in ignored_issues: - return True - - for prefix in [i[:1] for i in ignored_issues if i.endswith('*')]: - if issue.startswith(prefix): - return True - - for suffix in [i[1:] for i in ignored_issues if i.startswith('*')]: - if issue.endswith(suffix): - return True - - return False - - def _register_issue(self, path: str, line_number: int, issue: str, line: int) -> None: - if not self._is_ignored(path, issue): + def _register_issue(self, path: str, line_number: int, issue: str, line: str) -> None: + if not _is_ignored(self._ignored_issues, path, issue): if path and line_number and not line: line = linecache.getline(path, line_number).strip()

@@ -575,6 +569,65 @@ def pyflakes_issues(paths: Sequence[str]) -> Mapping[str, 'stem.util.test_tools. return issues

+def type_issues(paths: Sequence[str]) -> Mapping[str, 'stem.util.test_tools.Issue']: + """ + Performs type checks via mypy. False positives can be ignored via + 'mypy.ignore' entries in our 'test' config. For instance... + + :: + + mypy.ignore stem/util/system.py => Incompatible types in assignment* + + :param list paths: paths to search for problems + + :returns: dict of paths list of :class:`stem.util.test_tools.Issue` instances + """ + + issues = {} + + if is_mypy_available(): + import mypy.api + + ignored_issues = {} + + for line in CONFIG['mypy.ignore']: + path, issue = line.split('=>') + ignored_issues.setdefault(path.strip(), []).append(issue.strip()) + + lines = mypy.api.run(paths)[0].splitlines() # mypy returns (report, errors, exit_status) + + for line in lines: + # example: + # stem/util/__init__.py:89: error: Incompatible return value type (got "Union[bytes, str]", expected "bytes") + + if line.startswith('Found ') and line.endswith(' source files)'): + continue # ex. "Found 1786 errors in 45 files (checked 49 source files)" + elif line.count(':') < 3: + raise ValueError('Failed to parse mypy line: %s' % line) + + path, line_number, _, issue = line.split(':', 3) + issue = issue.strip() + + if line_number.isdigit(): + line_number = int(line_number) + else: + raise ValueError('Malformed line number on: %s' % line) + + if _is_ignored(ignored_issues, path, issue): + continue + + # skip getting code if there's too many reported issues + + if len(lines) < 25: + line = linecache.getline(path, line_number).strip() + else: + line = '' + + issues.setdefault(path, []).append(Issue(line_number, issue, line)) + + return issues + + def _module_exists(module_name: str) -> bool: """ Checks if a module exists. @@ -603,3 +656,20 @@ def _python_files(paths: Sequence[str]) -> Iterator[str]:

if not skip: yield file_path + + +def _is_ignored(config: Mapping[str, Sequence[str]], path: str, issue: str) -> bool: + for ignored_path, ignored_issues in config.items(): + if path.endswith(ignored_path): + if issue in ignored_issues: + return True + + for prefix in [i[:1] for i in ignored_issues if i.endswith('*')]: + if issue.startswith(prefix): + return True + + for suffix in [i[1:] for i in ignored_issues if i.startswith('*')]: + if issue.endswith(suffix): + return True + + return False diff --git a/test/task.py b/test/task.py index 939e263c..2366564c 100644 --- a/test/task.py +++ b/test/task.py @@ -16,12 +16,14 @@ |- CRYPTO_VERSION - checks our version of cryptography |- PYFLAKES_VERSION - checks our version of pyflakes |- PYCODESTYLE_VERSION - checks our version of pycodestyle + |- MYPY_VERSION - checks our version of mypy |- CLEAN_PYC - removes any *.pyc without a corresponding *.py |- REMOVE_TOR_DATA_DIR - removes our tor data directory |- IMPORT_TESTS - ensure all test modules have been imported |- UNUSED_TESTS - checks to see if any tests are missing from our settings |- PYFLAKES_TASK - static checks - +- PYCODESTYLE_TASK - style checks + |- PYCODESTYLE_TASK - style checks + +- MYPY_TASK - type checks """

import importlib @@ -60,12 +62,12 @@ SRC_PATHS = [os.path.join(test.STEM_BASE, path) for path in ( 'cache_fallback_directories.py', 'setup.py', 'tor-prompt', - os.path.join('docs', 'republish.py'), os.path.join('docs', 'roles.py'), )]

PYFLAKES_UNAVAILABLE = 'Static error checking requires pyflakes version 0.7.3 or later. Please install it from ...\n https://pypi.org/project/pyflakes/%5Cn' PYCODESTYLE_UNAVAILABLE = 'Style checks require pycodestyle version 1.4.2 or later. Please install it from...\n https://pypi.org/project/pycodestyle/%5Cn' +MYPY_UNAVAILABLE = 'Type checks require mypy. Please install it from...\n http://mypy-lang.org/%5Cn'

def _check_stem_version(): @@ -324,6 +326,7 @@ PLATFORM_VERSION = Task('operating system', _check_platform_version) CRYPTO_VERSION = ModuleVersion('cryptography version', 'cryptography', lambda: test.require.CRYPTOGRAPHY_AVAILABLE) PYFLAKES_VERSION = ModuleVersion('pyflakes version', 'pyflakes') PYCODESTYLE_VERSION = ModuleVersion('pycodestyle version', ['pycodestyle', 'pep8']) +MYPY_VERSION = ModuleVersion('mypy version', 'mypy.version') CLEAN_PYC = Task('checking for orphaned .pyc files', _clean_orphaned_pyc, (SRC_PATHS,), print_runtime = True) REMOVE_TOR_DATA_DIR = Task('emptying our tor data directory', _remove_tor_data_dir) IMPORT_TESTS = Task('importing test modules', _import_tests, print_runtime = True) @@ -348,3 +351,11 @@ PYCODESTYLE_TASK = StaticCheckTask( is_available = stem.util.test_tools.is_pycodestyle_available(), unavailable_msg = PYCODESTYLE_UNAVAILABLE, ) + +MYPY_TASK = StaticCheckTask( + 'running mypy', + stem.util.test_tools.type_issues, + args = ([os.path.join(test.STEM_BASE, 'stem')],), + is_available = stem.util.test_tools.is_mypy_available(), + unavailable_msg = MYPY_UNAVAILABLE, +)