feat: add the type-checking check: dynamically typed code is type checked

README.md

@@ -71,6 +71,7 @@ Exit codes: `0` everything passed, `1` a check failed or raised an error, `2` us
 | `code-formatting` | C/C++, Python, Rust, Go | the formatting style is defined (`.clang-format`; ruff or black settings; `rustfmt.toml`) and CI runs the formatter. Python, Rust and Go (gofmt, gofumpt, goimports) formatters have a default style, so CI enforcement alone passes. Warns when the style is defined but CI does not enforce it |
 | `function-limits` | C/C++, Python, Rust, Go | function length is limited: lizard runs with a length limit (`lizard --length 30`), or for C/C++, `.clang-tidy` enables `readability-function-size` with a threshold and clang-tidy runs, or for Go, golangci-lint runs with `funlen` enabled |
 | `static-analysis` | C/C++, Python, Rust, Go | static analyzers run. C/C++: both clang-tidy (`CMAKE_CXX_CLANG_TIDY`, `run-clang-tidy`, or `clang-tidy` in CI) with its checks chosen in a `.clang-tidy`, and cppcheck (`CMAKE_CXX_CPPCHECK` or `cppcheck` in CI); Python: `ruff check`, pylint or flake8, in CI or as a pre-commit hook; Rust: `cargo clippy`; Go: golangci-lint with linters enabled beyond its defaults (`enable:` or `default: all` in a `.golangci.yml`). Warns when an analyzer is configured but does not run, or runs short of that; fails when none runs |
+| `type-checking` | Python | CI (or a pre-commit hook) runs a static type checker: mypy, pyright or basedpyright, pyrefly, or ty. Strict mode is a bonus (`strict = true` in `[tool.mypy]` or a `[mypy]` section, `mypy --strict`, or pyright's `typeCheckingMode` `strict`); without it the result suggests it. Warns when a type checker is configured but does not run; fails when none runs. Compiled languages, whose compilers check types, are not judged |
 | `action-pinning` | all | every action and reusable workflow CI uses from another repository is pinned to a full commit SHA (`actions/checkout@<SHA> # v4.2.2`), and every `docker://` image to a digest. Fails when one is pinned to a branch or not at all; warns when one is pinned to a tag or an abbreviated commit (`--allow-official-tags` accepts tags for `actions/` and `github/`). Skipped without GitHub Actions |
 | `ci-lint` | all | CI (or a pre-commit hook) lints the configuration of each CI provider it uses: GitHub Actions with actionlint or zizmor, GitLab CI with `glab ci lint` or `gitlab-ci-local --preview`, CircleCI with `circleci config validate`. Warns otherwise; skipped without CI, or when no linter is known for the providers used |
 | `copy-paste-detection` | all | CI runs a duplicate code detector (PMD CPD, jscpd, Duplo, Simian, CCFinderX or SonarQube); warns otherwise |

project_validator/checks/static_analysis.py

@@ -3,31 +3,18 @@
 from project_validator.core import Context, Result
 from project_validator.ecosystems import practice
 from project_validator.rules import Analyzer, StaticAnalysis
-
-
-def _named(analyzer: Analyzer, context: Context) -> str:
-    config = analyzer.configured(context)
-    return f"{analyzer.name} ({config})" if config else analyzer.name
-
-
-def _not_running(analyzer: Analyzer, context: Context) -> str:
-    config = analyzer.configured(context)
-    return f"{analyzer.name} is configured ({config}) but does not run" if config else f"{analyzer.name} does not run"
-
-
-def _joined(names: list[str]) -> str:
-    return names[0] if len(names) == 1 else f"{', '.join(names[:-1])} and {names[-1]}"
+from project_validator.text import joined
 
 
 def _none_running(rule: StaticAnalysis, context: Context) -> Result:
-    if configured := [_not_running(analyzer, context) for analyzer in rule.analyzers if analyzer.configured(context)]:
+    if configured := [analyzer.not_running(context) for analyzer in rule.analyzers if analyzer.configured(context)]:
         return Result.warned("; ".join(configured), rule.hint)
     return Result.failed("no static analyzer runs", rule.hint)
 
 
 def _problems(rule: StaticAnalysis, running: list[Analyzer], context: Context) -> list[str]:
     """Analyzers that are required but do not run, and the shortfalls of those that run."""
-    idle = [_not_running(analyzer, context) for analyzer in rule.analyzers if analyzer not in running]
+    idle = [analyzer.not_running(context) for analyzer in rule.analyzers if analyzer not in running]
     shortfalls = [shortfall for analyzer in running if (shortfall := analyzer.shortfall(context))]
     return [*(idle if rule.all_required else ()), *shortfalls]
 
@@ -39,10 +26,10 @@ def _problems(rule: StaticAnalysis, running: list[Analyzer], context: Context) -
 )
 def static_analysis(rule: StaticAnalysis, context: Context) -> Result:
     texts = rule.texts(context)
-    running = [analyzer for analyzer in rule.analyzers if any(analyzer.runs.search(text) for text in texts)]
+    running = [analyzer for analyzer in rule.analyzers if analyzer.runs_in(texts)]
     if not running:
         return _none_running(rule, context)
-    ran = f"{_joined([_named(analyzer, context) for analyzer in running])} {'runs' if len(running) == 1 else 'run'}"
+    ran = f"{joined([analyzer.named(context) for analyzer in running])} {'runs' if len(running) == 1 else 'run'}"
     if problems := _problems(rule, running, context):
         return Result.warned(f"{ran}, but {'; '.join(problems)}", rule.hint)
     return Result.passed(ran)

project_validator/checks/type_checking.py

@@ -0,0 +1,24 @@
+"""Dynamically typed code is statically type checked, so type errors are found before the code runs."""
+
+from project_validator.core import Context, Result
+from project_validator.ecosystems import practice
+from project_validator.rules import TypeChecking
+from project_validator.text import joined
+
+
+@practice(
+    name="type-checking",
+    description="CI runs a static type checker on dynamically typed code (e.g. mypy, pyright), ideally strictly",
+    rule=TypeChecking,
+)
+def type_checking(rule: TypeChecking, context: Context) -> Result:
+    texts = rule.texts(context)
+    running = [checker for checker in rule.checkers if checker.runs_in(texts)]
+    if not running:
+        if configured := [checker.not_running(context) for checker in rule.checkers if checker.configured(context)]:
+            return Result.warned("; ".join(configured), rule.hint)
+        return Result.failed("no type checker runs", rule.hint)
+    ran = f"{joined([checker.named(context) for checker in running])} {'runs' if len(running) == 1 else 'run'}"
+    if strict := rule.strict(context):
+        return Result.passed(f"{ran}; strict mode bonus: {strict}")
+    return Result.passed(ran, rule.strict_hint)

project_validator/ecosystems/python.py

@@ -1,12 +1,13 @@
 """Python: packaged with pyproject.toml or setup.py, judged by the interpreter versions CI runs."""
 
 import re
+from collections.abc import Callable
 from fnmatch import fnmatch
 from pathlib import PurePosixPath
 
 from project_validator.core import Context, Result
 from project_validator.ecosystems.model import BuildSystem, Ecosystem, ToolchainPatterns, versions_axis
-from project_validator.evidence import ci_and_hook_texts, configuration, existing_texts, first_existing
+from project_validator.evidence import ci_and_hook_texts, configuration, existing_texts, first_existing, toml_value
 from project_validator.rules import (
     MULTI_ECOSYSTEM_AUDITS,
     Analyzer,
@@ -18,6 +19,7 @@
     SecurityLints,
     StaticAnalysis,
     TestSuite,
+    TypeChecking,
     WarningPolicy,
     directories,
     ignored_at_top_level,
@@ -187,6 +189,59 @@ def _section(name: str) -> re.Pattern[str]:
     hint="consider running ruff check (or pylint or flake8) in CI",
 )
 
+# mypy reads its [mypy] section from these files, and [tool.mypy] from pyproject.toml.
+MYPY_INI_FILES = ("mypy.ini", ".mypy.ini", "setup.cfg")
+MYPY_STRICT_SETTING = re.compile(r"^\[mypy\][^\[]*^\s*strict\s*=\s*(?:true|1|yes|on)\b", re.MULTILINE | re.IGNORECASE)
+MYPY_STRICT_FLAG = re.compile(r"\bmypy\b[^\n]*\s--strict\b")
+PYRIGHT_STRICT_MODE = re.compile(r"""\btypeCheckingMode["']?\s*[:=]\s*["'](?:strict|all)["']""")
+
+
+def _strict_type_checking(context: Context) -> str | None:
+    """Where strict type checking is enabled: mypy's strict setting or flag, or pyright's strict mode."""
+    if toml_value(context, "pyproject.toml", "tool", "mypy", "strict") is True:
+        return "mypy strict = true in pyproject.toml"
+    strict_files = (
+        *((path, MYPY_STRICT_SETTING, "mypy strict = true") for path in MYPY_INI_FILES),
+        *((path, PYRIGHT_STRICT_MODE, "pyright typeCheckingMode") for path in ("pyrightconfig.json", "pyproject.toml")),
+    )
+    for path, setting, name in strict_files:
+        if context.project.exists(path) and setting.search(context.project.read_text(path)):
+            return f"{name} in {path}"
+    return "mypy --strict" if any(map(MYPY_STRICT_FLAG.search, ci_and_hook_texts(context))) else None
+
+
+def _tool_configuration(name: str, *files: str) -> Callable[[Context], str | None]:
+    return configuration(files, {"pyproject.toml": _section(f"tool.{name}")})
+
+
+TYPE_CHECKING = TypeChecking(
+    checkers=(
+        Analyzer(
+            name="mypy",
+            runs=re.compile(r"\b(?:d?mypy|mirrors-mypy)\b"),
+            configured=configuration(
+                ("mypy.ini", ".mypy.ini"), {"pyproject.toml": _section("tool.mypy"), "setup.cfg": _section("mypy")}
+            ),
+        ),
+        Analyzer(
+            name="pyright",
+            runs=re.compile(r"\b(?:based)?pyright\b"),
+            configured=configuration(
+                ("pyrightconfig.json",),
+                {"pyproject.toml": re.compile(r"^\s*\[tool\.(?:based)?pyright\b", re.MULTILINE)},
+            ),
+        ),
+        Analyzer(
+            name="pyrefly", runs=re.compile(r"\bpyrefly\b"), configured=_tool_configuration("pyrefly", "pyrefly.toml")
+        ),
+        Analyzer(name="ty", runs=re.compile(r"\bty\s+check\b"), configured=_tool_configuration("ty", "ty.toml")),
+    ),
+    texts=ci_and_hook_texts,
+    strict=_strict_type_checking,
+    hint="consider running a type checker in CI: mypy, pyright (or basedpyright), pyrefly or ty",
+    strict_hint="consider enabling strict mode: strict = true in [tool.mypy], or pyright's typeCheckingMode = strict",
+)
+
 PYTHON = Ecosystem(
     label="Python",
     languages=frozenset({"Python"}),
@@ -213,6 +268,7 @@ def _section(name: str) -> re.Pattern[str]:
         DEPENDENCY_AUDIT,
         FunctionLimits(),
         STATIC_ANALYSIS,
+        TYPE_CHECKING,
         BYTECODE,
         LICENSE,
     ),

project_validator/rules.py

@@ -181,6 +181,18 @@ class Analyzer:
     configured: Callable[[Context], str | None] = _nothing
     shortfall: Callable[[Context], str | None] = _nothing
 
+    def runs_in(self, texts: Iterable[str]) -> bool:
+        return any(self.runs.search(text) for text in texts)
+
+    def named(self, context: Context) -> str:
+        """Its name, and the file that configures it."""
+        config = self.configured(context)
+        return f"{self.name} ({config})" if config else self.name
+
+    def not_running(self, context: Context) -> str:
+        config = self.configured(context)
+        return f"{self.name} is configured ({config}) but does not run" if config else f"{self.name} does not run"
+
 
 @dataclass(frozen=True, slots=True, kw_only=True)
 class StaticAnalysis:
@@ -195,6 +207,20 @@ class StaticAnalysis:
     all_required: bool = False
 
 
+@dataclass(frozen=True, slots=True, kw_only=True)
+class TypeChecking:
+    """The static type checkers of a dynamically typed ecosystem, and where its strict mode, a bonus, is enabled.
+
+    `strict` names where strict checking is enabled, if anywhere.
+    """
+
+    checkers: tuple[Analyzer, ...]
+    texts: Evidence
+    strict: Callable[[Context], str | None]
+    hint: str
+    strict_hint: str
+
+
 # Scanners that audit the dependencies of many ecosystems at once, from their manifests and lockfiles.
 MULTI_ECOSYSTEM_AUDITS = (
     pattern("osv-scanner", r"\bosv-scanner\b"),

project_validator/text.py

@@ -22,6 +22,11 @@ def matching_names(texts: Iterable[str], patterns: Iterable[Pattern]) -> list[st
     return list(dict.fromkeys(found))
 
 
+def joined(names: list[str]) -> str:
+    """`names` as a sentence lists them: "a", "a and b", "a, b and c"."""
+    return names[0] if len(names) == 1 else f"{', '.join(names[:-1])} and {names[-1]}"
+
+
 def found_and_missing(texts: Iterable[str], patterns: Iterable[Pattern]) -> tuple[list[str], list[str]]:
     """Names of the patterns found in any text, and of those found in none, both in pattern order."""
     texts = tuple(texts)

tests/checks/test_type_checking.py

@@ -0,0 +1,105 @@
+import unittest
+
+from project_validator.checks.type_checking import type_checking
+from project_validator.core import Result, Status
+from project_validator.ecosystems import C_AND_CXX, GO, PYTHON, RUST
+from project_validator.ecosystems.python import TYPE_CHECKING
+from tests.support import FIXTURES, WELL_CONFIGURED_PYTHON, github_job, make_context, run_as
+
+HINT = TYPE_CHECKING.hint
+STRICT_HINT = TYPE_CHECKING.strict_hint
+
+
+def _ci(*commands: str) -> dict[str, str]:
+    return github_job("ubuntu-latest", *commands)
+
+
+class TypeCheckingTests(unittest.TestCase):
+    def _run(self, files: dict[str, str]) -> Result:
+        return run_as(self, type_checking, files, PYTHON)
+
+    def test_applies_to_python_only_of_the_built_in_ecosystems(self) -> None:
+        self.assertIn("Python", type_checking.languages)
+        for ecosystem in (C_AND_CXX, RUST, GO):
+            with self.subTest(ecosystem.label):
+                self.assertFalse(ecosystem.languages & type_checking.languages)
+
+    def test_the_well_configured_project_passes_with_the_bonus(self) -> None:
+        expected = Result.passed("mypy (pyproject.toml) runs; strict mode bonus: mypy strict = true in pyproject.toml")
+        self.assertEqual(self._run(WELL_CONFIGURED_PYTHON), expected)
+
+    def test_compiled_ecosystems_are_skipped(self) -> None:
+        self.assertEqual(type_checking.run(make_context(FIXTURES / "well_configured")).status, Status.SKIP)
+
+    def test_fails_when_no_type_checker_runs(self) -> None:
+        self.assertEqual(self._run(_ci("pytest")), Result.failed("no type checker runs", HINT))
+
+    def test_any_type_checker_passes_and_suggests_strict_mode(self) -> None:
+        cases = {
+            "mypy src": "mypy",
+            "basedpyright": "pyright",
+            "npx pyright": "pyright",
+            "pyrefly check": "pyrefly",
+            "uvx ty check": "ty",
+        }
+        for command, checker in cases.items():
+            with self.subTest(command):
+                self.assertEqual(self._run(_ci(command)), Result.passed(f"{checker} runs", STRICT_HINT))
+
+    def test_ty_must_be_checking(self) -> None:
+        self.assertEqual(self._run(_ci("pip install ty")).status, Status.FAIL)
+
+    def test_a_pre_commit_hook_runs_mypy(self) -> None:
+        hooks = {".pre-commit-config.yaml": "repos:\n  - repo: https://github.com/pre-commit/mirrors-mypy\n"}
+        self.assertEqual(self._run(hooks), Result.passed("mypy runs", STRICT_HINT))
+
+    def test_names_every_checker_that_runs(self) -> None:
+        self.assertEqual(self._run(_ci("mypy .", "pyright")), Result.passed("mypy and pyright run", STRICT_HINT))
+
+    def test_warns_when_a_checker_is_configured_but_not_run(self) -> None:
+        cases = {
+            "mypy.ini": ("[mypy]\n", "mypy"),
+            "setup.cfg": ("[mypy]\nwarn_unused_ignores = True\n", "mypy"),
+            "pyrightconfig.json": ("{}", "pyright"),
+            "pyrefly.toml": ("", "pyrefly"),
+            "ty.toml": ("", "ty"),
+        }
+        for path, (text, checker) in cases.items():
+            with self.subTest(path):
+                expected = Result.warned(f"{checker} is configured ({path}) but does not run", HINT)
+                self.assertEqual(self._run({path: text}), expected)
+
+    def test_pyproject_sections_configure_checkers(self) -> None:
+        pyproject = "[project]\nname = 'x'\n\n[tool.basedpyright]\n\n[tool.ty.rules]\n"
+        expected = Result.warned(
+            "pyright is configured (pyproject.toml) but does not run; "
+            "ty is configured (pyproject.toml) but does not run",
+            HINT,
+        )
+        self.assertEqual(self._run({"pyproject.toml": pyproject}), expected)
+
+    def test_strict_mode_is_found_where_it_is_enabled(self) -> None:
+        cases = {
+            "mypy strict = true in pyproject.toml": {"pyproject.toml": "[tool.mypy]\nstrict = true\n"},
+            "mypy strict = true in mypy.ini": {"mypy.ini": "[mypy]\npython_version = 3.13\nstrict = True\n"},
+            "mypy strict = true in setup.cfg": {"setup.cfg": "[mypy]\nstrict = 1\n"},
+            "pyright typeCheckingMode in pyrightconfig.json": {"pyrightconfig.json": '{"typeCheckingMode": "strict"}'},
+            "pyright typeCheckingMode in pyproject.toml": {
+                "pyproject.toml": '[tool.basedpyright]\ntypeCheckingMode = "all"\n'
+            },
+            "mypy --strict": {},
+        }
+        for where, files in cases.items():
+            with self.subTest(where):
+                result = self._run({**files, **_ci("mypy --strict src")})
+                self.assertEqual(result.status, Status.PASS)
+                self.assertTrue(result.message.endswith(f"; strict mode bonus: {where}"), result.message)
+
+    def test_strict_settings_elsewhere_are_not_strict_mode(self) -> None:
+        files = {
+            "pyproject.toml": "[tool.mypy]\nstrict = false\n",
+            "setup.cfg": "[mypy]\nwarn_unused_ignores = True\n\n[other]\nstrict = true\n",
+            "pyrightconfig.json": '{"typeCheckingMode": "basic"}',
+        }
+        result = self._run({**files, **_ci("mypy src")})
+        self.assertEqual(result.details, (STRICT_HINT,))

tests/fixtures/well_configured_python/.github/workflows/ci.yml

@@ -15,5 +15,6 @@ jobs:
       - run: pip-audit -r requirements-dev.txt
       - run: ruff format --check
       - run: ruff check
+      - run: mypy src
       - run: lizard --length 30 --warnings_only .
       - run: python -W error -m coverage run -m pytest

tests/fixtures/well_configured_python/pyproject.toml

@@ -4,3 +4,6 @@ license = 'MIT'
 
 [tool.coverage.run]
 branch = true
+
+[tool.mypy]
+strict = true

tests/snapshots/bare_go.json

@@ -172,6 +172,11 @@
       "message": "no Go tests found (Test functions in *_test.go files)",
       "status": "FAIL"
     },
+    "type-checking": {
+      "details": [],
+      "message": "not applicable to Go (applies to Python)",
+      "status": "SKIP"
+    },
     "vcs-ignore": {
       "details": [
         "consider adding a .gitignore (or your version control system's equivalent) that ignores build outputs"

tests/snapshots/bare_project.json

@@ -162,6 +162,11 @@
       "message": "CTest is never enabled; call enable_testing() or include(CTest)",
       "status": "FAIL"
     },
+    "type-checking": {
+      "details": [],
+      "message": "not applicable to C, C++ (applies to Python)",
+      "status": "SKIP"
+    },
     "vcs-ignore": {
       "details": [
         "consider adding a .gitignore (or your version control system's equivalent) that ignores build outputs"

tests/snapshots/bare_python.json

@@ -165,6 +165,13 @@
       "message": "no Python tests found (test_*.py or *_test.py files)",
       "status": "FAIL"
     },
+    "type-checking": {
+      "details": [
+        "consider running a type checker in CI: mypy, pyright (or basedpyright), pyrefly or ty"
+      ],
+      "message": "no type checker runs",
+      "status": "FAIL"
+    },
     "vcs-ignore": {
       "details": [
         "consider adding a .gitignore (or your version control system's equivalent) that ignores build outputs"

tests/snapshots/bare_rust.json

@@ -165,6 +165,11 @@
       "message": "no Rust tests found (#[test] functions)",
       "status": "FAIL"
     },
+    "type-checking": {
+      "details": [],
+      "message": "not applicable to Rust (applies to Python)",
+      "status": "SKIP"
+    },
     "vcs-ignore": {
       "details": [
         "consider adding a .gitignore (or your version control system's equivalent) that ignores build outputs"