From 5984d874de126864da1419327fa558db6461c669 Mon Sep 17 00:00:00 2001 From: leftibot Date: Thu, 5 Nov 2026 23:24:17 -0700 Subject: [PATCH 1/6] feat: add a JavaScript/TypeScript ecosystem profile npm-compatible (package.json) and Deno (deno.json) projects are judged by prettier, biome, deno fmt or dprint; branch coverage from c8, nyc, jest, vitest, deno test or node's test runner; *.test.*, *.spec.* and __tests__/ files run by any common runner; eslint, biome or deno lint as the linter, failing on warnings with --max-warnings 0; npm, pnpm, yarn or bun audits; eslint-plugin-security or semgrep; eslint's max-lines-per-function; tsc (strict in tsconfig.json) or deno check; committed lockfiles; package.json's license; an ignored node_modules/; and the Node.js, Deno and Bun versions CI runs. A package.json alone is the tooling of many projects in other languages, so it makes a JavaScript project only when it names entry points (main, module, exports, bin); JavaScript and TypeScript sources do too. JavaScript tooling mostly runs through package scripts, so the scripts and Deno tasks that CI or a pre-commit hook runs (npm run lint, pnpm lint, deno task fmt, with their pre/post scripts and the scripts they run) count as CI configuration. For that, the Formatter rule gets an optional texts, CI configuration by default, which code-formatting reads; no other check changes. The JSON-with-comments reader moves from data-wellformed to text.py, so profiles can read tsconfig.json and deno.jsonc with evidence.json_value. Closes #65 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ByTXNqDWdM7HzezrFtEvQt --- README.md | 34 +- project_validator/checks/code_formatting.py | 2 +- project_validator/checks/data_wellformed.py | 18 +- project_validator/ecosystems/__init__.py | 4 +- project_validator/ecosystems/javascript.py | 414 ++++++++++++++++++ project_validator/evidence.py | 30 +- project_validator/rules.py | 9 +- project_validator/text.py | 14 + tests/checks/test_warnings_as_errors.py | 5 +- .../bare_javascript/coverage/lcov.info | 2 + tests/fixtures/bare_javascript/package.json | 8 + tests/fixtures/bare_javascript/src/index.js | 3 + .../.github/dependabot.yml | 6 + .../.github/workflows/ci-lint.yml | 9 + .../.github/workflows/ci.yml | 18 + .../workflows/copy-paste-detection.yml | 6 + .../.github/workflows/function-limits.yml | 7 + .../well_configured_javascript/.gitignore | 3 + .../well_configured_javascript/LICENSE | 21 + .../well_configured_javascript/README.md | 4 + .../well_configured_javascript/SECURITY.md | 3 + .../eslint.config.js | 3 + .../package-lock.json | 13 + .../well_configured_javascript/package.json | 22 + .../src/add.test.ts | 7 + .../well_configured_javascript/src/add.ts | 4 + .../well_configured_javascript/tsconfig.json | 10 + tests/snapshots/bare_go.json | 4 +- tests/snapshots/bare_javascript.json | 202 +++++++++ tests/snapshots/bare_project.json | 4 +- tests/snapshots/bare_python.json | 2 +- tests/snapshots/bare_rust.json | 4 +- tests/snapshots/buildable.json | 4 +- tests/snapshots/well_configured.json | 4 +- tests/snapshots/well_configured_go.json | 4 +- .../snapshots/well_configured_javascript.json | 311 +++++++++++++ tests/snapshots/well_configured_python.json | 2 +- tests/snapshots/well_configured_rust.json | 4 +- tests/support.py | 13 + tests/test_cli.py | 15 +- tests/test_ecosystem_contract.py | 12 +- tests/test_javascript.py | 305 +++++++++++++ tests/test_plugins.py | 10 +- 43 files changed, 1513 insertions(+), 66 deletions(-) create mode 100644 project_validator/ecosystems/javascript.py create mode 100644 tests/fixtures/bare_javascript/coverage/lcov.info create mode 100644 tests/fixtures/bare_javascript/package.json create mode 100644 tests/fixtures/bare_javascript/src/index.js create mode 100644 tests/fixtures/well_configured_javascript/.github/dependabot.yml create mode 100644 tests/fixtures/well_configured_javascript/.github/workflows/ci-lint.yml create mode 100644 tests/fixtures/well_configured_javascript/.github/workflows/ci.yml create mode 100644 tests/fixtures/well_configured_javascript/.github/workflows/copy-paste-detection.yml create mode 100644 tests/fixtures/well_configured_javascript/.github/workflows/function-limits.yml create mode 100644 tests/fixtures/well_configured_javascript/.gitignore create mode 100644 tests/fixtures/well_configured_javascript/LICENSE create mode 100644 tests/fixtures/well_configured_javascript/README.md create mode 100644 tests/fixtures/well_configured_javascript/SECURITY.md create mode 100644 tests/fixtures/well_configured_javascript/eslint.config.js create mode 100644 tests/fixtures/well_configured_javascript/package-lock.json create mode 100644 tests/fixtures/well_configured_javascript/package.json create mode 100644 tests/fixtures/well_configured_javascript/src/add.test.ts create mode 100644 tests/fixtures/well_configured_javascript/src/add.ts create mode 100644 tests/fixtures/well_configured_javascript/tsconfig.json create mode 100644 tests/snapshots/bare_javascript.json create mode 100644 tests/snapshots/well_configured_javascript.json create mode 100644 tests/test_javascript.py diff --git a/README.md b/README.md index ceeda93..6ae46a3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A tool to validate a project's build, CI and tooling against best practices. It recognises C and C++ (CMake or Meson), Python (`pyproject.toml` or `setup.py`), -Rust (Cargo) and Go (Go modules) projects, and judges each language by its own ecosystem's tools: a C++ +Rust (Cargo), Go (Go modules) and JavaScript/TypeScript (`package.json` or `deno.json`) projects, and judges each language by its own ecosystem's tools: a C++ project needs branch coverage from gcovr or llvm-cov, a Python project from coverage.py, a Rust project from cargo-llvm-cov. Checks that do not apply to any of the project's languages are reported as `SKIP`. A project using several languages is judged for each @@ -60,33 +60,33 @@ Exit codes: `0` everything passed, `1` a check failed or raised an error, `2` us | Check | Applies to | Passes when | |---|---|---| | `ci-present` | all | at least one CI configuration is found | -| `tests-run` | C/C++, Python, Rust, Go | tests exist and CI runs them (`ctest-runs` is accepted as an old name). C/C++ with CMake: `enable_testing()` or `include(CTest)` is called, tests are registered (`add_test`, `*_discover_tests`, ...) and CI runs `ctest`; with `--build`, the project is really configured and built, and `ctest` must list at least one test. C/C++ with Meson: `test()` is called in a `meson.build` and CI runs `meson test`. Python: `test_*.py` / `*_test.py` files run by pytest, unittest, tox or nox. Rust: `#[test]` functions run by `cargo test` or nextest. Go: `Test` functions in `*_test.go` files run by `go test`. Warns when CI never runs the tests | -| `coverage` | C/C++, Python, Rust, Go | CI reports branch coverage (Go's tools measure statements only, so Go needs statement coverage: `go test -cover` or `-coverprofile`). C/C++: gcovr, lcov, genhtml, gcov or llvm-cov branch options; Python: `coverage run --branch`, `--cov-branch` or `branch = true`; Rust: cargo-llvm-cov, cargo-tarpaulin or grcov with `--branch`. Path or MC/DC coverage (`-fcondition-coverage`, `-fcoverage-mcdc`, `--mcdc`, ...) is reported as a bonus | -| `platform-matrix` | C/C++, Python, Rust, Go | CI covers at least 2 operating systems, 2 toolchains (compilers, Python versions, Rust toolchains or Go versions) and 3 OS/toolchain combinations | +| `tests-run` | C/C++, Python, Rust, Go, JavaScript/TypeScript | tests exist and CI runs them (`ctest-runs` is accepted as an old name). C/C++ with CMake: `enable_testing()` or `include(CTest)` is called, tests are registered (`add_test`, `*_discover_tests`, ...) and CI runs `ctest`; with `--build`, the project is really configured and built, and `ctest` must list at least one test. C/C++ with Meson: `test()` is called in a `meson.build` and CI runs `meson test`. Python: `test_*.py` / `*_test.py` files run by pytest, unittest, tox or nox. Rust: `#[test]` functions run by `cargo test` or nextest. Go: `Test` functions in `*_test.go` files run by `go test`. JavaScript/TypeScript: `*.test.*`, `*.spec.*`, `*_test.*` or `__tests__/` files run by jest, vitest, mocha, ava, `node --test`, `deno test`, `bun test`, playwright or `npm test`. Warns when CI never runs the tests | +| `coverage` | C/C++, Python, Rust, Go, JavaScript/TypeScript | CI reports branch coverage (Go's tools measure statements only, so Go needs statement coverage: `go test -cover` or `-coverprofile`). C/C++: gcovr, lcov, genhtml, gcov or llvm-cov branch options; Python: `coverage run --branch`, `--cov-branch` or `branch = true`; Rust: cargo-llvm-cov, cargo-tarpaulin or grcov with `--branch`; JavaScript/TypeScript: c8, nyc, `jest --coverage` (or `collectCoverage: true`), `vitest --coverage` (or coverage `enabled: true`), `deno test --coverage` or `node --experimental-test-coverage`, in CI, the package scripts it runs, or the test runner's configuration. Path or MC/DC coverage (`-fcondition-coverage`, `-fcoverage-mcdc`, `--mcdc`, ...) is reported as a bonus | +| `platform-matrix` | C/C++, Python, Rust, Go, JavaScript/TypeScript | CI covers at least 2 operating systems, 2 toolchains (compilers, Python versions, Rust toolchains, Go versions, or Node.js, Deno or Bun versions) and 3 OS/toolchain combinations | | `ci-build-types` | C/C++, Rust | every OS/toolchain combination in CI builds Debug and Release (Rust: debug and release profiles), each with and without sanitizers | -| `compiler-warnings` | C/C++, Python, Rust, Go | a strict warning set is enabled. C/C++: `-Wall -Wextra -Wpedantic -Wshadow -Wconversion`, plus `/W4` and `/permissive-` when MSVC is in CI; Python: warnings are errors (`-W error`, `PYTHONWARNINGS=error`, pytest `filterwarnings = error`); Rust: `cargo clippy`, `-D warnings` and `clippy::pedantic`; Go: `go vet` and staticcheck (golangci-lint runs both). Warns listing what is missing | -| `warnings-as-errors` | C/C++ | `-Werror`, `/WX` or `COMPILE_WARNING_AS_ERROR` (Meson: `werror=true`) is enabled in at least one configuration | +| `compiler-warnings` | C/C++, Python, Rust, Go, JavaScript/TypeScript | a strict warning set is enabled. C/C++: `-Wall -Wextra -Wpedantic -Wshadow -Wconversion`, plus `/W4` and `/permissive-` when MSVC is in CI; Python: warnings are errors (`-W error`, `PYTHONWARNINGS=error`, pytest `filterwarnings = error`); Rust: `cargo clippy`, `-D warnings` and `clippy::pedantic`; Go: `go vet` and staticcheck (golangci-lint runs both); JavaScript/TypeScript: a linter runs (eslint, `biome lint` / `biome ci`, or `deno lint`). Warns listing what is missing | +| `warnings-as-errors` | C/C++, JavaScript/TypeScript | C/C++: `-Werror`, `/WX` or `COMPILE_WARNING_AS_ERROR` (Meson: `werror=true`) is enabled in at least one configuration; JavaScript/TypeScript: a lint warning fails CI (`eslint --max-warnings 0`, `biome ci` or `--error-on-warnings`, or `deno lint`) | | `cxx-extensions` | C/C++ | `CMAKE_CXX_EXTENSIONS` (or `CMAKE_C_EXTENSIONS` for C) is turned off, or with Meson `cpp_std` (`c_std`) names a standard such as `c++20` rather than `gnu++20`, so `-std=c++NN` is used rather than `-std=gnu++NN` | -| `hardening` | C/C++, Rust, Go | binary hardening is enabled: exploit mitigations that the build inserts into the binary (static analysis and dependency audits are judged by `security-scanning` and `dependency-audit`). C/C++: `_FORTIFY_SOURCE`, `-fstack-protector-strong`, `-fcf-protection`, `-fstack-clash-protection` and `_GLIBCXX_ASSERTIONS` or `_LIBCPP_HARDENING_MODE` (or GCC's `-fhardened`), plus `/sdl /guard:cf /DYNAMICBASE /CETCOMPAT` when MSVC is in CI; Rust: `overflow-checks` in release builds; Go: programs are built as position-independent executables (`-buildmode=pie`, not the default on Linux), and a module with no `package main` is skipped. Python builds no binary, so it is not judged. Warns listing what is missing | -| `code-formatting` | C/C++, Python, Rust, Go | the formatting style is defined (`.clang-format`; ruff or black settings; `rustfmt.toml`) and CI runs the formatter. Python, Rust and Go (gofmt, gofumpt, goimports) formatters have a default style, so CI enforcement alone passes. Warns when the style is defined but CI does not enforce it | -| `function-limits` | C/C++, Python, Rust, Go | function length is limited: lizard runs with a length limit (`lizard --length 30`), or for C/C++, `.clang-tidy` enables `readability-function-size` with a threshold and clang-tidy runs, or for Go, golangci-lint runs with `funlen` enabled | -| `static-analysis` | C/C++, Python, Rust, Go | static analyzers run. C/C++: both clang-tidy (`CMAKE_CXX_CLANG_TIDY`, `run-clang-tidy`, or `clang-tidy` in CI) with its checks chosen in a `.clang-tidy`, and cppcheck (`CMAKE_CXX_CPPCHECK` or `cppcheck` in CI); Python: `ruff check`, pylint or flake8, in CI or as a pre-commit hook; Rust: `cargo clippy`; Go: golangci-lint with linters enabled beyond its defaults (`enable:` or `default: all` in a `.golangci.yml`). Warns when an analyzer is configured but does not run, or runs short of that; fails when none runs | -| `type-checking` | Python | CI (or a pre-commit hook) runs a static type checker: mypy, pyright or basedpyright, pyrefly, or ty. Strict mode is a bonus (`strict = true` in `[tool.mypy]` or a `[mypy]` section, `mypy --strict`, or pyright's `typeCheckingMode` `strict`); without it the result suggests it. Warns when a type checker is configured but does not run; fails when none runs. Compiled languages, whose compilers check types, are not judged | +| `hardening` | C/C++, Rust, Go | binary hardening is enabled: exploit mitigations that the build inserts into the binary (static analysis and dependency audits are judged by `security-scanning` and `dependency-audit`). C/C++: `_FORTIFY_SOURCE`, `-fstack-protector-strong`, `-fcf-protection`, `-fstack-clash-protection` and `_GLIBCXX_ASSERTIONS` or `_LIBCPP_HARDENING_MODE` (or GCC's `-fhardened`), plus `/sdl /guard:cf /DYNAMICBASE /CETCOMPAT` when MSVC is in CI; Rust: `overflow-checks` in release builds; Go: programs are built as position-independent executables (`-buildmode=pie`, not the default on Linux), and a module with no `package main` is skipped. Python and JavaScript/TypeScript build no binary, so they are not judged. Warns listing what is missing | +| `code-formatting` | C/C++, Python, Rust, Go, JavaScript/TypeScript | the formatting style is defined (`.clang-format`; ruff or black settings; `rustfmt.toml`) and CI runs the formatter. Python, Rust, Go (gofmt, gofumpt, goimports) and JavaScript/TypeScript (`prettier --check`, `biome format` / `biome ci`, `deno fmt --check` or `dprint check`, in CI or the package scripts it runs) formatters have a default style, so CI enforcement alone passes. Warns when the style is defined but CI does not enforce it | +| `function-limits` | C/C++, Python, Rust, Go, JavaScript/TypeScript | function length is limited: lizard runs with a length limit (`lizard --length 30`), or for C/C++, `.clang-tidy` enables `readability-function-size` with a threshold and clang-tidy runs, or for Go, golangci-lint runs with `funlen` enabled, or for JavaScript/TypeScript, eslint runs with `max-lines-per-function` enabled | +| `static-analysis` | C/C++, Python, Rust, Go, JavaScript/TypeScript | static analyzers run. C/C++: both clang-tidy (`CMAKE_CXX_CLANG_TIDY`, `run-clang-tidy`, or `clang-tidy` in CI) with its checks chosen in a `.clang-tidy`, and cppcheck (`CMAKE_CXX_CPPCHECK` or `cppcheck` in CI); Python: `ruff check`, pylint or flake8, in CI or as a pre-commit hook; Rust: `cargo clippy`; Go: golangci-lint with linters enabled beyond its defaults (`enable:` or `default: all` in a `.golangci.yml`); JavaScript/TypeScript: eslint, `biome lint` or `deno lint`, in CI, a pre-commit hook or the package scripts CI runs. Warns when an analyzer is configured but does not run, or runs short of that; fails when none runs | +| `type-checking` | Python, JavaScript/TypeScript | CI (or a pre-commit hook) runs a static type checker. Python: mypy, pyright or basedpyright, pyrefly, or ty; JavaScript/TypeScript: `tsc` (JavaScript with `checkJs`), or `deno check` / `deno test`, also through the package scripts CI runs. Strict mode is a bonus (`strict = true` in `[tool.mypy]` or a `[mypy]` section, `mypy --strict`, pyright's `typeCheckingMode` `strict`, `"strict": true` or `@tsconfig/strictest` in `tsconfig.json`, or Deno, strict by default); without it the result suggests it. Warns when a type checker is configured but does not run; fails when none runs. Compiled languages, whose compilers check types, are not judged | | `action-pinning` | all | every action and reusable workflow CI uses from another repository is pinned to a full commit SHA (`actions/checkout@ # v4.2.2`), and every `docker://` image to a digest. Fails when one is pinned to a branch or not at all; warns when one is pinned to a tag or an abbreviated commit (`--allow-official-tags` accepts tags for `actions/` and `github/`). Skipped without GitHub Actions | | `ci-lint` | all | CI (or a pre-commit hook) lints the configuration of each CI provider it uses: GitHub Actions with actionlint or zizmor, GitLab CI with `glab ci lint` or `gitlab-ci-local --preview`, CircleCI with `circleci config validate`. Warns otherwise; skipped without CI, or when no linter is known for the providers used | | `copy-paste-detection` | all | CI runs a duplicate code detector (PMD CPD, jscpd, Duplo, Simian, CCFinderX or SonarQube); warns otherwise | -| `security-scanning` | all | CI runs code security scanning (CodeQL or GitLab SAST), or every language in the project that has security lints enables them: Python: bandit (or ruff's `S` rules); Rust: `unsafe_code` forbidden crate-wide; Go: gosec. Warns otherwise, naming the languages left unscanned | +| `security-scanning` | all | CI runs code security scanning (CodeQL or GitLab SAST), or every language in the project that has security lints enables them: Python: bandit (or ruff's `S` rules); Rust: `unsafe_code` forbidden crate-wide; Go: gosec; JavaScript/TypeScript: `eslint-plugin-security` (or `@microsoft/eslint-plugin-sdl`), or semgrep with a JavaScript ruleset. Warns otherwise, naming the languages left unscanned | | `secret-scanning` | all | CI or a pre-commit hook runs a secret scanner: gitleaks, TruffleHog, detect-secrets, ggshield or Nosey Parker. Warns otherwise, and says so when a scanner is configured (`.gitleaks.toml`, `.secrets.baseline`, `.gitguardian.yaml`) but nothing runs it. GitHub's own secret scanning is a repository setting, which cannot be seen from a checkout | -| `dependency-management` | C/C++, Python, Rust, Go | dependencies come from a package manager and are locked. C/C++: CPM, FetchContent, vcpkg (`vcpkg.json`), Conan (`conanfile.txt` / `conanfile.py`) or Meson wraps (`subprojects/*.wrap`); Python: dependencies declared in `pyproject.toml`, `setup.py` or requirements files are locked by a committed `uv.lock`, `poetry.lock`, `pdm.lock`, `Pipfile.lock` or `pylock.toml`, or by requirements pinned with `--hash`; Rust: a binary crate commits `Cargo.lock` (a library need not); Go: a module that requires others commits `go.sum`. Passes with a note when no dependencies are declared. Warns when they are not locked, or when third-party code is committed (`third_party/`, `external/`, `extern/`, `vendor/` other than `go mod vendor` output, or git submodules) | -| `dependency-audit` | Python, Rust, Go | CI checks the dependencies against known vulnerabilities. Python: pip-audit or safety; Rust: cargo audit or `cargo deny check` (advisories); Go: govulncheck. osv-scanner, `trivy fs` and `snyk test` count for every language. Warns otherwise | +| `dependency-management` | C/C++, Python, Rust, Go, JavaScript/TypeScript | dependencies come from a package manager and are locked. C/C++: CPM, FetchContent, vcpkg (`vcpkg.json`), Conan (`conanfile.txt` / `conanfile.py`) or Meson wraps (`subprojects/*.wrap`); Python: dependencies declared in `pyproject.toml`, `setup.py` or requirements files are locked by a committed `uv.lock`, `poetry.lock`, `pdm.lock`, `Pipfile.lock` or `pylock.toml`, or by requirements pinned with `--hash`; Rust: a binary crate commits `Cargo.lock` (a library need not); Go: a module that requires others commits `go.sum`; JavaScript/TypeScript: dependencies declared in `package.json` (or `imports` in `deno.json`) are locked by a committed `package-lock.json`, `npm-shrinkwrap.json`, `pnpm-lock.yaml`, `yarn.lock`, `bun.lock`, `bun.lockb` or `deno.lock`, libraries included. Passes with a note when no dependencies are declared. Warns when they are not locked, or when third-party code is committed (`third_party/`, `external/`, `extern/`, `vendor/` other than `go mod vendor` output, or git submodules) | +| `dependency-audit` | Python, Rust, Go, JavaScript/TypeScript | CI checks the dependencies against known vulnerabilities. Python: pip-audit or safety; Rust: cargo audit or `cargo deny check` (advisories); Go: govulncheck; JavaScript/TypeScript: `npm audit` (not `npm audit signatures`), `pnpm audit`, `yarn npm audit` / `yarn audit` or `bun audit`. osv-scanner, `trivy fs` and `snyk test` count for every language. Warns otherwise | | `data-wellformed` | all | every JSON, TOML, YAML and XML file parses, including the configuration in `.github/`, `.gitlab/`, `.circleci/`, `.buildkite/`, `.devcontainer/` and `.vscode/`. JSON that tools read with comments (`*.jsonc`, `tsconfig*.json`, `jsconfig.json`, `devcontainer.json`, `.vscode/*.json`) may have comments and trailing commas; YAML tags of any tool (GitLab's `!reference`) are accepted; XML external entities are never read. Files over 5 MiB are not parsed. Fails listing each file that does not parse, with the line and column | | `large-files` | all | no committed file is larger than 5 MiB (`--max-file-size`) unless Git LFS stores it (`filter=lfs` in a `.gitattributes`) or DVC tracks it (a `.dvc` file beside it). In a git checkout only the files git tracks are judged. Warns listing the largest files | | `security-policy` | all | a `SECURITY.md` (or `.rst`, `.txt`, ...) at the root, in `.github/` or in `docs/` says how to report a vulnerability: an email address, a link, or the forge's private vulnerability reporting. Warns otherwise | -| `license` | all | a license file (`LICENSE`, `LICENSE.md`, `LICENSE-MIT`, `COPYING`, `UNLICENSE`, ... or REUSE's `LICENSES/` directory) holds a recognised license: an `SPDX-License-Identifier` line, or the standard text of AGPL, LGPL, GPL, Apache-2.0, MPL-2.0, EPL-2.0, BSL-1.0, Unlicense, CC0-1.0, MIT, BSD, ISC, 0BSD or Zlib. Warns when the license is not recognised, or when a manifest declares another one (Python: `[project] license`; Rust: `[package] license`). Fails without a license file | +| `license` | all | a license file (`LICENSE`, `LICENSE.md`, `LICENSE-MIT`, `COPYING`, `UNLICENSE`, ... or REUSE's `LICENSES/` directory) holds a recognised license: an `SPDX-License-Identifier` line, or the standard text of AGPL, LGPL, GPL, Apache-2.0, MPL-2.0, EPL-2.0, BSL-1.0, Unlicense, CC0-1.0, MIT, BSD, ISC, 0BSD or Zlib. Warns when the license is not recognised, or when a manifest declares another one (Python: `[project] license`; Rust: `[package] license`; JavaScript/TypeScript: `license` in `package.json`). Fails without a license file | | `dependency-updates` | all | Dependabot or Renovate is configured; warns otherwise | | `status-badges` | all | the README (at the root, in `.github/` or in `docs/`) shows a CI status badge of any supported CI provider (GitHub Actions `badge.svg`, GitLab pipeline, CircleCI, ... or their shields.io badges) and a quality badge (Codecov, Coveralls, SonarCloud, Codacy, Code Climate, DeepSource, Coverity Scan, OpenSSF Scorecard or Best Practices). Warns when only one kind is shown; fails when there is no README or no badge | -| `committed-artifacts` | C/C++, Python, Rust, Go | no build output is committed. C/C++: object files and libraries (`*.o`, `*.obj`, `*.a`, `*.lib`, `*.so`, `*.dylib`, `*.dll`), `*.exe`, `*.pdb`, `CMakeCache.txt`, `CMakeFiles/`, `CMakeUserPresets.json`, and a top-level `build/`, `out/` or `builddir/`; Python: `__pycache__/`, `*.py[co]`, `*.egg-info/`, a top-level `dist/` or `build/`; Rust: a top-level `target/`; Go: `*.test`, `*.out` and a top-level `bin/`. In a git checkout only the files git tracks are judged; `--exclude` keeps fixtures out. Fails listing the outputs | -| `vcs-ignore` | C/C++, Python, Rust, Go | a `.gitignore`, `.hgignore`, `.bzrignore` or `.fossil-settings/ignore-glob` ignores the build outputs (`build/` or `out/`, `__pycache__/` or `*.py[cod]`, `/target`, Go's `*.test` or `*.out`). Warns when an ignore file misses them | +| `committed-artifacts` | C/C++, Python, Rust, Go, JavaScript/TypeScript | no build output is committed. C/C++: object files and libraries (`*.o`, `*.obj`, `*.a`, `*.lib`, `*.so`, `*.dylib`, `*.dll`), `*.exe`, `*.pdb`, `CMakeCache.txt`, `CMakeFiles/`, `CMakeUserPresets.json`, and a top-level `build/`, `out/` or `builddir/`; Python: `__pycache__/`, `*.py[co]`, `*.egg-info/`, a top-level `dist/` or `build/`; Rust: a top-level `target/`; Go: `*.test`, `*.out` and a top-level `bin/`; JavaScript/TypeScript: `node_modules/`, a top-level `coverage/` and `*.tsbuildinfo` (not `dist/`, which a JavaScript GitHub Action commits). In a git checkout only the files git tracks are judged; `--exclude` keeps fixtures out. Fails listing the outputs | +| `vcs-ignore` | C/C++, Python, Rust, Go, JavaScript/TypeScript | a `.gitignore`, `.hgignore`, `.bzrignore` or `.fossil-settings/ignore-glob` ignores the build outputs (`build/` or `out/`, `__pycache__/` or `*.py[cod]`, `/target`, Go's `*.test` or `*.out`, `node_modules/`). Warns when an ignore file misses them | CI detection is pluggable too. GitHub Actions workflows (including `strategy.matrix` with `include`/`exclude`) and GitLab CI pipelines (including `parallel: matrix:`) diff --git a/project_validator/checks/code_formatting.py b/project_validator/checks/code_formatting.py index 2911f09..be37ff6 100644 --- a/project_validator/checks/code_formatting.py +++ b/project_validator/checks/code_formatting.py @@ -11,7 +11,7 @@ ) def code_formatting(formatter: Formatter, context: Context) -> Result: config = formatter.config_file(context) - tools = matching_names(context.ci_texts, formatter.runs) + tools = matching_names(formatter.texts(context), formatter.runs) if tools and config is not None: return Result.passed(f"{config} is defined and CI runs {', '.join(tools)}") if tools and formatter.default_style: diff --git a/project_validator/checks/data_wellformed.py b/project_validator/checks/data_wellformed.py index c6497a1..5debd4f 100644 --- a/project_validator/checks/data_wellformed.py +++ b/project_validator/checks/data_wellformed.py @@ -1,7 +1,6 @@ """Every JSON, TOML, YAML and XML file parses: real validation, since a broken data file is a bug.""" import json -import re import tomllib from collections import Counter from collections.abc import Callable, Mapping @@ -11,6 +10,7 @@ from project_validator import yaml_support from project_validator.core import Context, Result, check +from project_validator.text import strip_json_comments # Hidden directories that hold configuration worth parsing (CI, editors, dev containers). CONFIGURATION_DIRECTORIES = frozenset({".github", ".gitlab", ".circleci", ".buildkite", ".devcontainer", ".vscode"}) @@ -33,24 +33,10 @@ def _decoded(data: bytes) -> str: raise _ParseError(f"not UTF-8 ({error.reason} at byte {error.start})") from error -# A string, a comment, or a comma that only whitespace separates from the closing bracket. -_JSONC_TOKENS = re.compile(r'"(?:\\.|[^"\\])*"|//[^\n]*|/\*.*?\*/|,(?=\s*[}\]])', re.DOTALL) - - -def _without_comments(text: str) -> str: - """JSON with comments and trailing commas as plain JSON; comments become blanks, so positions are kept.""" - - def blank(match: re.Match[str]) -> str: - token = match[0] - return token if token.startswith('"') else re.sub(r"[^\n]", " ", token) - - return _JSONC_TOKENS.sub(blank, _JSONC_TOKENS.sub(blank, text)) - - def _parse_json(data: bytes, *, comments: bool = False) -> None: text = _decoded(data) try: - json.loads(_without_comments(text) if comments else text) + json.loads(strip_json_comments(text) if comments else text) except json.JSONDecodeError as error: raise _ParseError(error.msg, error.lineno, error.colno) from error diff --git a/project_validator/ecosystems/__init__.py b/project_validator/ecosystems/__init__.py index 6bc1dcc..de3058f 100644 --- a/project_validator/ecosystems/__init__.py +++ b/project_validator/ecosystems/__init__.py @@ -9,6 +9,7 @@ from project_validator.core import Check, CheckFunction, Context, Result, Status from project_validator.ecosystems.c_cpp import C_AND_CXX from project_validator.ecosystems.go import GO +from project_validator.ecosystems.javascript import JAVASCRIPT_AND_TYPESCRIPT from project_validator.ecosystems.model import ( C_FAMILY, BuildSystem, @@ -24,7 +25,7 @@ from project_validator.ecosystems.python import PYTHON from project_validator.ecosystems.rust import RUST -BUILT_IN = (C_AND_CXX, PYTHON, RUST, GO) +BUILT_IN = (C_AND_CXX, PYTHON, RUST, GO, JAVASCRIPT_AND_TYPESCRIPT) _registered: list[Ecosystem] = list(BUILT_IN) @@ -204,6 +205,7 @@ def is_build_tree(directory: Path) -> bool: "C_AND_CXX", "C_FAMILY", "GO", + "JAVASCRIPT_AND_TYPESCRIPT", "PYTHON", "RUST", "BuildSystem", diff --git a/project_validator/ecosystems/javascript.py b/project_validator/ecosystems/javascript.py new file mode 100644 index 0000000..ba55bcd --- /dev/null +++ b/project_validator/ecosystems/javascript.py @@ -0,0 +1,414 @@ +"""JavaScript and TypeScript: packaged with package.json (npm, pnpm, yarn, bun) or deno.json, judged by the Node.js, +Deno and Bun versions CI runs. + +Much of a JavaScript project's tooling runs through the scripts of its package.json (`npm run lint`), so the scripts +CI runs count as CI configuration wherever a rule reads more than CI configuration. +""" + +import json +import re +from collections.abc import Iterator + +from project_validator.core import Context, Result +from project_validator.ecosystems.model import BuildSystem, Ecosystem, ToolchainPatterns, versions_axis +from project_validator.evidence import ci_and_hook_texts, configuration, existing_texts, first_existing, json_value +from project_validator.rules import ( + MULTI_ECOSYSTEM_AUDITS, + SUGGESTED_LIZARD, + Analyzer, + BuildOutputs, + CoverageTools, + DeclaredLicense, + DependencyAudit, + DependencyManagement, + Formatter, + FunctionLimits, + SecurityLints, + StaticAnalysis, + TestSuite, + TypeChecking, + WarningPolicy, + WarningsAsErrors, + directories, + ignored_at_top_level, +) +from project_validator.text import Pattern, matching_names, pattern + +JAVASCRIPT = "JavaScript" +TYPESCRIPT = "TypeScript" +PACKAGE_JSON = "package.json" +DENO_CONFIG_FILES = ("deno.json", "deno.jsonc") +# A package.json that names what it provides is a JavaScript package, even before any source is found. +ENTRY_POINTS = ("main", "module", "exports", "bin") + + +def _declared_by_package(text: str) -> frozenset[str]: + """package.json is everywhere, as the tooling of projects in other languages; only a package's entry points (or + JavaScript sources, recognised by their suffixes) make a project a JavaScript one.""" + try: + manifest = json.loads(text) + except json.JSONDecodeError: + return frozenset() + is_package = isinstance(manifest, dict) and any(key in manifest for key in ENTRY_POINTS) + return frozenset({JAVASCRIPT}) if is_package else frozenset() + + +NPM = BuildSystem(name="npm", files=(PACKAGE_JSON,), declared_languages=_declared_by_package) +DENO = BuildSystem(name="Deno", files=DENO_CONFIG_FILES, languages=frozenset({TYPESCRIPT})) + + +def _json_strings(value: object) -> dict[str, str]: + return {key: item for key, item in value.items() if isinstance(item, str)} if isinstance(value, dict) else {} + + +def _scripts(context: Context) -> dict[str, str]: + """The package.json scripts and deno.json tasks, by name.""" + tasks = (json_value(context, path, "tasks", comments=True) for path in DENO_CONFIG_FILES) + found = {name: script for value in tasks for name, script in _json_strings(value).items()} + return {**found, **_json_strings(json_value(context, PACKAGE_JSON, "scripts"))} + + +# `npm test`, `npm run lint`, `pnpm lint`, `yarn run lint`, `bun run lint`, `deno task lint`. +RUNS_SCRIPT = re.compile( + r"\bnpm\s+(?:run(?:-script)?\s+(?P[\w:.-]+)|(?Ptest|t|start)\b)" + r"|\b(?:pnpm|yarn)\s+(?:run\s+)?(?P[\w:.-]+)" + r"|\bbun\s+run\s+(?P[\w:.-]+)" + r"|\bdeno\s+task\s+(?P[\w:.-]+)" +) +NPM_LIFECYCLE_ALIASES = {"t": "test"} + + +def _named_scripts(text: str) -> Iterator[str]: + for match in RUNS_SCRIPT.finditer(text): + name = next(group for group in match.groups() if group) + yield NPM_LIFECYCLE_ALIASES.get(name, name) + + +def _scripts_run(context: Context, texts: tuple[str, ...]) -> tuple[str, ...]: + """The scripts that `texts` run, with the scripts those run in turn and their pre and post scripts.""" + scripts = _scripts(context) + pending = [name for text in texts for name in _named_scripts(text)] + run: dict[str, None] = {} + while pending: + name = pending.pop() + if name in run or name not in scripts: + continue + run[name] = None + pending += [f"pre{name}", f"post{name}", *_named_scripts(scripts[name])] + return tuple(scripts[name] for name in run) + + +def _run_texts(context: Context) -> tuple[str, ...]: + """CI configuration, pre-commit hooks, and the package scripts they run.""" + texts = ci_and_hook_texts(context) + return (*texts, *_scripts_run(context, texts)) + + +PRETTIER_CONFIG_FILES = tuple( + f"{stem}{suffix}" + for stem, suffixes in ( + (".prettierrc", ("", ".json", ".json5", ".yaml", ".yml", ".toml", ".js", ".cjs", ".mjs", ".ts")), + ("prettier.config", (".js", ".cjs", ".mjs", ".ts", ".cts", ".mts")), + ) + for suffix in suffixes +) +FORMATTER_CONFIG_FILES = (*PRETTIER_CONFIG_FILES, "biome.json", "biome.jsonc", "dprint.json", ".dprint.json") + + +def _formatter_config(context: Context) -> str | None: + if found := first_existing(context, FORMATTER_CONFIG_FILES): + return found + return PACKAGE_JSON if json_value(context, PACKAGE_JSON, "prettier") is not None else None + + +# Every one of them has a default style, so a configuration file is not needed. +FORMATTERS = Formatter( + name="prettier, biome, deno fmt or dprint", + config_file=_formatter_config, + runs=( + pattern("prettier", r"\bprettier\b[^\n]*\s(?:--check|-c|--list-different|-l)\b"), + pattern("biome", r"\bbiome\s+(?:format|check|ci)\b"), + pattern("deno fmt", r"\bdeno\s+fmt\b[^\n]*\s--check\b"), + pattern("dprint", r"\bdprint\s+check\b"), + ), + hint="consider a CI job that runs prettier --check . (or biome ci, deno fmt --check or dprint check)", + default_style=True, + texts=_run_texts, +) + +JS_SUFFIXES = (".js", ".mjs", ".cjs", ".jsx") +TS_SUFFIXES = (".ts", ".mts", ".cts", ".tsx") +SOURCE_SUFFIXES = (*JS_SUFFIXES, *TS_SUFFIXES) +# app.test.ts, app.spec.js, app_test.ts (Deno), or any source in a __tests__ directory (Jest). +TEST_FILE = re.compile(r"(?:^|/)(?:[^/]*(?:[._]test|\.spec)\.[^/]+|__tests__/.+)$") +RUNS_TESTS = ( + pattern( + "JavaScript tests", + r"\b(?:jest|vitest|mocha|ava|playwright\s+test)\b|\bnode\s+(?:[^\n]*\s)?--test\b|\b(?:deno|bun)\s+test\b" + r"|\bnpm\s+(?:t|test|run\s+test)\b|\b(?:pnpm|yarn)\s+(?:run\s+)?test\b", + ), +) + + +def _javascript_tests(context: Context) -> Result: + tests = [path for path in context.project.files() if path.endswith(SOURCE_SUFFIXES) and TEST_FILE.search(path)] + if not tests: + return Result.failed("no JavaScript or TypeScript tests found (*.test.*, *.spec.* or __tests__/ files)") + return Result.passed(f"JavaScript or TypeScript tests found in {len(tests)} file(s)") + + +TESTS = TestSuite( + found=_javascript_tests, + runs=RUNS_TESTS, + runner="jest, vitest, mocha, node --test, deno test, bun test or playwright (or npm test)", +) + +# Istanbul (nyc, jest, vitest's istanbul provider) and V8 coverage (c8, vitest, node, deno) both report branches. +COVERAGE_CONFIG_FILES = tuple( + f"{tool}.config.{suffix}" for tool in ("jest", "vitest", "vite") for suffix in ("js", "cjs", "mjs", "ts", "mts") +) + + +def _coverage_texts(context: Context) -> tuple[str, ...]: + return (*_run_texts(context), *existing_texts(context, (PACKAGE_JSON, *COVERAGE_CONFIG_FILES))) + + +COVERAGE = CoverageTools( + searched="CI configuration, the package scripts it runs, or the test runner's settings", + texts=_coverage_texts, + branch=( + pattern("c8", r"(? tuple[Pattern, ...]: + """One linter is enough: the ones that run, or eslint when none does.""" + texts = _run_texts(context) + return tuple(linter for linter in LINTERS if matching_names(texts, (linter,))) or LINTERS[:1] + + +# JavaScript has no compiler warnings; its linters report what other compilers warn about. +WARNINGS = WarningPolicy( + kind="JavaScript linters", recognised=LINTERS, required=_linters_required, texts=_run_texts, separator=", " +) + +WARNINGS_AS_ERRORS = WarningsAsErrors( + flags=( + pattern("eslint --max-warnings 0", rf"{RUNS_ESLINT}[^\n]*\s--max-warnings[=\s]+0\b"), + pattern("biome ci", r"\bbiome\s+ci\b|\bbiome\b[^\n]*\s--error-on-warnings\b"), + # deno lint fails on any finding. + pattern("deno lint", RUNS_DENO_LINT), + ), + texts=_run_texts, + searched="CI configuration or the package scripts it runs", + hint="run eslint with --max-warnings 0 (or biome ci, or deno lint), so that a lint warning fails CI", +) + + +def _security_texts(context: Context) -> tuple[str, ...]: + return (*_run_texts(context), *existing_texts(context, (PACKAGE_JSON, *ESLINT_CONFIG_FILES))) + + +SECURITY_LINTS = SecurityLints( + texts=_security_texts, + lints=( + pattern("eslint-plugin-security", r"\beslint-plugin-security\b|@microsoft/eslint-plugin-sdl\b"), + pattern( + "semgrep JavaScript rules", + r"\bsemgrep\b[^\n]*--config[=\s]+[\"']?(?:p/)?(?:javascript|typescript|nodejs|react|expressjs)\b", + ), + ), +) + +DEPENDENCY_AUDIT = DependencyAudit( + tools=( + # npm audit signatures checks provenance, not known vulnerabilities. + pattern("npm audit", r"\bnpm\s+audit\b(?!\s+signatures)"), + pattern("pnpm audit", r"\bpnpm\s+audit\b"), + pattern("yarn audit", r"\byarn\s+(?:npm\s+)?audit\b"), + pattern("bun audit", r"\bbun\s+audit\b"), + *MULTI_ECOSYSTEM_AUDITS, + ), + hint="consider running npm audit (or pnpm audit, yarn npm audit or bun audit) in CI", +) + +# The rule's options are "off", 0, or anything else (a severity, or a severity and its limit). +MAX_LINES_PER_FUNCTION = re.compile( + r"""["']?max-lines-per-function["']?\s*:(?!\s*\[?\s*(?:["']off["']|0\b))""", re.IGNORECASE +) + + +def _max_lines_per_function(context: Context) -> Result: + """eslint's max-lines-per-function rule limits function length, when it is enabled and eslint runs.""" + config = first_existing(context, ESLINT_CONFIG_FILES) + if config is None or not MAX_LINES_PER_FUNCTION.search(context.project.read_text(config)): + return Result.failed( + "function length is never limited; enable eslint's max-lines-per-function rule " + f"or run {SUGGESTED_LIZARD} in CI" + ) + if not matching_names(_run_texts(context), LINTERS[:1]): + return Result.warned(f"max-lines-per-function is enabled in {config} but CI does not run eslint") + return Result.passed(f"function length is limited with eslint max-lines-per-function ({config})") + + +BIOME_CONFIG = configuration(("biome.json", "biome.jsonc")) +DENO_CONFIG = configuration(DENO_CONFIG_FILES) + +STATIC_ANALYSIS = StaticAnalysis( + analyzers=( + Analyzer(name="eslint", runs=re.compile(RUNS_ESLINT), configured=configuration(ESLINT_CONFIG_FILES)), + Analyzer(name="biome lint", runs=re.compile(RUNS_BIOME_LINT), configured=BIOME_CONFIG), + Analyzer(name="deno lint", runs=re.compile(RUNS_DENO_LINT), configured=DENO_CONFIG), + ), + texts=_run_texts, + hint="consider running eslint (or biome lint, or deno lint) in CI", +) + +TSCONFIG_FILES = ("tsconfig.json", "jsconfig.json") +# @tsconfig/strictest turns on strict, and more. +STRICTEST_BASE = re.compile(r"@tsconfig/strictest\b") + + +def _strict_type_checking(context: Context) -> str | None: + """Where strict type checking is enabled: a tsconfig.json, or Deno, which checks strictly unless told not to.""" + for path in TSCONFIG_FILES: + if json_value(context, path, "compilerOptions", "strict", comments=True) is True: + return f'"strict": true in {path}' + extends = json_value(context, path, "extends", comments=True) + bases = extends if isinstance(extends, list) else [extends] + if any(isinstance(base, str) and STRICTEST_BASE.search(base) for base in bases): + return f"@tsconfig/strictest in {path}" + deno = first_existing(context, DENO_CONFIG_FILES) + if deno is not None and json_value(context, deno, "compilerOptions", "strict", comments=True) is not False: + return f"Deno's default strict mode ({deno})" + return None + + +TYPE_CHECKING = TypeChecking( + checkers=( + Analyzer(name="tsc", runs=re.compile(r"(? str | None: + if any(json_value(context, PACKAGE_JSON, field) for field in DEPENDENCY_FIELDS): + return f"declared in {PACKAGE_JSON}" + deno = next((path for path in DENO_CONFIG_FILES if json_value(context, path, "imports", comments=True)), None) + return f"declared in {deno}" if deno else None + + +def _lockfile(context: Context) -> str | None: + return next((path for path in LOCKFILES if context.committed(path)), None) + + +# A package's lockfile is not published, so even a library commits one: its CI and contributors then install the +# dependencies it was tested with. +DEPENDENCIES = DependencyManagement( + managed=_declared_dependencies, + locked=_lockfile, + lock_hint="commit the package manager's lockfile (package-lock.json, pnpm-lock.yaml, yarn.lock, bun.lock or " + "deno.lock) and install from it in CI (npm ci, pnpm install --frozen-lockfile, yarn install --immutable)", +) + + +def _package_license(context: Context) -> str | None: + license_ = json_value(context, PACKAGE_JSON, "license") + return license_ if isinstance(license_, str) else None + + +LICENSE = DeclaredLicense(manifest=PACKAGE_JSON, declared=_package_license) + +NODE_MODULES = BuildOutputs( + name="node_modules", + globs=ignored_at_top_level("node_modules"), + paths=directories("node_modules"), + hint="consider ignoring node_modules/ (and coverage/ and dist/)", + # dist/ is left out: a GitHub Action written in JavaScript must commit its bundle. + artifacts=("node_modules/", "/coverage/", "*.tsbuildinfo"), +) + +_VERSION = r"(\d+(?:\.(?:\d+|x))*" +RUNTIME_VERSIONS = ( + re.compile(rf"\bnode-version\W+{_VERSION}|lts/[\w*-]+|latest|current|node)"), + re.compile(r"\bnode:(\d+(?:\.\d+)*)"), + re.compile(rf"\bdeno-version\W+v?{_VERSION}|canary|lts|rc)"), + re.compile(r"\bdenoland/deno:(?:\w+-)?(\d+(?:\.\d+)*)"), + re.compile(rf"\bbun-version\W+{_VERSION}|latest|canary)"), + re.compile(r"\boven/bun:(\d+(?:\.\d+)*)"), +) + +JAVASCRIPT_AND_TYPESCRIPT = Ecosystem( + label="JavaScript/TypeScript", + languages=frozenset({JAVASCRIPT, TYPESCRIPT}), + build_systems=(NPM, DENO), + source_suffixes={ + **dict.fromkeys(JS_SUFFIXES, JAVASCRIPT), + **dict.fromkeys(TS_SUFFIXES, TYPESCRIPT), + }, + toolchain=ToolchainPatterns( + language="JavaScript runtime", + versions=RUNTIME_VERSIONS, + # Not npx alone: other ecosystems' CI runs tools such as jscpd with it. + usage=re.compile( + r"\bnpm\s+(?:ci|install|i|test|t|run)\b|\b(?:pnpm|yarn)\s+\w|\bnode\s+(?:-\S|[\w./-]+\.[mc]?[jt]s\b)|\bsetup-(?:deno|bun)\b" + r"|\bdeno\s+(?:test|task|run|check|lint|fmt)\b|\bbun\s+(?:install|test|run)\b" + ), + ), + rules=( + versions_axis("runtime version", "JavaScript runtime"), + FORMATTERS, + COVERAGE, + TESTS, + WARNINGS, + WARNINGS_AS_ERRORS, + SECURITY_LINTS, + DEPENDENCY_AUDIT, + FunctionLimits(without_lizard=_max_lines_per_function), + STATIC_ANALYSIS, + TYPE_CHECKING, + DEPENDENCIES, + NODE_MODULES, + LICENSE, + ), +) diff --git a/project_validator/evidence.py b/project_validator/evidence.py index 1fef0a6..8865130 100644 --- a/project_validator/evidence.py +++ b/project_validator/evidence.py @@ -1,10 +1,12 @@ """Gathering the files whose text a rule searches for evidence.""" +import json import re import tomllib from collections.abc import Callable, Iterable, Mapping from project_validator.core import Context +from project_validator.text import strip_json_comments def first_existing(context: Context, names: Iterable[str]) -> str | None: @@ -15,6 +17,15 @@ def existing_texts(context: Context, names: Iterable[str]) -> tuple[str, ...]: return tuple(context.project.read_text(name) for name in names if context.project.exists(name)) +def _at(value: object, keys: tuple[str, ...]) -> object | None: + """The value at `keys` in nested tables; None when one is missing.""" + for key in keys: + if not isinstance(value, dict): + return None + value = value.get(key) + return value + + def toml_value(context: Context, path: str, *keys: str) -> object | None: """The value at `keys` in the TOML file `path`; None when the file, or the value, is missing or does not parse.""" if not context.project.exists(path): @@ -23,11 +34,20 @@ def toml_value(context: Context, path: str, *keys: str) -> object | None: value: object = tomllib.loads(context.project.read_text(path)) except tomllib.TOMLDecodeError: return None - for key in keys: - if not isinstance(value, dict): - return None - value = value.get(key) - return value + return _at(value, keys) + + +def json_value(context: Context, path: str, *keys: str, comments: bool = False) -> object | None: + """The value at `keys` in the JSON file `path` (with `comments`, JSON with comments and trailing commas); None + when the file, or the value, is missing or does not parse.""" + if not context.project.exists(path): + return None + text = context.project.read_text(path) + try: + value: object = json.loads(strip_json_comments(text) if comments else text) + except json.JSONDecodeError: + return None + return _at(value, keys) # pre-commit's configuration: a hook it runs counts as evidence much as a CI step does. diff --git a/project_validator/rules.py b/project_validator/rules.py index bb40dd6..e783578 100644 --- a/project_validator/rules.py +++ b/project_validator/rules.py @@ -61,11 +61,17 @@ def of_job(job: Job) -> frozenset[str]: return ToolchainAxis(noun=noun, of_job=of_job, runner_default=_never) +def ci_configuration(context: Context) -> tuple[str, ...]: + """The texts of the project's CI configuration.""" + return context.ci_texts + + @dataclass(frozen=True, slots=True, kw_only=True) class Formatter: """Where an ecosystem's formatting style is defined, and the CI commands that enforce it. - A formatter with a `default_style` defines the formatting even without a configuration file. + A formatter with a `default_style` defines the formatting even without a configuration file. Its commands are + searched in `texts`: CI configuration, unless the ecosystem's CI runs them through scripts it can read. """ name: str @@ -73,6 +79,7 @@ class Formatter: runs: tuple[Pattern, ...] hint: str default_style: bool = False + texts: Evidence = ci_configuration @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/project_validator/text.py b/project_validator/text.py index f25ce0e..9ffb5f7 100644 --- a/project_validator/text.py +++ b/project_validator/text.py @@ -54,3 +54,17 @@ def _keep_unless_comment(match: re.Match[str]) -> str: def strip_cmake_comments(text: str) -> str: return _CMAKE_TOKENS.sub(_keep_unless_comment, text) + + +# A string, a comment, or a comma that only whitespace separates from the closing bracket. +_JSONC_TOKENS = re.compile(r'"(?:\\.|[^"\\])*"|//[^\n]*|/\*.*?\*/|,(?=\s*[}\]])', re.DOTALL) + + +def _blank_unless_string(match: re.Match[str]) -> str: + token = match[0] + return token if token.startswith('"') else re.sub(r"[^\n]", " ", token) + + +def strip_json_comments(text: str) -> str: + """JSON with comments and trailing commas as plain JSON; comments become blanks, so positions are kept.""" + return _JSONC_TOKENS.sub(_blank_unless_string, _JSONC_TOKENS.sub(_blank_unless_string, text)) diff --git a/tests/checks/test_warnings_as_errors.py b/tests/checks/test_warnings_as_errors.py index f977e62..bbaadc2 100644 --- a/tests/checks/test_warnings_as_errors.py +++ b/tests/checks/test_warnings_as_errors.py @@ -19,9 +19,10 @@ def _run(self, files: dict[str, str]) -> Result: self, warnings_as_errors, files, c_family_layouts_built_with(warnings_as_errors, "CMake") ) - # -Werror and /WX are C/C++ compiler flags; other languages' equivalents are judged by compiler-warnings. + # -Werror and /WX are C/C++ compiler flags, and eslint --max-warnings 0 fails JavaScript linting on a warning; + # other languages' equivalents are judged by compiler-warnings. def test_declares_the_languages_it_judges(self) -> None: - self.assertEqual(warnings_as_errors.languages, C_FAMILY) + self.assertEqual(warnings_as_errors.languages, C_FAMILY | {"JavaScript", "TypeScript"}) def test_fails_without_warnings_as_errors(self) -> None: result = self._run(BARE_PROJECT) diff --git a/tests/fixtures/bare_javascript/coverage/lcov.info b/tests/fixtures/bare_javascript/coverage/lcov.info new file mode 100644 index 0000000..fde885d --- /dev/null +++ b/tests/fixtures/bare_javascript/coverage/lcov.info @@ -0,0 +1,2 @@ +TN: +end_of_record diff --git a/tests/fixtures/bare_javascript/package.json b/tests/fixtures/bare_javascript/package.json new file mode 100644 index 0000000..3bf3146 --- /dev/null +++ b/tests/fixtures/bare_javascript/package.json @@ -0,0 +1,8 @@ +{ + "name": "app", + "version": "0.1.0", + "main": "src/index.js", + "dependencies": { + "left-pad": "^1.3.0" + } +} diff --git a/tests/fixtures/bare_javascript/src/index.js b/tests/fixtures/bare_javascript/src/index.js new file mode 100644 index 0000000..7d65831 --- /dev/null +++ b/tests/fixtures/bare_javascript/src/index.js @@ -0,0 +1,3 @@ +export function add(a, b) { + return a + b; +} diff --git a/tests/fixtures/well_configured_javascript/.github/dependabot.yml b/tests/fixtures/well_configured_javascript/.github/dependabot.yml new file mode 100644 index 0000000..ca79ca5 --- /dev/null +++ b/tests/fixtures/well_configured_javascript/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly diff --git a/tests/fixtures/well_configured_javascript/.github/workflows/ci-lint.yml b/tests/fixtures/well_configured_javascript/.github/workflows/ci-lint.yml new file mode 100644 index 0000000..ac9333b --- /dev/null +++ b/tests/fixtures/well_configured_javascript/.github/workflows/ci-lint.yml @@ -0,0 +1,9 @@ +on: [push, pull_request] +jobs: + ci-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - run: pipx run zizmor .github + - uses: rhysd/actionlint@914e7df21a07ef503a81201c76d2b11c789d3fca # v1.7.12 + - run: gitleaks git --redact . diff --git a/tests/fixtures/well_configured_javascript/.github/workflows/ci.yml b/tests/fixtures/well_configured_javascript/.github/workflows/ci.yml new file mode 100644 index 0000000..6784df7 --- /dev/null +++ b/tests/fixtures/well_configured_javascript/.github/workflows/ci.yml @@ -0,0 +1,18 @@ +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + node: ["22", "24"] + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: ${{ matrix.node }} + - run: npm ci + - run: npm run format:check + - run: npm run lint + - run: npm run typecheck + - run: npm test + - run: npm audit --audit-level=moderate diff --git a/tests/fixtures/well_configured_javascript/.github/workflows/copy-paste-detection.yml b/tests/fixtures/well_configured_javascript/.github/workflows/copy-paste-detection.yml new file mode 100644 index 0000000..f7f288a --- /dev/null +++ b/tests/fixtures/well_configured_javascript/.github/workflows/copy-paste-detection.yml @@ -0,0 +1,6 @@ +jobs: + copy-paste-detection: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - run: npx --yes jscpd@4 --threshold 0 . diff --git a/tests/fixtures/well_configured_javascript/.github/workflows/function-limits.yml b/tests/fixtures/well_configured_javascript/.github/workflows/function-limits.yml new file mode 100644 index 0000000..0b6c011 --- /dev/null +++ b/tests/fixtures/well_configured_javascript/.github/workflows/function-limits.yml @@ -0,0 +1,7 @@ +jobs: + function-limits: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - run: python -m pip install lizard + - run: lizard --length 30 --warnings_only . diff --git a/tests/fixtures/well_configured_javascript/.gitignore b/tests/fixtures/well_configured_javascript/.gitignore new file mode 100644 index 0000000..06c3eac --- /dev/null +++ b/tests/fixtures/well_configured_javascript/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +coverage/ +dist/ diff --git a/tests/fixtures/well_configured_javascript/LICENSE b/tests/fixtures/well_configured_javascript/LICENSE new file mode 100644 index 0000000..1b3edae --- /dev/null +++ b/tests/fixtures/well_configured_javascript/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Example Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tests/fixtures/well_configured_javascript/README.md b/tests/fixtures/well_configured_javascript/README.md new file mode 100644 index 0000000..ec54841 --- /dev/null +++ b/tests/fixtures/well_configured_javascript/README.md @@ -0,0 +1,4 @@ +# app + +[![ci](https://github.com/example/app/actions/workflows/ci.yml/badge.svg)](https://github.com/example/app/actions/workflows/ci.yml) +[![codecov](https://codecov.io/gh/example/app/graph/badge.svg)](https://codecov.io/gh/example/app) diff --git a/tests/fixtures/well_configured_javascript/SECURITY.md b/tests/fixtures/well_configured_javascript/SECURITY.md new file mode 100644 index 0000000..0b161bd --- /dev/null +++ b/tests/fixtures/well_configured_javascript/SECURITY.md @@ -0,0 +1,3 @@ +# Security policy + +Report vulnerabilities to security@example.com. diff --git a/tests/fixtures/well_configured_javascript/eslint.config.js b/tests/fixtures/well_configured_javascript/eslint.config.js new file mode 100644 index 0000000..23c0bd0 --- /dev/null +++ b/tests/fixtures/well_configured_javascript/eslint.config.js @@ -0,0 +1,3 @@ +import security from "eslint-plugin-security"; + +export default [security.configs.recommended]; diff --git a/tests/fixtures/well_configured_javascript/package-lock.json b/tests/fixtures/well_configured_javascript/package-lock.json new file mode 100644 index 0000000..afc63a1 --- /dev/null +++ b/tests/fixtures/well_configured_javascript/package-lock.json @@ -0,0 +1,13 @@ +{ + "name": "app", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "app", + "version": "0.1.0", + "license": "MIT" + } + } +} diff --git a/tests/fixtures/well_configured_javascript/package.json b/tests/fixtures/well_configured_javascript/package.json new file mode 100644 index 0000000..ca94253 --- /dev/null +++ b/tests/fixtures/well_configured_javascript/package.json @@ -0,0 +1,22 @@ +{ + "name": "app", + "version": "0.1.0", + "license": "MIT", + "type": "module", + "main": "dist/add.js", + "scripts": { + "build": "tsc", + "format:check": "prettier --check .", + "lint": "eslint --max-warnings 0 .", + "typecheck": "tsc --noEmit", + "test": "vitest run --coverage" + }, + "devDependencies": { + "@vitest/coverage-v8": "^3.2.0", + "eslint": "^9.30.0", + "eslint-plugin-security": "^3.0.1", + "prettier": "^3.6.0", + "typescript": "^5.9.0", + "vitest": "^3.2.0" + } +} diff --git a/tests/fixtures/well_configured_javascript/src/add.test.ts b/tests/fixtures/well_configured_javascript/src/add.test.ts new file mode 100644 index 0000000..3a44af4 --- /dev/null +++ b/tests/fixtures/well_configured_javascript/src/add.test.ts @@ -0,0 +1,7 @@ +import { expect, test } from "vitest"; + +import { add } from "./add.js"; + +test("adds", () => { + expect(add(1, 2)).toBe(3); +}); diff --git a/tests/fixtures/well_configured_javascript/src/add.ts b/tests/fixtures/well_configured_javascript/src/add.ts new file mode 100644 index 0000000..1527df3 --- /dev/null +++ b/tests/fixtures/well_configured_javascript/src/add.ts @@ -0,0 +1,4 @@ +/** The sum of a and b. */ +export function add(a: number, b: number): number { + return a + b; +} diff --git a/tests/fixtures/well_configured_javascript/tsconfig.json b/tests/fixtures/well_configured_javascript/tsconfig.json new file mode 100644 index 0000000..a7ffb1b --- /dev/null +++ b/tests/fixtures/well_configured_javascript/tsconfig.json @@ -0,0 +1,10 @@ +{ + // Type check everything; the build writes dist/. + "compilerOptions": { + "target": "es2022", + "module": "nodenext", + "strict": true, + "outDir": "dist", + }, + "include": ["src"], +} diff --git a/tests/snapshots/bare_go.json b/tests/snapshots/bare_go.json index c92abe3..15d4618 100644 --- a/tests/snapshots/bare_go.json +++ b/tests/snapshots/bare_go.json @@ -181,7 +181,7 @@ }, "type-checking": { "details": [], - "message": "not applicable to Go (applies to Python)", + "message": "not applicable to Go (applies to JavaScript, Python, TypeScript)", "status": "SKIP" }, "vcs-ignore": { @@ -193,7 +193,7 @@ }, "warnings-as-errors": { "details": [], - "message": "not applicable to Go (applies to C, C++)", + "message": "not applicable to Go (applies to C, C++, JavaScript, TypeScript)", "status": "SKIP" } } diff --git a/tests/snapshots/bare_javascript.json b/tests/snapshots/bare_javascript.json new file mode 100644 index 0000000..1f6ac99 --- /dev/null +++ b/tests/snapshots/bare_javascript.json @@ -0,0 +1,202 @@ +{ + "map": { + "build_systems": [ + "npm" + ], + "ci": { + "files": [], + "jobs": [], + "providers": [] + }, + "ecosystems": { + "JavaScript/TypeScript": [ + "build file package.json", + ".js files" + ] + }, + "languages": [ + "JavaScript" + ], + "unrecognised_suffixes": [ + { + "files": 1, + "suffix": ".info" + } + ] + }, + "results": { + "action-pinning": { + "details": [], + "message": "no CI configuration with reusable steps", + "status": "SKIP" + }, + "ci-build-types": { + "details": [], + "message": "not applicable to JavaScript (applies to C, C++, Rust)", + "status": "SKIP" + }, + "ci-lint": { + "details": [], + "message": "no CI configuration", + "status": "SKIP" + }, + "ci-present": { + "details": [], + "message": "no CI configuration found (looked for GitHub Actions, GitLab CI, Azure Pipelines, AppVeyor, CircleCI, Travis CI, Jenkins, Buildkite, Drone, Cirrus CI, Bitbucket Pipelines, Woodpecker)", + "status": "FAIL" + }, + "code-formatting": { + "details": [ + "consider a CI job that runs prettier --check . (or biome ci, deno fmt --check or dprint check)" + ], + "message": "CI never runs prettier, biome, deno fmt or dprint; its formatting is not enforced", + "status": "FAIL" + }, + "committed-artifacts": { + "details": [ + "coverage/", + "remove them from version control (git rm -r --cached ) and ignore them" + ], + "message": "1 build output is committed", + "status": "FAIL" + }, + "compiler-warnings": { + "details": [], + "message": "no JavaScript linters found; enable at least eslint", + "status": "FAIL" + }, + "copy-paste-detection": { + "details": [ + "consider adding PMD CPD (pmd cpd --language cpp), jscpd, Duplo or Simian" + ], + "message": "no copy-paste detection found in CI configuration", + "status": "WARN" + }, + "coverage": { + "details": [], + "message": "no branch coverage reporting found in CI configuration, the package scripts it runs, or the test runner's settings", + "status": "FAIL" + }, + "cxx-extensions": { + "details": [], + "message": "not applicable to JavaScript (applies to C, C++)", + "status": "SKIP" + }, + "data-wellformed": { + "details": [], + "message": "1 data file parses (JSON: 1)", + "status": "PASS" + }, + "dependency-audit": { + "details": [ + "consider running npm audit (or pnpm audit, yarn npm audit or bun audit) in CI" + ], + "message": "no dependency vulnerability audit found in CI configuration", + "status": "WARN" + }, + "dependency-management": { + "details": [ + "commit the package manager's lockfile (package-lock.json, pnpm-lock.yaml, yarn.lock, bun.lock or deno.lock) and install from it in CI (npm ci, pnpm install --frozen-lockfile, yarn install --immutable)" + ], + "message": "dependencies are declared in package.json but not locked", + "status": "WARN" + }, + "dependency-updates": { + "details": [ + "consider adding .github/dependabot.yml or a Renovate configuration (renovate.json)" + ], + "message": "no automated dependency updates configured", + "status": "WARN" + }, + "function-limits": { + "details": [], + "message": "function length is never limited; enable eslint's max-lines-per-function rule or run lizard --length 30 in CI", + "status": "FAIL" + }, + "hardening": { + "details": [], + "message": "not applicable to JavaScript (applies to C, C++, Go, Rust)", + "status": "SKIP" + }, + "large-files": { + "details": [], + "message": "no committed file is larger than 5 MiB", + "status": "PASS" + }, + "license": { + "details": [ + "add a LICENSE file with the text of a license (see https://choosealicense.com)" + ], + "message": "no license file", + "status": "FAIL" + }, + "platform-matrix": { + "details": [], + "message": "no CI configuration found", + "status": "FAIL" + }, + "secret-scanning": { + "details": [ + "consider running gitleaks, TruffleHog (trufflesecurity/trufflehog) or detect-secrets in CI or as a pre-commit hook; GitHub's secret scanning and push protection are repository settings, which cannot be seen from here" + ], + "message": "no secret scanning found in CI configuration", + "status": "WARN" + }, + "security-policy": { + "details": [ + "add a SECURITY.md that says how to report a vulnerability privately: an email address, or GitHub's private vulnerability reporting (Settings > Code security, then link to /security/advisories/new)" + ], + "message": "no security policy", + "status": "WARN" + }, + "security-scanning": { + "details": [ + "consider adding CodeQL (github/codeql-action) or GitLab SAST (Security/SAST.gitlab-ci.yml)", + "or enable each language's security lints (JavaScript/TypeScript: eslint-plugin-security, semgrep JavaScript rules)" + ], + "message": "no code security scanning found in CI configuration", + "status": "WARN" + }, + "static-analysis": { + "details": [ + "consider running eslint (or biome lint, or deno lint) in CI" + ], + "message": "no static analyzer runs", + "status": "FAIL" + }, + "status-badges": { + "details": [ + "add a CI status badge, e.g. ![ci](https://github.com///actions/workflows/ci.yml/badge.svg)", + "add a quality badge: coverage (Codecov, Coveralls) or analysis (SonarCloud, Codacy, OpenSSF Scorecard)" + ], + "message": "no README", + "status": "FAIL" + }, + "tests-run": { + "details": [], + "message": "no JavaScript or TypeScript tests found (*.test.*, *.spec.* or __tests__/ files)", + "status": "FAIL" + }, + "type-checking": { + "details": [ + "consider running tsc --noEmit in CI (JavaScript is checked with checkJs in a jsconfig.json, or deno check)" + ], + "message": "no type checker runs", + "status": "FAIL" + }, + "vcs-ignore": { + "details": [ + "consider adding a .gitignore (or your version control system's equivalent) that ignores build outputs" + ], + "message": "no version control ignore file found (.gitignore, .hgignore, .bzrignore, .fossil-settings/ignore-glob); build outputs can be committed by accident", + "status": "FAIL" + }, + "warnings-as-errors": { + "details": [ + "run eslint with --max-warnings 0 (or biome ci, or deno lint), so that a lint warning fails CI" + ], + "message": "warnings are not treated as errors in CI configuration or the package scripts it runs", + "status": "FAIL" + } + } +} diff --git a/tests/snapshots/bare_project.json b/tests/snapshots/bare_project.json index c256dd8..aff9196 100644 --- a/tests/snapshots/bare_project.json +++ b/tests/snapshots/bare_project.json @@ -85,7 +85,7 @@ }, "dependency-audit": { "details": [], - "message": "not applicable to C, C++ (applies to Go, Python, Rust)", + "message": "not applicable to C, C++ (applies to Go, JavaScript, Python, Rust, TypeScript)", "status": "SKIP" }, "dependency-management": { @@ -173,7 +173,7 @@ }, "type-checking": { "details": [], - "message": "not applicable to C, C++ (applies to Python)", + "message": "not applicable to C, C++ (applies to JavaScript, Python, TypeScript)", "status": "SKIP" }, "vcs-ignore": { diff --git a/tests/snapshots/bare_python.json b/tests/snapshots/bare_python.json index d77c926..45474e5 100644 --- a/tests/snapshots/bare_python.json +++ b/tests/snapshots/bare_python.json @@ -188,7 +188,7 @@ }, "warnings-as-errors": { "details": [], - "message": "not applicable to Python (applies to C, C++)", + "message": "not applicable to Python (applies to C, C++, JavaScript, TypeScript)", "status": "SKIP" } } diff --git a/tests/snapshots/bare_rust.json b/tests/snapshots/bare_rust.json index 4be526a..7f70708 100644 --- a/tests/snapshots/bare_rust.json +++ b/tests/snapshots/bare_rust.json @@ -174,7 +174,7 @@ }, "type-checking": { "details": [], - "message": "not applicable to Rust (applies to Python)", + "message": "not applicable to Rust (applies to JavaScript, Python, TypeScript)", "status": "SKIP" }, "vcs-ignore": { @@ -186,7 +186,7 @@ }, "warnings-as-errors": { "details": [], - "message": "not applicable to Rust (applies to C, C++)", + "message": "not applicable to Rust (applies to C, C++, JavaScript, TypeScript)", "status": "SKIP" } } diff --git a/tests/snapshots/buildable.json b/tests/snapshots/buildable.json index 27bb013..92f1b52 100644 --- a/tests/snapshots/buildable.json +++ b/tests/snapshots/buildable.json @@ -81,7 +81,7 @@ }, "dependency-audit": { "details": [], - "message": "not applicable to C, C++ (applies to Go, Python, Rust)", + "message": "not applicable to C, C++ (applies to Go, JavaScript, Python, Rust, TypeScript)", "status": "SKIP" }, "dependency-management": { @@ -168,7 +168,7 @@ }, "type-checking": { "details": [], - "message": "not applicable to C, C++ (applies to Python)", + "message": "not applicable to C, C++ (applies to JavaScript, Python, TypeScript)", "status": "SKIP" }, "vcs-ignore": { diff --git a/tests/snapshots/well_configured.json b/tests/snapshots/well_configured.json index d7784ba..5368fee 100644 --- a/tests/snapshots/well_configured.json +++ b/tests/snapshots/well_configured.json @@ -414,7 +414,7 @@ }, "dependency-audit": { "details": [], - "message": "not applicable to C, C++ (applies to Go, Python, Rust)", + "message": "not applicable to C, C++ (applies to Go, JavaScript, Python, Rust, TypeScript)", "status": "SKIP" }, "dependency-management": { @@ -513,7 +513,7 @@ }, "type-checking": { "details": [], - "message": "not applicable to C, C++ (applies to Python)", + "message": "not applicable to C, C++ (applies to JavaScript, Python, TypeScript)", "status": "SKIP" }, "vcs-ignore": { diff --git a/tests/snapshots/well_configured_go.json b/tests/snapshots/well_configured_go.json index 08bef07..1cc2079 100644 --- a/tests/snapshots/well_configured_go.json +++ b/tests/snapshots/well_configured_go.json @@ -305,7 +305,7 @@ }, "type-checking": { "details": [], - "message": "not applicable to Go (applies to Python)", + "message": "not applicable to Go (applies to JavaScript, Python, TypeScript)", "status": "SKIP" }, "vcs-ignore": { @@ -315,7 +315,7 @@ }, "warnings-as-errors": { "details": [], - "message": "not applicable to Go (applies to C, C++)", + "message": "not applicable to Go (applies to C, C++, JavaScript, TypeScript)", "status": "SKIP" } } diff --git a/tests/snapshots/well_configured_javascript.json b/tests/snapshots/well_configured_javascript.json new file mode 100644 index 0000000..60b8cd1 --- /dev/null +++ b/tests/snapshots/well_configured_javascript.json @@ -0,0 +1,311 @@ +{ + "map": { + "build_systems": [ + "npm" + ], + "ci": { + "files": [ + ".github/workflows/ci-lint.yml", + ".github/workflows/ci.yml", + ".github/workflows/copy-paste-detection.yml", + ".github/workflows/function-limits.yml" + ], + "jobs": [ + { + "compilers": [ + "gcc" + ], + "configurations": [], + "ecosystems": [], + "name": ".github/workflows/ci-lint.yml: ci-lint", + "os": "linux", + "toolchains": [] + }, + { + "compilers": [ + "gcc" + ], + "configurations": [], + "ecosystems": [ + "JavaScript/TypeScript" + ], + "name": ".github/workflows/ci.yml: test (ubuntu-latest, 22)", + "os": "linux", + "toolchains": [ + "JavaScript runtime 22" + ] + }, + { + "compilers": [ + "gcc" + ], + "configurations": [], + "ecosystems": [ + "JavaScript/TypeScript" + ], + "name": ".github/workflows/ci.yml: test (ubuntu-latest, 24)", + "os": "linux", + "toolchains": [ + "JavaScript runtime 24" + ] + }, + { + "compilers": [ + "appleclang" + ], + "configurations": [], + "ecosystems": [ + "JavaScript/TypeScript" + ], + "name": ".github/workflows/ci.yml: test (macos-latest, 22)", + "os": "macos", + "toolchains": [ + "JavaScript runtime 22" + ] + }, + { + "compilers": [ + "appleclang" + ], + "configurations": [], + "ecosystems": [ + "JavaScript/TypeScript" + ], + "name": ".github/workflows/ci.yml: test (macos-latest, 24)", + "os": "macos", + "toolchains": [ + "JavaScript runtime 24" + ] + }, + { + "compilers": [ + "msvc" + ], + "configurations": [], + "ecosystems": [ + "JavaScript/TypeScript" + ], + "name": ".github/workflows/ci.yml: test (windows-latest, 22)", + "os": "windows", + "toolchains": [ + "JavaScript runtime 22" + ] + }, + { + "compilers": [ + "msvc" + ], + "configurations": [], + "ecosystems": [ + "JavaScript/TypeScript" + ], + "name": ".github/workflows/ci.yml: test (windows-latest, 24)", + "os": "windows", + "toolchains": [ + "JavaScript runtime 24" + ] + }, + { + "compilers": [ + "gcc" + ], + "configurations": [], + "ecosystems": [], + "name": ".github/workflows/copy-paste-detection.yml: copy-paste-detection", + "os": "linux", + "toolchains": [] + }, + { + "compilers": [ + "gcc" + ], + "configurations": [], + "ecosystems": [ + "Python" + ], + "name": ".github/workflows/function-limits.yml: function-limits", + "os": "linux", + "toolchains": [ + "Python default" + ] + } + ], + "providers": [ + "GitHub Actions" + ] + }, + "ecosystems": { + "JavaScript/TypeScript": [ + "build file package.json", + ".js files", + ".ts files" + ] + }, + "languages": [ + "JavaScript", + "TypeScript" + ], + "unrecognised_suffixes": [] + }, + "results": { + "action-pinning": { + "details": [], + "message": "all 6 actions are pinned to a commit", + "status": "PASS" + }, + "ci-build-types": { + "details": [], + "message": "not applicable to JavaScript, TypeScript (applies to C, C++, Rust)", + "status": "SKIP" + }, + "ci-lint": { + "details": [], + "message": "CI lints its configuration (GitHub Actions: actionlint, zizmor)", + "status": "PASS" + }, + "ci-present": { + "details": [ + ".github/workflows/ci-lint.yml", + ".github/workflows/ci.yml", + ".github/workflows/copy-paste-detection.yml", + ".github/workflows/function-limits.yml" + ], + "message": "CI configuration found for GitHub Actions", + "status": "PASS" + }, + "code-formatting": { + "details": [], + "message": "CI enforces the prettier style", + "status": "PASS" + }, + "committed-artifacts": { + "details": [], + "message": "no build outputs are committed", + "status": "PASS" + }, + "compiler-warnings": { + "details": [], + "message": "strict JavaScript linters are enabled: eslint", + "status": "PASS" + }, + "copy-paste-detection": { + "details": [], + "message": "CI runs copy-paste detection with jscpd", + "status": "PASS" + }, + "coverage": { + "details": [], + "message": "branch coverage is reported with vitest --coverage", + "status": "PASS" + }, + "cxx-extensions": { + "details": [], + "message": "not applicable to JavaScript, TypeScript (applies to C, C++)", + "status": "SKIP" + }, + "data-wellformed": { + "details": [], + "message": "8 data files parse (JSON: 3, YAML: 5)", + "status": "PASS" + }, + "dependency-audit": { + "details": [], + "message": "CI audits the dependencies with npm audit", + "status": "PASS" + }, + "dependency-management": { + "details": [], + "message": "dependencies are declared in package.json and locked in package-lock.json", + "status": "PASS" + }, + "dependency-updates": { + "details": [ + ".github/dependabot.yml" + ], + "message": "automated dependency updates are configured with Dependabot", + "status": "PASS" + }, + "function-limits": { + "details": [], + "message": "function length is limited with lizard --length 30", + "status": "PASS" + }, + "hardening": { + "details": [], + "message": "not applicable to JavaScript, TypeScript (applies to C, C++, Go, Rust)", + "status": "SKIP" + }, + "large-files": { + "details": [], + "message": "no committed file is larger than 5 MiB", + "status": "PASS" + }, + "license": { + "details": [ + "LICENSE" + ], + "message": "the project is licensed under MIT", + "status": "PASS" + }, + "platform-matrix": { + "details": [ + ".github/workflows/ci-lint.yml: ci-lint: linux / no runtime version", + ".github/workflows/ci.yml: test (ubuntu-latest, 22): linux / 22", + ".github/workflows/ci.yml: test (ubuntu-latest, 24): linux / 24", + ".github/workflows/ci.yml: test (macos-latest, 22): macos / 22", + ".github/workflows/ci.yml: test (macos-latest, 24): macos / 24", + ".github/workflows/ci.yml: test (windows-latest, 22): windows / 22", + ".github/workflows/ci.yml: test (windows-latest, 24): windows / 24", + ".github/workflows/copy-paste-detection.yml: copy-paste-detection: linux / no runtime version", + ".github/workflows/function-limits.yml: function-limits: linux / no runtime version" + ], + "message": "CI covers 3 OS(es) (linux, macos, windows), 2 runtime version(s) (22, 24), 6 combination(s)", + "status": "PASS" + }, + "secret-scanning": { + "details": [], + "message": "secrets are scanned for with gitleaks", + "status": "PASS" + }, + "security-policy": { + "details": [], + "message": "SECURITY.md says how to report a vulnerability", + "status": "PASS" + }, + "security-scanning": { + "details": [], + "message": "security lints are enabled: eslint-plugin-security", + "status": "PASS" + }, + "static-analysis": { + "details": [], + "message": "eslint (eslint.config.js) runs", + "status": "PASS" + }, + "status-badges": { + "details": [], + "message": "README.md shows CI (GitHub Actions) and quality (Codecov) badges", + "status": "PASS" + }, + "tests-run": { + "details": [], + "message": "JavaScript or TypeScript tests found in 1 file(s)", + "status": "PASS" + }, + "type-checking": { + "details": [], + "message": "tsc (tsconfig.json) runs; strict mode bonus: \"strict\": true in tsconfig.json", + "status": "PASS" + }, + "vcs-ignore": { + "details": [], + "message": ".gitignore ignores node_modules/", + "status": "PASS" + }, + "warnings-as-errors": { + "details": [], + "message": "warnings are treated as errors with eslint --max-warnings 0", + "status": "PASS" + } + } +} diff --git a/tests/snapshots/well_configured_python.json b/tests/snapshots/well_configured_python.json index 7e644d5..672f503 100644 --- a/tests/snapshots/well_configured_python.json +++ b/tests/snapshots/well_configured_python.json @@ -298,7 +298,7 @@ }, "warnings-as-errors": { "details": [], - "message": "not applicable to Python (applies to C, C++)", + "message": "not applicable to Python (applies to C, C++, JavaScript, TypeScript)", "status": "SKIP" } } diff --git a/tests/snapshots/well_configured_rust.json b/tests/snapshots/well_configured_rust.json index 716ba6b..8a88d5e 100644 --- a/tests/snapshots/well_configured_rust.json +++ b/tests/snapshots/well_configured_rust.json @@ -431,7 +431,7 @@ }, "type-checking": { "details": [], - "message": "not applicable to Rust (applies to Python)", + "message": "not applicable to Rust (applies to JavaScript, Python, TypeScript)", "status": "SKIP" }, "vcs-ignore": { @@ -441,7 +441,7 @@ }, "warnings-as-errors": { "details": [], - "message": "not applicable to Rust (applies to C, C++)", + "message": "not applicable to Rust (applies to C, C++, JavaScript, TypeScript)", "status": "SKIP" } } diff --git a/tests/support.py b/tests/support.py index a37e36d..730d39b 100644 --- a/tests/support.py +++ b/tests/support.py @@ -133,6 +133,18 @@ def gets_c_family_checks(self) -> bool: languages=frozenset({"Go"}), build_systems=frozenset({"Go modules"}), ), + ProjectLayout( + name="npm JavaScript", + files={"package.json": '{"main": "index.js"}\n', "index.js": ""}, + languages=frozenset({"JavaScript"}), + build_systems=frozenset({"npm"}), + ), + ProjectLayout( + name="Deno TypeScript", + files={"deno.json": "{}\n", "main.ts": ""}, + languages=frozenset({"TypeScript"}), + build_systems=frozenset({"Deno"}), + ), ProjectLayout( name="Unrecognised language", files={"README.md": "", "src/main.zig": ""}, @@ -226,6 +238,7 @@ def run_in_every_layout( WELL_CONFIGURED_PYTHON = fixture_files("well_configured_python") WELL_CONFIGURED_RUST = fixture_files("well_configured_rust") WELL_CONFIGURED_GO = fixture_files("well_configured_go") +WELL_CONFIGURED_JAVASCRIPT = fixture_files("well_configured_javascript") UNSTRUCTURED_CI_FILES = { provider: path for provider, path in CI_FILES.items() if provider not in {"GitHub Actions", "GitLab CI"} diff --git a/tests/test_cli.py b/tests/test_cli.py index 53cb037..eac0247 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -21,6 +21,7 @@ PROJECT_LAYOUTS, WELL_CONFIGURED, WELL_CONFIGURED_GO, + WELL_CONFIGURED_JAVASCRIPT, WELL_CONFIGURED_PYTHON, WELL_CONFIGURED_RUST, ProjectLayout, @@ -33,7 +34,7 @@ # Language -> the language-specific checks that do not apply to it. NOT_APPLICABLE = { language: {found.name for found in LANGUAGE_SPECIFIC_CHECKS if language not in found.languages} - for language in ("Python", "Rust", "Go") + for language in ("Python", "Rust", "Go", "JavaScript", "TypeScript") } @@ -63,7 +64,13 @@ def test_well_configured_project_passes(self) -> None: self.assertIn("[PASS] coverage:", output) def test_well_configured_projects_in_other_languages_pass_strictly(self) -> None: - projects = {"Python": WELL_CONFIGURED_PYTHON, "Rust": WELL_CONFIGURED_RUST, "Go": WELL_CONFIGURED_GO} + projects = { + "Python": WELL_CONFIGURED_PYTHON, + "Rust": WELL_CONFIGURED_RUST, + "Go": WELL_CONFIGURED_GO, + "JavaScript": WELL_CONFIGURED_JAVASCRIPT, + "TypeScript": WELL_CONFIGURED_JAVASCRIPT, + } for layout in OTHER_LANGUAGE_LAYOUTS: [language] = layout.languages with layout_project(self, layout, projects[language]) as root: @@ -223,6 +230,6 @@ def test_list_as_json_is_the_support_matrix(self) -> None: self.assertEqual(code, EXIT_SUCCESS) matrix = {row["check"]: row["applies_to"] for row in json.loads(output)} self.assertEqual(matrix["ci-present"], []) - self.assertEqual(matrix["coverage"], ["C/C++", "Python", "Rust", "Go"]) - self.assertEqual(matrix["warnings-as-errors"], ["C/C++"]) + self.assertEqual(matrix["coverage"], ["C/C++", "Python", "Rust", "Go", "JavaScript/TypeScript"]) + self.assertEqual(matrix["warnings-as-errors"], ["C/C++", "JavaScript/TypeScript"]) self.assertEqual(matrix["ci-build-types"], ["C/C++", "Rust"]) diff --git a/tests/test_ecosystem_contract.py b/tests/test_ecosystem_contract.py index c53f6ba..cd1c913 100644 --- a/tests/test_ecosystem_contract.py +++ b/tests/test_ecosystem_contract.py @@ -6,7 +6,16 @@ import unittest from project_validator.core import Status -from project_validator.ecosystems import BUILT_IN, C_AND_CXX, GO, PYTHON, RUST, Ecosystem, applies_to +from project_validator.ecosystems import ( + BUILT_IN, + C_AND_CXX, + GO, + JAVASCRIPT_AND_TYPESCRIPT, + PYTHON, + RUST, + Ecosystem, + applies_to, +) from project_validator.plugins import discover from project_validator.runner import run_checks from tests.support import FIXTURES, make_context @@ -17,6 +26,7 @@ PYTHON: ("well_configured_python", "bare_python"), RUST: ("well_configured_rust", "bare_rust"), GO: ("well_configured_go", "bare_go"), + JAVASCRIPT_AND_TYPESCRIPT: ("well_configured_javascript", "bare_javascript"), } diff --git a/tests/test_javascript.py b/tests/test_javascript.py new file mode 100644 index 0000000..9db7c55 --- /dev/null +++ b/tests/test_javascript.py @@ -0,0 +1,305 @@ +"""The JavaScript/TypeScript profile's rules, judged by the generic checks in every JavaScript project layout.""" + +import json +import unittest + +from project_validator.checks.code_formatting import code_formatting +from project_validator.checks.committed_artifacts import committed_artifacts +from project_validator.checks.compiler_warnings import compiler_warnings +from project_validator.checks.coverage import coverage +from project_validator.checks.dependency_audit import dependency_audit +from project_validator.checks.dependency_management import dependency_management +from project_validator.checks.function_limits import function_limits +from project_validator.checks.license import license_ +from project_validator.checks.platform_matrix import platform_matrix +from project_validator.checks.security_scanning import security_scanning +from project_validator.checks.static_analysis import static_analysis +from project_validator.checks.tests_run import tests_run +from project_validator.checks.type_checking import type_checking +from project_validator.checks.vcs_ignore import vcs_ignore +from project_validator.checks.warnings_as_errors import warnings_as_errors +from project_validator.core import Check, Result, Status +from project_validator.ecosystems import JAVASCRIPT_AND_TYPESCRIPT +from project_validator.ecosystems.javascript import DEPENDENCIES, NODE_MODULES, TYPE_CHECKING +from tests.support import ( + GITHUB_WORKFLOW, + PROJECT_LAYOUTS, + github_job, + make_context, + run_as, + run_in_every_layout, + temp_project, +) + +STRICT_TSCONFIG = {"tsconfig.json": '{\n // checked strictly\n "compilerOptions": {"strict": true,},\n}\n'} + + +def _ci(*steps: str) -> dict[str, str]: + return github_job("ubuntu-latest", *steps) + + +def _package(**fields: object) -> dict[str, str]: + return {"package.json": json.dumps(fields)} + + +def _scripts(**scripts: str) -> dict[str, str]: + return _package(main="index.js", scripts=scripts) + + +class RecognitionTests(unittest.TestCase): + def _languages(self, files: dict[str, str]) -> frozenset[str]: + with temp_project(files) as root: + return make_context(root).structure.languages + + # package.json is the tooling of projects in many languages. + def test_a_package_json_alone_is_not_a_javascript_project(self) -> None: + self.assertEqual(self._languages({"package.json": '{"devDependencies": {"jscpd": "4"}}'}), frozenset()) + self.assertEqual(self._languages({"package.json": "not json"}), frozenset()) + + def test_a_package_with_entry_points_or_sources_is(self) -> None: + for files in (_package(bin={"app": "cli.js"}), {"package.json": "{}", "src/app.mjs": ""}): + with self.subTest(files=files): + self.assertEqual(self._languages(files), frozenset({"JavaScript"})) + self.assertEqual(self._languages({"deno.jsonc": "{}"}), frozenset({"TypeScript"})) + self.assertEqual(self._languages({"app.tsx": ""}), frozenset({"TypeScript"})) + + def test_sources_in_node_modules_do_not_count(self) -> None: + self.assertEqual(self._languages({"node_modules/left-pad/index.js": ""}), frozenset()) + + +class JavaScriptTests(unittest.TestCase): + def _run(self, selected: Check, files: dict[str, str]) -> Result: + return run_as(self, selected, files, JAVASCRIPT_AND_TYPESCRIPT) + + def test_formatting_is_enforced_by_any_formatter_in_ci_or_a_script_it_runs(self) -> None: + for files in ( + _ci("npx prettier --check ."), + _ci("biome ci"), + _ci("deno fmt --check"), + _ci("dprint check"), + {**_scripts(fmt="prettier -c src"), **_ci("pnpm fmt")}, + {**_scripts(check="npm run fmt", fmt="prettier -l ."), **_ci("yarn run check")}, + ): + with self.subTest(files=files): + self.assertEqual(self._run(code_formatting, files).status, Status.PASS) + self.assertEqual(self._run(code_formatting, _ci("npx prettier --write .")).status, Status.FAIL) + # A script CI never runs enforces nothing. + self.assertEqual(self._run(code_formatting, _scripts(fmt="prettier --check .")).status, Status.FAIL) + + def test_a_formatter_configuration_without_ci_warns(self) -> None: + for files in ({".prettierrc.json": "{}"}, _package(main="index.js", prettier={})): + with self.subTest(files=files): + self.assertEqual(self._run(code_formatting, files).status, Status.WARN) + + def test_scripts_run_their_pre_and_post_scripts_and_deno_tasks(self) -> None: + pre = {**_scripts(pretest="prettier --check .", test="node --test"), **_ci("npm t")} + self.assertEqual(self._run(code_formatting, pre).status, Status.PASS) + deno = {"deno.json": '{"tasks": {"fmt": "deno fmt --check"}}', **_ci("deno task fmt")} + self.assertEqual(self._run(code_formatting, deno).status, Status.PASS) + loop = {**_scripts(a="npm run b", b="npm run a"), **_ci("npm run a")} + self.assertEqual(self._run(code_formatting, loop).status, Status.FAIL) + + def test_tests_are_test_and_spec_files(self) -> None: + for path in ("src/app.test.ts", "app.spec.jsx", "lib/app_test.mjs", "src/__tests__/app.js"): + with self.subTest(path=path): + result = self._run(tests_run, {path: "", **_ci("npx vitest run")}) + self.assertEqual(result, Result.passed("JavaScript or TypeScript tests found in 1 file(s)")) + for path in ("src/latest.js", "test.md", "app.test.py"): + with self.subTest(path=path): + self.assertEqual(self._run(tests_run, {path: ""}).status, Status.FAIL) + + def test_ci_must_run_the_tests(self) -> None: + self.assertEqual(self._run(tests_run, {"a.test.js": ""}).status, Status.WARN) + for command in ("npm test", "pnpm test", "node --experimental-strip-types --test", "bun test", "jest"): + with self.subTest(command=command): + self.assertEqual(self._run(tests_run, {"a.test.js": "", **_ci(command)}).status, Status.PASS) + + def test_coverage_is_found_in_ci_scripts_or_runner_settings(self) -> None: + cases = { + "c8": _ci("npx c8 node --test"), + "nyc": _scripts(test="nyc mocha") | _ci("npm test"), + "jest --coverage": {"jest.config.js": "export default { collectCoverage: true };\n"}, + "vitest --coverage": {"vitest.config.ts": "test: { coverage: { provider: 'v8', enabled: true } }\n"}, + "deno test --coverage": _ci("deno test --coverage=cov"), + "node --experimental-test-coverage": _ci("node --test --experimental-test-coverage"), + } + for tool, files in cases.items(): + with self.subTest(tool=tool): + self.assertEqual(self._run(coverage, files), Result.passed(f"branch coverage is reported with {tool}")) + self.assertEqual(self._run(coverage, _ci("npx vitest run", "git checkout c8a1b2")).status, Status.FAIL) + + def test_one_linter_is_enough(self) -> None: + for command, linter in (("npx eslint .", "eslint"), ("biome ci", "biome lint"), ("deno lint", "deno lint")): + with self.subTest(command=command): + result = self._run(compiler_warnings, _ci(command)) + self.assertEqual(result, Result.passed(f"strict JavaScript linters are enabled: {linter}")) + result = self._run(compiler_warnings, _ci("npm ci")) + self.assertEqual(result, Result.failed("no JavaScript linters found; enable at least eslint")) + + def test_a_lint_warning_must_fail_ci(self) -> None: + for command in ("eslint --max-warnings=0 .", "biome lint --error-on-warnings", "biome ci", "deno lint"): + with self.subTest(command=command): + self.assertEqual(self._run(warnings_as_errors, _ci(command)).status, Status.PASS) + for command in ("eslint .", "eslint --max-warnings 10 .", "biome lint"): + with self.subTest(command=command): + self.assertEqual(self._run(warnings_as_errors, _ci(command)).status, Status.FAIL) + + def test_package_manager_audits_count_but_not_signature_checks(self) -> None: + for command in ("npm audit --omit=dev", "pnpm audit", "yarn npm audit", "bun audit", "osv-scanner ."): + with self.subTest(command=command): + self.assertEqual(self._run(dependency_audit, _ci(command)).status, Status.PASS) + self.assertEqual(self._run(dependency_audit, _ci("npm audit signatures")).status, Status.WARN) + + +class FunctionLimitTests(unittest.TestCase): + def _run(self, files: dict[str, str]) -> Result: + return run_as(self, function_limits, files, JAVASCRIPT_AND_TYPESCRIPT) + + def test_eslint_max_lines_per_function_limits_functions_when_eslint_runs(self) -> None: + config = {"eslint.config.js": 'export default [{ rules: { "max-lines-per-function": ["error", 40] } }];\n'} + expected = Result.passed("function length is limited with eslint max-lines-per-function (eslint.config.js)") + self.assertEqual(self._run({**config, **_ci("npx eslint .")}), expected) + warned = Result.warned("max-lines-per-function is enabled in eslint.config.js but CI does not run eslint") + self.assertEqual(self._run(config), warned) + + def test_a_disabled_rule_limits_nothing(self) -> None: + for setting in ('"off"', "0", '["off"]'): + with self.subTest(setting=setting): + config = {".eslintrc.json": f'{{"rules": {{"max-lines-per-function": {setting}}}}}'} + result = self._run({**config, **_ci("eslint .")}) + self.assertEqual(result.status, Status.FAIL) + self.assertIn("max-lines-per-function", result.message) + + +def _in_layout(test: unittest.TestCase, name: str, selected: Check, files: dict[str, str]) -> Result: + """The result of `selected` in a project of the layout called `name` with `files` added.""" + layouts = tuple(layout for layout in PROJECT_LAYOUTS if layout.name == name) + return run_in_every_layout(test, selected, files, layouts) + + +class StaticAnalysisTests(unittest.TestCase): + def _run(self, selected: Check, files: dict[str, str]) -> Result: + return run_as(self, selected, files, JAVASCRIPT_AND_TYPESCRIPT) + + def test_a_linter_is_the_static_analyzer(self) -> None: + files = {"eslint.config.js": "export default [];\n", **_scripts(lint="eslint ."), **_ci("npm run lint")} + self.assertEqual(self._run(static_analysis, files), Result.passed("eslint (eslint.config.js) runs")) + warned = _in_layout(self, "npm JavaScript", static_analysis, {"biome.json": "{}"}) + self.assertEqual(warned.message, "biome lint is configured (biome.json) but does not run") + + def test_security_lints_are_a_plugin_or_semgrep_rules(self) -> None: + for files in ( + {"eslint.config.mjs": 'import security from "eslint-plugin-security";\n'}, + _ci("semgrep scan --config p/typescript"), + ): + with self.subTest(files=files): + self.assertEqual(_in_layout(self, "npm JavaScript", security_scanning, files).status, Status.PASS) + self.assertEqual( + _in_layout(self, "npm JavaScript", security_scanning, _ci("semgrep scan --config p/python")).status, + Status.WARN, + ) + + +class TypeCheckingTests(unittest.TestCase): + def _npm(self, files: dict[str, str]) -> Result: + return _in_layout(self, "npm JavaScript", type_checking, files) + + def _deno(self, files: dict[str, str]) -> Result: + return _in_layout(self, "Deno TypeScript", type_checking, files) + + def test_tsc_type_checks_and_strict_mode_is_the_bonus(self) -> None: + typecheck = {**_scripts(typecheck="tsc --noEmit"), **_ci("pnpm typecheck")} + expected = 'tsc (tsconfig.json) runs; strict mode bonus: "strict": true in tsconfig.json' + self.assertEqual(self._npm({**STRICT_TSCONFIG, **typecheck}), Result.passed(expected)) + + def test_the_strictest_base_configuration_is_strict(self) -> None: + for extends in ('"@tsconfig/strictest/tsconfig.json"', '["@tsconfig/node22", "@tsconfig/strictest"]'): + with self.subTest(extends=extends): + files = {"jsconfig.json": f'{{"extends": {extends}}}', **_ci("npx tsc -p jsconfig.json")} + self.assertIn("@tsconfig/strictest in jsconfig.json", self._npm(files).message) + + def test_without_strict_mode_it_is_suggested(self) -> None: + files = {"tsconfig.json": '{"extends": "./base.json"}', **_ci("npx vue-tsc --noEmit")} + self.assertEqual(self._npm(files), Result.passed("tsc (tsconfig.json) runs", TYPE_CHECKING.strict_hint)) + self.assertEqual(self._npm(_ci("npm ci")), Result.failed("no type checker runs", TYPE_CHECKING.hint)) + + def test_deno_checks_strictly_unless_told_not_to(self) -> None: + strict = "deno check (deno.json) runs; strict mode bonus: Deno's default strict mode (deno.json)" + self.assertEqual(self._deno(_ci("deno test")), Result.passed(strict)) + lax = {"deno.json": '{"compilerOptions": {"strict": false}}', **_ci("deno check main.ts")} + self.assertEqual(self._deno(lax).details, (TYPE_CHECKING.strict_hint,)) + + +class DependencyTests(unittest.TestCase): + def _npm(self, selected: Check, files: dict[str, str]) -> Result: + return _in_layout(self, "npm JavaScript", selected, files) + + def test_declared_dependencies_must_be_locked_even_in_a_library(self) -> None: + declared = _package(main="index.js", peerDependencies={"react": "^19"}) + expected = Result.warned("dependencies are declared in package.json but not locked", DEPENDENCIES.lock_hint) + self.assertEqual(self._npm(dependency_management, declared), expected) + locked = self._npm(dependency_management, {**declared, "pnpm-lock.yaml": "lockfileVersion: '9.0'\n"}) + self.assertEqual( + locked, Result.passed("dependencies are declared in package.json and locked in pnpm-lock.yaml") + ) + + def test_deno_imports_are_dependencies(self) -> None: + files = {"deno.jsonc": '{"imports": {"@std/assert": "jsr:@std/assert@1"}}', "deno.lock": "{}"} + expected = Result.passed("dependencies are declared in deno.jsonc and locked in deno.lock") + self.assertEqual(_in_layout(self, "Deno TypeScript", dependency_management, files), expected) + + def test_without_dependencies_there_is_nothing_to_lock(self) -> None: + self.assertEqual(self._npm(dependency_management, {}), Result.passed("no dependencies are declared")) + # A package.json that does not parse declares nothing. + broken = {"package.json": '{"main": "index.js", "dependencies": {', "index.js": ""} + self.assertEqual(self._npm(dependency_management, broken), Result.passed("no dependencies are declared")) + + def test_the_package_license_must_match_the_license_file(self) -> None: + mit = "Permission is hereby granted, free of charge, to any person obtaining a copy\n" + files = {"LICENSE": mit, **_package(main="index.js", license="Apache-2.0")} + result = self._npm(license_, files) + self.assertEqual(result.status, Status.WARN) + self.assertIn("package.json declares Apache-2.0", result.details[0]) + self.assertEqual(self._npm(license_, {**files, **_package(license="MIT", main="a.js")}).status, Status.PASS) + + +class OutputTests(unittest.TestCase): + def _run(self, selected: Check, files: dict[str, str]) -> Result: + return run_as(self, selected, files, JAVASCRIPT_AND_TYPESCRIPT) + + def test_node_modules_must_be_ignored(self) -> None: + for entry in ("node_modules", "/node_modules/", "**/node_modules"): + with self.subTest(entry=entry): + self.assertEqual(self._run(vcs_ignore, {".gitignore": f"{entry}\n"}).status, Status.PASS) + self.assertEqual(self._run(vcs_ignore, {".gitignore": "dist/\n"}).status, Status.WARN) + + def test_dependencies_coverage_and_build_info_are_outputs_but_dist_is_not(self) -> None: + result = self._run(committed_artifacts, {"coverage/lcov.info": "", "tsconfig.tsbuildinfo": ""}) + self.assertEqual(result.message, "2 build outputs are committed") + self.assertEqual(self._run(committed_artifacts, {"dist/index.js": ""}).status, Status.PASS) + # Only git, not a walk of the files, sees a committed node_modules. + self.assertEqual(NODE_MODULES.artifact("packages/a/node_modules/x/index.js"), "packages/a/node_modules/") + + def test_the_platform_matrix_counts_node_deno_and_bun_versions(self) -> None: + files = { + GITHUB_WORKFLOW: """\ + jobs: + node: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + node: ["22", lts/*] + steps: + - uses: actions/setup-node@v5 + with: + node-version: ${{ matrix.node }} + deno: + runs-on: macos-latest + container: denoland/deno:alpine-2.5.0 + steps: + - run: deno test + """ + } + result = self._run(platform_matrix, files) + self.assertIn("3 runtime version(s) (2.5.0, 22, lts/*)", result.message) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 94d11ef..665bae0 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -133,7 +133,10 @@ def _discover_zig(self) -> Plugins: def test_a_plugin_ecosystem_is_registered_and_practices_apply_to_it(self) -> None: plugins = self._discover_zig() - self.assertEqual([ecosystem.label for ecosystem in registered()], ["C/C++", "Python", "Rust", "Go", "Zig"]) + self.assertEqual( + [ecosystem.label for ecosystem in registered()], + ["C/C++", "Python", "Rust", "Go", "JavaScript/TypeScript", "Zig"], + ) formatting = next(found for found in plugins.checks if found.name == "code-formatting") self.assertIn("Zig", formatting.languages) self.assertIn("Zig", applies_to(formatting)) @@ -154,7 +157,10 @@ def test_a_plugin_ecosystem_is_detected_and_judged(self) -> None: def test_loading_a_plugin_again_replaces_its_ecosystem(self) -> None: self._discover_zig() self._discover_zig() - self.assertEqual([ecosystem.label for ecosystem in registered()], ["C/C++", "Python", "Rust", "Go", "Zig"]) + self.assertEqual( + [ecosystem.label for ecosystem in registered()], + ["C/C++", "Python", "Rust", "Go", "JavaScript/TypeScript", "Zig"], + ) def test_rejects_an_ecosystem_with_a_built_in_label(self) -> None: clash = ECOSYSTEM_PLUGIN.replace('label="Zig"', 'label="Python"') From 3d67cae24109bef2ef29a523487d9f1280421afa Mon Sep 17 00:00:00 2001 From: leftibot Date: Thu, 5 Nov 2026 23:50:22 -0700 Subject: [PATCH 2/6] feat: add a JVM ecosystem profile (Java and Kotlin, with Maven and Gradle) Maven and Gradle projects are judged by Spotless (or fmt-maven-plugin, google-java-format, ktfmt, ktlint); JaCoCo or Kover branch coverage; @Test methods and Kotest specs under src/test run by mvn or gradle; javac's -Xlint:all (Kotlin warns by default); -Werror or Kotlin's allWarningsAsErrors; OWASP dependency-check; Find Security Bugs or PMD's security rules; Checkstyle MethodLength or detekt LongMethod; Error Prone, SpotBugs, PMD, Checkstyle or detekt; Gradle dependency verification, a gradle.lockfile or Maven's --strict-checksums; ignored target/ (Maven) or build/ (Gradle); and the JDK versions CI runs. JVM builds wire their tools in: a Maven plugin binds a goal to a phase, a Gradle plugin adds its task to check. So the build files count as CI configuration when CI runs the build far enough: any compiling phase or task for compiler flags, mvn verify or gradle check/build for the verification plugins. A Maven plugin counts only when an execution binds the goal. Two language-agnostic changes support it: DependencyAudit gets the optional texts that Formatter got (dependency-check bound in pom.xml), and vcs-ignore judges the BuildOutputs of the build system a project uses, as committed-artifacts already did (judge_each_ecosystem). Tests look up checks by name (checks_named) and layouts by name (run_in_layout). Closes #66 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ByTXNqDWdM7HzezrFtEvQt --- README.md | 32 +- project_validator/checks/dependency_audit.py | 2 +- project_validator/checks/vcs_ignore.py | 4 +- project_validator/ecosystems/__init__.py | 24 +- project_validator/ecosystems/jvm.py | 381 ++++++++++++++++++ project_validator/rules.py | 7 +- tests/checks/test_warnings_as_errors.py | 6 +- tests/fixtures/bare_jvm/pom.xml | 15 + .../src/main/java/com/example/App.java | 11 + .../target/classes/com/example/App.class | 0 .../.github/dependabot.yml | 6 + .../.github/workflows/ci-lint.yml | 9 + .../.github/workflows/ci.yml | 14 + .../workflows/copy-paste-detection.yml | 6 + .../.github/workflows/function-limits.yml | 7 + tests/fixtures/well_configured_jvm/.gitignore | 1 + tests/fixtures/well_configured_jvm/LICENSE | 21 + tests/fixtures/well_configured_jvm/README.md | 4 + .../fixtures/well_configured_jvm/SECURITY.md | 3 + tests/fixtures/well_configured_jvm/pom.xml | 92 +++++ .../src/main/java/com/example/App.java | 11 + .../src/test/java/com/example/AppTest.java | 12 + tests/snapshots/bare_go.json | 2 +- tests/snapshots/bare_jvm.json | 194 +++++++++ tests/snapshots/bare_project.json | 2 +- tests/snapshots/bare_python.json | 2 +- tests/snapshots/bare_rust.json | 2 +- tests/snapshots/buildable.json | 2 +- tests/snapshots/well_configured.json | 2 +- tests/snapshots/well_configured_go.json | 2 +- tests/snapshots/well_configured_jvm.json | 308 ++++++++++++++ tests/snapshots/well_configured_python.json | 2 +- tests/snapshots/well_configured_rust.json | 2 +- tests/support.py | 26 ++ tests/test_cli.py | 9 +- tests/test_ecosystem_contract.py | 2 + tests/test_javascript.py | 23 +- tests/test_jvm.py | 263 ++++++++++++ tests/test_plugins.py | 4 +- 39 files changed, 1454 insertions(+), 61 deletions(-) create mode 100644 project_validator/ecosystems/jvm.py create mode 100644 tests/fixtures/bare_jvm/pom.xml create mode 100644 tests/fixtures/bare_jvm/src/main/java/com/example/App.java create mode 100644 tests/fixtures/bare_jvm/target/classes/com/example/App.class create mode 100644 tests/fixtures/well_configured_jvm/.github/dependabot.yml create mode 100644 tests/fixtures/well_configured_jvm/.github/workflows/ci-lint.yml create mode 100644 tests/fixtures/well_configured_jvm/.github/workflows/ci.yml create mode 100644 tests/fixtures/well_configured_jvm/.github/workflows/copy-paste-detection.yml create mode 100644 tests/fixtures/well_configured_jvm/.github/workflows/function-limits.yml create mode 100644 tests/fixtures/well_configured_jvm/.gitignore create mode 100644 tests/fixtures/well_configured_jvm/LICENSE create mode 100644 tests/fixtures/well_configured_jvm/README.md create mode 100644 tests/fixtures/well_configured_jvm/SECURITY.md create mode 100644 tests/fixtures/well_configured_jvm/pom.xml create mode 100644 tests/fixtures/well_configured_jvm/src/main/java/com/example/App.java create mode 100644 tests/fixtures/well_configured_jvm/src/test/java/com/example/AppTest.java create mode 100644 tests/snapshots/bare_jvm.json create mode 100644 tests/snapshots/well_configured_jvm.json create mode 100644 tests/test_jvm.py diff --git a/README.md b/README.md index 6ae46a3..fbcf9f0 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,13 @@ A tool to validate a project's build, CI and tooling against best practices. It recognises C and C++ (CMake or Meson), Python (`pyproject.toml` or `setup.py`), -Rust (Cargo), Go (Go modules) and JavaScript/TypeScript (`package.json` or `deno.json`) projects, and judges each language by its own ecosystem's tools: a C++ +Rust (Cargo), Go (Go modules), JavaScript/TypeScript (`package.json` or `deno.json`) and JVM (Java and Kotlin, with Maven or Gradle) projects, and judges each language by its own ecosystem's tools: a C++ project needs branch coverage from gcovr or llvm-cov, a Python project from coverage.py, a Rust project from cargo-llvm-cov. Checks that do not apply to any of the project's languages are reported as `SKIP`. A project using several languages is judged for each of them and gets the worst result. In a project of no recognised language, only the checks that apply to every project run. A C/C++ project is judged by the rules of the -build system it uses (CMake or Meson); one using neither is judged as a CMake project. +build system it uses (CMake or Meson); one using neither is judged as a CMake project. A JVM project's Maven or Gradle build counts as CI configuration when CI runs it (`mvn verify`, `gradle check`), since that is where its plugins are wired in. More ecosystems can be added with plugins (see [Adding an ecosystem](#adding-an-ecosystem)). The validator itself is pure Python and has no build step; it only inspects @@ -60,33 +60,33 @@ Exit codes: `0` everything passed, `1` a check failed or raised an error, `2` us | Check | Applies to | Passes when | |---|---|---| | `ci-present` | all | at least one CI configuration is found | -| `tests-run` | C/C++, Python, Rust, Go, JavaScript/TypeScript | tests exist and CI runs them (`ctest-runs` is accepted as an old name). C/C++ with CMake: `enable_testing()` or `include(CTest)` is called, tests are registered (`add_test`, `*_discover_tests`, ...) and CI runs `ctest`; with `--build`, the project is really configured and built, and `ctest` must list at least one test. C/C++ with Meson: `test()` is called in a `meson.build` and CI runs `meson test`. Python: `test_*.py` / `*_test.py` files run by pytest, unittest, tox or nox. Rust: `#[test]` functions run by `cargo test` or nextest. Go: `Test` functions in `*_test.go` files run by `go test`. JavaScript/TypeScript: `*.test.*`, `*.spec.*`, `*_test.*` or `__tests__/` files run by jest, vitest, mocha, ava, `node --test`, `deno test`, `bun test`, playwright or `npm test`. Warns when CI never runs the tests | -| `coverage` | C/C++, Python, Rust, Go, JavaScript/TypeScript | CI reports branch coverage (Go's tools measure statements only, so Go needs statement coverage: `go test -cover` or `-coverprofile`). C/C++: gcovr, lcov, genhtml, gcov or llvm-cov branch options; Python: `coverage run --branch`, `--cov-branch` or `branch = true`; Rust: cargo-llvm-cov, cargo-tarpaulin or grcov with `--branch`; JavaScript/TypeScript: c8, nyc, `jest --coverage` (or `collectCoverage: true`), `vitest --coverage` (or coverage `enabled: true`), `deno test --coverage` or `node --experimental-test-coverage`, in CI, the package scripts it runs, or the test runner's configuration. Path or MC/DC coverage (`-fcondition-coverage`, `-fcoverage-mcdc`, `--mcdc`, ...) is reported as a bonus | -| `platform-matrix` | C/C++, Python, Rust, Go, JavaScript/TypeScript | CI covers at least 2 operating systems, 2 toolchains (compilers, Python versions, Rust toolchains, Go versions, or Node.js, Deno or Bun versions) and 3 OS/toolchain combinations | +| `tests-run` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | tests exist and CI runs them (`ctest-runs` is accepted as an old name). C/C++ with CMake: `enable_testing()` or `include(CTest)` is called, tests are registered (`add_test`, `*_discover_tests`, ...) and CI runs `ctest`; with `--build`, the project is really configured and built, and `ctest` must list at least one test. C/C++ with Meson: `test()` is called in a `meson.build` and CI runs `meson test`. Python: `test_*.py` / `*_test.py` files run by pytest, unittest, tox or nox. Rust: `#[test]` functions run by `cargo test` or nextest. Go: `Test` functions in `*_test.go` files run by `go test`. JavaScript/TypeScript: `*.test.*`, `*.spec.*`, `*_test.*` or `__tests__/` files run by jest, vitest, mocha, ava, `node --test`, `deno test`, `bun test`, playwright or `npm test`. JVM: `@Test` methods or Kotest specs under `src/test/java` or `src/test/kotlin`, run by `mvn test` / `verify` or `gradle test` / `check` / `build`. Warns when CI never runs the tests | +| `coverage` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | CI reports branch coverage (Go's tools measure statements only, so Go needs statement coverage: `go test -cover` or `-coverprofile`). C/C++: gcovr, lcov, genhtml, gcov or llvm-cov branch options; Python: `coverage run --branch`, `--cov-branch` or `branch = true`; Rust: cargo-llvm-cov, cargo-tarpaulin or grcov with `--branch`; JavaScript/TypeScript: c8, nyc, `jest --coverage` (or `collectCoverage: true`), `vitest --coverage` (or coverage `enabled: true`), `deno test --coverage` or `node --experimental-test-coverage`, in CI, the package scripts it runs, or the test runner's configuration; JVM: JaCoCo (`jacoco-maven-plugin` with its `report` goal, `jacocoTestReport`) or Kover. Path or MC/DC coverage (`-fcondition-coverage`, `-fcoverage-mcdc`, `--mcdc`, ...) is reported as a bonus | +| `platform-matrix` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | CI covers at least 2 operating systems, 2 toolchains (compilers, Python versions, Rust toolchains, Go versions, Node.js, Deno or Bun versions, or JDK versions) and 3 OS/toolchain combinations | | `ci-build-types` | C/C++, Rust | every OS/toolchain combination in CI builds Debug and Release (Rust: debug and release profiles), each with and without sanitizers | -| `compiler-warnings` | C/C++, Python, Rust, Go, JavaScript/TypeScript | a strict warning set is enabled. C/C++: `-Wall -Wextra -Wpedantic -Wshadow -Wconversion`, plus `/W4` and `/permissive-` when MSVC is in CI; Python: warnings are errors (`-W error`, `PYTHONWARNINGS=error`, pytest `filterwarnings = error`); Rust: `cargo clippy`, `-D warnings` and `clippy::pedantic`; Go: `go vet` and staticcheck (golangci-lint runs both); JavaScript/TypeScript: a linter runs (eslint, `biome lint` / `biome ci`, or `deno lint`). Warns listing what is missing | -| `warnings-as-errors` | C/C++, JavaScript/TypeScript | C/C++: `-Werror`, `/WX` or `COMPILE_WARNING_AS_ERROR` (Meson: `werror=true`) is enabled in at least one configuration; JavaScript/TypeScript: a lint warning fails CI (`eslint --max-warnings 0`, `biome ci` or `--error-on-warnings`, or `deno lint`) | +| `compiler-warnings` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | a strict warning set is enabled. C/C++: `-Wall -Wextra -Wpedantic -Wshadow -Wconversion`, plus `/W4` and `/permissive-` when MSVC is in CI; Python: warnings are errors (`-W error`, `PYTHONWARNINGS=error`, pytest `filterwarnings = error`); Rust: `cargo clippy`, `-D warnings` and `clippy::pedantic`; Go: `go vet` and staticcheck (golangci-lint runs both); JavaScript/TypeScript: a linter runs (eslint, `biome lint` / `biome ci`, or `deno lint`); JVM: javac's `-Xlint:all` (a Kotlin-only project passes on Kotlin's default warnings). Warns listing what is missing | +| `warnings-as-errors` | C/C++, JavaScript/TypeScript, JVM | C/C++: `-Werror`, `/WX` or `COMPILE_WARNING_AS_ERROR` (Meson: `werror=true`) is enabled in at least one configuration; JavaScript/TypeScript: a lint warning fails CI (`eslint --max-warnings 0`, `biome ci` or `--error-on-warnings`, or `deno lint`); JVM: `-Werror` for javac or kotlinc, or Kotlin's `allWarningsAsErrors` | | `cxx-extensions` | C/C++ | `CMAKE_CXX_EXTENSIONS` (or `CMAKE_C_EXTENSIONS` for C) is turned off, or with Meson `cpp_std` (`c_std`) names a standard such as `c++20` rather than `gnu++20`, so `-std=c++NN` is used rather than `-std=gnu++NN` | -| `hardening` | C/C++, Rust, Go | binary hardening is enabled: exploit mitigations that the build inserts into the binary (static analysis and dependency audits are judged by `security-scanning` and `dependency-audit`). C/C++: `_FORTIFY_SOURCE`, `-fstack-protector-strong`, `-fcf-protection`, `-fstack-clash-protection` and `_GLIBCXX_ASSERTIONS` or `_LIBCPP_HARDENING_MODE` (or GCC's `-fhardened`), plus `/sdl /guard:cf /DYNAMICBASE /CETCOMPAT` when MSVC is in CI; Rust: `overflow-checks` in release builds; Go: programs are built as position-independent executables (`-buildmode=pie`, not the default on Linux), and a module with no `package main` is skipped. Python and JavaScript/TypeScript build no binary, so they are not judged. Warns listing what is missing | -| `code-formatting` | C/C++, Python, Rust, Go, JavaScript/TypeScript | the formatting style is defined (`.clang-format`; ruff or black settings; `rustfmt.toml`) and CI runs the formatter. Python, Rust, Go (gofmt, gofumpt, goimports) and JavaScript/TypeScript (`prettier --check`, `biome format` / `biome ci`, `deno fmt --check` or `dprint check`, in CI or the package scripts it runs) formatters have a default style, so CI enforcement alone passes. Warns when the style is defined but CI does not enforce it | -| `function-limits` | C/C++, Python, Rust, Go, JavaScript/TypeScript | function length is limited: lizard runs with a length limit (`lizard --length 30`), or for C/C++, `.clang-tidy` enables `readability-function-size` with a threshold and clang-tidy runs, or for Go, golangci-lint runs with `funlen` enabled, or for JavaScript/TypeScript, eslint runs with `max-lines-per-function` enabled | -| `static-analysis` | C/C++, Python, Rust, Go, JavaScript/TypeScript | static analyzers run. C/C++: both clang-tidy (`CMAKE_CXX_CLANG_TIDY`, `run-clang-tidy`, or `clang-tidy` in CI) with its checks chosen in a `.clang-tidy`, and cppcheck (`CMAKE_CXX_CPPCHECK` or `cppcheck` in CI); Python: `ruff check`, pylint or flake8, in CI or as a pre-commit hook; Rust: `cargo clippy`; Go: golangci-lint with linters enabled beyond its defaults (`enable:` or `default: all` in a `.golangci.yml`); JavaScript/TypeScript: eslint, `biome lint` or `deno lint`, in CI, a pre-commit hook or the package scripts CI runs. Warns when an analyzer is configured but does not run, or runs short of that; fails when none runs | +| `hardening` | C/C++, Rust, Go | binary hardening is enabled: exploit mitigations that the build inserts into the binary (static analysis and dependency audits are judged by `security-scanning` and `dependency-audit`). C/C++: `_FORTIFY_SOURCE`, `-fstack-protector-strong`, `-fcf-protection`, `-fstack-clash-protection` and `_GLIBCXX_ASSERTIONS` or `_LIBCPP_HARDENING_MODE` (or GCC's `-fhardened`), plus `/sdl /guard:cf /DYNAMICBASE /CETCOMPAT` when MSVC is in CI; Rust: `overflow-checks` in release builds; Go: programs are built as position-independent executables (`-buildmode=pie`, not the default on Linux), and a module with no `package main` is skipped. Python, JavaScript/TypeScript and the JVM build no native binary, so they are not judged. Warns listing what is missing | +| `code-formatting` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | the formatting style is defined (`.clang-format`; ruff or black settings; `rustfmt.toml`) and CI runs the formatter. Python, Rust, Go (gofmt, gofumpt, goimports) and JavaScript/TypeScript (`prettier --check`, `biome format` / `biome ci`, `deno fmt --check` or `dprint check`, in CI or the package scripts it runs) and JVM (Spotless, fmt-maven-plugin, google-java-format, ktfmt or ktlint) formatters have a default style, so CI enforcement alone passes. Warns when the style is defined but CI does not enforce it | +| `function-limits` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | function length is limited: lizard runs with a length limit (`lizard --length 30`), or for C/C++, `.clang-tidy` enables `readability-function-size` with a threshold and clang-tidy runs, or for Go, golangci-lint runs with `funlen` enabled, or for JavaScript/TypeScript, eslint runs with `max-lines-per-function` enabled, or for the JVM, Checkstyle runs with `MethodLength` or detekt runs with `LongMethod` (on by default) | +| `static-analysis` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | static analyzers run. C/C++: both clang-tidy (`CMAKE_CXX_CLANG_TIDY`, `run-clang-tidy`, or `clang-tidy` in CI) with its checks chosen in a `.clang-tidy`, and cppcheck (`CMAKE_CXX_CPPCHECK` or `cppcheck` in CI); Python: `ruff check`, pylint or flake8, in CI or as a pre-commit hook; Rust: `cargo clippy`; Go: golangci-lint with linters enabled beyond its defaults (`enable:` or `default: all` in a `.golangci.yml`); JavaScript/TypeScript: eslint, `biome lint` or `deno lint`, in CI, a pre-commit hook or the package scripts CI runs; JVM: Error Prone, SpotBugs, PMD, Checkstyle or detekt. Warns when an analyzer is configured but does not run, or runs short of that; fails when none runs | | `type-checking` | Python, JavaScript/TypeScript | CI (or a pre-commit hook) runs a static type checker. Python: mypy, pyright or basedpyright, pyrefly, or ty; JavaScript/TypeScript: `tsc` (JavaScript with `checkJs`), or `deno check` / `deno test`, also through the package scripts CI runs. Strict mode is a bonus (`strict = true` in `[tool.mypy]` or a `[mypy]` section, `mypy --strict`, pyright's `typeCheckingMode` `strict`, `"strict": true` or `@tsconfig/strictest` in `tsconfig.json`, or Deno, strict by default); without it the result suggests it. Warns when a type checker is configured but does not run; fails when none runs. Compiled languages, whose compilers check types, are not judged | | `action-pinning` | all | every action and reusable workflow CI uses from another repository is pinned to a full commit SHA (`actions/checkout@ # v4.2.2`), and every `docker://` image to a digest. Fails when one is pinned to a branch or not at all; warns when one is pinned to a tag or an abbreviated commit (`--allow-official-tags` accepts tags for `actions/` and `github/`). Skipped without GitHub Actions | | `ci-lint` | all | CI (or a pre-commit hook) lints the configuration of each CI provider it uses: GitHub Actions with actionlint or zizmor, GitLab CI with `glab ci lint` or `gitlab-ci-local --preview`, CircleCI with `circleci config validate`. Warns otherwise; skipped without CI, or when no linter is known for the providers used | | `copy-paste-detection` | all | CI runs a duplicate code detector (PMD CPD, jscpd, Duplo, Simian, CCFinderX or SonarQube); warns otherwise | -| `security-scanning` | all | CI runs code security scanning (CodeQL or GitLab SAST), or every language in the project that has security lints enables them: Python: bandit (or ruff's `S` rules); Rust: `unsafe_code` forbidden crate-wide; Go: gosec; JavaScript/TypeScript: `eslint-plugin-security` (or `@microsoft/eslint-plugin-sdl`), or semgrep with a JavaScript ruleset. Warns otherwise, naming the languages left unscanned | +| `security-scanning` | all | CI runs code security scanning (CodeQL or GitLab SAST), or every language in the project that has security lints enables them: Python: bandit (or ruff's `S` rules); Rust: `unsafe_code` forbidden crate-wide; Go: gosec; JavaScript/TypeScript: `eslint-plugin-security` (or `@microsoft/eslint-plugin-sdl`), or semgrep with a JavaScript ruleset; JVM: SpotBugs with Find Security Bugs, or PMD's `category/java/security.xml` rules. Warns otherwise, naming the languages left unscanned | | `secret-scanning` | all | CI or a pre-commit hook runs a secret scanner: gitleaks, TruffleHog, detect-secrets, ggshield or Nosey Parker. Warns otherwise, and says so when a scanner is configured (`.gitleaks.toml`, `.secrets.baseline`, `.gitguardian.yaml`) but nothing runs it. GitHub's own secret scanning is a repository setting, which cannot be seen from a checkout | -| `dependency-management` | C/C++, Python, Rust, Go, JavaScript/TypeScript | dependencies come from a package manager and are locked. C/C++: CPM, FetchContent, vcpkg (`vcpkg.json`), Conan (`conanfile.txt` / `conanfile.py`) or Meson wraps (`subprojects/*.wrap`); Python: dependencies declared in `pyproject.toml`, `setup.py` or requirements files are locked by a committed `uv.lock`, `poetry.lock`, `pdm.lock`, `Pipfile.lock` or `pylock.toml`, or by requirements pinned with `--hash`; Rust: a binary crate commits `Cargo.lock` (a library need not); Go: a module that requires others commits `go.sum`; JavaScript/TypeScript: dependencies declared in `package.json` (or `imports` in `deno.json`) are locked by a committed `package-lock.json`, `npm-shrinkwrap.json`, `pnpm-lock.yaml`, `yarn.lock`, `bun.lock`, `bun.lockb` or `deno.lock`, libraries included. Passes with a note when no dependencies are declared. Warns when they are not locked, or when third-party code is committed (`third_party/`, `external/`, `extern/`, `vendor/` other than `go mod vendor` output, or git submodules) | -| `dependency-audit` | Python, Rust, Go, JavaScript/TypeScript | CI checks the dependencies against known vulnerabilities. Python: pip-audit or safety; Rust: cargo audit or `cargo deny check` (advisories); Go: govulncheck; JavaScript/TypeScript: `npm audit` (not `npm audit signatures`), `pnpm audit`, `yarn npm audit` / `yarn audit` or `bun audit`. osv-scanner, `trivy fs` and `snyk test` count for every language. Warns otherwise | +| `dependency-management` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | dependencies come from a package manager and are locked. C/C++: CPM, FetchContent, vcpkg (`vcpkg.json`), Conan (`conanfile.txt` / `conanfile.py`) or Meson wraps (`subprojects/*.wrap`); Python: dependencies declared in `pyproject.toml`, `setup.py` or requirements files are locked by a committed `uv.lock`, `poetry.lock`, `pdm.lock`, `Pipfile.lock` or `pylock.toml`, or by requirements pinned with `--hash`; Rust: a binary crate commits `Cargo.lock` (a library need not); Go: a module that requires others commits `go.sum`; JavaScript/TypeScript: dependencies declared in `package.json` (or `imports` in `deno.json`) are locked by a committed `package-lock.json`, `npm-shrinkwrap.json`, `pnpm-lock.yaml`, `yarn.lock`, `bun.lock`, `bun.lockb` or `deno.lock`, libraries included; JVM: dependencies declared in `pom.xml` or a Gradle build are verified by a committed `gradle/verification-metadata.xml` or `gradle.lockfile`, or Maven runs with `--strict-checksums`. Passes with a note when no dependencies are declared. Warns when they are not locked, or when third-party code is committed (`third_party/`, `external/`, `extern/`, `vendor/` other than `go mod vendor` output, or git submodules) | +| `dependency-audit` | Python, Rust, Go, JavaScript/TypeScript, JVM | CI checks the dependencies against known vulnerabilities. Python: pip-audit or safety; Rust: cargo audit or `cargo deny check` (advisories); Go: govulncheck; JavaScript/TypeScript: `npm audit` (not `npm audit signatures`), `pnpm audit`, `yarn npm audit` / `yarn audit` or `bun audit`; JVM: OWASP dependency-check (`dependency-check-maven` bound to `check`, or `dependencyCheckAnalyze`). osv-scanner, `trivy fs` and `snyk test` count for every language. Warns otherwise | | `data-wellformed` | all | every JSON, TOML, YAML and XML file parses, including the configuration in `.github/`, `.gitlab/`, `.circleci/`, `.buildkite/`, `.devcontainer/` and `.vscode/`. JSON that tools read with comments (`*.jsonc`, `tsconfig*.json`, `jsconfig.json`, `devcontainer.json`, `.vscode/*.json`) may have comments and trailing commas; YAML tags of any tool (GitLab's `!reference`) are accepted; XML external entities are never read. Files over 5 MiB are not parsed. Fails listing each file that does not parse, with the line and column | | `large-files` | all | no committed file is larger than 5 MiB (`--max-file-size`) unless Git LFS stores it (`filter=lfs` in a `.gitattributes`) or DVC tracks it (a `.dvc` file beside it). In a git checkout only the files git tracks are judged. Warns listing the largest files | | `security-policy` | all | a `SECURITY.md` (or `.rst`, `.txt`, ...) at the root, in `.github/` or in `docs/` says how to report a vulnerability: an email address, a link, or the forge's private vulnerability reporting. Warns otherwise | | `license` | all | a license file (`LICENSE`, `LICENSE.md`, `LICENSE-MIT`, `COPYING`, `UNLICENSE`, ... or REUSE's `LICENSES/` directory) holds a recognised license: an `SPDX-License-Identifier` line, or the standard text of AGPL, LGPL, GPL, Apache-2.0, MPL-2.0, EPL-2.0, BSL-1.0, Unlicense, CC0-1.0, MIT, BSD, ISC, 0BSD or Zlib. Warns when the license is not recognised, or when a manifest declares another one (Python: `[project] license`; Rust: `[package] license`; JavaScript/TypeScript: `license` in `package.json`). Fails without a license file | | `dependency-updates` | all | Dependabot or Renovate is configured; warns otherwise | | `status-badges` | all | the README (at the root, in `.github/` or in `docs/`) shows a CI status badge of any supported CI provider (GitHub Actions `badge.svg`, GitLab pipeline, CircleCI, ... or their shields.io badges) and a quality badge (Codecov, Coveralls, SonarCloud, Codacy, Code Climate, DeepSource, Coverity Scan, OpenSSF Scorecard or Best Practices). Warns when only one kind is shown; fails when there is no README or no badge | -| `committed-artifacts` | C/C++, Python, Rust, Go, JavaScript/TypeScript | no build output is committed. C/C++: object files and libraries (`*.o`, `*.obj`, `*.a`, `*.lib`, `*.so`, `*.dylib`, `*.dll`), `*.exe`, `*.pdb`, `CMakeCache.txt`, `CMakeFiles/`, `CMakeUserPresets.json`, and a top-level `build/`, `out/` or `builddir/`; Python: `__pycache__/`, `*.py[co]`, `*.egg-info/`, a top-level `dist/` or `build/`; Rust: a top-level `target/`; Go: `*.test`, `*.out` and a top-level `bin/`; JavaScript/TypeScript: `node_modules/`, a top-level `coverage/` and `*.tsbuildinfo` (not `dist/`, which a JavaScript GitHub Action commits). In a git checkout only the files git tracks are judged; `--exclude` keeps fixtures out. Fails listing the outputs | -| `vcs-ignore` | C/C++, Python, Rust, Go, JavaScript/TypeScript | a `.gitignore`, `.hgignore`, `.bzrignore` or `.fossil-settings/ignore-glob` ignores the build outputs (`build/` or `out/`, `__pycache__/` or `*.py[cod]`, `/target`, Go's `*.test` or `*.out`, `node_modules/`). Warns when an ignore file misses them | +| `committed-artifacts` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | no build output is committed. C/C++: object files and libraries (`*.o`, `*.obj`, `*.a`, `*.lib`, `*.so`, `*.dylib`, `*.dll`), `*.exe`, `*.pdb`, `CMakeCache.txt`, `CMakeFiles/`, `CMakeUserPresets.json`, and a top-level `build/`, `out/` or `builddir/`; Python: `__pycache__/`, `*.py[co]`, `*.egg-info/`, a top-level `dist/` or `build/`; Rust: a top-level `target/`; Go: `*.test`, `*.out` and a top-level `bin/`; JavaScript/TypeScript: `node_modules/`, a top-level `coverage/` and `*.tsbuildinfo` (not `dist/`, which a JavaScript GitHub Action commits); JVM: `*.class`, plus Maven's `target/` or Gradle's `build/` and `.gradle/`. In a git checkout only the files git tracks are judged; `--exclude` keeps fixtures out. Fails listing the outputs | +| `vcs-ignore` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | a `.gitignore`, `.hgignore`, `.bzrignore` or `.fossil-settings/ignore-glob` ignores the build outputs (`build/` or `out/`, `__pycache__/` or `*.py[cod]`, `/target`, Go's `*.test` or `*.out`, `node_modules/`, Maven's `target/` or Gradle's `build/`). Warns when an ignore file misses them | CI detection is pluggable too. GitHub Actions workflows (including `strategy.matrix` with `include`/`exclude`) and GitLab CI pipelines (including `parallel: matrix:`) diff --git a/project_validator/checks/dependency_audit.py b/project_validator/checks/dependency_audit.py index c87f7f4..436ba8a 100644 --- a/project_validator/checks/dependency_audit.py +++ b/project_validator/checks/dependency_audit.py @@ -10,6 +10,6 @@ rule=DependencyAudit, ) def dependency_audit(audit: DependencyAudit, context: Context) -> Result: - if tools := matching_names(context.ci_texts, audit.tools): + if tools := matching_names(audit.texts(context), audit.tools): return Result.passed(f"CI audits the dependencies with {', '.join(tools)}") return Result.warned("no dependency vulnerability audit found in CI configuration", audit.hint) diff --git a/project_validator/checks/vcs_ignore.py b/project_validator/checks/vcs_ignore.py index 7715cca..8545fea 100644 --- a/project_validator/checks/vcs_ignore.py +++ b/project_validator/checks/vcs_ignore.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from project_validator.core import Context, Result -from project_validator.ecosystems import check_each_ecosystem, ecosystem_check, rules_of +from project_validator.ecosystems import ecosystem_check, judge_each_ecosystem from project_validator.rules import BuildOutputs @@ -180,4 +180,4 @@ def vcs_ignore(context: Context) -> Result: f"no version control ignore file found ({IGNORE_FILE_NAMES}); build outputs can be committed by accident", "consider adding a .gitignore (or your version control system's equivalent) that ignores build outputs", ) - return check_each_ecosystem(context, rules_of(BuildOutputs), _outputs_ignored) + return judge_each_ecosystem(context, BuildOutputs, _outputs_ignored) diff --git a/project_validator/ecosystems/__init__.py b/project_validator/ecosystems/__init__.py index de3058f..ada1f1e 100644 --- a/project_validator/ecosystems/__init__.py +++ b/project_validator/ecosystems/__init__.py @@ -10,6 +10,7 @@ from project_validator.ecosystems.c_cpp import C_AND_CXX from project_validator.ecosystems.go import GO from project_validator.ecosystems.javascript import JAVASCRIPT_AND_TYPESCRIPT +from project_validator.ecosystems.jvm import JVM from project_validator.ecosystems.model import ( C_FAMILY, BuildSystem, @@ -25,7 +26,7 @@ from project_validator.ecosystems.python import PYTHON from project_validator.ecosystems.rust import RUST -BUILT_IN = (C_AND_CXX, PYTHON, RUST, GO, JAVASCRIPT_AND_TYPESCRIPT) +BUILT_IN = (C_AND_CXX, PYTHON, RUST, GO, JAVASCRIPT_AND_TYPESCRIPT, JVM) _registered: list[Ecosystem] = list(BUILT_IN) @@ -169,6 +170,16 @@ def judge_ecosystem[R]( return Result(status=Status.SKIP, message=f"not applicable to {ecosystem.label} built with {used}") +def judge_each_ecosystem[R](context: Context, kind: type[R], judge: Judge[R]) -> Result: + """`judge` of the rule of type `kind` of each ecosystem the project uses, for the build systems it uses (see + `judge_ecosystem`): the worst result wins, and findings are labelled.""" + structure = context.structure + used = ecosystems_for(structure.languages, providers(kind)) + return combined( + {ecosystem: judge_ecosystem(ecosystem, kind, judge, context, structure.build_systems) for ecosystem in used} + ) + + def practice[R]( *, name: str, description: str, rule: type[R], aliases: tuple[str, ...] = () ) -> Callable[[Judge[R]], Check]: @@ -180,14 +191,7 @@ def practice[R]( def wrap(judge: Judge[R]) -> Check: def run(context: Context) -> Result: - structure = context.structure - used = ecosystems_for(structure.languages, providers(rule)) - return combined( - { - ecosystem: judge_ecosystem(ecosystem, rule, judge, context, structure.build_systems) - for ecosystem in used - } - ) + return judge_each_ecosystem(context, rule, judge) languages = languages_of(providers(rule)) return Check(name=name, description=description, run=run, languages=languages, rule=rule, aliases=aliases) @@ -206,6 +210,7 @@ def is_build_tree(directory: Path) -> bool: "C_FAMILY", "GO", "JAVASCRIPT_AND_TYPESCRIPT", + "JVM", "PYTHON", "RUST", "BuildSystem", @@ -222,6 +227,7 @@ def is_build_tree(directory: Path) -> bool: "ecosystems_for", "is_build_tree", "isolated_registry", + "judge_each_ecosystem", "judge_ecosystem", "languages_of", "practice", diff --git a/project_validator/ecosystems/jvm.py b/project_validator/ecosystems/jvm.py new file mode 100644 index 0000000..5058e4a --- /dev/null +++ b/project_validator/ecosystems/jvm.py @@ -0,0 +1,381 @@ +"""Java and Kotlin on the JVM: built with Maven or Gradle, judged by the JDK versions CI runs. + +Maven and Gradle projects wire their tools into the build: a Maven plugin binds a goal to a lifecycle phase +(`check` runs in `mvn verify`), and a Gradle plugin adds its task to `check`. So the build files count as +CI configuration when CI runs the build far enough to run them. +""" + +import re +from collections.abc import Callable +from pathlib import Path, PurePosixPath + +from project_validator.core import Context, Result +from project_validator.ecosystems.model import BuildSystem, Ecosystem, ToolchainPatterns, versions_axis +from project_validator.evidence import ci_and_hook_texts, configuration, first_existing +from project_validator.rules import ( + MULTI_ECOSYSTEM_AUDITS, + SUGGESTED_LIZARD, + Analyzer, + BuildOutputs, + CoverageTools, + DependencyAudit, + DependencyManagement, + Formatter, + FunctionLimits, + SecurityLints, + StaticAnalysis, + TestSuite, + WarningPolicy, + WarningsAsErrors, + directories, + ignored_at_top_level, +) +from project_validator.text import Pattern, matching_names, pattern + +JAVA = "Java" +KOTLIN = "Kotlin" +POM = "pom.xml" +GRADLE_BUILD_FILES = ("build.gradle", "build.gradle.kts", "settings.gradle", "settings.gradle.kts") + + +def _output_beside(name: str, build_files: tuple[str, ...]) -> Callable[[Path], bool]: + def is_build_tree(directory: Path) -> bool: + return directory.name == name and any((directory.parent / build_file).is_file() for build_file in build_files) + + return is_build_tree + + +def _build_outputs(name: str, directory: str, *more: str) -> BuildOutputs: + return BuildOutputs( + name=f"the {name} directory", + globs=ignored_at_top_level(directory), + paths=directories(directory), + hint=f"consider ignoring {' and '.join((f'{directory}/', *more))}", + artifacts=(f"{directory}/", *more, "*.class"), + ) + + +MAVEN = BuildSystem( + name="Maven", + files=(POM,), + is_build_tree=_output_beside("target", (POM,)), + rules=(_build_outputs("target", "target"),), +) +GRADLE = BuildSystem( + name="Gradle", + files=GRADLE_BUILD_FILES, + is_build_tree=_output_beside("build", GRADLE_BUILD_FILES), + rules=(_build_outputs("build", "build", ".gradle/"),), +) + +# How far CI runs the build: a phase or task that compiles, and one that also runs the verification plugins. +MAVEN_COMMAND = r"(? tuple[str, ...]: + names = {POM, *GRADLE_BUILD_FILES} + return tuple(path for path in context.project.files() if PurePosixPath(path).name in names) + + +def _texts_when(runs: re.Pattern[str]) -> Callable[[Context], tuple[str, ...]]: + """CI configuration and pre-commit hooks, with the build files when CI runs the build as far as `runs`.""" + + def texts(context: Context) -> tuple[str, ...]: + found = ci_and_hook_texts(context) + if not any(runs.search(text) for text in found): + return found + return (*found, *(context.project.read_text(path) for path in _build_files(context))) + + return texts + + +_build_texts = _texts_when(RUNS_BUILD) +_verification_texts = _texts_when(RUNS_VERIFICATION) + + +def _bound_goal(artifact: str, goals: str) -> str: + """A Maven plugin whose execution binds one of `goals`, so that the lifecycle runs it.""" + return rf"{artifact}(?:(?!).)*?(?:{goals})" + + +def _setting(name: str, regex: str) -> Pattern: + return Pattern(name, re.compile(regex, re.IGNORECASE | re.DOTALL)) + + +GRADLE_PLUGIN = r"""(?:id\s*\(?\s*|apply\s+plugin:\s*|plugins\.apply\s*\(\s*)["']{}["']""" + + +def _gradle_plugin(plugin_id: str) -> str: + return GRADLE_PLUGIN.format(re.escape(plugin_id)) + + +def _spotless_config(context: Context) -> str | None: + return next((path for path in _build_files(context) if "spotless" in context.project.read_text(path)), None) + + +# Every formatter here has a built-in style (Spotless applies one of them). +FORMATTERS = Formatter( + name="Spotless, google-java-format, ktfmt or ktlint", + config_file=_spotless_config, + runs=( + _setting( + "Spotless", + rf"\bspotless:check\b|\bspotlessCheck\b|{_bound_goal('spotless-maven-plugin', 'check')}" + rf"|{_gradle_plugin('com.diffplug.spotless')}", + ), + _setting("fmt-maven-plugin", _bound_goal("fmt-maven-plugin", "check")), + _setting("google-java-format", r"\bgoogle-java-format\b[^\n]*--set-exit-if-changed\b"), + _setting("ktfmt", r"\bktfmt\b[^\n]*--set-exit-if-changed\b"), + _setting( + "ktlint", + rf"(? Result: + tests = (path for path in context.project.files() if TEST_SOURCES.search(path)) + count = sum(1 for path in tests if TEST_CASE.search(context.project.read_text(path))) + if count == 0: + return Result.failed( + "no JVM tests found (@Test methods or Kotest specs under src/test/java or src/test/kotlin)" + ) + return Result.passed(f"JVM tests found in {count} file(s)") + + +TESTS = TestSuite(found=_jvm_tests, runs=RUNS_TESTS, runner="mvn test or gradle test") + +# JaCoCo and Kover count branches; neither reports path coverage. +COVERAGE = CoverageTools( + searched="CI configuration, or the Maven or Gradle build CI runs", + texts=_verification_texts, + branch=( + _setting( + "JaCoCo", rf"\bjacoco:report\b|\bjacocoTestReport\b|{_bound_goal('jacoco-maven-plugin', 'report[\\w-]*')}" + ), + _setting( + "Kover", + r"\bkover\w*Report\b|\borg\.jetbrains\.kotlinx\.kover\b|kover-maven-plugin", + ), + ), +) + +XLINT = _setting("-Xlint:all", r"-Xlint(?::all)?(?![:\w-])") +# The Kotlin compiler warns by default; a Kotlin-only project is judged by warnings-as-errors. +KOTLIN_WARNINGS = _setting( + "Kotlin's default warnings", r"\bkotlin-maven-plugin\b|\borg\.jetbrains\.kotlin\b|\bkotlin\s*\(\s*[\"']jvm[\"']" +) + + +def _required_warnings(context: Context) -> tuple[Pattern, ...]: + return (XLINT,) if JAVA in context.structure.languages else (KOTLIN_WARNINGS,) + + +WARNINGS = WarningPolicy( + kind="JVM compiler warnings", + recognised=(XLINT, KOTLIN_WARNINGS), + required=_required_warnings, + texts=_build_texts, + separator=", ", +) + +WARNINGS_AS_ERRORS = WarningsAsErrors( + flags=( + _setting("-Werror", r"-Werror\b"), + _setting("allWarningsAsErrors", r"\ballWarningsAsErrors\b(?:\s*(?:=|\.set\s*\()\s*|>\s*)true\b"), + ), + texts=_build_texts, + searched="the Maven or Gradle build, or CI configuration", + hint="add -Werror to javac's compilerArgs (Kotlin: allWarningsAsErrors = true)", +) + +SECURITY_LINTS = SecurityLints( + texts=_verification_texts, + lints=( + _setting("Find Security Bugs", r"\bfindsecbugs-plugin\b|\bcom\.h3xstream\.findsecbugs\b"), + _setting("PMD security rules", r"\bcategory/java/security\.xml\b"), + ), +) + +DEPENDENCY_AUDIT = DependencyAudit( + tools=( + _setting( + "OWASP dependency-check", + rf"\bdependency-check:(?:check|aggregate)\b|\bdependencyCheck(?:Analyze|Aggregate)\b" + rf"|\bdependency-check\.sh\b|\bdependency-check/Dependency-Check_Action\b" + rf"|{_bound_goal('dependency-check-maven', 'check|aggregate')}", + ), + *MULTI_ECOSYSTEM_AUDITS, + ), + hint="consider OWASP dependency-check in CI (dependency-check-maven bound to check, or gradle " + "dependencyCheckAnalyze)", + texts=_verification_texts, +) + + +def _checkstyle_configs(context: Context) -> list[str]: + return [ + path + for path in context.project.files() + if "checkstyle" in PurePosixPath(path).name.lower() and path.endswith(".xml") + ] + + +DETEKT_CONFIG_FILES = ("detekt.yml", "config/detekt/detekt.yml", "detekt-config.yml") +METHOD_LENGTH = re.compile(r"""name\s*=\s*["']MethodLength["']""") +# detekt's LongMethod rule is active by default. +LONG_METHOD_OFF = re.compile(r"^\s*LongMethod:\s*\n\s*active:\s*false\b", re.MULTILINE) +RUNS_CHECKSTYLE = pattern("Checkstyle", r"\bcheckstyle\b") +RUNS_DETEKT = pattern("detekt", r"\bdetekt\b") + + +def _method_length_check(context: Context) -> str | None: + """The Checkstyle configuration that enables the MethodLength check, if any.""" + configs = _checkstyle_configs(context) + return next((path for path in configs if METHOD_LENGTH.search(context.project.read_text(path))), None) + + +def _long_method_active(context: Context) -> bool: + config = first_existing(context, DETEKT_CONFIG_FILES) + return config is None or not LONG_METHOD_OFF.search(context.project.read_text(config)) + + +def _method_length_enforced(context: Context) -> Result: + """Checkstyle's MethodLength check, or detekt's LongMethod rule, limits method length when the tool runs.""" + texts = _verification_texts(context) + checkstyle = _method_length_check(context) + if checkstyle and matching_names(texts, (RUNS_CHECKSTYLE,)): + return Result.passed(f"function length is limited with Checkstyle MethodLength ({checkstyle})") + if _long_method_active(context) and matching_names(texts, (RUNS_DETEKT,)): + return Result.passed("function length is limited with detekt LongMethod") + if checkstyle: + return Result.warned(f"MethodLength is enabled in {checkstyle} but Checkstyle does not run") + return Result.failed( + "function length is never limited; enable Checkstyle's MethodLength check, run detekt, " + f"or run {SUGGESTED_LIZARD} in CI" + ) + + +def _checkstyle_config(context: Context) -> str | None: + return next(iter(_checkstyle_configs(context)), None) + + +STATIC_ANALYSIS = StaticAnalysis( + analyzers=( + Analyzer(name="Error Prone", runs=re.compile(r"\berror_?prone\b|\bnet\.ltgt\.errorprone\b", re.IGNORECASE)), + Analyzer(name="SpotBugs", runs=re.compile(r"\bspotbugs\b", re.IGNORECASE)), + Analyzer( + name="PMD", + runs=re.compile( + rf"maven-pmd-plugin|\bpmd:check\b|\bpmd\s+check\b|{_gradle_plugin('pmd')}" + ), + ), + Analyzer(name="Checkstyle", runs=RUNS_CHECKSTYLE.regex, configured=_checkstyle_config), + Analyzer(name="detekt", runs=RUNS_DETEKT.regex, configured=configuration(DETEKT_CONFIG_FILES)), + ), + texts=_verification_texts, + hint="consider Error Prone, SpotBugs, PMD or Checkstyle (detekt for Kotlin), run by mvn verify or gradle check", +) + +MAVEN_DEPENDENCY = re.compile(r"") +GRADLE_DEPENDENCY = re.compile(r"^\s*(?:implementation|api|compileOnly|runtimeOnly|testImplementation)\b", re.MULTILINE) +GRADLE_LOCKS = ("gradle/verification-metadata.xml", "gradle.lockfile") +MAVEN_CONFIG = ".mvn/maven.config" +STRICT_CHECKSUMS = re.compile(rf"{MVN}(?:--strict-checksums|\s-C\b)") + + +def _managed(context: Context) -> str | None: + for path in _build_files(context): + text = context.project.read_text(path) + if PurePosixPath(path).name == POM and MAVEN_DEPENDENCY.search(text): + return "managed with Maven" + if PurePosixPath(path).name != POM and GRADLE_DEPENDENCY.search(text): + return "managed with Gradle" + return None + + +def _verified(context: Context) -> str | None: + """Gradle's dependency verification metadata or lockfile, or Maven failing on a bad checksum.""" + if lock := next((path for path in GRADLE_LOCKS if context.committed(path)), None): + return lock + if context.project.exists(MAVEN_CONFIG) and "--strict-checksums" in context.project.read_text(MAVEN_CONFIG): + return f"{MAVEN_CONFIG} (--strict-checksums)" + return "CI (mvn --strict-checksums)" if any(map(STRICT_CHECKSUMS.search, context.ci_texts)) else None + + +DEPENDENCIES = DependencyManagement( + managed=_managed, + locked=_verified, + lock_hint="verify them: commit gradle/verification-metadata.xml (gradle --write-verification-metadata sha256) " + "or a gradle.lockfile; with Maven, run with --strict-checksums", +) + +CLASS_FILES = BuildOutputs( + name="compiled classes", + globs=ignored_at_top_level(r"\*\.class"), + paths=("App.class",), + hint="consider ignoring *.class", + artifacts=("*.class",), +) + +_JDK = r"(\d+(?:\.\d+)*)" +JDK_VERSIONS = ( + re.compile(rf"\bjava-version\W+{_JDK}"), + re.compile(rf"\b(?:eclipse-temurin|openjdk|amazoncorretto|sapmachine|azul/zulu-openjdk\w*):{_JDK}"), + re.compile(rf"\bmaven:[\d.]+-[a-z-]+?-{_JDK}\b"), + re.compile(rf"\bgradle:[\w.-]*?jdk-?{_JDK}\b"), +) + +JVM = Ecosystem( + label="JVM", + languages=frozenset({JAVA, KOTLIN}), + build_systems=(MAVEN, GRADLE), + source_suffixes={".java": JAVA, ".kt": KOTLIN}, + # Kotlin and Groovy scripts are mostly Gradle builds and their helpers, which name no language. + ambiguous_suffixes={".kts": None, ".gradle": None}, + toolchain=ToolchainPatterns( + language="JDK", + versions=JDK_VERSIONS, + usage=re.compile(rf"{MAVEN_COMMAND}|{GRADLE_COMMAND}|\bsetup-java\b|\bjavac\b"), + ), + rules=( + versions_axis("JDK version", "JDK"), + FORMATTERS, + COVERAGE, + TESTS, + WARNINGS, + WARNINGS_AS_ERRORS, + SECURITY_LINTS, + DEPENDENCY_AUDIT, + FunctionLimits(without_lizard=_method_length_enforced), + STATIC_ANALYSIS, + DEPENDENCIES, + CLASS_FILES, + ), +) diff --git a/project_validator/rules.py b/project_validator/rules.py index e783578..e673a13 100644 --- a/project_validator/rules.py +++ b/project_validator/rules.py @@ -165,10 +165,15 @@ class SecurityLints: @dataclass(frozen=True, slots=True, kw_only=True) class DependencyAudit: - """The tools that check an ecosystem's dependencies against known vulnerabilities when CI runs them.""" + """The tools that check an ecosystem's dependencies against known vulnerabilities when CI runs them. + + They are searched in `texts`: CI configuration, unless the ecosystem's CI runs them through files it can read, + such as a build that runs its plugins. + """ tools: tuple[Pattern, ...] hint: str + texts: Evidence = ci_configuration def _nothing(_: Context) -> str | None: diff --git a/tests/checks/test_warnings_as_errors.py b/tests/checks/test_warnings_as_errors.py index bbaadc2..45ea210 100644 --- a/tests/checks/test_warnings_as_errors.py +++ b/tests/checks/test_warnings_as_errors.py @@ -19,10 +19,10 @@ def _run(self, files: dict[str, str]) -> Result: self, warnings_as_errors, files, c_family_layouts_built_with(warnings_as_errors, "CMake") ) - # -Werror and /WX are C/C++ compiler flags, and eslint --max-warnings 0 fails JavaScript linting on a warning; - # other languages' equivalents are judged by compiler-warnings. + # -Werror and /WX are C/C++ (and JVM) compiler flags, and eslint --max-warnings 0 fails JavaScript linting on a + # warning; other languages' equivalents are judged by compiler-warnings. def test_declares_the_languages_it_judges(self) -> None: - self.assertEqual(warnings_as_errors.languages, C_FAMILY | {"JavaScript", "TypeScript"}) + self.assertEqual(warnings_as_errors.languages, C_FAMILY | {"JavaScript", "TypeScript", "Java", "Kotlin"}) def test_fails_without_warnings_as_errors(self) -> None: result = self._run(BARE_PROJECT) diff --git a/tests/fixtures/bare_jvm/pom.xml b/tests/fixtures/bare_jvm/pom.xml new file mode 100644 index 0000000..606889c --- /dev/null +++ b/tests/fixtures/bare_jvm/pom.xml @@ -0,0 +1,15 @@ + + + 4.0.0 + com.example + app + 0.1.0 + + + + com.google.guava + guava + 33.4.8-jre + + + diff --git a/tests/fixtures/bare_jvm/src/main/java/com/example/App.java b/tests/fixtures/bare_jvm/src/main/java/com/example/App.java new file mode 100644 index 0000000..c2fa141 --- /dev/null +++ b/tests/fixtures/bare_jvm/src/main/java/com/example/App.java @@ -0,0 +1,11 @@ +package com.example; + +/** Adds numbers. */ +public final class App { + private App() {} + + /** The sum of a and b. */ + public static int add(int a, int b) { + return a + b; + } +} diff --git a/tests/fixtures/bare_jvm/target/classes/com/example/App.class b/tests/fixtures/bare_jvm/target/classes/com/example/App.class new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/well_configured_jvm/.github/dependabot.yml b/tests/fixtures/well_configured_jvm/.github/dependabot.yml new file mode 100644 index 0000000..ca79ca5 --- /dev/null +++ b/tests/fixtures/well_configured_jvm/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly diff --git a/tests/fixtures/well_configured_jvm/.github/workflows/ci-lint.yml b/tests/fixtures/well_configured_jvm/.github/workflows/ci-lint.yml new file mode 100644 index 0000000..ac9333b --- /dev/null +++ b/tests/fixtures/well_configured_jvm/.github/workflows/ci-lint.yml @@ -0,0 +1,9 @@ +on: [push, pull_request] +jobs: + ci-lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - run: pipx run zizmor .github + - uses: rhysd/actionlint@914e7df21a07ef503a81201c76d2b11c789d3fca # v1.7.12 + - run: gitleaks git --redact . diff --git a/tests/fixtures/well_configured_jvm/.github/workflows/ci.yml b/tests/fixtures/well_configured_jvm/.github/workflows/ci.yml new file mode 100644 index 0000000..ed92661 --- /dev/null +++ b/tests/fixtures/well_configured_jvm/.github/workflows/ci.yml @@ -0,0 +1,14 @@ +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + java: ["21", "25"] + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4 + with: + distribution: temurin + java-version: ${{ matrix.java }} + - run: mvn --batch-mode --strict-checksums verify diff --git a/tests/fixtures/well_configured_jvm/.github/workflows/copy-paste-detection.yml b/tests/fixtures/well_configured_jvm/.github/workflows/copy-paste-detection.yml new file mode 100644 index 0000000..f7f288a --- /dev/null +++ b/tests/fixtures/well_configured_jvm/.github/workflows/copy-paste-detection.yml @@ -0,0 +1,6 @@ +jobs: + copy-paste-detection: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - run: npx --yes jscpd@4 --threshold 0 . diff --git a/tests/fixtures/well_configured_jvm/.github/workflows/function-limits.yml b/tests/fixtures/well_configured_jvm/.github/workflows/function-limits.yml new file mode 100644 index 0000000..0b6c011 --- /dev/null +++ b/tests/fixtures/well_configured_jvm/.github/workflows/function-limits.yml @@ -0,0 +1,7 @@ +jobs: + function-limits: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - run: python -m pip install lizard + - run: lizard --length 30 --warnings_only . diff --git a/tests/fixtures/well_configured_jvm/.gitignore b/tests/fixtures/well_configured_jvm/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/tests/fixtures/well_configured_jvm/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/tests/fixtures/well_configured_jvm/LICENSE b/tests/fixtures/well_configured_jvm/LICENSE new file mode 100644 index 0000000..1b3edae --- /dev/null +++ b/tests/fixtures/well_configured_jvm/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Example Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tests/fixtures/well_configured_jvm/README.md b/tests/fixtures/well_configured_jvm/README.md new file mode 100644 index 0000000..ec54841 --- /dev/null +++ b/tests/fixtures/well_configured_jvm/README.md @@ -0,0 +1,4 @@ +# app + +[![ci](https://github.com/example/app/actions/workflows/ci.yml/badge.svg)](https://github.com/example/app/actions/workflows/ci.yml) +[![codecov](https://codecov.io/gh/example/app/graph/badge.svg)](https://codecov.io/gh/example/app) diff --git a/tests/fixtures/well_configured_jvm/SECURITY.md b/tests/fixtures/well_configured_jvm/SECURITY.md new file mode 100644 index 0000000..0b161bd --- /dev/null +++ b/tests/fixtures/well_configured_jvm/SECURITY.md @@ -0,0 +1,3 @@ +# Security policy + +Report vulnerabilities to security@example.com. diff --git a/tests/fixtures/well_configured_jvm/pom.xml b/tests/fixtures/well_configured_jvm/pom.xml new file mode 100644 index 0000000..e032fbb --- /dev/null +++ b/tests/fixtures/well_configured_jvm/pom.xml @@ -0,0 +1,92 @@ + + + 4.0.0 + com.example + app + 0.1.0 + + + 21 + + + + + org.junit.jupiter + junit-jupiter + 5.13.4 + test + + + + + + + maven-compiler-plugin + + + -Xlint:all + -Werror + + + + + com.diffplug.spotless + spotless-maven-plugin + + + + + + + + + check + + + + + + org.jacoco + jacoco-maven-plugin + + + + prepare-agent + report + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + + com.h3xstream.findsecbugs + findsecbugs-plugin + + + + + + + check + + + + + + org.owasp + dependency-check-maven + + + + check + + + + + + + diff --git a/tests/fixtures/well_configured_jvm/src/main/java/com/example/App.java b/tests/fixtures/well_configured_jvm/src/main/java/com/example/App.java new file mode 100644 index 0000000..c2fa141 --- /dev/null +++ b/tests/fixtures/well_configured_jvm/src/main/java/com/example/App.java @@ -0,0 +1,11 @@ +package com.example; + +/** Adds numbers. */ +public final class App { + private App() {} + + /** The sum of a and b. */ + public static int add(int a, int b) { + return a + b; + } +} diff --git a/tests/fixtures/well_configured_jvm/src/test/java/com/example/AppTest.java b/tests/fixtures/well_configured_jvm/src/test/java/com/example/AppTest.java new file mode 100644 index 0000000..dfb73d9 --- /dev/null +++ b/tests/fixtures/well_configured_jvm/src/test/java/com/example/AppTest.java @@ -0,0 +1,12 @@ +package com.example; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class AppTest { + @Test + void adds() { + assertEquals(3, App.add(1, 2)); + } +} diff --git a/tests/snapshots/bare_go.json b/tests/snapshots/bare_go.json index 15d4618..bae6481 100644 --- a/tests/snapshots/bare_go.json +++ b/tests/snapshots/bare_go.json @@ -193,7 +193,7 @@ }, "warnings-as-errors": { "details": [], - "message": "not applicable to Go (applies to C, C++, JavaScript, TypeScript)", + "message": "not applicable to Go (applies to C, C++, Java, JavaScript, Kotlin, TypeScript)", "status": "SKIP" } } diff --git a/tests/snapshots/bare_jvm.json b/tests/snapshots/bare_jvm.json new file mode 100644 index 0000000..801d024 --- /dev/null +++ b/tests/snapshots/bare_jvm.json @@ -0,0 +1,194 @@ +{ + "map": { + "build_systems": [ + "Maven" + ], + "ci": { + "files": [], + "jobs": [], + "providers": [] + }, + "ecosystems": { + "JVM": [ + ".java files" + ] + }, + "languages": [ + "Java" + ], + "unrecognised_suffixes": [] + }, + "results": { + "action-pinning": { + "details": [], + "message": "no CI configuration with reusable steps", + "status": "SKIP" + }, + "ci-build-types": { + "details": [], + "message": "not applicable to Java (applies to C, C++, Rust)", + "status": "SKIP" + }, + "ci-lint": { + "details": [], + "message": "no CI configuration", + "status": "SKIP" + }, + "ci-present": { + "details": [], + "message": "no CI configuration found (looked for GitHub Actions, GitLab CI, Azure Pipelines, AppVeyor, CircleCI, Travis CI, Jenkins, Buildkite, Drone, Cirrus CI, Bitbucket Pipelines, Woodpecker)", + "status": "FAIL" + }, + "code-formatting": { + "details": [ + "consider Spotless (spotless:check bound in pom.xml, or the com.diffplug.spotless Gradle plugin, run by mvn verify or gradle check) with google-java-format, palantir-java-format, ktfmt or ktlint" + ], + "message": "CI never runs Spotless, google-java-format, ktfmt or ktlint; its formatting is not enforced", + "status": "FAIL" + }, + "committed-artifacts": { + "details": [ + "target/", + "remove them from version control (git rm -r --cached ) and ignore them" + ], + "message": "1 build output is committed", + "status": "FAIL" + }, + "compiler-warnings": { + "details": [], + "message": "no JVM compiler warnings found; enable at least -Xlint:all", + "status": "FAIL" + }, + "copy-paste-detection": { + "details": [ + "consider adding PMD CPD (pmd cpd --language cpp), jscpd, Duplo or Simian" + ], + "message": "no copy-paste detection found in CI configuration", + "status": "WARN" + }, + "coverage": { + "details": [], + "message": "no branch coverage reporting found in CI configuration, or the Maven or Gradle build CI runs", + "status": "FAIL" + }, + "cxx-extensions": { + "details": [], + "message": "not applicable to Java (applies to C, C++)", + "status": "SKIP" + }, + "data-wellformed": { + "details": [], + "message": "1 data file parses (XML: 1)", + "status": "PASS" + }, + "dependency-audit": { + "details": [ + "consider OWASP dependency-check in CI (dependency-check-maven bound to check, or gradle dependencyCheckAnalyze)" + ], + "message": "no dependency vulnerability audit found in CI configuration", + "status": "WARN" + }, + "dependency-management": { + "details": [ + "verify them: commit gradle/verification-metadata.xml (gradle --write-verification-metadata sha256) or a gradle.lockfile; with Maven, run with --strict-checksums" + ], + "message": "dependencies are managed with Maven but not locked", + "status": "WARN" + }, + "dependency-updates": { + "details": [ + "consider adding .github/dependabot.yml or a Renovate configuration (renovate.json)" + ], + "message": "no automated dependency updates configured", + "status": "WARN" + }, + "function-limits": { + "details": [], + "message": "function length is never limited; enable Checkstyle's MethodLength check, run detekt, or run lizard --length 30 in CI", + "status": "FAIL" + }, + "hardening": { + "details": [], + "message": "not applicable to Java (applies to C, C++, Go, Rust)", + "status": "SKIP" + }, + "large-files": { + "details": [], + "message": "no committed file is larger than 5 MiB", + "status": "PASS" + }, + "license": { + "details": [ + "add a LICENSE file with the text of a license (see https://choosealicense.com)" + ], + "message": "no license file", + "status": "FAIL" + }, + "platform-matrix": { + "details": [], + "message": "no CI configuration found", + "status": "FAIL" + }, + "secret-scanning": { + "details": [ + "consider running gitleaks, TruffleHog (trufflesecurity/trufflehog) or detect-secrets in CI or as a pre-commit hook; GitHub's secret scanning and push protection are repository settings, which cannot be seen from here" + ], + "message": "no secret scanning found in CI configuration", + "status": "WARN" + }, + "security-policy": { + "details": [ + "add a SECURITY.md that says how to report a vulnerability privately: an email address, or GitHub's private vulnerability reporting (Settings > Code security, then link to /security/advisories/new)" + ], + "message": "no security policy", + "status": "WARN" + }, + "security-scanning": { + "details": [ + "consider adding CodeQL (github/codeql-action) or GitLab SAST (Security/SAST.gitlab-ci.yml)", + "or enable each language's security lints (JVM: Find Security Bugs, PMD security rules)" + ], + "message": "no code security scanning found in CI configuration", + "status": "WARN" + }, + "static-analysis": { + "details": [ + "consider Error Prone, SpotBugs, PMD or Checkstyle (detekt for Kotlin), run by mvn verify or gradle check" + ], + "message": "no static analyzer runs", + "status": "FAIL" + }, + "status-badges": { + "details": [ + "add a CI status badge, e.g. ![ci](https://github.com///actions/workflows/ci.yml/badge.svg)", + "add a quality badge: coverage (Codecov, Coveralls) or analysis (SonarCloud, Codacy, OpenSSF Scorecard)" + ], + "message": "no README", + "status": "FAIL" + }, + "tests-run": { + "details": [], + "message": "no JVM tests found (@Test methods or Kotest specs under src/test/java or src/test/kotlin)", + "status": "FAIL" + }, + "type-checking": { + "details": [], + "message": "not applicable to Java (applies to JavaScript, Python, TypeScript)", + "status": "SKIP" + }, + "vcs-ignore": { + "details": [ + "consider adding a .gitignore (or your version control system's equivalent) that ignores build outputs" + ], + "message": "no version control ignore file found (.gitignore, .hgignore, .bzrignore, .fossil-settings/ignore-glob); build outputs can be committed by accident", + "status": "FAIL" + }, + "warnings-as-errors": { + "details": [ + "add -Werror to javac's compilerArgs (Kotlin: allWarningsAsErrors = true)" + ], + "message": "warnings are not treated as errors in the Maven or Gradle build, or CI configuration", + "status": "FAIL" + } + } +} diff --git a/tests/snapshots/bare_project.json b/tests/snapshots/bare_project.json index aff9196..62952b4 100644 --- a/tests/snapshots/bare_project.json +++ b/tests/snapshots/bare_project.json @@ -85,7 +85,7 @@ }, "dependency-audit": { "details": [], - "message": "not applicable to C, C++ (applies to Go, JavaScript, Python, Rust, TypeScript)", + "message": "not applicable to C, C++ (applies to Go, Java, JavaScript, Kotlin, Python, Rust, TypeScript)", "status": "SKIP" }, "dependency-management": { diff --git a/tests/snapshots/bare_python.json b/tests/snapshots/bare_python.json index 45474e5..c945a7e 100644 --- a/tests/snapshots/bare_python.json +++ b/tests/snapshots/bare_python.json @@ -188,7 +188,7 @@ }, "warnings-as-errors": { "details": [], - "message": "not applicable to Python (applies to C, C++, JavaScript, TypeScript)", + "message": "not applicable to Python (applies to C, C++, Java, JavaScript, Kotlin, TypeScript)", "status": "SKIP" } } diff --git a/tests/snapshots/bare_rust.json b/tests/snapshots/bare_rust.json index 7f70708..57445d8 100644 --- a/tests/snapshots/bare_rust.json +++ b/tests/snapshots/bare_rust.json @@ -186,7 +186,7 @@ }, "warnings-as-errors": { "details": [], - "message": "not applicable to Rust (applies to C, C++, JavaScript, TypeScript)", + "message": "not applicable to Rust (applies to C, C++, Java, JavaScript, Kotlin, TypeScript)", "status": "SKIP" } } diff --git a/tests/snapshots/buildable.json b/tests/snapshots/buildable.json index 92f1b52..201d262 100644 --- a/tests/snapshots/buildable.json +++ b/tests/snapshots/buildable.json @@ -81,7 +81,7 @@ }, "dependency-audit": { "details": [], - "message": "not applicable to C, C++ (applies to Go, JavaScript, Python, Rust, TypeScript)", + "message": "not applicable to C, C++ (applies to Go, Java, JavaScript, Kotlin, Python, Rust, TypeScript)", "status": "SKIP" }, "dependency-management": { diff --git a/tests/snapshots/well_configured.json b/tests/snapshots/well_configured.json index 5368fee..76f6f40 100644 --- a/tests/snapshots/well_configured.json +++ b/tests/snapshots/well_configured.json @@ -414,7 +414,7 @@ }, "dependency-audit": { "details": [], - "message": "not applicable to C, C++ (applies to Go, JavaScript, Python, Rust, TypeScript)", + "message": "not applicable to C, C++ (applies to Go, Java, JavaScript, Kotlin, Python, Rust, TypeScript)", "status": "SKIP" }, "dependency-management": { diff --git a/tests/snapshots/well_configured_go.json b/tests/snapshots/well_configured_go.json index 1cc2079..65dc99c 100644 --- a/tests/snapshots/well_configured_go.json +++ b/tests/snapshots/well_configured_go.json @@ -315,7 +315,7 @@ }, "warnings-as-errors": { "details": [], - "message": "not applicable to Go (applies to C, C++, JavaScript, TypeScript)", + "message": "not applicable to Go (applies to C, C++, Java, JavaScript, Kotlin, TypeScript)", "status": "SKIP" } } diff --git a/tests/snapshots/well_configured_jvm.json b/tests/snapshots/well_configured_jvm.json new file mode 100644 index 0000000..3e9596d --- /dev/null +++ b/tests/snapshots/well_configured_jvm.json @@ -0,0 +1,308 @@ +{ + "map": { + "build_systems": [ + "Maven" + ], + "ci": { + "files": [ + ".github/workflows/ci-lint.yml", + ".github/workflows/ci.yml", + ".github/workflows/copy-paste-detection.yml", + ".github/workflows/function-limits.yml" + ], + "jobs": [ + { + "compilers": [ + "gcc" + ], + "configurations": [], + "ecosystems": [], + "name": ".github/workflows/ci-lint.yml: ci-lint", + "os": "linux", + "toolchains": [] + }, + { + "compilers": [ + "gcc" + ], + "configurations": [], + "ecosystems": [ + "JVM" + ], + "name": ".github/workflows/ci.yml: test (ubuntu-latest, 21)", + "os": "linux", + "toolchains": [ + "JDK 21" + ] + }, + { + "compilers": [ + "gcc" + ], + "configurations": [], + "ecosystems": [ + "JVM" + ], + "name": ".github/workflows/ci.yml: test (ubuntu-latest, 25)", + "os": "linux", + "toolchains": [ + "JDK 25" + ] + }, + { + "compilers": [ + "appleclang" + ], + "configurations": [], + "ecosystems": [ + "JVM" + ], + "name": ".github/workflows/ci.yml: test (macos-latest, 21)", + "os": "macos", + "toolchains": [ + "JDK 21" + ] + }, + { + "compilers": [ + "appleclang" + ], + "configurations": [], + "ecosystems": [ + "JVM" + ], + "name": ".github/workflows/ci.yml: test (macos-latest, 25)", + "os": "macos", + "toolchains": [ + "JDK 25" + ] + }, + { + "compilers": [ + "msvc" + ], + "configurations": [], + "ecosystems": [ + "JVM" + ], + "name": ".github/workflows/ci.yml: test (windows-latest, 21)", + "os": "windows", + "toolchains": [ + "JDK 21" + ] + }, + { + "compilers": [ + "msvc" + ], + "configurations": [], + "ecosystems": [ + "JVM" + ], + "name": ".github/workflows/ci.yml: test (windows-latest, 25)", + "os": "windows", + "toolchains": [ + "JDK 25" + ] + }, + { + "compilers": [ + "gcc" + ], + "configurations": [], + "ecosystems": [], + "name": ".github/workflows/copy-paste-detection.yml: copy-paste-detection", + "os": "linux", + "toolchains": [] + }, + { + "compilers": [ + "gcc" + ], + "configurations": [], + "ecosystems": [ + "Python" + ], + "name": ".github/workflows/function-limits.yml: function-limits", + "os": "linux", + "toolchains": [ + "Python default" + ] + } + ], + "providers": [ + "GitHub Actions" + ] + }, + "ecosystems": { + "JVM": [ + ".java files" + ] + }, + "languages": [ + "Java" + ], + "unrecognised_suffixes": [] + }, + "results": { + "action-pinning": { + "details": [], + "message": "all 6 actions are pinned to a commit", + "status": "PASS" + }, + "ci-build-types": { + "details": [], + "message": "not applicable to Java (applies to C, C++, Rust)", + "status": "SKIP" + }, + "ci-lint": { + "details": [], + "message": "CI lints its configuration (GitHub Actions: actionlint, zizmor)", + "status": "PASS" + }, + "ci-present": { + "details": [ + ".github/workflows/ci-lint.yml", + ".github/workflows/ci.yml", + ".github/workflows/copy-paste-detection.yml", + ".github/workflows/function-limits.yml" + ], + "message": "CI configuration found for GitHub Actions", + "status": "PASS" + }, + "code-formatting": { + "details": [], + "message": "pom.xml is defined and CI runs Spotless", + "status": "PASS" + }, + "committed-artifacts": { + "details": [], + "message": "no build outputs are committed", + "status": "PASS" + }, + "compiler-warnings": { + "details": [], + "message": "strict JVM compiler warnings are enabled: -Xlint:all", + "status": "PASS" + }, + "copy-paste-detection": { + "details": [], + "message": "CI runs copy-paste detection with jscpd", + "status": "PASS" + }, + "coverage": { + "details": [], + "message": "branch coverage is reported with JaCoCo", + "status": "PASS" + }, + "cxx-extensions": { + "details": [], + "message": "not applicable to Java (applies to C, C++)", + "status": "SKIP" + }, + "data-wellformed": { + "details": [], + "message": "6 data files parse (XML: 1, YAML: 5)", + "status": "PASS" + }, + "dependency-audit": { + "details": [], + "message": "CI audits the dependencies with OWASP dependency-check", + "status": "PASS" + }, + "dependency-management": { + "details": [], + "message": "dependencies are managed with Maven and locked in CI (mvn --strict-checksums)", + "status": "PASS" + }, + "dependency-updates": { + "details": [ + ".github/dependabot.yml" + ], + "message": "automated dependency updates are configured with Dependabot", + "status": "PASS" + }, + "function-limits": { + "details": [], + "message": "function length is limited with lizard --length 30", + "status": "PASS" + }, + "hardening": { + "details": [], + "message": "not applicable to Java (applies to C, C++, Go, Rust)", + "status": "SKIP" + }, + "large-files": { + "details": [], + "message": "no committed file is larger than 5 MiB", + "status": "PASS" + }, + "license": { + "details": [ + "LICENSE" + ], + "message": "the project is licensed under MIT", + "status": "PASS" + }, + "platform-matrix": { + "details": [ + ".github/workflows/ci-lint.yml: ci-lint: linux / no JDK version", + ".github/workflows/ci.yml: test (ubuntu-latest, 21): linux / 21", + ".github/workflows/ci.yml: test (ubuntu-latest, 25): linux / 25", + ".github/workflows/ci.yml: test (macos-latest, 21): macos / 21", + ".github/workflows/ci.yml: test (macos-latest, 25): macos / 25", + ".github/workflows/ci.yml: test (windows-latest, 21): windows / 21", + ".github/workflows/ci.yml: test (windows-latest, 25): windows / 25", + ".github/workflows/copy-paste-detection.yml: copy-paste-detection: linux / no JDK version", + ".github/workflows/function-limits.yml: function-limits: linux / no JDK version" + ], + "message": "CI covers 3 OS(es) (linux, macos, windows), 2 JDK version(s) (21, 25), 6 combination(s)", + "status": "PASS" + }, + "secret-scanning": { + "details": [], + "message": "secrets are scanned for with gitleaks", + "status": "PASS" + }, + "security-policy": { + "details": [], + "message": "SECURITY.md says how to report a vulnerability", + "status": "PASS" + }, + "security-scanning": { + "details": [], + "message": "security lints are enabled: Find Security Bugs", + "status": "PASS" + }, + "static-analysis": { + "details": [], + "message": "SpotBugs runs", + "status": "PASS" + }, + "status-badges": { + "details": [], + "message": "README.md shows CI (GitHub Actions) and quality (Codecov) badges", + "status": "PASS" + }, + "tests-run": { + "details": [], + "message": "JVM tests found in 1 file(s)", + "status": "PASS" + }, + "type-checking": { + "details": [], + "message": "not applicable to Java (applies to JavaScript, Python, TypeScript)", + "status": "SKIP" + }, + "vcs-ignore": { + "details": [], + "message": ".gitignore ignores target/", + "status": "PASS" + }, + "warnings-as-errors": { + "details": [], + "message": "warnings are treated as errors with -Werror", + "status": "PASS" + } + } +} diff --git a/tests/snapshots/well_configured_python.json b/tests/snapshots/well_configured_python.json index 672f503..c2822e5 100644 --- a/tests/snapshots/well_configured_python.json +++ b/tests/snapshots/well_configured_python.json @@ -298,7 +298,7 @@ }, "warnings-as-errors": { "details": [], - "message": "not applicable to Python (applies to C, C++, JavaScript, TypeScript)", + "message": "not applicable to Python (applies to C, C++, Java, JavaScript, Kotlin, TypeScript)", "status": "SKIP" } } diff --git a/tests/snapshots/well_configured_rust.json b/tests/snapshots/well_configured_rust.json index 8a88d5e..d8c1e97 100644 --- a/tests/snapshots/well_configured_rust.json +++ b/tests/snapshots/well_configured_rust.json @@ -441,7 +441,7 @@ }, "warnings-as-errors": { "details": [], - "message": "not applicable to Rust (applies to C, C++, JavaScript, TypeScript)", + "message": "not applicable to Rust (applies to C, C++, Java, JavaScript, Kotlin, TypeScript)", "status": "SKIP" } } diff --git a/tests/support.py b/tests/support.py index 730d39b..ffc3a05 100644 --- a/tests/support.py +++ b/tests/support.py @@ -145,6 +145,18 @@ def gets_c_family_checks(self) -> bool: languages=frozenset({"TypeScript"}), build_systems=frozenset({"Deno"}), ), + ProjectLayout( + name="Maven Java", + files={"pom.xml": "\n", "src/main/java/App.java": ""}, + languages=frozenset({"Java"}), + build_systems=frozenset({"Maven"}), + ), + ProjectLayout( + name="Gradle Kotlin", + files={"settings.gradle.kts": "", "build.gradle.kts": "", "src/main/kotlin/App.kt": ""}, + languages=frozenset({"Kotlin"}), + build_systems=frozenset({"Gradle"}), + ), ProjectLayout( name="Unrecognised language", files={"README.md": "", "src/main.zig": ""}, @@ -239,6 +251,7 @@ def run_in_every_layout( WELL_CONFIGURED_RUST = fixture_files("well_configured_rust") WELL_CONFIGURED_GO = fixture_files("well_configured_go") WELL_CONFIGURED_JAVASCRIPT = fixture_files("well_configured_javascript") +WELL_CONFIGURED_JVM = fixture_files("well_configured_jvm") UNSTRUCTURED_CI_FILES = { provider: path for provider, path in CI_FILES.items() if provider not in {"GitHub Actions", "GitLab CI"} @@ -259,6 +272,19 @@ def run_as( return run_in_every_layout(test, selected, files, layouts, options) +def run_in_layout(test: unittest.TestCase, name: str, selected: Check, files: Mapping[str, str]) -> Result: + """The result of `selected` in a project of the layout called `name` with `files` added.""" + layouts = tuple(layout for layout in PROJECT_LAYOUTS if layout.name == name) + test.assertTrue(layouts, f"no layout is called {name}") + return run_in_every_layout(test, selected, files, layouts) + + +def checks_named(*names: str) -> tuple[Check, ...]: + """The checks called `names`, in that order.""" + found = {found.name: found for found in discover().checks} + return tuple(found[name] for name in names) + + def github_job(runs_on: str, *steps: str, matrix: Mapping[str, object] | None = None) -> dict[str, str]: job: dict[str, object] = {"runs-on": runs_on, "steps": [{"run": step} for step in steps]} if matrix is not None: diff --git a/tests/test_cli.py b/tests/test_cli.py index eac0247..e70fd4c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -22,6 +22,7 @@ WELL_CONFIGURED, WELL_CONFIGURED_GO, WELL_CONFIGURED_JAVASCRIPT, + WELL_CONFIGURED_JVM, WELL_CONFIGURED_PYTHON, WELL_CONFIGURED_RUST, ProjectLayout, @@ -34,7 +35,7 @@ # Language -> the language-specific checks that do not apply to it. NOT_APPLICABLE = { language: {found.name for found in LANGUAGE_SPECIFIC_CHECKS if language not in found.languages} - for language in ("Python", "Rust", "Go", "JavaScript", "TypeScript") + for language in ("Python", "Rust", "Go", "JavaScript", "TypeScript", "Java", "Kotlin") } @@ -70,6 +71,8 @@ def test_well_configured_projects_in_other_languages_pass_strictly(self) -> None "Go": WELL_CONFIGURED_GO, "JavaScript": WELL_CONFIGURED_JAVASCRIPT, "TypeScript": WELL_CONFIGURED_JAVASCRIPT, + "Java": WELL_CONFIGURED_JVM, + "Kotlin": WELL_CONFIGURED_JVM, } for layout in OTHER_LANGUAGE_LAYOUTS: [language] = layout.languages @@ -230,6 +233,6 @@ def test_list_as_json_is_the_support_matrix(self) -> None: self.assertEqual(code, EXIT_SUCCESS) matrix = {row["check"]: row["applies_to"] for row in json.loads(output)} self.assertEqual(matrix["ci-present"], []) - self.assertEqual(matrix["coverage"], ["C/C++", "Python", "Rust", "Go", "JavaScript/TypeScript"]) - self.assertEqual(matrix["warnings-as-errors"], ["C/C++", "JavaScript/TypeScript"]) + self.assertEqual(matrix["coverage"], ["C/C++", "Python", "Rust", "Go", "JavaScript/TypeScript", "JVM"]) + self.assertEqual(matrix["warnings-as-errors"], ["C/C++", "JavaScript/TypeScript", "JVM"]) self.assertEqual(matrix["ci-build-types"], ["C/C++", "Rust"]) diff --git a/tests/test_ecosystem_contract.py b/tests/test_ecosystem_contract.py index cd1c913..e5faa33 100644 --- a/tests/test_ecosystem_contract.py +++ b/tests/test_ecosystem_contract.py @@ -11,6 +11,7 @@ C_AND_CXX, GO, JAVASCRIPT_AND_TYPESCRIPT, + JVM, PYTHON, RUST, Ecosystem, @@ -27,6 +28,7 @@ RUST: ("well_configured_rust", "bare_rust"), GO: ("well_configured_go", "bare_go"), JAVASCRIPT_AND_TYPESCRIPT: ("well_configured_javascript", "bare_javascript"), + JVM: ("well_configured_jvm", "bare_jvm"), } diff --git a/tests/test_javascript.py b/tests/test_javascript.py index 9db7c55..192f7bb 100644 --- a/tests/test_javascript.py +++ b/tests/test_javascript.py @@ -23,11 +23,10 @@ from project_validator.ecosystems.javascript import DEPENDENCIES, NODE_MODULES, TYPE_CHECKING from tests.support import ( GITHUB_WORKFLOW, - PROJECT_LAYOUTS, github_job, make_context, run_as, - run_in_every_layout, + run_in_layout, temp_project, ) @@ -171,12 +170,6 @@ def test_a_disabled_rule_limits_nothing(self) -> None: self.assertIn("max-lines-per-function", result.message) -def _in_layout(test: unittest.TestCase, name: str, selected: Check, files: dict[str, str]) -> Result: - """The result of `selected` in a project of the layout called `name` with `files` added.""" - layouts = tuple(layout for layout in PROJECT_LAYOUTS if layout.name == name) - return run_in_every_layout(test, selected, files, layouts) - - class StaticAnalysisTests(unittest.TestCase): def _run(self, selected: Check, files: dict[str, str]) -> Result: return run_as(self, selected, files, JAVASCRIPT_AND_TYPESCRIPT) @@ -184,7 +177,7 @@ def _run(self, selected: Check, files: dict[str, str]) -> Result: def test_a_linter_is_the_static_analyzer(self) -> None: files = {"eslint.config.js": "export default [];\n", **_scripts(lint="eslint ."), **_ci("npm run lint")} self.assertEqual(self._run(static_analysis, files), Result.passed("eslint (eslint.config.js) runs")) - warned = _in_layout(self, "npm JavaScript", static_analysis, {"biome.json": "{}"}) + warned = run_in_layout(self, "npm JavaScript", static_analysis, {"biome.json": "{}"}) self.assertEqual(warned.message, "biome lint is configured (biome.json) but does not run") def test_security_lints_are_a_plugin_or_semgrep_rules(self) -> None: @@ -193,19 +186,19 @@ def test_security_lints_are_a_plugin_or_semgrep_rules(self) -> None: _ci("semgrep scan --config p/typescript"), ): with self.subTest(files=files): - self.assertEqual(_in_layout(self, "npm JavaScript", security_scanning, files).status, Status.PASS) + self.assertEqual(run_in_layout(self, "npm JavaScript", security_scanning, files).status, Status.PASS) self.assertEqual( - _in_layout(self, "npm JavaScript", security_scanning, _ci("semgrep scan --config p/python")).status, + run_in_layout(self, "npm JavaScript", security_scanning, _ci("semgrep scan --config p/python")).status, Status.WARN, ) class TypeCheckingTests(unittest.TestCase): def _npm(self, files: dict[str, str]) -> Result: - return _in_layout(self, "npm JavaScript", type_checking, files) + return run_in_layout(self, "npm JavaScript", type_checking, files) def _deno(self, files: dict[str, str]) -> Result: - return _in_layout(self, "Deno TypeScript", type_checking, files) + return run_in_layout(self, "Deno TypeScript", type_checking, files) def test_tsc_type_checks_and_strict_mode_is_the_bonus(self) -> None: typecheck = {**_scripts(typecheck="tsc --noEmit"), **_ci("pnpm typecheck")} @@ -232,7 +225,7 @@ def test_deno_checks_strictly_unless_told_not_to(self) -> None: class DependencyTests(unittest.TestCase): def _npm(self, selected: Check, files: dict[str, str]) -> Result: - return _in_layout(self, "npm JavaScript", selected, files) + return run_in_layout(self, "npm JavaScript", selected, files) def test_declared_dependencies_must_be_locked_even_in_a_library(self) -> None: declared = _package(main="index.js", peerDependencies={"react": "^19"}) @@ -246,7 +239,7 @@ def test_declared_dependencies_must_be_locked_even_in_a_library(self) -> None: def test_deno_imports_are_dependencies(self) -> None: files = {"deno.jsonc": '{"imports": {"@std/assert": "jsr:@std/assert@1"}}', "deno.lock": "{}"} expected = Result.passed("dependencies are declared in deno.jsonc and locked in deno.lock") - self.assertEqual(_in_layout(self, "Deno TypeScript", dependency_management, files), expected) + self.assertEqual(run_in_layout(self, "Deno TypeScript", dependency_management, files), expected) def test_without_dependencies_there_is_nothing_to_lock(self) -> None: self.assertEqual(self._npm(dependency_management, {}), Result.passed("no dependencies are declared")) diff --git a/tests/test_jvm.py b/tests/test_jvm.py new file mode 100644 index 0000000..0287b75 --- /dev/null +++ b/tests/test_jvm.py @@ -0,0 +1,263 @@ +"""The JVM profile's rules, judged by the generic checks in the Maven and Gradle project layouts.""" + +import unittest + +from project_validator.core import Check, Result, Status +from project_validator.ecosystems import JVM +from project_validator.ecosystems.jvm import DEPENDENCIES +from tests.support import ( + GITHUB_WORKFLOW, + checks_named, + github_job, + make_context, + run_as, + run_in_layout, + temp_project, +) + +( + CODE_FORMATTING, + COMMITTED_ARTIFACTS, + COMPILER_WARNINGS, + COVERAGE, + DEPENDENCY_AUDIT, + DEPENDENCY_MANAGEMENT, + FUNCTION_LIMITS, + PLATFORM_MATRIX, + SECURITY_SCANNING, + STATIC_ANALYSIS, + TESTS_RUN, + VCS_IGNORE, + WARNINGS_AS_ERRORS, +) = checks_named( + "code-formatting", + "committed-artifacts", + "compiler-warnings", + "coverage", + "dependency-audit", + "dependency-management", + "function-limits", + "platform-matrix", + "security-scanning", + "static-analysis", + "tests-run", + "vcs-ignore", + "warnings-as-errors", +) +MAVEN_VERIFY = "./mvnw -B verify" +GRADLE_CHECK = "./gradlew check" + + +def _ci(*steps: str) -> dict[str, str]: + return github_job("ubuntu-latest", *steps) + + +def _pom(plugins: str = "", dependencies: str = "") -> dict[str, str]: + return { + "pom.xml": f"{dependencies}" + f"{plugins}\n" + } + + +def _plugin(artifact: str, *goals: str) -> str: + bound = "".join(f"{goal}" for goal in goals) + executions = f"{bound}" + return f"{artifact}{executions}" + + +def _gradle(build: str) -> dict[str, str]: + return {"build.gradle.kts": build} + + +def _maven(test: unittest.TestCase, selected: Check, files: dict[str, str]) -> Result: + return run_in_layout(test, "Maven Java", selected, files) + + +def _gradle_kotlin(test: unittest.TestCase, selected: Check, files: dict[str, str]) -> Result: + return run_in_layout(test, "Gradle Kotlin", selected, files) + + +class RecognitionTests(unittest.TestCase): + def _structure(self, files: dict[str, str]) -> tuple[frozenset[str], tuple[tuple[str, int], ...]]: + with temp_project(files) as root: + structure = make_context(root).structure + return structure.languages, structure.unrecognised_suffixes + + def test_kotlin_and_groovy_scripts_add_no_language(self) -> None: + files = {"build.gradle.kts": "", "gradle/publish.gradle": "", "src/main/java/App.java": "", "x.kts": ""} + self.assertEqual(self._structure(files), (frozenset({"Java"}), ())) + + def test_build_output_beside_a_build_file_is_not_inspected(self) -> None: + files = {"pom.xml": "", "target/generated/App.kt": "", "lib/build.gradle": "", "lib/build/Gen.kt": ""} + self.assertEqual(self._structure(files)[0], frozenset()) + + +class BuildLifecycleTests(unittest.TestCase): + def _run(self, selected: Check, files: dict[str, str]) -> Result: + return run_as(self, selected, files, JVM) + + def test_a_bound_maven_plugin_runs_when_ci_verifies(self) -> None: + spotless = _pom(_plugin("spotless-maven-plugin", "check")) + self.assertEqual(_maven(self, CODE_FORMATTING, {**spotless, **_ci(MAVEN_VERIFY)}).status, Status.PASS) + # mvn test stops before the verify phase, and an unbound plugin never runs. + self.assertEqual(_maven(self, CODE_FORMATTING, {**spotless, **_ci("mvn test")}).status, Status.WARN) + unbound = _pom("spotless-maven-plugin") + self.assertEqual(_maven(self, CODE_FORMATTING, {**unbound, **_ci(MAVEN_VERIFY)}).status, Status.WARN) + + def test_a_gradle_plugin_runs_with_check(self) -> None: + build = _gradle('plugins {\n id("com.diffplug.spotless") version "7.0.0"\n}\n') + self.assertEqual(_gradle_kotlin(self, CODE_FORMATTING, {**build, **_ci(GRADLE_CHECK)}).status, Status.PASS) + self.assertEqual(_gradle_kotlin(self, CODE_FORMATTING, {**build, **_ci("./gradlew test")}).status, Status.WARN) + + def test_formatters_run_directly(self) -> None: + for command in ("mvn spotless:check", "./gradlew spotlessCheck", "ktlint", "ktfmt --set-exit-if-changed src"): + with self.subTest(command=command): + self.assertEqual(self._run(CODE_FORMATTING, _ci(command)).status, Status.PASS) + self.assertEqual(self._run(CODE_FORMATTING, _ci("ktlint -F")).status, Status.FAIL) + + def test_tests_are_annotated_or_kotest_specs_under_src_test(self) -> None: + kotest = {"src/test/kotlin/AppSpec.kt": "class AppSpec : FunSpec({})\n"} + self.assertEqual(self._run(TESTS_RUN, {**kotest, **_ci(GRADLE_CHECK)}).status, Status.PASS) + self.assertEqual(self._run(TESTS_RUN, kotest).status, Status.WARN) + self.assertEqual( + self._run(TESTS_RUN, {"src/main/java/AppTest.java": "@Test void a() {}\n"}).status, Status.FAIL + ) + + def test_coverage_is_jacoco_or_kover(self) -> None: + for files in ( + _ci("./gradlew test jacocoTestReport"), + _ci("./gradlew koverXmlReport"), + {**_pom(_plugin("jacoco-maven-plugin", "prepare-agent", "report")), **_ci(MAVEN_VERIFY)}, + ): + with self.subTest(files=files): + self.assertEqual(self._run(COVERAGE, files).status, Status.PASS) + prepared_only = {**_pom(_plugin("jacoco-maven-plugin", "prepare-agent")), **_ci(MAVEN_VERIFY)} + self.assertEqual(self._run(COVERAGE, prepared_only).status, Status.FAIL) + + +class WarningTests(unittest.TestCase): + def test_java_needs_xlint_all(self) -> None: + flags = _pom("-Xlint:all-Werror") + result = _maven(self, COMPILER_WARNINGS, {**flags, **_ci("mvn test")}) + self.assertEqual(result, Result.passed("strict JVM compiler warnings are enabled: -Xlint:all")) + self.assertEqual(_maven(self, WARNINGS_AS_ERRORS, {**flags, **_ci("mvn test")}).status, Status.PASS) + # The build files count only when CI builds the project. + self.assertEqual(_maven(self, COMPILER_WARNINGS, flags).status, Status.FAIL) + + def test_kotlin_warns_by_default_and_fails_on_warnings_when_told(self) -> None: + build = _gradle('plugins { kotlin("jvm") }\nkotlin { compilerOptions { allWarningsAsErrors = true } }\n') + files = {**build, **_ci("./gradlew build")} + expected = Result.passed("strict JVM compiler warnings are enabled: Kotlin's default warnings") + self.assertEqual(_gradle_kotlin(self, COMPILER_WARNINGS, files), expected) + result = _gradle_kotlin(self, WARNINGS_AS_ERRORS, files) + self.assertEqual(result, Result.passed("warnings are treated as errors with allWarningsAsErrors")) + + def test_mixed_java_and_kotlin_still_need_xlint(self) -> None: + files = {"src/main/java/A.java": "", **_gradle('plugins { kotlin("jvm") }\n'), **_ci("./gradlew build")} + result = _gradle_kotlin(self, COMPILER_WARNINGS, files) + self.assertEqual(result.message, "missing JVM compiler warnings: -Xlint:all") + + +class SecurityTests(unittest.TestCase): + def test_dependency_check_runs_bound_or_as_a_task(self) -> None: + bound = {**_pom(_plugin("dependency-check-maven", "aggregate")), **_ci(MAVEN_VERIFY)} + for files in (bound, _ci("./gradlew dependencyCheckAnalyze")): + with self.subTest(files=files): + self.assertEqual(run_as(self, DEPENDENCY_AUDIT, files, JVM).status, Status.PASS) + # The Gradle plugin adds no task to check. + applied = {**_gradle('plugins { id("org.owasp.dependencycheck") }\n'), **_ci(GRADLE_CHECK)} + self.assertEqual(run_as(self, DEPENDENCY_AUDIT, applied, JVM).status, Status.WARN) + + def test_security_lints_are_find_security_bugs_or_pmd_security_rules(self) -> None: + pmd = "maven-pmd-plugincategory/java/security.xml" + files = {**_pom(pmd), **_ci(MAVEN_VERIFY)} + result = _maven(self, SECURITY_SCANNING, files) + self.assertEqual(result, Result.passed("security lints are enabled: PMD security rules")) + self.assertEqual(_maven(self, STATIC_ANALYSIS, files), Result.passed("PMD runs")) + + def test_any_analyzer_counts_as_static_analysis(self) -> None: + error_prone = {**_gradle('plugins { id("net.ltgt.errorprone") }\n'), **_ci(GRADLE_CHECK)} + for files in (error_prone, _ci("mvn spotbugs:check"), _ci("./gradlew detekt")): + with self.subTest(files=files): + self.assertEqual(run_as(self, STATIC_ANALYSIS, files, JVM).status, Status.PASS) + # PMD's copy-paste detector is not its static analysis. + self.assertEqual(run_as(self, STATIC_ANALYSIS, _ci("pmd cpd --minimum-tokens 100"), JVM).status, Status.FAIL) + + +CHECKSTYLE = { + "config/checkstyle/checkstyle.xml": '' +} + + +class FunctionLimitTests(unittest.TestCase): + def _run(self, files: dict[str, str]) -> Result: + return run_as(self, FUNCTION_LIMITS, files, JVM) + + def test_checkstyle_method_length_limits_methods_when_checkstyle_runs(self) -> None: + result = self._run({**CHECKSTYLE, **_ci("mvn checkstyle:check")}) + expected = "function length is limited with Checkstyle MethodLength (config/checkstyle/checkstyle.xml)" + self.assertEqual(result, Result.passed(expected)) + warned = self._run(CHECKSTYLE) + self.assertEqual( + warned.message, "MethodLength is enabled in config/checkstyle/checkstyle.xml but Checkstyle does not run" + ) + + def test_detekt_limits_methods_unless_long_method_is_off(self) -> None: + self.assertEqual( + self._run(_ci("./gradlew detekt")), Result.passed("function length is limited with detekt LongMethod") + ) + off = {"config/detekt/detekt.yml": "complexity:\n LongMethod:\n active: false\n", **_ci("./gradlew detekt")} + self.assertEqual(self._run(off).status, Status.FAIL) + + +class DependencyAndOutputTests(unittest.TestCase): + def test_declared_dependencies_must_be_verified(self) -> None: + declared = _pom(dependencies="guava") + expected = Result.warned("dependencies are managed with Maven but not locked", DEPENDENCIES.lock_hint) + self.assertEqual(_maven(self, DEPENDENCY_MANAGEMENT, declared), expected) + config = {**declared, ".mvn/maven.config": "--strict-checksums\n"} + locked = _maven(self, DEPENDENCY_MANAGEMENT, config) + self.assertEqual( + locked.message, "dependencies are managed with Maven and locked in .mvn/maven.config (--strict-checksums)" + ) + + def test_gradle_verification_metadata_locks_dependencies(self) -> None: + build = _gradle('dependencies {\n implementation("com.google.guava:guava:33.4.8-jre")\n}\n') + files = {**build, "gradle/verification-metadata.xml": ""} + result = _gradle_kotlin(self, DEPENDENCY_MANAGEMENT, files) + self.assertEqual( + result.message, "dependencies are managed with Gradle and locked in gradle/verification-metadata.xml" + ) + self.assertEqual(_gradle_kotlin(self, DEPENDENCY_MANAGEMENT, {}).message, "no dependencies are declared") + + def test_each_build_system_ignores_its_own_output(self) -> None: + self.assertEqual(_maven(self, VCS_IGNORE, {".gitignore": "target/\n"}).status, Status.PASS) + self.assertEqual(_maven(self, VCS_IGNORE, {".gitignore": "build/\n"}).status, Status.WARN) + self.assertEqual(_gradle_kotlin(self, VCS_IGNORE, {".gitignore": "/build\n"}).status, Status.PASS) + self.assertEqual(_gradle_kotlin(self, VCS_IGNORE, {".gitignore": "target/\n"}).status, Status.WARN) + + def test_class_files_are_outputs_without_a_build_system(self) -> None: + with temp_project({"src/App.java": "", "src/App.class": ""}) as root: + result = COMMITTED_ARTIFACTS.run(make_context(root)) + self.assertEqual(result.message, "1 build output is committed") + + def test_the_platform_matrix_counts_jdk_versions(self) -> None: + files = { + GITHUB_WORKFLOW: """\ + jobs: + linux: + runs-on: ubuntu-latest + container: maven:3.9-eclipse-temurin-21 + steps: + - run: mvn verify + windows: + runs-on: windows-latest + steps: + - uses: actions/setup-java@v4 + with: + java-version: "17" + - run: ./gradlew.bat build + """ + } + result = run_as(self, PLATFORM_MATRIX, files, JVM) + self.assertIn("2 JDK version(s) (17, 21)", result.message) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 665bae0..22e0c9e 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -135,7 +135,7 @@ def test_a_plugin_ecosystem_is_registered_and_practices_apply_to_it(self) -> Non plugins = self._discover_zig() self.assertEqual( [ecosystem.label for ecosystem in registered()], - ["C/C++", "Python", "Rust", "Go", "JavaScript/TypeScript", "Zig"], + ["C/C++", "Python", "Rust", "Go", "JavaScript/TypeScript", "JVM", "Zig"], ) formatting = next(found for found in plugins.checks if found.name == "code-formatting") self.assertIn("Zig", formatting.languages) @@ -159,7 +159,7 @@ def test_loading_a_plugin_again_replaces_its_ecosystem(self) -> None: self._discover_zig() self.assertEqual( [ecosystem.label for ecosystem in registered()], - ["C/C++", "Python", "Rust", "Go", "JavaScript/TypeScript", "Zig"], + ["C/C++", "Python", "Rust", "Go", "JavaScript/TypeScript", "JVM", "Zig"], ) def test_rejects_an_ecosystem_with_a_built_in_label(self) -> None: From 6a5384d14cd1ca3f87617d7bbe7512bad327e590 Mon Sep 17 00:00:00 2001 From: leftibot Date: Fri, 6 Nov 2026 00:19:50 -0700 Subject: [PATCH 3/6] feat: add a PHP ecosystem profile Composer projects are judged by PHP-CS-Fixer, PHP_CodeSniffer or Pint; PHPUnit --path-coverage, which reports branch and path coverage together (so it earns the path bonus, and line coverage alone counts no branches); *Test.php files with PHPUnit test cases or Pest tests; error_reporting E_ALL while testing; PHPUnit failing on warnings, notices and deprecations; composer audit or roave/security-advisories; Psalm taint analysis or phpcs-security-audit; PHPMD's ExcessiveMethodLength; PHPStan, Psalm, PHPMD or Phan; PHPStan or Psalm as type checkers, strict at PHPStan level 8+ or Psalm errorLevel 1-2; composer.lock for anything not declared a library; composer.json's license; an ignored vendor/; and the PHP versions CI runs. PHP tooling runs through Composer scripts, so the scripts CI runs (composer test, composer run-script x, @x and @composer x inside a script) count as CI configuration. Following scripts moves to evidence.scripts_run, shared with the JavaScript profile. TestSuite gets the optional texts that Formatter and DependencyAudit have, which tests-run reads, so tests run through a script count; the JavaScript profile uses it too. jscpd ignores parenthesised import blocks, which every full profile shares with the others. Closes #69 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01ByTXNqDWdM7HzezrFtEvQt --- .jscpd.json | 1 + README.md | 34 +- project_validator/checks/tests_run.py | 2 +- project_validator/ecosystems/__init__.py | 4 +- project_validator/ecosystems/javascript.py | 26 +- project_validator/ecosystems/php.py | 353 ++++++++++++++++++ project_validator/evidence.py | 28 +- project_validator/rules.py | 7 +- tests/checks/test_warnings_as_errors.py | 7 +- tests/fixtures/bare_php/.phpunit.result.cache | 1 + tests/fixtures/bare_php/composer.json | 7 + tests/fixtures/bare_php/src/Calculator.php | 13 + .../.github/dependabot.yml | 6 + .../.github/workflows/ci-lint.yml | 9 + .../.github/workflows/ci.yml | 16 + .../workflows/copy-paste-detection.yml | 6 + .../.github/workflows/function-limits.yml | 7 + tests/fixtures/well_configured_php/.gitignore | 2 + tests/fixtures/well_configured_php/LICENSE | 21 ++ tests/fixtures/well_configured_php/README.md | 4 + .../fixtures/well_configured_php/SECURITY.md | 3 + .../well_configured_php/composer.json | 30 ++ .../well_configured_php/composer.lock | 5 + .../well_configured_php/phpstan.neon.dist | 5 + .../well_configured_php/phpunit.xml.dist | 19 + .../well_configured_php/src/Calculator.php | 13 + .../tests/CalculatorTest.php | 16 + tests/snapshots/bare_go.json | 4 +- tests/snapshots/bare_jvm.json | 2 +- tests/snapshots/bare_php.json | 202 ++++++++++ tests/snapshots/bare_project.json | 4 +- tests/snapshots/bare_python.json | 2 +- tests/snapshots/bare_rust.json | 4 +- tests/snapshots/buildable.json | 4 +- tests/snapshots/well_configured.json | 4 +- tests/snapshots/well_configured_go.json | 4 +- tests/snapshots/well_configured_jvm.json | 2 +- tests/snapshots/well_configured_php.json | 314 ++++++++++++++++ tests/snapshots/well_configured_python.json | 2 +- tests/snapshots/well_configured_rust.json | 4 +- tests/support.py | 7 + tests/test_cli.py | 8 +- tests/test_ecosystem_contract.py | 2 + tests/test_php.py | 231 ++++++++++++ tests/test_plugins.py | 4 +- 45 files changed, 1388 insertions(+), 61 deletions(-) create mode 100644 project_validator/ecosystems/php.py create mode 100644 tests/fixtures/bare_php/.phpunit.result.cache create mode 100644 tests/fixtures/bare_php/composer.json create mode 100644 tests/fixtures/bare_php/src/Calculator.php create mode 100644 tests/fixtures/well_configured_php/.github/dependabot.yml create mode 100644 tests/fixtures/well_configured_php/.github/workflows/ci-lint.yml create mode 100644 tests/fixtures/well_configured_php/.github/workflows/ci.yml create mode 100644 tests/fixtures/well_configured_php/.github/workflows/copy-paste-detection.yml create mode 100644 tests/fixtures/well_configured_php/.github/workflows/function-limits.yml create mode 100644 tests/fixtures/well_configured_php/.gitignore create mode 100644 tests/fixtures/well_configured_php/LICENSE create mode 100644 tests/fixtures/well_configured_php/README.md create mode 100644 tests/fixtures/well_configured_php/SECURITY.md create mode 100644 tests/fixtures/well_configured_php/composer.json create mode 100644 tests/fixtures/well_configured_php/composer.lock create mode 100644 tests/fixtures/well_configured_php/phpstan.neon.dist create mode 100644 tests/fixtures/well_configured_php/phpunit.xml.dist create mode 100644 tests/fixtures/well_configured_php/src/Calculator.php create mode 100644 tests/fixtures/well_configured_php/tests/CalculatorTest.php create mode 100644 tests/snapshots/bare_php.json create mode 100644 tests/snapshots/well_configured_php.json create mode 100644 tests/test_php.py diff --git a/.jscpd.json b/.jscpd.json index e4a2fc7..1657631 100644 --- a/.jscpd.json +++ b/.jscpd.json @@ -4,5 +4,6 @@ "format": ["python"], "gitignore": true, "ignore": ["**/__pycache__/**", "**/.venv/**", "**/*.egg-info/**"], + "ignorePattern": ["from [\\w.]+ import \\([^)]*\\)"], "path": ["project_validator", "tests"] } diff --git a/README.md b/README.md index fbcf9f0..3fb1cf2 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A tool to validate a project's build, CI and tooling against best practices. It recognises C and C++ (CMake or Meson), Python (`pyproject.toml` or `setup.py`), -Rust (Cargo), Go (Go modules), JavaScript/TypeScript (`package.json` or `deno.json`) and JVM (Java and Kotlin, with Maven or Gradle) projects, and judges each language by its own ecosystem's tools: a C++ +Rust (Cargo), Go (Go modules), JavaScript/TypeScript (`package.json` or `deno.json`) JVM (Java and Kotlin, with Maven or Gradle) and PHP (Composer) projects, and judges each language by its own ecosystem's tools: a C++ project needs branch coverage from gcovr or llvm-cov, a Python project from coverage.py, a Rust project from cargo-llvm-cov. Checks that do not apply to any of the project's languages are reported as `SKIP`. A project using several languages is judged for each @@ -60,33 +60,33 @@ Exit codes: `0` everything passed, `1` a check failed or raised an error, `2` us | Check | Applies to | Passes when | |---|---|---| | `ci-present` | all | at least one CI configuration is found | -| `tests-run` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | tests exist and CI runs them (`ctest-runs` is accepted as an old name). C/C++ with CMake: `enable_testing()` or `include(CTest)` is called, tests are registered (`add_test`, `*_discover_tests`, ...) and CI runs `ctest`; with `--build`, the project is really configured and built, and `ctest` must list at least one test. C/C++ with Meson: `test()` is called in a `meson.build` and CI runs `meson test`. Python: `test_*.py` / `*_test.py` files run by pytest, unittest, tox or nox. Rust: `#[test]` functions run by `cargo test` or nextest. Go: `Test` functions in `*_test.go` files run by `go test`. JavaScript/TypeScript: `*.test.*`, `*.spec.*`, `*_test.*` or `__tests__/` files run by jest, vitest, mocha, ava, `node --test`, `deno test`, `bun test`, playwright or `npm test`. JVM: `@Test` methods or Kotest specs under `src/test/java` or `src/test/kotlin`, run by `mvn test` / `verify` or `gradle test` / `check` / `build`. Warns when CI never runs the tests | -| `coverage` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | CI reports branch coverage (Go's tools measure statements only, so Go needs statement coverage: `go test -cover` or `-coverprofile`). C/C++: gcovr, lcov, genhtml, gcov or llvm-cov branch options; Python: `coverage run --branch`, `--cov-branch` or `branch = true`; Rust: cargo-llvm-cov, cargo-tarpaulin or grcov with `--branch`; JavaScript/TypeScript: c8, nyc, `jest --coverage` (or `collectCoverage: true`), `vitest --coverage` (or coverage `enabled: true`), `deno test --coverage` or `node --experimental-test-coverage`, in CI, the package scripts it runs, or the test runner's configuration; JVM: JaCoCo (`jacoco-maven-plugin` with its `report` goal, `jacocoTestReport`) or Kover. Path or MC/DC coverage (`-fcondition-coverage`, `-fcoverage-mcdc`, `--mcdc`, ...) is reported as a bonus | -| `platform-matrix` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | CI covers at least 2 operating systems, 2 toolchains (compilers, Python versions, Rust toolchains, Go versions, Node.js, Deno or Bun versions, or JDK versions) and 3 OS/toolchain combinations | +| `tests-run` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM, PHP | tests exist and CI runs them (`ctest-runs` is accepted as an old name). C/C++ with CMake: `enable_testing()` or `include(CTest)` is called, tests are registered (`add_test`, `*_discover_tests`, ...) and CI runs `ctest`; with `--build`, the project is really configured and built, and `ctest` must list at least one test. C/C++ with Meson: `test()` is called in a `meson.build` and CI runs `meson test`. Python: `test_*.py` / `*_test.py` files run by pytest, unittest, tox or nox. Rust: `#[test]` functions run by `cargo test` or nextest. Go: `Test` functions in `*_test.go` files run by `go test`. JavaScript/TypeScript: `*.test.*`, `*.spec.*`, `*_test.*` or `__tests__/` files run by jest, vitest, mocha, ava, `node --test`, `deno test`, `bun test`, playwright or `npm test`. JVM: `@Test` methods or Kotest specs under `src/test/java` or `src/test/kotlin`, run by `mvn test` / `verify` or `gradle test` / `check` / `build`. PHP: `*Test.php` files with PHPUnit test cases or Pest tests, run by phpunit, pest, paratest or `composer test`, in CI or the Composer scripts it runs. Warns when CI never runs the tests | +| `coverage` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM, PHP | CI reports branch coverage (Go's tools measure statements only, so Go needs statement coverage: `go test -cover` or `-coverprofile`). C/C++: gcovr, lcov, genhtml, gcov or llvm-cov branch options; Python: `coverage run --branch`, `--cov-branch` or `branch = true`; Rust: cargo-llvm-cov, cargo-tarpaulin or grcov with `--branch`; JavaScript/TypeScript: c8, nyc, `jest --coverage` (or `collectCoverage: true`), `vitest --coverage` (or coverage `enabled: true`), `deno test --coverage` or `node --experimental-test-coverage`, in CI, the package scripts it runs, or the test runner's configuration; JVM: JaCoCo (`jacoco-maven-plugin` with its `report` goal, `jacocoTestReport`) or Kover; PHP: PHPUnit `--path-coverage` (or `pathCoverage="true"`), which reports branch and path coverage together, so it also earns the path bonus (line coverage alone counts no branches). Path or MC/DC coverage (`-fcondition-coverage`, `-fcoverage-mcdc`, `--mcdc`, ...) is reported as a bonus | +| `platform-matrix` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM, PHP | CI covers at least 2 operating systems, 2 toolchains (compilers, Python versions, Rust toolchains, Go versions, Node.js, Deno or Bun versions, JDK versions, or PHP versions) and 3 OS/toolchain combinations | | `ci-build-types` | C/C++, Rust | every OS/toolchain combination in CI builds Debug and Release (Rust: debug and release profiles), each with and without sanitizers | -| `compiler-warnings` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | a strict warning set is enabled. C/C++: `-Wall -Wextra -Wpedantic -Wshadow -Wconversion`, plus `/W4` and `/permissive-` when MSVC is in CI; Python: warnings are errors (`-W error`, `PYTHONWARNINGS=error`, pytest `filterwarnings = error`); Rust: `cargo clippy`, `-D warnings` and `clippy::pedantic`; Go: `go vet` and staticcheck (golangci-lint runs both); JavaScript/TypeScript: a linter runs (eslint, `biome lint` / `biome ci`, or `deno lint`); JVM: javac's `-Xlint:all` (a Kotlin-only project passes on Kotlin's default warnings). Warns listing what is missing | -| `warnings-as-errors` | C/C++, JavaScript/TypeScript, JVM | C/C++: `-Werror`, `/WX` or `COMPILE_WARNING_AS_ERROR` (Meson: `werror=true`) is enabled in at least one configuration; JavaScript/TypeScript: a lint warning fails CI (`eslint --max-warnings 0`, `biome ci` or `--error-on-warnings`, or `deno lint`); JVM: `-Werror` for javac or kotlinc, or Kotlin's `allWarningsAsErrors` | +| `compiler-warnings` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM, PHP | a strict warning set is enabled. C/C++: `-Wall -Wextra -Wpedantic -Wshadow -Wconversion`, plus `/W4` and `/permissive-` when MSVC is in CI; Python: warnings are errors (`-W error`, `PYTHONWARNINGS=error`, pytest `filterwarnings = error`); Rust: `cargo clippy`, `-D warnings` and `clippy::pedantic`; Go: `go vet` and staticcheck (golangci-lint runs both); JavaScript/TypeScript: a linter runs (eslint, `biome lint` / `biome ci`, or `deno lint`); JVM: javac's `-Xlint:all` (a Kotlin-only project passes on Kotlin's default warnings); PHP: every error is reported while testing (`error_reporting` `-1` or `E_ALL` in `phpunit.xml`, `php.ini` or CI). Warns listing what is missing | +| `warnings-as-errors` | C/C++, JavaScript/TypeScript, JVM, PHP | C/C++: `-Werror`, `/WX` or `COMPILE_WARNING_AS_ERROR` (Meson: `werror=true`) is enabled in at least one configuration; JavaScript/TypeScript: a lint warning fails CI (`eslint --max-warnings 0`, `biome ci` or `--error-on-warnings`, or `deno lint`); JVM: `-Werror` for javac or kotlinc, or Kotlin's `allWarningsAsErrors`; PHP: PHPUnit's `failOnWarning`, `failOnNotice` and `failOnDeprecation` (or `failOnAllIssues`) | | `cxx-extensions` | C/C++ | `CMAKE_CXX_EXTENSIONS` (or `CMAKE_C_EXTENSIONS` for C) is turned off, or with Meson `cpp_std` (`c_std`) names a standard such as `c++20` rather than `gnu++20`, so `-std=c++NN` is used rather than `-std=gnu++NN` | -| `hardening` | C/C++, Rust, Go | binary hardening is enabled: exploit mitigations that the build inserts into the binary (static analysis and dependency audits are judged by `security-scanning` and `dependency-audit`). C/C++: `_FORTIFY_SOURCE`, `-fstack-protector-strong`, `-fcf-protection`, `-fstack-clash-protection` and `_GLIBCXX_ASSERTIONS` or `_LIBCPP_HARDENING_MODE` (or GCC's `-fhardened`), plus `/sdl /guard:cf /DYNAMICBASE /CETCOMPAT` when MSVC is in CI; Rust: `overflow-checks` in release builds; Go: programs are built as position-independent executables (`-buildmode=pie`, not the default on Linux), and a module with no `package main` is skipped. Python, JavaScript/TypeScript and the JVM build no native binary, so they are not judged. Warns listing what is missing | -| `code-formatting` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | the formatting style is defined (`.clang-format`; ruff or black settings; `rustfmt.toml`) and CI runs the formatter. Python, Rust, Go (gofmt, gofumpt, goimports) and JavaScript/TypeScript (`prettier --check`, `biome format` / `biome ci`, `deno fmt --check` or `dprint check`, in CI or the package scripts it runs) and JVM (Spotless, fmt-maven-plugin, google-java-format, ktfmt or ktlint) formatters have a default style, so CI enforcement alone passes. Warns when the style is defined but CI does not enforce it | -| `function-limits` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | function length is limited: lizard runs with a length limit (`lizard --length 30`), or for C/C++, `.clang-tidy` enables `readability-function-size` with a threshold and clang-tidy runs, or for Go, golangci-lint runs with `funlen` enabled, or for JavaScript/TypeScript, eslint runs with `max-lines-per-function` enabled, or for the JVM, Checkstyle runs with `MethodLength` or detekt runs with `LongMethod` (on by default) | -| `static-analysis` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | static analyzers run. C/C++: both clang-tidy (`CMAKE_CXX_CLANG_TIDY`, `run-clang-tidy`, or `clang-tidy` in CI) with its checks chosen in a `.clang-tidy`, and cppcheck (`CMAKE_CXX_CPPCHECK` or `cppcheck` in CI); Python: `ruff check`, pylint or flake8, in CI or as a pre-commit hook; Rust: `cargo clippy`; Go: golangci-lint with linters enabled beyond its defaults (`enable:` or `default: all` in a `.golangci.yml`); JavaScript/TypeScript: eslint, `biome lint` or `deno lint`, in CI, a pre-commit hook or the package scripts CI runs; JVM: Error Prone, SpotBugs, PMD, Checkstyle or detekt. Warns when an analyzer is configured but does not run, or runs short of that; fails when none runs | -| `type-checking` | Python, JavaScript/TypeScript | CI (or a pre-commit hook) runs a static type checker. Python: mypy, pyright or basedpyright, pyrefly, or ty; JavaScript/TypeScript: `tsc` (JavaScript with `checkJs`), or `deno check` / `deno test`, also through the package scripts CI runs. Strict mode is a bonus (`strict = true` in `[tool.mypy]` or a `[mypy]` section, `mypy --strict`, pyright's `typeCheckingMode` `strict`, `"strict": true` or `@tsconfig/strictest` in `tsconfig.json`, or Deno, strict by default); without it the result suggests it. Warns when a type checker is configured but does not run; fails when none runs. Compiled languages, whose compilers check types, are not judged | +| `hardening` | C/C++, Rust, Go | binary hardening is enabled: exploit mitigations that the build inserts into the binary (static analysis and dependency audits are judged by `security-scanning` and `dependency-audit`). C/C++: `_FORTIFY_SOURCE`, `-fstack-protector-strong`, `-fcf-protection`, `-fstack-clash-protection` and `_GLIBCXX_ASSERTIONS` or `_LIBCPP_HARDENING_MODE` (or GCC's `-fhardened`), plus `/sdl /guard:cf /DYNAMICBASE /CETCOMPAT` when MSVC is in CI; Rust: `overflow-checks` in release builds; Go: programs are built as position-independent executables (`-buildmode=pie`, not the default on Linux), and a module with no `package main` is skipped. Python, JavaScript/TypeScript, the JVM and PHP build no native binary, so they are not judged. Warns listing what is missing | +| `code-formatting` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM, PHP | the formatting style is defined (`.clang-format`; ruff or black settings; `rustfmt.toml`) and CI runs the formatter. Python, Rust, Go (gofmt, gofumpt, goimports) and JavaScript/TypeScript (`prettier --check`, `biome format` / `biome ci`, `deno fmt --check` or `dprint check`, in CI or the package scripts it runs) and JVM (Spotless, fmt-maven-plugin, google-java-format, ktfmt or ktlint) and PHP (`php-cs-fixer check` / `--dry-run`, phpcs or `pint --test`, in CI or the Composer scripts it runs) formatters have a default style, so CI enforcement alone passes. Warns when the style is defined but CI does not enforce it | +| `function-limits` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM, PHP | function length is limited: lizard runs with a length limit (`lizard --length 30`), or for C/C++, `.clang-tidy` enables `readability-function-size` with a threshold and clang-tidy runs, or for Go, golangci-lint runs with `funlen` enabled, or for JavaScript/TypeScript, eslint runs with `max-lines-per-function` enabled, or for the JVM, Checkstyle runs with `MethodLength` or detekt runs with `LongMethod` (on by default), or for PHP, PHPMD runs with its `codesize` rules (`ExcessiveMethodLength`) | +| `static-analysis` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM, PHP | static analyzers run. C/C++: both clang-tidy (`CMAKE_CXX_CLANG_TIDY`, `run-clang-tidy`, or `clang-tidy` in CI) with its checks chosen in a `.clang-tidy`, and cppcheck (`CMAKE_CXX_CPPCHECK` or `cppcheck` in CI); Python: `ruff check`, pylint or flake8, in CI or as a pre-commit hook; Rust: `cargo clippy`; Go: golangci-lint with linters enabled beyond its defaults (`enable:` or `default: all` in a `.golangci.yml`); JavaScript/TypeScript: eslint, `biome lint` or `deno lint`, in CI, a pre-commit hook or the package scripts CI runs; JVM: Error Prone, SpotBugs, PMD, Checkstyle or detekt; PHP: PHPStan, Psalm, PHPMD or Phan. Warns when an analyzer is configured but does not run, or runs short of that; fails when none runs | +| `type-checking` | Python, JavaScript/TypeScript, PHP | CI (or a pre-commit hook) runs a static type checker. Python: mypy, pyright or basedpyright, pyrefly, or ty; JavaScript/TypeScript: `tsc` (JavaScript with `checkJs`), or `deno check` / `deno test`, also through the package scripts CI runs; PHP: PHPStan or Psalm, also through the Composer scripts CI runs. Strict mode is a bonus (`strict = true` in `[tool.mypy]` or a `[mypy]` section, `mypy --strict`, pyright's `typeCheckingMode` `strict`, `"strict": true` or `@tsconfig/strictest` in `tsconfig.json`, Deno, strict by default, PHPStan `level` 8 or higher, or Psalm `errorLevel` 1 or 2); without it the result suggests it. Warns when a type checker is configured but does not run; fails when none runs. Compiled languages, whose compilers check types, are not judged | | `action-pinning` | all | every action and reusable workflow CI uses from another repository is pinned to a full commit SHA (`actions/checkout@ # v4.2.2`), and every `docker://` image to a digest. Fails when one is pinned to a branch or not at all; warns when one is pinned to a tag or an abbreviated commit (`--allow-official-tags` accepts tags for `actions/` and `github/`). Skipped without GitHub Actions | | `ci-lint` | all | CI (or a pre-commit hook) lints the configuration of each CI provider it uses: GitHub Actions with actionlint or zizmor, GitLab CI with `glab ci lint` or `gitlab-ci-local --preview`, CircleCI with `circleci config validate`. Warns otherwise; skipped without CI, or when no linter is known for the providers used | | `copy-paste-detection` | all | CI runs a duplicate code detector (PMD CPD, jscpd, Duplo, Simian, CCFinderX or SonarQube); warns otherwise | -| `security-scanning` | all | CI runs code security scanning (CodeQL or GitLab SAST), or every language in the project that has security lints enables them: Python: bandit (or ruff's `S` rules); Rust: `unsafe_code` forbidden crate-wide; Go: gosec; JavaScript/TypeScript: `eslint-plugin-security` (or `@microsoft/eslint-plugin-sdl`), or semgrep with a JavaScript ruleset; JVM: SpotBugs with Find Security Bugs, or PMD's `category/java/security.xml` rules. Warns otherwise, naming the languages left unscanned | +| `security-scanning` | all | CI runs code security scanning (CodeQL or GitLab SAST), or every language in the project that has security lints enables them: Python: bandit (or ruff's `S` rules); Rust: `unsafe_code` forbidden crate-wide; Go: gosec; JavaScript/TypeScript: `eslint-plugin-security` (or `@microsoft/eslint-plugin-sdl`), or semgrep with a JavaScript ruleset; JVM: SpotBugs with Find Security Bugs, or PMD's `category/java/security.xml` rules; PHP: Psalm's taint analysis or phpcs-security-audit. Warns otherwise, naming the languages left unscanned | | `secret-scanning` | all | CI or a pre-commit hook runs a secret scanner: gitleaks, TruffleHog, detect-secrets, ggshield or Nosey Parker. Warns otherwise, and says so when a scanner is configured (`.gitleaks.toml`, `.secrets.baseline`, `.gitguardian.yaml`) but nothing runs it. GitHub's own secret scanning is a repository setting, which cannot be seen from a checkout | -| `dependency-management` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | dependencies come from a package manager and are locked. C/C++: CPM, FetchContent, vcpkg (`vcpkg.json`), Conan (`conanfile.txt` / `conanfile.py`) or Meson wraps (`subprojects/*.wrap`); Python: dependencies declared in `pyproject.toml`, `setup.py` or requirements files are locked by a committed `uv.lock`, `poetry.lock`, `pdm.lock`, `Pipfile.lock` or `pylock.toml`, or by requirements pinned with `--hash`; Rust: a binary crate commits `Cargo.lock` (a library need not); Go: a module that requires others commits `go.sum`; JavaScript/TypeScript: dependencies declared in `package.json` (or `imports` in `deno.json`) are locked by a committed `package-lock.json`, `npm-shrinkwrap.json`, `pnpm-lock.yaml`, `yarn.lock`, `bun.lock`, `bun.lockb` or `deno.lock`, libraries included; JVM: dependencies declared in `pom.xml` or a Gradle build are verified by a committed `gradle/verification-metadata.xml` or `gradle.lockfile`, or Maven runs with `--strict-checksums`. Passes with a note when no dependencies are declared. Warns when they are not locked, or when third-party code is committed (`third_party/`, `external/`, `extern/`, `vendor/` other than `go mod vendor` output, or git submodules) | -| `dependency-audit` | Python, Rust, Go, JavaScript/TypeScript, JVM | CI checks the dependencies against known vulnerabilities. Python: pip-audit or safety; Rust: cargo audit or `cargo deny check` (advisories); Go: govulncheck; JavaScript/TypeScript: `npm audit` (not `npm audit signatures`), `pnpm audit`, `yarn npm audit` / `yarn audit` or `bun audit`; JVM: OWASP dependency-check (`dependency-check-maven` bound to `check`, or `dependencyCheckAnalyze`). osv-scanner, `trivy fs` and `snyk test` count for every language. Warns otherwise | +| `dependency-management` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM, PHP | dependencies come from a package manager and are locked. C/C++: CPM, FetchContent, vcpkg (`vcpkg.json`), Conan (`conanfile.txt` / `conanfile.py`) or Meson wraps (`subprojects/*.wrap`); Python: dependencies declared in `pyproject.toml`, `setup.py` or requirements files are locked by a committed `uv.lock`, `poetry.lock`, `pdm.lock`, `Pipfile.lock` or `pylock.toml`, or by requirements pinned with `--hash`; Rust: a binary crate commits `Cargo.lock` (a library need not); Go: a module that requires others commits `go.sum`; JavaScript/TypeScript: dependencies declared in `package.json` (or `imports` in `deno.json`) are locked by a committed `package-lock.json`, `npm-shrinkwrap.json`, `pnpm-lock.yaml`, `yarn.lock`, `bun.lock`, `bun.lockb` or `deno.lock`, libraries included; JVM: dependencies declared in `pom.xml` or a Gradle build are verified by a committed `gradle/verification-metadata.xml` or `gradle.lockfile`, or Maven runs with `--strict-checksums`; PHP: dependencies required in `composer.json` (besides `php` and extensions) are locked by a committed `composer.lock`, unless the package is a `"type": "library"`. Passes with a note when no dependencies are declared. Warns when they are not locked, or when third-party code is committed (`third_party/`, `external/`, `extern/`, `vendor/` other than `go mod vendor` output, or git submodules) | +| `dependency-audit` | Python, Rust, Go, JavaScript/TypeScript, JVM, PHP | CI checks the dependencies against known vulnerabilities. Python: pip-audit or safety; Rust: cargo audit or `cargo deny check` (advisories); Go: govulncheck; JavaScript/TypeScript: `npm audit` (not `npm audit signatures`), `pnpm audit`, `yarn npm audit` / `yarn audit` or `bun audit`; JVM: OWASP dependency-check (`dependency-check-maven` bound to `check`, or `dependencyCheckAnalyze`); PHP: `composer audit`, or `roave/security-advisories` required. osv-scanner, `trivy fs` and `snyk test` count for every language. Warns otherwise | | `data-wellformed` | all | every JSON, TOML, YAML and XML file parses, including the configuration in `.github/`, `.gitlab/`, `.circleci/`, `.buildkite/`, `.devcontainer/` and `.vscode/`. JSON that tools read with comments (`*.jsonc`, `tsconfig*.json`, `jsconfig.json`, `devcontainer.json`, `.vscode/*.json`) may have comments and trailing commas; YAML tags of any tool (GitLab's `!reference`) are accepted; XML external entities are never read. Files over 5 MiB are not parsed. Fails listing each file that does not parse, with the line and column | | `large-files` | all | no committed file is larger than 5 MiB (`--max-file-size`) unless Git LFS stores it (`filter=lfs` in a `.gitattributes`) or DVC tracks it (a `.dvc` file beside it). In a git checkout only the files git tracks are judged. Warns listing the largest files | | `security-policy` | all | a `SECURITY.md` (or `.rst`, `.txt`, ...) at the root, in `.github/` or in `docs/` says how to report a vulnerability: an email address, a link, or the forge's private vulnerability reporting. Warns otherwise | -| `license` | all | a license file (`LICENSE`, `LICENSE.md`, `LICENSE-MIT`, `COPYING`, `UNLICENSE`, ... or REUSE's `LICENSES/` directory) holds a recognised license: an `SPDX-License-Identifier` line, or the standard text of AGPL, LGPL, GPL, Apache-2.0, MPL-2.0, EPL-2.0, BSL-1.0, Unlicense, CC0-1.0, MIT, BSD, ISC, 0BSD or Zlib. Warns when the license is not recognised, or when a manifest declares another one (Python: `[project] license`; Rust: `[package] license`; JavaScript/TypeScript: `license` in `package.json`). Fails without a license file | +| `license` | all | a license file (`LICENSE`, `LICENSE.md`, `LICENSE-MIT`, `COPYING`, `UNLICENSE`, ... or REUSE's `LICENSES/` directory) holds a recognised license: an `SPDX-License-Identifier` line, or the standard text of AGPL, LGPL, GPL, Apache-2.0, MPL-2.0, EPL-2.0, BSL-1.0, Unlicense, CC0-1.0, MIT, BSD, ISC, 0BSD or Zlib. Warns when the license is not recognised, or when a manifest declares another one (Python: `[project] license`; Rust: `[package] license`; JavaScript/TypeScript: `license` in `package.json`; PHP: `license` in `composer.json`). Fails without a license file | | `dependency-updates` | all | Dependabot or Renovate is configured; warns otherwise | | `status-badges` | all | the README (at the root, in `.github/` or in `docs/`) shows a CI status badge of any supported CI provider (GitHub Actions `badge.svg`, GitLab pipeline, CircleCI, ... or their shields.io badges) and a quality badge (Codecov, Coveralls, SonarCloud, Codacy, Code Climate, DeepSource, Coverity Scan, OpenSSF Scorecard or Best Practices). Warns when only one kind is shown; fails when there is no README or no badge | -| `committed-artifacts` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | no build output is committed. C/C++: object files and libraries (`*.o`, `*.obj`, `*.a`, `*.lib`, `*.so`, `*.dylib`, `*.dll`), `*.exe`, `*.pdb`, `CMakeCache.txt`, `CMakeFiles/`, `CMakeUserPresets.json`, and a top-level `build/`, `out/` or `builddir/`; Python: `__pycache__/`, `*.py[co]`, `*.egg-info/`, a top-level `dist/` or `build/`; Rust: a top-level `target/`; Go: `*.test`, `*.out` and a top-level `bin/`; JavaScript/TypeScript: `node_modules/`, a top-level `coverage/` and `*.tsbuildinfo` (not `dist/`, which a JavaScript GitHub Action commits); JVM: `*.class`, plus Maven's `target/` or Gradle's `build/` and `.gradle/`. In a git checkout only the files git tracks are judged; `--exclude` keeps fixtures out. Fails listing the outputs | -| `vcs-ignore` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM | a `.gitignore`, `.hgignore`, `.bzrignore` or `.fossil-settings/ignore-glob` ignores the build outputs (`build/` or `out/`, `__pycache__/` or `*.py[cod]`, `/target`, Go's `*.test` or `*.out`, `node_modules/`, Maven's `target/` or Gradle's `build/`). Warns when an ignore file misses them | +| `committed-artifacts` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM, PHP | no build output is committed. C/C++: object files and libraries (`*.o`, `*.obj`, `*.a`, `*.lib`, `*.so`, `*.dylib`, `*.dll`), `*.exe`, `*.pdb`, `CMakeCache.txt`, `CMakeFiles/`, `CMakeUserPresets.json`, and a top-level `build/`, `out/` or `builddir/`; Python: `__pycache__/`, `*.py[co]`, `*.egg-info/`, a top-level `dist/` or `build/`; Rust: a top-level `target/`; Go: `*.test`, `*.out` and a top-level `bin/`; JavaScript/TypeScript: `node_modules/`, a top-level `coverage/` and `*.tsbuildinfo` (not `dist/`, which a JavaScript GitHub Action commits); JVM: `*.class`, plus Maven's `target/` or Gradle's `build/` and `.gradle/`; PHP: a top-level `vendor/`, `.phpunit.cache/` and `.phpunit.result.cache`. In a git checkout only the files git tracks are judged; `--exclude` keeps fixtures out. Fails listing the outputs | +| `vcs-ignore` | C/C++, Python, Rust, Go, JavaScript/TypeScript, JVM, PHP | a `.gitignore`, `.hgignore`, `.bzrignore` or `.fossil-settings/ignore-glob` ignores the build outputs (`build/` or `out/`, `__pycache__/` or `*.py[cod]`, `/target`, Go's `*.test` or `*.out`, `node_modules/`, Maven's `target/` or Gradle's `build/`, PHP's `vendor/`). Warns when an ignore file misses them | CI detection is pluggable too. GitHub Actions workflows (including `strategy.matrix` with `include`/`exclude`) and GitLab CI pipelines (including `parallel: matrix:`) diff --git a/project_validator/checks/tests_run.py b/project_validator/checks/tests_run.py index c8e2950..fba88f8 100644 --- a/project_validator/checks/tests_run.py +++ b/project_validator/checks/tests_run.py @@ -12,6 +12,6 @@ ) def tests_run(suite: TestSuite, context: Context) -> Result: result = suite.found(context) - if result.status != Status.PASS or matching_names(context.ci_texts, suite.runs): + if result.status != Status.PASS or matching_names(suite.texts(context), suite.runs): return result return Result.warned(f"{result.message}, but no CI configuration runs {suite.runner}", *result.details) diff --git a/project_validator/ecosystems/__init__.py b/project_validator/ecosystems/__init__.py index ada1f1e..253c854 100644 --- a/project_validator/ecosystems/__init__.py +++ b/project_validator/ecosystems/__init__.py @@ -23,10 +23,11 @@ languages_of, toolchain_versions, ) +from project_validator.ecosystems.php import PHP from project_validator.ecosystems.python import PYTHON from project_validator.ecosystems.rust import RUST -BUILT_IN = (C_AND_CXX, PYTHON, RUST, GO, JAVASCRIPT_AND_TYPESCRIPT, JVM) +BUILT_IN = (C_AND_CXX, PYTHON, RUST, GO, JAVASCRIPT_AND_TYPESCRIPT, JVM, PHP) _registered: list[Ecosystem] = list(BUILT_IN) @@ -211,6 +212,7 @@ def is_build_tree(directory: Path) -> bool: "GO", "JAVASCRIPT_AND_TYPESCRIPT", "JVM", + "PHP", "PYTHON", "RUST", "BuildSystem", diff --git a/project_validator/ecosystems/javascript.py b/project_validator/ecosystems/javascript.py index ba55bcd..1b91d6a 100644 --- a/project_validator/ecosystems/javascript.py +++ b/project_validator/ecosystems/javascript.py @@ -11,7 +11,14 @@ from project_validator.core import Context, Result from project_validator.ecosystems.model import BuildSystem, Ecosystem, ToolchainPatterns, versions_axis -from project_validator.evidence import ci_and_hook_texts, configuration, existing_texts, first_existing, json_value +from project_validator.evidence import ( + ci_and_hook_texts, + configuration, + existing_texts, + first_existing, + json_value, + scripts_run, +) from project_validator.rules import ( MULTI_ECOSYSTEM_AUDITS, SUGGESTED_LIZARD, @@ -84,24 +91,14 @@ def _named_scripts(text: str) -> Iterator[str]: yield NPM_LIFECYCLE_ALIASES.get(name, name) -def _scripts_run(context: Context, texts: tuple[str, ...]) -> tuple[str, ...]: - """The scripts that `texts` run, with the scripts those run in turn and their pre and post scripts.""" - scripts = _scripts(context) - pending = [name for text in texts for name in _named_scripts(text)] - run: dict[str, None] = {} - while pending: - name = pending.pop() - if name in run or name not in scripts: - continue - run[name] = None - pending += [f"pre{name}", f"post{name}", *_named_scripts(scripts[name])] - return tuple(scripts[name] for name in run) +def _pre_and_post(name: str) -> tuple[str, ...]: + return (f"pre{name}", f"post{name}") def _run_texts(context: Context) -> tuple[str, ...]: """CI configuration, pre-commit hooks, and the package scripts they run.""" texts = ci_and_hook_texts(context) - return (*texts, *_scripts_run(context, texts)) + return (*texts, *scripts_run(texts, _scripts(context), _named_scripts, _pre_and_post)) PRETTIER_CONFIG_FILES = tuple( @@ -161,6 +158,7 @@ def _javascript_tests(context: Context) -> Result: found=_javascript_tests, runs=RUNS_TESTS, runner="jest, vitest, mocha, node --test, deno test, bun test or playwright (or npm test)", + texts=_run_texts, ) # Istanbul (nyc, jest, vitest's istanbul provider) and V8 coverage (c8, vitest, node, deno) both report branches. diff --git a/project_validator/ecosystems/php.py b/project_validator/ecosystems/php.py new file mode 100644 index 0000000..a8a9ebe --- /dev/null +++ b/project_validator/ecosystems/php.py @@ -0,0 +1,353 @@ +"""PHP: managed with Composer, judged by the PHP versions CI runs. + +PHP projects run their tools through Composer scripts (`composer test`, `composer cs-check`), so the scripts CI runs +count as CI configuration, as package scripts do for JavaScript. +""" + +import re +from collections.abc import Iterator +from pathlib import PurePosixPath + +from project_validator.core import Context, Result +from project_validator.ecosystems.model import BuildSystem, Ecosystem, ToolchainPatterns, versions_axis +from project_validator.evidence import ( + ci_and_hook_texts, + configuration, + existing_texts, + first_existing, + json_value, + scripts_run, +) +from project_validator.rules import ( + MULTI_ECOSYSTEM_AUDITS, + SUGGESTED_LIZARD, + Analyzer, + BuildOutputs, + CoverageTools, + DeclaredLicense, + DependencyAudit, + DependencyManagement, + Evidence, + Formatter, + FunctionLimits, + SecurityLints, + StaticAnalysis, + TestSuite, + TypeChecking, + WarningPolicy, + WarningsAsErrors, + directories, + ignored_at_top_level, +) +from project_validator.text import Pattern, pattern + +PHP_LANGUAGE = "PHP" +COMPOSER_JSON = "composer.json" +COMPOSER = BuildSystem(name="Composer", files=(COMPOSER_JSON,), languages=frozenset({PHP_LANGUAGE})) +PHPUNIT_CONFIG_FILES = ("phpunit.xml", "phpunit.xml.dist", "phpunit.dist.xml") + + +def _scripts(context: Context) -> dict[str, str]: + """The Composer scripts, by name; a script of several commands is one command per line.""" + scripts = json_value(context, COMPOSER_JSON, "scripts") + if not isinstance(scripts, dict): + return {} + commands = {name: [value] if isinstance(value, str) else value for name, value in scripts.items()} + return { + name: "\n".join(item for item in value if isinstance(item, str)) + for name, value in commands.items() + if isinstance(value, list) + } + + +# `composer test`, `composer run-script lint`, and `@lint` or `@composer lint` inside a script. +CALLS_SCRIPT = re.compile( + r"\bcomposer\s+(?:run(?:-script)?\s+)?(?P[\w:.-]+)|(?:^|\s)@(?!composer\b|php\b)(?P