diff --git a/tools/sbom-diff-and-risk/docs/dependency-risk-heuristics.md b/tools/sbom-diff-and-risk/docs/dependency-risk-heuristics.md index 529d087..abf3799 100644 --- a/tools/sbom-diff-and-risk/docs/dependency-risk-heuristics.md +++ b/tools/sbom-diff-and-risk/docs/dependency-risk-heuristics.md @@ -17,6 +17,14 @@ The current rules are intentionally conservative: - `new_package` is a change signal, not a vulnerability claim. - `major_upgrade` fires only when both versions look reliably parseable as strict SemVer. +- Eligibility follows the [SemVer 2.0.0 grammar](https://semver.org/spec/v2.0.0.html): + ASCII `major.minor.patch`, optional prerelease identifiers and optional build + metadata. Empty identifiers and leading zeroes in core or numeric prerelease + identifiers are ineligible; numeric build identifiers may have leading zeroes. + Outer whitespace and one lowercase `v` prefix remain supported tool extensions. +- Only the validated major integers are compared. Prerelease-to-final, + final-to-prerelease, build-only changes and major decreases remain unclassified + when the canonical strings differ. No package-manager precedence is inferred. - uncertain version changes fall back to `version_change_unclassified`. - Version comparison uses the same canonical identity as the diff: a PURL version is evidence even when the explicit version field is absent. Conflicting explicit diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/risk.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/risk.py index 3b25cdd..cd65eb6 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/risk.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/risk.py @@ -12,10 +12,12 @@ _UNKNOWN_LICENSES = {"", "unknown", "noassertion"} _SEMVER_RE = re.compile( - r"^v?(?P0|[1-9]\d*)" - r"\.(?P0|[1-9]\d*)" - r"\.(?P0|[1-9]\d*)" - r"(?:[-+][0-9A-Za-z.-]+)?$" + r"v?(?P0|[1-9][0-9]*)" + r"\.(?:0|[1-9][0-9]*)" + r"\.(?:0|[1-9][0-9]*)" + r"(?:-(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)" + r"(?:\.(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?" + r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?" ) @@ -61,8 +63,7 @@ def _version_change_findings(change: ComponentChange) -> list[RiskFinding]: if before_version == after_version: return [] - semver_delta = _semver_major_delta(before_version, after_version) - if semver_delta is not None and semver_delta > 0: + if _is_semver_major_increase(before_version, after_version): return [ RiskFinding( bucket=RiskBucket.MAJOR_UPGRADE, @@ -123,21 +124,23 @@ def _is_unknown_license(license_id: str | None) -> bool: return license_id.strip().lower() in _UNKNOWN_LICENSES -def _semver_major_delta(before: str | None, after: str | None) -> int | None: +def _is_semver_major_increase(before: str | None, after: str | None) -> bool: before_major = _semver_major(before) after_major = _semver_major(after) if before_major is None or after_major is None: - return None - return after_major - before_major + return False + # Validated ASCII integers have no leading zeroes. Avoid int conversion's + # digit limit while comparing their numeric magnitude, not version precedence. + return (len(after_major), after_major) > (len(before_major), before_major) -def _semver_major(version: str | None) -> int | None: +def _semver_major(version: str | None) -> str | None: if not version: return None - match = _SEMVER_RE.match(version.strip()) + match = _SEMVER_RE.fullmatch(version.strip()) if not match: return None - return int(match.group("major")) + return match.group("major") def _is_suspicious_source(component: Component, allowlist: set[str]) -> bool: diff --git a/tools/sbom-diff-and-risk/tests/test_version_decision_reports.py b/tools/sbom-diff-and-risk/tests/test_version_decision_reports.py new file mode 100644 index 0000000..6bafd5e --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_version_decision_reports.py @@ -0,0 +1,208 @@ +"""Public CLI controls for syntax admission, policy and restricted SARIF projection.""" + +from __future__ import annotations + +from collections import Counter +import json +from pathlib import Path +import socket + +import pytest +from sbom_diff_risk import cli + + +@pytest.mark.parametrize("input_format", ["cyclonedx", "spdx"]) +@pytest.mark.parametrize( + "mode", + ["none", "unmatched", "block-major", "warn-major", "block-unclassified", "warn-unclassified", "mixed", "ignored"], +) +def test_version_decisions_across_cli_projections(tmp_path, monkeypatch, input_format, mode): + def reject_network(*args, **kwargs): + pytest.fail("Default comparison attempted enrichment or a network connection") + + monkeypatch.setattr(socket.socket, "connect", reject_network) + monkeypatch.setattr(socket, "create_connection", reject_network) + monkeypatch.setattr(cli, "PyPIProvenanceEnricher", reject_network) + monkeypatch.setattr(cli, "ScorecardEnricher", reject_network) + monkeypatch.chdir(tmp_path) + before_rows, after_rows = rows("before"), rows("after") + for side, items in (("before", before_rows), ("after", after_rows)): + write_sbom(Path(side + ".json"), input_format, items) + major, unclassified = "major_upgrade", "version_change_unclassified" + policy = {"version": 1, "block_on": [], "warn_on": [], "ignore_rules": []} + if mode in {"block-major", "mixed", "ignored"}: + policy["block_on"] = [major] + if mode == "block-unclassified": + policy["block_on"] = [unclassified] + if mode == "warn-major": + policy["warn_on"] = [major] + if mode in {"warn-unclassified", "mixed"}: + policy["warn_on"] = [unclassified] + if mode == "ignored": + policy["block_on"] = policy["ignore_rules"] = [major, unclassified] + if mode == "unmatched": + policy["block_on"] = ["suspicious_source"] + Path("policy.yml").write_text(json.dumps(policy), encoding="utf-8") + names = ("report.json", "summary.json", "policy.json", "report.md", "report.sarif") + args = ["compare", "--before", "before.json", "--after", "after.json"] + for flag, name in zip(("--out-json", "--summary-json", "--policy-json", "--out-md", "--out-sarif"), names): + args.extend((flag, name)) + if mode != "none": + args.extend(("--policy", "policy.yml")) + blocking = 1 if mode in {"block-major", "mixed"} else 2 if mode == "block-unclassified" else 0 + warnings = 1 if mode == "warn-major" else 2 if mode in {"warn-unclassified", "mixed"} else 0 + suppressed = 3 if mode == "ignored" else 0 + assert cli.main(args) == int(blocking > 0) + full = json.loads(Path("report.json").read_text(encoding="utf-8")) + summary = json.loads(Path("summary.json").read_text(encoding="utf-8")) + sidecar = json.loads(Path("policy.json").read_text(encoding="utf-8")) + markdown = Path("report.md").read_text(encoding="utf-8") + run = json.loads(Path("report.sarif").read_text(encoding="utf-8"))["runs"][0] + assert full["report_schema"] == "sbom-diff-risk.report.v1" + assert sidecar["policy_schema"] == "sbom-diff-risk.policy.v1" + assert summary == full["summary"] + assert (summary["added"], summary["removed"], summary["changed"]) == (1, 0, 4) + counts = Counter(f["bucket"] for f in full["risks"]) + assert {k: v for k, v in summary["risk_counts"].items() if v} == dict(counts) + assert counts == {major: 1, unclassified: 2, "new_package": 1, "unknown_license": 1, "not_evaluated": 5} + expected_risks = [ + (major, "alpha"), + (unclassified, "bravo"), + (unclassified, "echo"), + ("new_package", "new-lib"), + ("unknown_license", "new-lib"), + ] + expected_risks += [("not_evaluated", name) for name in ("alpha", "bravo", "delta", "echo", "new-lib")] + assert [(f["bucket"], f["component_key"]) for f in full["risks"]] == sorted( + (bucket, "purl:pkg:pypi/" + name) for bucket, name in expected_risks + ) + for section, count in ( + ("blocking_findings", blocking), + ("warning_findings", warnings), + ("suppressed_findings", suppressed), + ): + assert sidecar[section] == full[section] + assert len(full[section]) == count + expected_targets = [] + if section == "blocking_findings": + expected_targets = [(major, "alpha")] if mode in {"block-major", "mixed"} else [] + if mode == "block-unclassified": + expected_targets = [(unclassified, "bravo"), (unclassified, "echo")] + elif section == "warning_findings": + expected_targets = [(major, "alpha")] if mode == "warn-major" else [] + if mode in {"warn-unclassified", "mixed"}: + expected_targets = [(unclassified, "bravo"), (unclassified, "echo")] + elif mode == "ignored": + expected_targets = [(major, "alpha"), (unclassified, "bravo"), (unclassified, "echo")] + assert [(f["rule_id"], f["component_name"]) for f in full[section]] == expected_targets + for f in full[section]: + rule, key = f["rule_id"], f["component_key"] + assert f["matched_rule_id"] == f["policy_rule"] == rule + assert f["observed_value"] == f["finding_bucket"] == rule + assert f["exact_evidence"] == { + "component_key": key, + "finding_bucket": rule, + "matched_threshold": None, + "observed_value": rule, + } + assert f["decision_reason"] == "risk_finding_matched_policy_rule" + assert f["confidence_level"] == "policy_matched" + assert f["severity_source"] == ("warn_on" if section == "warning_findings" else "block_on") + assert f["level"] == ("warn" if section == "warning_findings" else "block") + if section == "suppressed_findings": + assert f["suppression_reason"] == "ignored_by_policy" + assert sidecar["policy_evaluation"] == full["policy_evaluation"] == full["metadata"]["policy_evaluation"] + assert sidecar["rule_catalog"] == full["rule_catalog"] + if mode != "none": + assert ( + sidecar["summary"]["policy"] + == summary["policy"] + == { + "blocking": blocking, + "warning": warnings, + "suppressed": suppressed, + "status": "fail" if blocking else "warn" if warnings else "pass", + } + ) + confidence = "policy_matched" if blocking + warnings + suppressed else "sbom_present" + assert full["evidence_confidence"] == summary["evidence_confidence"] == confidence + assert full["metadata"]["evidence_confidence"] == confidence + assert full["metadata"]["enrichment"]["network_access_performed"] is False + for bucket, count in counts.items(): + assert f"- {bucket}: {count}" in markdown + assert f"- Exit code: {int(blocking > 0)}" in markdown + assert f"- Evidence confidence: {confidence}" in markdown + assert "| alpha | | | version_changed |" in markdown + for change in full["components"]["changed"]: + assert change["classification"] == "version_changed" + for side in ("before", "after"): + original_rows = before_rows if side == "before" else after_rows + row = next(r for r in original_rows if r["name"] == change[side]["name"]) + assert change[side]["version"] == row.get("version") + raw_key = "component" if input_format == "cyclonedx" else "package" + document = json.loads(Path(side + ".json").read_text(encoding="utf-8")) + items = document["components" if input_format == "cyclonedx" else "packages"] + assert change[side]["evidence"][raw_key] == next(r for r in items if r["name"] == row["name"]) + results = run["results"] + level = "error" if mode in {"block-major", "mixed"} else "warning" if mode == "warn-major" else "note" + expected_order = [("sdr.major_upgrade", level), ("sdr.unknown_license", "warning")] + if level != "error": + expected_order.reverse() + assert [(r["ruleId"], r["level"]) for r in results] == expected_order + assert [r["properties"]["component_name"] for r in results] == ( + ["alpha", "new-lib"] if level == "error" else ["new-lib", "alpha"] + ) + major_result = next(r for r in results if r["ruleId"] == "sdr.major_upgrade") + assert major_result["properties"]["policy_matched"] is (mode in {"block-major", "warn-major", "mixed"}) + assert major_result["properties"]["policy_blocking"] is (level == "error") + assert major_result["partialFingerprints"]["componentKey"] == "purl:pkg:pypi/alpha" + assert len({(r["ruleId"], r["properties"]["component_key"]) for r in results}) == len(results) + assert run["properties"]["sbom_diff_risk"]["truncated"] is False + # Suppression removes policy escalation, not the independent major heuristic note. + # Unclassified/not_evaluated are deliberately absent from the SARIF subset. + first = {name: Path(name).read_bytes() for name in names} + assert cli.main(args) == int(blocking > 0) + assert first == {name: Path(name).read_bytes() for name in names} + write_sbom(Path("before.json"), input_format, list(reversed(before_rows))) + write_sbom(Path("after.json"), input_format, list(reversed(after_rows))) + assert cli.main(args) == int(blocking > 0) + assert first == {name: Path(name).read_bytes() for name in names} + + +def rows(side): + after = side == "after" + items = [ + {"name": "alpha", "purl": "pkg:pypi/alpha@" + ("2.0.0-rc.1+build.01" if after else "1.2.3")}, + {"name": "bravo", "purl": "pkg:pypi/bravo", "version": "2.0.0-01" if after else "1.2.3"}, + {"name": "delta", "purl": "pkg:pypi/delta", **({"version": "2.0.0"} if after else {})}, + { + "name": "echo", + "purl": "pkg:pypi/echo@" + ("2.0.0" if after else "3.0.0"), + "version": "2.0.0" if after else "3.0.0", + }, + {"name": "quiet", "purl": "pkg:pypi/quiet@1.2.3", "version": "1.2.3"}, + ] + if after: + items.append({"name": "new-lib", "purl": "pkg:pypi/new-lib@1.0.0"}) + return items + + +def write_sbom(path, input_format, items): + if input_format == "cyclonedx": + components = [ + {**r, **({"licenses": [{"license": {"id": "MIT"}}]} if r["name"] != "new-lib" else {})} for r in items + ] + payload = {"bomFormat": "CycloneDX", "specVersion": "1.5", "components": components} + else: + packages = [ + { + "name": r["name"], + "SPDXID": "SPDXRef-" + r["name"], + "externalRefs": [{"referenceType": "purl", "referenceLocator": r["purl"]}], + **({"versionInfo": r["version"]} if "version" in r else {}), + **({"licenseConcluded": "MIT"} if r["name"] != "new-lib" else {}), + } + for r in items + ] + payload = {"spdxVersion": "SPDX-2.3", "packages": packages} + path.write_text(json.dumps(payload), encoding="utf-8") diff --git a/tools/sbom-diff-and-risk/tests/test_version_decisions.py b/tools/sbom-diff-and-risk/tests/test_version_decisions.py new file mode 100644 index 0000000..a15a4ed --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_version_decisions.py @@ -0,0 +1,180 @@ +"""Independent SemVer 2.0.0 grammar oracle; no production parser is reused. + +https://semver.org/spec/v2.0.0.html, rules 2, 9, 10 and the BNF. +The existing tool extensions are outer whitespace and one lowercase ``v``. +Only major numbers are compared; this is not ecosystem version precedence. +""" + +from __future__ import annotations + +from dataclasses import asdict +from string import ascii_letters, digits + +import pytest +from sbom_diff_risk.diffing import diff_components +from sbom_diff_risk.models import Component +from sbom_diff_risk.risk import evaluate_risks + + +def reference_major(value: str | None) -> str | None: + if not value: + return None + value = value.strip().removeprefix("v") + core_and_pre, plus, build = value.partition("+") + core, dash, pre = core_and_pre.partition("-") + numbers = core.split(".") + if len(numbers) != 3 or any(not numeric_id(part) for part in numbers): + return None + for present, suffix, numeric_zero_rule in ((dash, pre, True), (plus, build, False)): + if not present: + continue + for part in suffix.split("."): + if not part or any(c not in ascii_letters + digits + "-" for c in part): + return None + if numeric_zero_rule and all(c in digits for c in part) and not numeric_id(part): + return None + return numbers[0] + + +def numeric_id(value: str) -> bool: + return bool(value) and all(c in digits for c in value) and (value == "0" or value[0] != "0") + + +def expected_bucket(before: str | None, after: str | None) -> str | None: + before, after = (value.strip() or None if value is not None else None for value in (before, after)) + if not before or not after or before == after: + return None + left, right = reference_major(before), reference_major(after) + if left is not None and right is not None and (len(right), right) > (len(left), left): + return "major_upgrade" + return "version_change_unclassified" + + +SYNTAX_CASES = [ + ("zero", "0.0.0", True), + ("plain", "1.2.3", True), + ("major", "2.0.0", True), + ("prefix", "v2.0.0", True), + ("outer-space", " \t2.0.0\n", True), + ("major-zero", "02.0.0", False), + ("minor-zero", "2.01.0", False), + ("patch-zero", "2.0.01", False), + ("missing", "2.0", False), + ("extra-core", "2.0.0.0", False), + ("inner-space", "2. 0.0", False), + ("unicode-digit", "2.0.1\u0662", False), + ("uppercase-prefix", "V2.0.0", False), + ("pre", "2.0.0-alpha.1", True), + ("pre-zero", "2.0.0-0", True), + ("pre-leading-zero", "2.0.0-01", False), + ("pre-alphanumeric", "2.0.0-01a", True), + ("pre-hyphens", "2.0.0-x-y.--", True), + ("empty-pre", "2.0.0-", False), + ("pre-empty-id", "2.0.0-alpha..1", False), + ("pre-trailing-dot", "2.0.0-alpha.", False), + ("pre-leading-dot", "2.0.0-.alpha", False), + ("pre-underscore", "2.0.0-alpha_1", False), + ("build", "2.0.0+build.01", True), + ("both", "2.0.0-rc.1+build.01", True), + ("empty-build", "2.0.0+", False), + ("build-empty-id", "2.0.0+a..b", False), + ("build-trailing-dot", "2.0.0+a.", False), + ("build-leading-dot", "2.0.0+.a", False), + ("double-plus", "2.0.0+a+b", False), + ("build-underscore", "2.0.0+a_b", False), + ("pep440", "2.0rc1", False), + ("maven", "2.0-SNAPSHOT", False), + ("nuget-four-part", "2.0.0.1", False), + ("opaque", "release-b", False), + ("huge-major", "9" * 5000 + ".0.0", True), + ("huge-minor", "2." + "9" * 5000 + ".0", True), + ("huge-patch", "2.0." + "9" * 5000, True), +] + + +@pytest.mark.parametrize("label,version,eligible", SYNTAX_CASES, ids=[row[0] for row in SYNTAX_CASES]) +def test_oracle_grammar_anchors(label: str, version: str, eligible: bool) -> None: + assert (reference_major(version) is not None) is eligible + + +@pytest.mark.parametrize("label,version,eligible", SYNTAX_CASES, ids=[row[0] for row in SYNTAX_CASES]) +@pytest.mark.parametrize("reverse", [False, True]) +def test_version_risk_matches_independent_grammar(label, version, eligible, reverse) -> None: + before, after = (version, "9.0.0") if reverse else ("0.0.0", version) + assert_transition(before, after, "explicit", "pypi") + + +TRANSITIONS = [ + ("equal", "1.2.3", "1.2.3"), + ("major-up", "1.2.3", "2.0.0"), + ("major-down", "2.0.0", "1.2.3"), + ("minor", "1.2.3", "1.3.0"), + ("patch", "1.2.3", "1.2.4"), + ("pre-final", "2.0.0-rc.1", "2.0.0"), + ("final-pre", "2.0.0", "2.0.0-rc.1"), + ("build-only", "2.0.0+a", "2.0.0+b"), + ("both-unknown", None, None), + ("lose-evidence", "1.2.3", None), + ("gain-evidence", None, "2.0.0"), + ("opaque", "release-a", "release-b"), + ("lose-semver", "1.2.3", "release-b"), + ("gain-semver", "release-a", "2.0.0"), + ("valid-suffix", "1.2.3-rc.1+a", "2.0.0-rc.2+b"), + ("invalid-suffix", "1.2.3", "2.0.0-01"), + ("prefix-only", "1.2.3", "v1.2.3"), + ("trim-only", " 1.2.3 ", "1.2.3"), + ("decimal-boundary", "9.9.9", "10.0.0"), +] + + +@pytest.mark.parametrize("label,before,after", TRANSITIONS, ids=[row[0] for row in TRANSITIONS]) +@pytest.mark.parametrize("source", ["explicit", "purl", "both"]) +def test_transition_families(label, before, after, source) -> None: + assert_transition(before, after, source, "pypi") + + +@pytest.mark.parametrize("ecosystem", ["pypi", "npm", "maven", "nuget", "generic", "custom"]) +@pytest.mark.parametrize("before,after", [("1.2.3", "2.0.0-rc.1+build"), ("2.0.0", "1.2.3")]) +def test_ecosystem_does_not_imply_version_precedence(ecosystem, before, after) -> None: + assert_transition(before, after, "both", ecosystem) + + +@pytest.mark.parametrize( + "purl", + [ + "pkg:pypi/sample_lib@2.0.0-rc.1%2Bbuild.01", + "pkg:pypi/sample-lib@2.0.0-rc.1+build.01?os=linux&arch=any", + ], +) +def test_equivalent_purl_representation_preserves_version_evidence(purl) -> None: + before = Component("sample-lib", None, "pypi", purl="pkg:pypi/sample-lib@1.2.3") + after = Component("sample-lib", None, "pypi", purl=purl) + added, removed, changed = diff_components([before], [after]) + findings = evaluate_risks(added, changed) + assert added == removed == [] + assert changed[0].key == "purl:pkg:pypi/sample-lib" + assert [f.bucket.value for f in findings if f.bucket.value == "major_upgrade"] == ["major_upgrade"] + assert after.purl == purl and after.version is None + + +def assert_transition(before_version, after_version, source, ecosystem) -> None: + components = [] + for version in (before_version, after_version): + purl = f"pkg:{ecosystem}/sample-lib" + if source != "explicit" and version and version.strip(): + purl += "@" + version.strip() + components.append( + Component("sample-lib", None if source == "purl" else version, ecosystem, purl=purl, license_id="MIT") + ) + original = [asdict(item) for item in components] + added, removed, changed = diff_components(components[:1], components[1:]) + findings = evaluate_risks(added, changed) + assert added == removed == [] + assert [asdict(item) for item in components] == original + expected = expected_bucket(before_version, after_version) + actual = [f.bucket.value for f in findings if f.bucket.value in {"major_upgrade", "version_change_unclassified"}] + assert actual == ([] if expected is None else [expected]) + if changed: + left, right = (v.strip() or None if v is not None else None for v in (before_version, after_version)) + assert changed[0].key == f"purl:pkg:{ecosystem}/sample-lib" + assert changed[0].classification == ("version_changed" if left != right else "metadata_changed")