From dce5082e3f943c0e796e74607aa65c02cf86f10c Mon Sep 17 00:00:00 2001 From: stacknil <219124580+stacknil@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:56:53 +0800 Subject: [PATCH] test(sbom): qualify evidence-policy decision stability Signed-off-by: stacknil <219124580+stacknil@users.noreply.github.com> --- .../sbom-diff-and-risk/docs/policy-schema.md | 13 + .../decision-stability/COUNTEREXAMPLES.md | 205 + .../DECISION-STABILITY-REPORT.md | 154 + .../EXECUTION-MANIFEST.json | 490 ++ .../decision-stability/INDEPENDENT-AUDIT.md | 30 + .../decision-stability/INVARIANT-CATALOG.md | 49 + .../decision-stability/NEGATIVE-CONTROLS.json | 760 +++ .../validation/decision-stability/PROGRESS.md | 21 + .../decision-stability/REPRODUCTION.md | 94 + .../decision-stability/REVERSAL-MATRIX.json | 4857 +++++++++++++++++ .../TRUST-BOUNDARY-MATRIX.md | 60 + .../scripts/qualify-decision-stability.py | 408 ++ .../src/sbom_diff_risk/cli.py | 7 +- .../src/sbom_diff_risk/diffing.py | 5 + .../src/sbom_diff_risk/enrichment.py | 13 +- .../src/sbom_diff_risk/policy_evaluator.py | 9 +- .../src/sbom_diff_risk/policy_parser.py | 13 +- .../src/sbom_diff_risk/pypi_provenance.py | 42 +- .../src/sbom_diff_risk/scorecard_client.py | 20 +- .../test_canonical_release_enrichment.py | 277 + .../tests/test_decision_stability.py | 270 + .../test_decision_stability_negatives.py | 185 + .../tests/test_decision_stability_oracle.py | 862 +++ .../tests/test_decision_stability_pairs.py | 641 +++ .../test_decision_stability_providers.py | 445 ++ .../tests/test_decision_stability_reports.py | 493 ++ .../tests/test_decision_stability_runner.py | 55 + .../tests/test_policy_package_equivalence.py | 175 + .../test_policy_validation_qualification.py | 94 + .../test_scorecard_evidence_integrity.py | 169 + 30 files changed, 10885 insertions(+), 31 deletions(-) create mode 100644 tools/sbom-diff-and-risk/docs/validation/decision-stability/COUNTEREXAMPLES.md create mode 100644 tools/sbom-diff-and-risk/docs/validation/decision-stability/DECISION-STABILITY-REPORT.md create mode 100644 tools/sbom-diff-and-risk/docs/validation/decision-stability/EXECUTION-MANIFEST.json create mode 100644 tools/sbom-diff-and-risk/docs/validation/decision-stability/INDEPENDENT-AUDIT.md create mode 100644 tools/sbom-diff-and-risk/docs/validation/decision-stability/INVARIANT-CATALOG.md create mode 100644 tools/sbom-diff-and-risk/docs/validation/decision-stability/NEGATIVE-CONTROLS.json create mode 100644 tools/sbom-diff-and-risk/docs/validation/decision-stability/PROGRESS.md create mode 100644 tools/sbom-diff-and-risk/docs/validation/decision-stability/REPRODUCTION.md create mode 100644 tools/sbom-diff-and-risk/docs/validation/decision-stability/REVERSAL-MATRIX.json create mode 100644 tools/sbom-diff-and-risk/docs/validation/decision-stability/TRUST-BOUNDARY-MATRIX.md create mode 100644 tools/sbom-diff-and-risk/scripts/qualify-decision-stability.py create mode 100644 tools/sbom-diff-and-risk/tests/test_canonical_release_enrichment.py create mode 100644 tools/sbom-diff-and-risk/tests/test_decision_stability.py create mode 100644 tools/sbom-diff-and-risk/tests/test_decision_stability_negatives.py create mode 100644 tools/sbom-diff-and-risk/tests/test_decision_stability_oracle.py create mode 100644 tools/sbom-diff-and-risk/tests/test_decision_stability_pairs.py create mode 100644 tools/sbom-diff-and-risk/tests/test_decision_stability_providers.py create mode 100644 tools/sbom-diff-and-risk/tests/test_decision_stability_reports.py create mode 100644 tools/sbom-diff-and-risk/tests/test_decision_stability_runner.py create mode 100644 tools/sbom-diff-and-risk/tests/test_policy_package_equivalence.py create mode 100644 tools/sbom-diff-and-risk/tests/test_policy_validation_qualification.py create mode 100644 tools/sbom-diff-and-risk/tests/test_scorecard_evidence_integrity.py diff --git a/tools/sbom-diff-and-risk/docs/policy-schema.md b/tools/sbom-diff-and-risk/docs/policy-schema.md index 71f4100..58e9041 100644 --- a/tools/sbom-diff-and-risk/docs/policy-schema.md +++ b/tools/sbom-diff-and-risk/docs/policy-schema.md @@ -104,8 +104,21 @@ Version `3` supports every version `1` and `2` rule id plus: - `require_attestations_for_new_packages` applies only to added PyPI packages. - `require_provenance_for_suspicious_sources` applies only when the component also triggered `suspicious_source`. - `allow_unattested_packages` is a narrow package-name override for explicit missing-attestation exceptions only. +- For PyPI components, both exception entries and component names use PEP 503 + normalized equality: case and runs of hyphens, underscores and dots are + equivalent. This preserves package-name scope across versions; it does not + authorize full PURLs, cross-ecosystem aliases or fuzzy name matches. Matching + does not rewrite the serialized policy spelling beyond existing parser + normalization. - `allow_unattested_packages` does not waive `provenance_unavailable` or `unverified_provenance`; those remain separate, reviewable policy decisions. +- Explicit PyPI enrichment uses validated canonical package names and release + versions, including versions present only in a valid PyPI PURL. Conflicting + or malformed identities fail before provider lookup. Missing versions remain + unsupported; no latest release is inferred. Cache entries use canonical + ecosystem/name/version coordinates. Observed component fields are preserved. + A PURL supplies coordinates, not authenticity or publisher authorization. + Offline defaults and policy/report schema versions remain unchanged. - `allow_provenance_publishers` and `allow_unattested_publishers` apply only when attestations exist and publisher kinds are available to verify. - when enrichment is disabled, deterministic local mode is unchanged unless a diff --git a/tools/sbom-diff-and-risk/docs/validation/decision-stability/COUNTEREXAMPLES.md b/tools/sbom-diff-and-risk/docs/validation/decision-stability/COUNTEREXAMPLES.md new file mode 100644 index 0000000..ba79b41 --- /dev/null +++ b/tools/sbom-diff-and-risk/docs/validation/decision-stability/COUNTEREXAMPLES.md @@ -0,0 +1,205 @@ +# Counterexamples and rejected hypotheses + +Baseline: `29203f9f14e2a3df2ad77a4e874bf752bf44c599`, tree `3e6f45e7968d4ac32f475d6842aaa523860fa4a9`. + +Status: baseline probes independently reproduced. C01/C04/C05 and the approved C02 clarification have focused RED/GREEN evidence. C03 now has explicit owner approval and executed repair evidence. All examples below are synthetic. Provider clients consumed in-memory byte streams, never live responses. Raw diagnostic receipts remain outside the checkout. + +## C01 — Boolean integer-policy fields + +```yaml +version: true +``` + +The loader accepts this as a version-1 policy because Python Boolean values are integer subclasses. `version: 1` with `max_added_packages: true` or `false` is also accepted, acting as thresholds 1 or 0. `version: false` is already rejected; it is not an accepted counterexample. + +Classification: **production defect**. The [policy schema](../../policy-schema.md) declares integer fields and rejects invalid types. Expected outcome is controlled `PolicyError` / CLI exit 2, preserving valid integer versions and non-negative integer thresholds. This does not require a new severity/default policy. + +The same regression group includes non-string YAML keys: integer-only unknown +keys raised `TypeError` while formatting diagnostics, and mixed string/integer +keys raised `TypeError` during sorting. The repair validates key types before +sorting. Boolean/invalid-key cases now fail through the existing `PolicyError` +boundary and CLI exit 2, without producing reports. + +## C02 — Canonical PyPI aliases and package exceptions + +Policy: + +```yaml +version: 2 +block_on: [missing_attestation] +allow_unattested_packages: [demo-pkg] +``` + +Added CycloneDX component, with synthetic successful metadata retrieval but no attestations: + +```json +{"name":"demo-pkg","version":"1.0.0","purl":"pkg:pypi/demo-pkg@1.0.0"} +``` + +Change only `name` to `Demo_Pkg` or `demo.pkg`. All three canonical identities are `purl:pkg:pypi/demo-pkg`, canonical version `1.0.0`; comparison between these representations produces no changes. Decisions are respectively **PASS, FAIL, FAIL**. The failures use `attestation_not_published`. + +First divergence: `policy_evaluator._package_is_unattested_allowed` matches the observed name after trim/lowercase, not the canonical PyPI name. The identity documents explicitly promise a comparison contract; package-exception documentation does not define alias equivalence. + +Classification: **CONTRACT_CLARIFIED_CANONICAL_NAME_EQUIVALENCE**. The owner +explicitly approved PEP 503 name equality for PyPI package exceptions. Both +policy entries and component names use the existing `packaging.utils.canonicalize_name` +semantics. The exception remains package-name scoped, PyPI-only and limited to +missing attestations; it cannot waive unavailable or unverified provenance. +No version/PURL/fuzzy/cross-ecosystem authorization is introduced. Serialized +policy spelling retains the existing parser's normalization only. + +The original reproducer exits `[0, 1, 1]` before correction and `[0, 0, 0]` +after correction. The spelling-only transition is **STABLE**, not a justified +policy reversal. The new 29-case boundary regression was 17 failed / 12 passed +before correction, then 29 passed. YAML-loaded and directly constructed policies, +non-equivalent names, unavailable/unverified evidence and ecosystem scope are +covered. The affected policy/report gate passed 216 tests. + +## C03 — PURL-only version and opt-in provenance eligibility + +Policy: + +```yaml +version: 2 +block_on: [provenance_unavailable] +``` + +Compare separate added-component runs: + +```json +{"name":"demo-pkg","purl":"pkg:pypi/demo-pkg@1.0.0"} +``` + +```json +{"name":"demo-pkg","version":"1.0.0","purl":"pkg:pypi/demo-pkg@1.0.0"} +``` + +Canonical identity/version are equal, and comparison between these representations produces no change. With the same synthetic provider fixture, the first run makes no transport calls, records `unsupported_for_package` and **FAILS** with `provenance_evidence_unavailable`. The second invokes the synthetic release/provenance transports, records fetched release metadata without attestations and **PASSES** this unavailable-only rule. Passing that rule does not mean an attestation exists or a package is verified. + +First divergence: `normalize_pypi_provenance` requires raw `Component.version`. The explicit canonical-PURL guarantee in the existing risk documentation applies to version comparison, not all enrichment eligibility. + +Classification: **CONTRACT_CLARIFIED_CANONICAL_RELEASE_VERSION** (owner approved). +The original FAIL/PASS reproducer and raw responses remain retained. The first +divergent stage was `normalize_pypi_provenance` rejecting an absent raw version; +the raw-field cache key also conflated distinct PURL-only releases. Canonical +ecosystem/name/version now drive lookup and caching without rewriting observed +components. Equivalent releases are **STABLE** (PASS/PASS with opt-in enabled; +FAIL/FAIL under the same unavailable-only policy offline). + +The corrected initial regression gate was 22 failed / 3 passed, then 49 affected +tests passed. A preliminary test invocation also contained mock-serialization +and policy-path comparison errors; it is not the authoritative RED receipt. +Final C03 coverage has 31 tests including direct APIs, CLI inputs, cache separation, +equivalent metadata, no version, malformed/conflicting metadata and error states. + +Fresh independent review found that per-item CLI validation still admitted +provider calls before rejecting duplicate/conflicting snapshot indexes. Two +additional RED cases confirmed calls despite eventual exit 2. CLI now reuses +the existing index validator on both sides before creating either enricher; +43 affected tests passed. Direct enrichment release batches can contain repeated +or different valid releases; CLI SBOM snapshots retain their existing uniqueness +contract. Contradictory identities are INVALID_INPUT, never equivalent. + +No latest-release fallback, authenticity claim, default network activity, schema +change or new evidence/publisher authorization is introduced. Independent final +review of the index correction remains pending after infrastructure failure. + +## C04 — Scorecard response identity laundering + +Request the synthetic repository `github.com/example/demo`; return: + +```json +{"repo":{"name":"github.com/other/unrelated","commit":"other-commit"},"score":9} +``` + +The response parser accepts the different identity. The enricher then reports the requested repository mapping while retaining the other repository's commit/score and `scorecard_available`. An explicitly active threshold-7 rule passes. + +Classification: **production evidence-binding defect**. An explicit contradiction must not become evidence for another requested repository. Missing repository/name fallback is a separate currently supported case and must remain compatible; case-only GitHub identity spelling is also a separate compatibility control. No redirect, path or universal alias inference is proposed. + +Independent review found that the first C04 repair used Unicode `casefold`, +which also equated sharp-s with `ss`, the Kelvin sign with `k`, and the `ff` +ligature with `ff`. All three regressions failed against that repair. +ASCII-only letter folding now preserves non-ASCII distinctions; the affected +gate passed 60 tests. Final independent re-review remains incomplete. + +## C05 — Invalid numeric Scorecard evidence and non-JSON output + +Synthetic overall scores `true` and `false` become 1.0 and 0.0. Overall `1e309`, `-1e309` and `NaN` become available nonfinite scores. The rendered full JSON contains `Infinity`, `-Infinity` or `NaN`, which a strict JSON reader rejects. + +Positive infinity passes a threshold-7 rule. **NaN blocks under the baseline evaluator**; there is no NaN-bypass claim. Invalid per-check infinity/NaN escape as conversion exceptions rather than becoming available evidence; Boolean check scores are accepted as integers. + +Classification: **production validation/report contract defect**. Boolean/nonfinite values are not usable finite numeric evidence. They should produce the existing controlled provider-error state, not a new risk bucket, arbitrary severity or a package-safety verdict. Valid finite evidence and currently supported sentinel values must remain compatible. + +## Repair evidence for C01/C04/C05 + +Against unchanged baseline production, the new 32-case regression group had +**25 failed, 7 passed, 0 errors, 0 skips**. All failures reached policy loading, +provider parsing or the actual CLI. After the narrow repairs, **32 passed** and +the then-complete suite passed **505** tests. No golden or schema was changed. + +- C01: reject Boolean integer fields and non-string policy keys with the + existing controlled error type; valid integer values remain accepted. +- C04: reject an explicitly conflicting Scorecard response repository. Missing + repository/name fallback and case-only GitHub spelling remain accepted. +- C05: reject Boolean/nonfinite numeric evidence with `ScorecardClientError`. + Existing enrichment error handling records the evidence gap. It does not + activate a new policy violation, so a rule that only enforces *available* + low scores can return PASS when malformed evidence becomes unavailable. + `scorecard_recorded` still describes a recorded error, never verification. + +## Rejected oracle assumptions + +The first representation gate had **51 passed, 2 failed**. Both failures were +in the qualification expectations, not production: + +1. R14 expected source input order instead of the required deterministic lexical + identity order. The corrected expectation sorts the independently specified + names. +2. R19 assumed exact manifest pins used coordinate fallback. Both existing + parsers explicitly call `build_pypi_purl`, so the expected identity is + `purl:pkg:pypi/qualification-alpha`. The test still checks the independently + specified package/version and warning rule. + +The corrected gate passed **53** tests. These failed experiments remain in +private receipts and are not counted as production defects. + +A further cache probe failed twice by returning different attestation states +for `1.0.0-rc` and `1.0.0-RC`. Independent review rejected this distinct-release +assumption: the installed PEP 440 implementation normalizes both to +`1.0.0rc0`. With identical provider evidence, both comparison orders keep the +same missing-attestation decision. The cache records the first lookup's version +spelling; each raw component keeps its own version. No existing field contract +requires a lookup record to echo every equivalent component spelling. + +Classification: **TEST_ORACLE_DEFECT**, not an established production cache +defect. The two failed probes and their source are retained privately, and the +invalid probe is excluded from the accepted regression suite. No cache change or +PEP 440 normalization of general component identity is warranted by this probe. + +## Qualification harness audit corrections + +Independent audit identified Boolean/numeric equality, coherent corruption of +serialized reason/observed/threshold values, and a Markdown message replacement +retaining the row prefix. Seven meaningful regressions failed before repair +because the expected rejection did not occur. The corrected harness compares +serialized rows against literal independent expectations, checks full Markdown +messages and distinguishes nested Boolean/numeric evidence. The affected gate +passed 153 tests. An earlier invocation had a quoting error (exit 4, no tests); +it is retained and is not counted as RED evidence. + +## Pre-existing lint limitation + +A full Ruff correctness scan reports `F821` at +`tests/test_policy_scorecard.py:58` (`Path` appears only in a postponed type +annotation). That file is byte-identical to the frozen base and was not changed. +Ruff correctness checks pass for the candidate's changed/new Python files. +The full scan is **not** reported as passing; this unrelated existing annotation +finding does not explain a runtime regression in the executed suite. + +## Boundaries on the findings + +- No change to the already merged strict-SemVer classifier is proposed. +- Recorded attestation/publisher metadata is not a cryptographic verifier, and qualification does not invent one. +- A recorded evidence-confidence label is a source category, not a probability or proof of safety. +- Missing optional Scorecard evidence does not by itself activate an existing score threshold rule. +- Contract gaps are not converted into passing invariants, defects or silently selected policy decisions. diff --git a/tools/sbom-diff-and-risk/docs/validation/decision-stability/DECISION-STABILITY-REPORT.md b/tools/sbom-diff-and-risk/docs/validation/decision-stability/DECISION-STABILITY-REPORT.md new file mode 100644 index 0000000..a73a9d9 --- /dev/null +++ b/tools/sbom-diff-and-risk/docs/validation/decision-stability/DECISION-STABILITY-REPORT.md @@ -0,0 +1,154 @@ +# Evidence-to-policy decision stability + +Status: **BLOCKED / AUDIT_PENDING**. C03 is explicitly approved and repaired. +All local requalification gates passed; fresh independent reviewers again stopped +with execution-service credit errors before completing final acceptance. + +## Scope and authority + +This campaign tests whether dependency identity, heuristic observations, +recorded trust evidence, explicit policy and consumer outputs stay distinct. +It is a bounded synthetic qualification, not a vulnerability scanner, generic +alias resolver, reputation model or security certification. + +The frozen main/base is `29203f9f14e2a3df2ad77a4e874bf752bf44c599`, tree +`3e6f45e7968d4ac32f475d6842aaa523860fa4a9`. The candidate remains local and +uncommitted. Issues #103 and #117 supplied claim/evidence context; this work +does not close, modify or claim completion of either issue. + +## Decision properties + +The expected-value module imports only the Python standard library. Its literal +microcases specify decisions, rule identities, severity source, reason codes, +thresholds, observed values, component keys, suppression evidence and confidence +labels. Product functions are used only by the observation adapters. Cross-file +agreement checks supplement that oracle; they are not described as a second +independent policy oracle. + +The catalog covers defaults, explicit block/warn priority, ignore precedence, +added-count boundaries, exact source hosts, unavailable provenance, known absent +attestations, unverified publishers, required provenance, package exceptions, +publisher allowlists, Scorecard activation/thresholds, policy version eligibility +and CLI merging/promotion. No blanket monotonicity rule is assumed. + +More evidence can reveal a policy failure. A lookup error dominates a recorded +attestation in the current evaluator. A successful metadata lookup without +attestations can satisfy an *unavailable-only* policy while failing a +missing-attestation policy. Losing an available low Scorecard score can remove +a threshold violation because the existing rule does not fabricate an unknown +score. Each transition retains its active policy and evidence-state explanation. + +Actual parsers and CLI paths additionally check PEP 503 spelling equivalence, +namespaced PURLs, PURL-only versions, missing references, opaque identities, +unsupported ecosystem equivalence, duplicates/conflicts, the shared CycloneDX/ +SPDX subset, overlapping exact manifest pins, order, ignored metadata, +serialization and incomplete/contradictory fields. + +The namespace/case tests deliberately preserve unsupported distinctions. An +equal policy outcome for two opaque identities does not establish identity +equivalence. A metadata change that enters `changed.after` policy scope can +legitimately expose an already disallowed source; ignored raw annotations do +not enter that scope. + +## Confirmed repairs and compatibility + +1. Policy parsing rejects Boolean integer fields and non-string keys through + `PolicyError` / CLI exit 2. Valid policy integer values and defaults remain. +2. Scorecard parsing rejects Boolean/nonfinite numbers and explicitly conflicting + response repositories. Missing repository/name fallback, ASCII case-only spelling, + finite values and the existing per-check `-1` sentinel remain compatible. +3. The explicitly approved C02 policy clarification applies PEP 503 equality + to both PyPI package exception entries and component names. It preserves + package-name scope, serialized spelling and the missing/unavailable/unverified + evidence boundaries. Spelling-only changes now classify as `STABLE`. + +Malformed Scorecard evidence becomes an existing error record. Consumers can +observe a policy PASS when an active threshold has no usable score to evaluate; +this is existing unknown-score policy behavior, not a new safety assertion. +Recorded-error confidence remains `scorecard_recorded`. + +Canonicalization, diff signatures, risk buckets, strict SemVer rules, report and +policy schema versions, offline defaults and ordinary policy severity rules are +unchanged. Goldens, committed example reports, release workflows, dependency +declarations and repository settings are unchanged. Rollback consists of removing +the narrow validation/matching repairs and their qualification files; no history +rewrite, data migration or remote rollback is involved. + +The baseline boundary regressions, C02 decision, rejected oracle assumptions and +original C03 reproducer and approved repair are retained in `COUNTEREXAMPLES.md`. Two wrong +representation expectations and an invalid distinct-PyPI-release cache probe +were rejected without changing production semantics. A first runner-startup +failure caused by a dynamic import table was corrected with an explicit API +mapping and a failing-to-passing actual worker regression. No failed or aborted +run is counted as a successful qualification round. + +## Output and consumer mapping + +| Consumer | Required correspondence | Deliberate limit | +| --- | --- | --- | +| Full JSON | Evaluation, rule identity, reason, exact evidence, confidence, raw component evidence | Recorded evidence is not verification | +| Summary JSON | Exact full-report summary; status/counts/confidence | No independent list of all finding evidence | +| Policy JSON | Full-report policy sections and policy summary | No component inventory or risk inventory | +| Markdown | Grouped rule/name/level/message, suppression reason, counts, exit/confidence | Does not serialize every JSON explanation field | +| SARIF | Explicit subset mapping, rule/component identity and emitted policy severity | Unsupported or superseded findings may be omitted | +| CLI exit | 1 for blocking; 0 for applied pass/warn; 2 for invalid input | No native needs-review exit | +| Example CI consumer | Preserve captured exit; attempt artifact upload with `if: always()` before enforcement | Invalid input can produce no policy JSON, so upload success is not guaranteed | + +The checked-in example workflow captures/enforces exit status and attempts artifact +upload; it does not parse policy JSON. The separate test demonstration consumer +reads warning counts and applied-policy state rather +than treating exit 0 as affirmative safety. No-policy or runtime-error states +remain consumer `needs-review`. Synthetic provider bodies are parsed by actual +clients; their configured lookup flags are distinguished from actual audited +network operations. + +## Execution and acceptance gate + +The clean baseline passed 473 package tests and the supported offline example +artifact check. The final C03 candidate passed 817 tests in each of three rounds +(seeds 103, 117 and 120), with no failures, errors or skips. The supported +artifact check passed, and audited network attempts were zero. See the manifest +for the exact source inventory and executed environment. Repeated rounds do not +substitute for the interrupted independent final audit. + +The final oracle checks literal expected evidence after serialization as well +as on the model, rejects Boolean/numeric substitutions recursively, and checks +complete Markdown messages. Seven deliberate corruptions were first accepted +incorrectly (seven failing regressions), then rejected after the harness repair. +The affected gate passed 153 tests. These are qualification-harness defects, +not additional product defects. + +Four protected repositories retained their original HEAD, branch, status, +staged and unstaged diff bytes, existing refs and worktree registrations. Only +the authorized candidate branch/worktree was added. Untracked filenames/status +were compared; there were no initial byte snapshots of untracked contents. + +Scoped Ruff correctness checks and pip check passed. The full Ruff scan retains +the documented pre-existing F821 finding. Native Linux CPython 3.11.17 on ext4 passed 817 tests plus a targeted rerun, +golden checks and the network-denial audit on the same semantic source. This was +maintainer execution after the reviewer interruption. Remote CI, live providers +and release builds were not executed for this candidate. + +The N01–N14 controls validate removed violations, wrong severity, lost +suppression evidence, unavailable/verified confusion, attestation-state confusion, +publisher acceptance, inactive Scorecard enforcement, threshold boundaries, +identity splitting/conflicts, sidecar disagreement, exit disagreement and +heuristic authority escalation. Each control reaches and records its named +detector after a validated normal observation. + +Completion requires executed regression evidence, stable or explained decisions, +detected negative controls, exact source identity, preserved existing work, +unchanged public contracts except the approved clarification, and no unresolved +material audit blocker. Repeated seeds and overlapping integration views are +not added together as independent semantic cases. + +## C03 release-coordinate boundary + +The approved contract is `CONTRACT_CLARIFIED_CANONICAL_RELEASE_VERSION`. +Canonicalization remains unchanged. Canonical release coordinates now govern +explicit PyPI enrichment and cache isolation; no-version cases remain unsupported. +Both CLI input indexes are validated before either provider is constructed. +Direct PyPI normalization validates the component, and batch enrichment validates +all items before its first lookup. Original component metadata remains intact. +Recorded metadata, attestation availability, verification and publisher permission +retain their prior distinct meanings. See the 31 focused C03 regressions. diff --git a/tools/sbom-diff-and-risk/docs/validation/decision-stability/EXECUTION-MANIFEST.json b/tools/sbom-diff-and-risk/docs/validation/decision-stability/EXECUTION-MANIFEST.json new file mode 100644 index 0000000..dc1e1cb --- /dev/null +++ b/tools/sbom-diff-and-risk/docs/validation/decision-stability/EXECUTION-MANIFEST.json @@ -0,0 +1,490 @@ +{ + "actual_network_attempts": 0, + "audit": { + "executed_independent_tests": { + "oracle": 129, + "production": 105 + }, + "independent_negative_controls": 14, + "independent_oracle_probes": 7, + "reason": "Fresh independent reviewers again interrupted by execution-service credit errors; final index preflight repair has not received completed independent review.", + "reviewed_intermediate_tree": "b9931ef405586683b6ce40e1a0d639ad63cb7906", + "status": "AUDIT_PENDING" + }, + "audit_status": "AUDIT_PENDING", + "base_commit": "29203f9f14e2a3df2ad77a4e874bf752bf44c599", + "base_root_tree": "3e6f45e7968d4ac32f475d6842aaa523860fa4a9", + "baseline": { + "example_artifacts": "PASS", + "full_suite": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 473 + } + }, + "branch": "stacknil/sbom-evidence-decision-stability", + "candidate_commit": "NOT_CREATED", + "candidate_semantic_tree": "0760a5543df366f3a8237c60a187cde7f3aae0b0", + "candidate_semantic_tree_scope": "Base plus exact implementation/test/runner/policy-contract allowlist; derived qualification evidence is excluded to avoid self-reference.", + "changed_implementation_files": [ + "tools/sbom-diff-and-risk/docs/policy-schema.md", + "tools/sbom-diff-and-risk/scripts/qualify-decision-stability.py", + "tools/sbom-diff-and-risk/src/sbom_diff_risk/cli.py", + "tools/sbom-diff-and-risk/src/sbom_diff_risk/diffing.py", + "tools/sbom-diff-and-risk/src/sbom_diff_risk/enrichment.py", + "tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_evaluator.py", + "tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_parser.py", + "tools/sbom-diff-and-risk/src/sbom_diff_risk/pypi_provenance.py", + "tools/sbom-diff-and-risk/src/sbom_diff_risk/scorecard_client.py", + "tools/sbom-diff-and-risk/tests/test_canonical_release_enrichment.py", + "tools/sbom-diff-and-risk/tests/test_decision_stability.py", + "tools/sbom-diff-and-risk/tests/test_decision_stability_negatives.py", + "tools/sbom-diff-and-risk/tests/test_decision_stability_oracle.py", + "tools/sbom-diff-and-risk/tests/test_decision_stability_pairs.py", + "tools/sbom-diff-and-risk/tests/test_decision_stability_providers.py", + "tools/sbom-diff-and-risk/tests/test_decision_stability_reports.py", + "tools/sbom-diff-and-risk/tests/test_decision_stability_runner.py", + "tools/sbom-diff-and-risk/tests/test_policy_package_equivalence.py", + "tools/sbom-diff-and-risk/tests/test_policy_validation_qualification.py", + "tools/sbom-diff-and-risk/tests/test_scorecard_evidence_integrity.py" + ], + "consumer_shell": { + "reason": "bash unavailable", + "status": "NOT_RUN" + }, + "consumer_shell_supplement": { + "exits": [ + 0, + 1, + 2 + ], + "platform": "Existing WSL bash", + "scope": "Synthetic capture/enforcement snippet only; no actual GitHub workflow or Linux Python execution", + "status": "PASS" + }, + "dependencies": { + "PyYAML": "6.0.3", + "packageurl-python": "0.17.6", + "packaging": "26.3", + "pytest": "9.1.1" + }, + "format_cleanup": { + "earlier_786_test_rounds_retained": true, + "final_source_retested": true, + "production_AST_unchanged": true + }, + "independent_cases_are_not_multiplied_by_round_count": true, + "lint": { + "changed_files": "PASS", + "correctness_selection": "E9,F63,F7,F82", + "full_output_exact_baseline_match": true, + "full_scan": "PRE_EXISTING_FAILURE", + "preexisting_finding": "F821 tests/test_policy_scorecard.py:58; byte-identical to base", + "tool": "ruff 0.17.2" + }, + "linux_execution": { + "actual_network_attempts": 0, + "consumer_shell": { + "kind": "local bash status propagation, no workflow execution", + "records": [ + { + "comparison_exit": 0, + "final_exit": 0, + "output": "exit_code=0" + }, + { + "comparison_exit": 1, + "final_exit": 1, + "output": "exit_code=1" + }, + { + "comparison_exit": 2, + "final_exit": 2, + "output": "exit_code=2" + } + ], + "status": "PASS" + }, + "dependencies": { + "PyYAML": "6.0.3", + "packageurl-python": "0.17.6", + "packaging": "26.3", + "pytest": "9.1.1" + }, + "independent_cases_are_not_multiplied_by_round_count": true, + "live_evidence_enrichment": "FORBIDDEN", + "os": "Linux", + "parallelism": 1, + "per_command_timeout_seconds": 300, + "publication": "NOT_PERFORMED", + "python": "3.11.17", + "python_implementation": "CPython", + "remote_ci": "NOT_RUN", + "rounds": [ + { + "counts": { + "classifications": { + "INVALID_INPUT": 7, + "JUSTIFIED_REVERSAL": 23, + "STABLE": 25 + }, + "cli_cases": 11, + "microcases": 72, + "negative_controls": 14, + "provider_cases": 22, + "transitions": 55 + }, + "full_suite": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 817 + }, + "seed": 103 + } + ], + "seeds": [ + 103 + ], + "source_inventory": { + "docs/policy-schema.md": "7994dae3af1182ea36484e940aeccdc53df24d5758d65ac33e335bdd0257201d", + "scripts/qualify-decision-stability.py": "dfc5d9f7a5548a92d78ed3cdc463b851c75de74bf72621c174a6e57caeeb14b9", + "src/sbom_diff_risk/__init__.py": "6a6775b4814db46e5b905de4de07befd569eb4e82168e59c555f55501e58e4cd", + "src/sbom_diff_risk/cli.py": "b1e5233c8d8d721122ed81096d38414e88eb05497de298a83ba79de10a6d20e7", + "src/sbom_diff_risk/component_identity.py": "52d820c5fd192ca84847ccecadc19e37e9b759f40de77d40558450e4730ec3af", + "src/sbom_diff_risk/diffing.py": "d2822a8c388d0e03c695a19ab3b229688de1094b1669f40db7aea76ebebc7815", + "src/sbom_diff_risk/enrichment.py": "d5f75dd0b7ab6baae78d9477066621a5412454996b20e46168d0639e232a837a", + "src/sbom_diff_risk/errors.py": "71c329b91f24b64e2f67de6f0a4013d7d5aa6e8e803a1029e9f70e11578061a1", + "src/sbom_diff_risk/evidence_confidence.py": "c5fc6281d34cbe2db9bf9af866e3667586cf38578eb3ba6491179a1359e348c7", + "src/sbom_diff_risk/models.py": "da27543546456476b511ef31cc29b06f8d66bf9308c47d9cf9f307f22bef75db", + "src/sbom_diff_risk/normalize.py": "9badd527b92318d783972a47bf29d90b4a22ab134c4c0d1372f0d9b4840e3472", + "src/sbom_diff_risk/parsers/__init__.py": "ff7132ba67e8c0f9951c1b6728c0ea05e12439fa24238f4186d73246bae5da7d", + "src/sbom_diff_risk/parsers/common.py": "322c09ddb3040f30844ed27b73f689f5eb4295ddc1ec6489109458a99554eea5", + "src/sbom_diff_risk/parsers/cyclonedx_json.py": "15bb8e1075f8115334b099d3c6025aa94d94195d8b4752a6f6c94a771ad3ec6c", + "src/sbom_diff_risk/parsers/pyproject_groups.py": "ec2267804d6f30b1fa0af884adab9a065e83ce587ad8852e5e4e9d7cbed2d1d5", + "src/sbom_diff_risk/parsers/pyproject_toml.py": "dcaa57a70d67ab5e355ab7aa8a21f4359592f74cff315635ba5a3a4f2ca00b37", + "src/sbom_diff_risk/parsers/requirements_rules.py": "d5569c31f8bcc7d53492ff6bc17ee21b361bcfcf8d1536624a618f990961fa3c", + "src/sbom_diff_risk/parsers/requirements_txt.py": "ca0bc71b870beb7e1e0fe3f4e9a40b5933180fba8ebbf184a9063ccda1549023", + "src/sbom_diff_risk/parsers/spdx_json.py": "75915904bafc0c884ff5f2d8933d7ea589c12162fd176cc8786d02c5f5854f93", + "src/sbom_diff_risk/policy_evaluator.py": "f68f76b40da0d863de267111de770573a1826db55284680b30dad947b5c36605", + "src/sbom_diff_risk/policy_models.py": "89a85c507e3a3a7ccb6c68603e1a83c79fef67a89196e83f820b2e1eade84361", + "src/sbom_diff_risk/policy_parser.py": "1eb7154dfd3626d2a38c944a206c714d4e840478c908c047df6b88b582f2fb3d", + "src/sbom_diff_risk/presentation.py": "41200758748fe2b7f283d9d7cc70377d03555c5aebf2493053c25a4ae77888b7", + "src/sbom_diff_risk/pypi_integrity_client.py": "115ac3f6facaacd698ab38a849903d778930457e339a0f86bd7f6e52e674f366", + "src/sbom_diff_risk/pypi_provenance.py": "09c711e53ee6fe2691d08c7f66f771c447c0fee333a0566618cf76e4a7413bb6", + "src/sbom_diff_risk/report_json.py": "535dabe37571ba8cda869243ce35d2a7a3b055e6fa2ea4a3194ee710259159b5", + "src/sbom_diff_risk/report_md.py": "36b078399013c790e6c893ade6b7256f50b609b77d35be3df699cb5102c9caa1", + "src/sbom_diff_risk/report_sarif.py": "a345c4ceafbbb78d135a885f981a2ce7a69050933e83ef43f85b19ea8a3fc4c3", + "src/sbom_diff_risk/repository_mapping.py": "4930a40b83746ec0521ed2ca4683435bf265ca874b9a9ee2da4cb669b7a27ab5", + "src/sbom_diff_risk/risk.py": "1532231014562961e294400033df2dca48abcaac51dc8db289aa062332511194", + "src/sbom_diff_risk/schema_versions.py": "0dd826c95e974feb6a7be7d8b4839071d8127a7b9025c1768b53db4448d14b7e", + "src/sbom_diff_risk/scorecard_client.py": "ac61b302ec7490362e9a0ba4c717e77b46a97f7a5e9e4379b81d972e154ca5e9", + "src/sbom_diff_risk/scorecard_enrichment.py": "ccb8ee5a33601a097d16d6141fcfa186255238f27e3b47849c3f236f19fbd55a", + "tests/conftest.py": "340b5fabccb10cc03e3b60bf8744e1f40a89b0cf625d3e4234de50a2a06ba015", + "tests/test_canonical_release_enrichment.py": "c34538e1b15abf201b5f4d342298d421839fa744507565ee5d71444ad4cfe3ad", + "tests/test_ci_consumer_demo.py": "5c0ad5aa334298166a8fac6db1c001857517d7a2d27163e3deb1162fe9a25035", + "tests/test_cli_exit_codes.py": "17611bfed88763fd6d5ef38bf9f42fe0244741bacbdf27db5152eb6a14d26d76", + "tests/test_cli_no_enrichment_regression.py": "26928304f936a770ca5cde95589cf416206d353016eee5a877fda39ae2a916ca", + "tests/test_cli_policy_json.py": "692c337c49370f72f587ad688e1edbfc7596dc9df6555c9bced2562ba1ffbda9", + "tests/test_cli_provenance_policy.py": "16bd95451c75a6a6aafb77938a0d436a8db449af796602c49ad39dba72b7d48a", + "tests/test_cli_summary_json.py": "b5a1beef4f5386ba0df7917944d68c7661a8caf4df8ded95412bfa5fc3715429", + "tests/test_component_identity.py": "ef86543a94ad1ce7761f4b16a1ead8bf569ea0b3fbb64ded46c3088a480203cd", + "tests/test_decision_stability.py": "718cb54a0300fa785449a080b02b9bca4b3bee100b3ac387cf69ecd8f1cd7278", + "tests/test_decision_stability_negatives.py": "6e443afaf96c37e445c4140798ca1c83aa08ed34f577d8fd9529f81f256817bb", + "tests/test_decision_stability_oracle.py": "2989a8bc7d806b24bb5a69011b0211b1a6bd997c3c4cf427db4bc95ab49d2a88", + "tests/test_decision_stability_pairs.py": "180b5ecad2bfdd18c0e009f578c2d644cb2c6c84167d8948016e27c5ae16bce3", + "tests/test_decision_stability_providers.py": "4ca598d0419ef69a6403f6e4d7baa353c35518240c53d8afe0c1bc5be388c5f9", + "tests/test_decision_stability_reports.py": "41f8273abfc9e7b0bd01e5f85fa4ebfd9839a3b6d1a71cf1e499a3a55019ad87", + "tests/test_decision_stability_runner.py": "d93e01e6c258da5efd325cdc9a74a3efe4bd22f60496f45b9a51069a9b4e62c8", + "tests/test_diffing.py": "2a8a1512456f9a13d740f47cba57f42da1494670b88371d61f27600306dc19ed", + "tests/test_evidence_confidence.py": "8351a7df5fc6b0a6d1c5d2554c6509c3dd96ab3e8d9b8f31deb83346ff845dec", + "tests/test_example_artifacts.py": "1c84e14cbdd1c479c431085064413d911adf2a36e27d6beca0eefa4220e1644a", + "tests/test_normalize.py": "b3d878efcfa5ea3fa974997f6ab419e0e5d262d1b1c0094da5435646a8aeeb81", + "tests/test_parsers.py": "9db22c37098c5f7744098dc883f82ca4eb4616e69ee0de260dc06eefda72e3bb", + "tests/test_policy.py": "0a7da0334df58262c96ea02bf083b24a16f7d114081210093b8b5c6480498083", + "tests/test_policy_decision_examples.py": "e35b45b1d230a90b5cd329ebce5722e84162701549fadf614329422004c2fb43", + "tests/test_policy_package_equivalence.py": "d60f90a197f9a6b47e4ecda74158e6e412e980c0646274145d39ed81aad237e4", + "tests/test_policy_provenance.py": "aedc127d83b15a871047a7ac4a3d869ac5893e944e977164b5b26cdfd1e7996b", + "tests/test_policy_scorecard.py": "9f5e84aba8296fe6bc4dc0502b853da0b19cae840335ac77ad17f6bfb36be929", + "tests/test_policy_validation_qualification.py": "c07a0a6609e3c4258b699b9d80b52d2a9ef7d35e8d72f536eac23a5f675c92ae", + "tests/test_policy_warning_reviewer_case.py": "ff4dd0d31fa162d16f678dff795ab2c13c5c27ed65502706d178753d349a6ca8", + "tests/test_provenance_reporting.py": "6e43b1091e07dbf5ef5ce3e50b3d32e0fba0fbd6ae991fbdf474092e1efbacf9", + "tests/test_pypi_enrichment.py": "695025ddc41dc92c340d9445ed1ef2badedffdcc233e496d23820f1c1c5d1b87", + "tests/test_pypi_integrity_client.py": "14e5506668d4cbfe539c559bb64062940b3e425e1f4ba4bfb627f8bd6c32514e", + "tests/test_release_workflow.py": "86675bcde521a5186a9568574303e457a8eb9aa02cee1c1c39d41d0420bc28c3", + "tests/test_report_schema_compatibility.py": "3e39cc210be1f05474eea7a33f9f32e5b0f9b31439024f4d4cc0b8f5da7b711f", + "tests/test_reports.py": "589829fd4e704efe6b56b344a79a9dc8e70b41990ef5ef500a79035d255be807", + "tests/test_repository_mapping.py": "ddf9a31ce8df50d17f9492c2d781932c5bce595f13db6487b7b85d48739c4f3f", + "tests/test_reproducible_sdist.py": "4edc52068416718a9a6f3b02729b156275413abc33e27e333e8246083a699394", + "tests/test_risk.py": "63223b579dcd789b35960d6a8f86791b63beff5bce61e1b0f7c621a8d295b2fa", + "tests/test_risk_model_boundary_docs.py": "9176fe213b42d3e4c98e28cb75c0abf63474f616ae8f78c0aa7fcb885080b917", + "tests/test_sarif.py": "4470631f14f5d512cd2b87f4124b2c657d2aaac37cb57f1a2570e281366385b8", + "tests/test_scorecard_client.py": "869ae476c9351f9b1584830305b1c4c225db8ab1fdb0cdd26e3636d3ff1ecdb8", + "tests/test_scorecard_enrichment.py": "f0d794557dda7a2647924acca37fa619488d188a2e5b06e63535a2ba25ce6ddf", + "tests/test_scorecard_evidence_integrity.py": "a805371c91fc31a6c2b1d5511afc2cfdffe6381370e9f5e4fae62fd695c6d6a1", + "tests/test_scorecard_reporting.py": "04878b10608fd6f702ab2fce8b280a953335b7536823ec45354b63bb8d4f5c1a", + "tests/test_version_decision_reports.py": "7636106f5d37374b876727b92f0b3681ca39c7515532c8d8cc6455fa2787bd1a", + "tests/test_version_decisions.py": "f7051388eb7207fc3642659dd379e70920bd53a744e2224756ecdf645b6a3c47", + "tests/test_version_evidence.py": "cb342258c11208af2af29088cbfdb621a3a6f681954f0b8d7a73d0a216735337" + }, + "status": "EXECUTED" + }, + "linux_source_git_blob_match": true, + "live_evidence_enrichment": "FORBIDDEN", + "local_platform_coverage": { + "Linux_executor": "Maintainer after reviewer interruption; not independent execution", + "Python": "3.11.17", + "Windows_Python_3_12": "PASS: 817 tests per seed 103/117/120", + "native_Linux_Python_3_11": "PASS: 817 tests, seed 103; additional targeted rerun", + "native_filesystem": "ext4 (findmnt; statfs reports ext2/ext3 family)" + }, + "os": "Windows", + "parallelism": 1, + "per_command_timeout_seconds": 300, + "policy_decisions": { + "C02": "CONTRACT_CLARIFIED_CANONICAL_NAME_EQUIVALENCE", + "C03": "CONTRACT_CLARIFIED_CANONICAL_RELEASE_VERSION" + }, + "preservation": { + "authorized_additions": "Candidate branch and worktree only", + "checked": [ + "HEAD", + "branch", + "status", + "staged diff bytes", + "unstaged diff bytes", + "original refs", + "original worktree registrations" + ], + "limit": "Untracked names/status checked; initial untracked content bytes were not captured.", + "repositories": 4, + "status": "PASS" + }, + "publication": "NOT_PERFORMED", + "python": "3.12.7", + "python_implementation": "CPython", + "red_green": { + "C03": { + "final_regressions": 31, + "index_preflight_green_affected": 43, + "index_preflight_red_failed": 2, + "initial_green_affected": 49, + "original_FAIL_to_PASS_reproducer_retained": true, + "red_failed": 22, + "red_passed": 3 + }, + "boundary": { + "green_passed": 32, + "red_failed": 25, + "red_passed": 7 + }, + "c02": { + "green_passed": 29, + "owner_decision": "CONTRACT_CLARIFIED_CANONICAL_NAME_EQUIVALENCE", + "red_failed": 17, + "red_passed": 12 + }, + "c04_unicode": { + "green_affected_passed": 60, + "red_failed": 3 + }, + "oracle_audit": { + "classification": "QUALIFICATION_HARNESS_DEFECT", + "green_affected_passed": 153, + "red_failed": 7 + }, + "runner": { + "green_passed": 1, + "red_failed": 1 + } + }, + "release_build": "OUT_OF_SCOPE", + "remote_ci": "NOT_RUN", + "retained_failures": [ + { + "classification": "TEST_ORACLE_DEFECT", + "failed": 2, + "label": "pairs-first", + "passed": 51 + }, + { + "accepted_as_production_defect": false, + "classification": "TEST_ORACLE_DEFECT", + "failed": 2, + "label": "release-cache-red" + }, + { + "cause": "Dynamic import map lost statically unused imports; repaired explicit map.", + "classification": "TEST_HARNESS_DEFECT", + "completed_rounds": 0, + "label": "campaign-v1" + }, + { + "classification": "INVOCATION_ERROR", + "counts_as_red": false, + "exit": 4, + "label": "oracle-audit-red", + "tests": 0 + }, + { + "classification": "MIXED_PRODUCTION_FAILURES_AND_TEST_HARNESS_DEFECTS", + "details": "Mock return values reached serialization; invocation-specific policy_path mismatch. Corrected before authoritative RED run.", + "label": "c03-owner-red", + "superseded_by": "c03-owner-red-corrected" + } + ], + "rounds": [ + { + "counts": { + "classifications": { + "INVALID_INPUT": 7, + "JUSTIFIED_REVERSAL": 23, + "STABLE": 25 + }, + "cli_cases": 11, + "microcases": 72, + "negative_controls": 14, + "provider_cases": 22, + "transitions": 55 + }, + "full_suite": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 817 + }, + "seed": 103 + }, + { + "counts": { + "classifications": { + "INVALID_INPUT": 7, + "JUSTIFIED_REVERSAL": 23, + "STABLE": 25 + }, + "cli_cases": 11, + "microcases": 72, + "negative_controls": 14, + "provider_cases": 22, + "transitions": 55 + }, + "full_suite": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 817 + }, + "seed": 117 + }, + { + "counts": { + "classifications": { + "INVALID_INPUT": 7, + "JUSTIFIED_REVERSAL": 23, + "STABLE": 25 + }, + "cli_cases": 11, + "microcases": 72, + "negative_controls": 14, + "provider_cases": 22, + "transitions": 55 + }, + "full_suite": { + "errors": 0, + "failures": 0, + "skipped": 0, + "tests": 817 + }, + "seed": 120 + } + ], + "seeds": [ + 103, + 117, + 120 + ], + "source_inventory": { + "docs/policy-schema.md": "7994dae3af1182ea36484e940aeccdc53df24d5758d65ac33e335bdd0257201d", + "scripts/qualify-decision-stability.py": "dfc5d9f7a5548a92d78ed3cdc463b851c75de74bf72621c174a6e57caeeb14b9", + "src/sbom_diff_risk/__init__.py": "6a6775b4814db46e5b905de4de07befd569eb4e82168e59c555f55501e58e4cd", + "src/sbom_diff_risk/cli.py": "b1e5233c8d8d721122ed81096d38414e88eb05497de298a83ba79de10a6d20e7", + "src/sbom_diff_risk/component_identity.py": "52d820c5fd192ca84847ccecadc19e37e9b759f40de77d40558450e4730ec3af", + "src/sbom_diff_risk/diffing.py": "d2822a8c388d0e03c695a19ab3b229688de1094b1669f40db7aea76ebebc7815", + "src/sbom_diff_risk/enrichment.py": "d5f75dd0b7ab6baae78d9477066621a5412454996b20e46168d0639e232a837a", + "src/sbom_diff_risk/errors.py": "71c329b91f24b64e2f67de6f0a4013d7d5aa6e8e803a1029e9f70e11578061a1", + "src/sbom_diff_risk/evidence_confidence.py": "c5fc6281d34cbe2db9bf9af866e3667586cf38578eb3ba6491179a1359e348c7", + "src/sbom_diff_risk/models.py": "da27543546456476b511ef31cc29b06f8d66bf9308c47d9cf9f307f22bef75db", + "src/sbom_diff_risk/normalize.py": "9badd527b92318d783972a47bf29d90b4a22ab134c4c0d1372f0d9b4840e3472", + "src/sbom_diff_risk/parsers/__init__.py": "ff7132ba67e8c0f9951c1b6728c0ea05e12439fa24238f4186d73246bae5da7d", + "src/sbom_diff_risk/parsers/common.py": "322c09ddb3040f30844ed27b73f689f5eb4295ddc1ec6489109458a99554eea5", + "src/sbom_diff_risk/parsers/cyclonedx_json.py": "15bb8e1075f8115334b099d3c6025aa94d94195d8b4752a6f6c94a771ad3ec6c", + "src/sbom_diff_risk/parsers/pyproject_groups.py": "ec2267804d6f30b1fa0af884adab9a065e83ce587ad8852e5e4e9d7cbed2d1d5", + "src/sbom_diff_risk/parsers/pyproject_toml.py": "dcaa57a70d67ab5e355ab7aa8a21f4359592f74cff315635ba5a3a4f2ca00b37", + "src/sbom_diff_risk/parsers/requirements_rules.py": "d5569c31f8bcc7d53492ff6bc17ee21b361bcfcf8d1536624a618f990961fa3c", + "src/sbom_diff_risk/parsers/requirements_txt.py": "ca0bc71b870beb7e1e0fe3f4e9a40b5933180fba8ebbf184a9063ccda1549023", + "src/sbom_diff_risk/parsers/spdx_json.py": "75915904bafc0c884ff5f2d8933d7ea589c12162fd176cc8786d02c5f5854f93", + "src/sbom_diff_risk/policy_evaluator.py": "f7920cdda0a9fd0ad162af09e652e970d4c6c432b48b08a3eec0068db9754414", + "src/sbom_diff_risk/policy_models.py": "89a85c507e3a3a7ccb6c68603e1a83c79fef67a89196e83f820b2e1eade84361", + "src/sbom_diff_risk/policy_parser.py": "30222b549e94a7bcbcbbb80d674de1dde89deb5e6f00d7467f8e72bb3b2c374e", + "src/sbom_diff_risk/presentation.py": "41200758748fe2b7f283d9d7cc70377d03555c5aebf2493053c25a4ae77888b7", + "src/sbom_diff_risk/pypi_integrity_client.py": "115ac3f6facaacd698ab38a849903d778930457e339a0f86bd7f6e52e674f366", + "src/sbom_diff_risk/pypi_provenance.py": "09c711e53ee6fe2691d08c7f66f771c447c0fee333a0566618cf76e4a7413bb6", + "src/sbom_diff_risk/report_json.py": "535dabe37571ba8cda869243ce35d2a7a3b055e6fa2ea4a3194ee710259159b5", + "src/sbom_diff_risk/report_md.py": "36b078399013c790e6c893ade6b7256f50b609b77d35be3df699cb5102c9caa1", + "src/sbom_diff_risk/report_sarif.py": "a345c4ceafbbb78d135a885f981a2ce7a69050933e83ef43f85b19ea8a3fc4c3", + "src/sbom_diff_risk/repository_mapping.py": "4930a40b83746ec0521ed2ca4683435bf265ca874b9a9ee2da4cb669b7a27ab5", + "src/sbom_diff_risk/risk.py": "1532231014562961e294400033df2dca48abcaac51dc8db289aa062332511194", + "src/sbom_diff_risk/schema_versions.py": "0dd826c95e974feb6a7be7d8b4839071d8127a7b9025c1768b53db4448d14b7e", + "src/sbom_diff_risk/scorecard_client.py": "0394f3a1e7c808d01bc5961a59c19a8e51dea65fa06b1fe252f76461c59cc8bc", + "src/sbom_diff_risk/scorecard_enrichment.py": "ccb8ee5a33601a097d16d6141fcfa186255238f27e3b47849c3f236f19fbd55a", + "tests/conftest.py": "340b5fabccb10cc03e3b60bf8744e1f40a89b0cf625d3e4234de50a2a06ba015", + "tests/test_canonical_release_enrichment.py": "c34538e1b15abf201b5f4d342298d421839fa744507565ee5d71444ad4cfe3ad", + "tests/test_ci_consumer_demo.py": "5c0ad5aa334298166a8fac6db1c001857517d7a2d27163e3deb1162fe9a25035", + "tests/test_cli_exit_codes.py": "17611bfed88763fd6d5ef38bf9f42fe0244741bacbdf27db5152eb6a14d26d76", + "tests/test_cli_no_enrichment_regression.py": "26928304f936a770ca5cde95589cf416206d353016eee5a877fda39ae2a916ca", + "tests/test_cli_policy_json.py": "692c337c49370f72f587ad688e1edbfc7596dc9df6555c9bced2562ba1ffbda9", + "tests/test_cli_provenance_policy.py": "16bd95451c75a6a6aafb77938a0d436a8db449af796602c49ad39dba72b7d48a", + "tests/test_cli_summary_json.py": "b5a1beef4f5386ba0df7917944d68c7661a8caf4df8ded95412bfa5fc3715429", + "tests/test_component_identity.py": "ef86543a94ad1ce7761f4b16a1ead8bf569ea0b3fbb64ded46c3088a480203cd", + "tests/test_decision_stability.py": "718cb54a0300fa785449a080b02b9bca4b3bee100b3ac387cf69ecd8f1cd7278", + "tests/test_decision_stability_negatives.py": "6e443afaf96c37e445c4140798ca1c83aa08ed34f577d8fd9529f81f256817bb", + "tests/test_decision_stability_oracle.py": "2989a8bc7d806b24bb5a69011b0211b1a6bd997c3c4cf427db4bc95ab49d2a88", + "tests/test_decision_stability_pairs.py": "180b5ecad2bfdd18c0e009f578c2d644cb2c6c84167d8948016e27c5ae16bce3", + "tests/test_decision_stability_providers.py": "4ca598d0419ef69a6403f6e4d7baa353c35518240c53d8afe0c1bc5be388c5f9", + "tests/test_decision_stability_reports.py": "41f8273abfc9e7b0bd01e5f85fa4ebfd9839a3b6d1a71cf1e499a3a55019ad87", + "tests/test_decision_stability_runner.py": "d93e01e6c258da5efd325cdc9a74a3efe4bd22f60496f45b9a51069a9b4e62c8", + "tests/test_diffing.py": "2a8a1512456f9a13d740f47cba57f42da1494670b88371d61f27600306dc19ed", + "tests/test_evidence_confidence.py": "8351a7df5fc6b0a6d1c5d2554c6509c3dd96ab3e8d9b8f31deb83346ff845dec", + "tests/test_example_artifacts.py": "1c84e14cbdd1c479c431085064413d911adf2a36e27d6beca0eefa4220e1644a", + "tests/test_normalize.py": "b3d878efcfa5ea3fa974997f6ab419e0e5d262d1b1c0094da5435646a8aeeb81", + "tests/test_parsers.py": "9db22c37098c5f7744098dc883f82ca4eb4616e69ee0de260dc06eefda72e3bb", + "tests/test_policy.py": "0a7da0334df58262c96ea02bf083b24a16f7d114081210093b8b5c6480498083", + "tests/test_policy_decision_examples.py": "e35b45b1d230a90b5cd329ebce5722e84162701549fadf614329422004c2fb43", + "tests/test_policy_package_equivalence.py": "d60f90a197f9a6b47e4ecda74158e6e412e980c0646274145d39ed81aad237e4", + "tests/test_policy_provenance.py": "aedc127d83b15a871047a7ac4a3d869ac5893e944e977164b5b26cdfd1e7996b", + "tests/test_policy_scorecard.py": "9f5e84aba8296fe6bc4dc0502b853da0b19cae840335ac77ad17f6bfb36be929", + "tests/test_policy_validation_qualification.py": "c07a0a6609e3c4258b699b9d80b52d2a9ef7d35e8d72f536eac23a5f675c92ae", + "tests/test_policy_warning_reviewer_case.py": "ff4dd0d31fa162d16f678dff795ab2c13c5c27ed65502706d178753d349a6ca8", + "tests/test_provenance_reporting.py": "6e43b1091e07dbf5ef5ce3e50b3d32e0fba0fbd6ae991fbdf474092e1efbacf9", + "tests/test_pypi_enrichment.py": "695025ddc41dc92c340d9445ed1ef2badedffdcc233e496d23820f1c1c5d1b87", + "tests/test_pypi_integrity_client.py": "14e5506668d4cbfe539c559bb64062940b3e425e1f4ba4bfb627f8bd6c32514e", + "tests/test_release_workflow.py": "86675bcde521a5186a9568574303e457a8eb9aa02cee1c1c39d41d0420bc28c3", + "tests/test_report_schema_compatibility.py": "3e39cc210be1f05474eea7a33f9f32e5b0f9b31439024f4d4cc0b8f5da7b711f", + "tests/test_reports.py": "589829fd4e704efe6b56b344a79a9dc8e70b41990ef5ef500a79035d255be807", + "tests/test_repository_mapping.py": "ddf9a31ce8df50d17f9492c2d781932c5bce595f13db6487b7b85d48739c4f3f", + "tests/test_reproducible_sdist.py": "4edc52068416718a9a6f3b02729b156275413abc33e27e333e8246083a699394", + "tests/test_risk.py": "63223b579dcd789b35960d6a8f86791b63beff5bce61e1b0f7c621a8d295b2fa", + "tests/test_risk_model_boundary_docs.py": "9176fe213b42d3e4c98e28cb75c0abf63474f616ae8f78c0aa7fcb885080b917", + "tests/test_sarif.py": "4470631f14f5d512cd2b87f4124b2c657d2aaac37cb57f1a2570e281366385b8", + "tests/test_scorecard_client.py": "869ae476c9351f9b1584830305b1c4c225db8ab1fdb0cdd26e3636d3ff1ecdb8", + "tests/test_scorecard_enrichment.py": "f0d794557dda7a2647924acca37fa619488d188a2e5b06e63535a2ba25ce6ddf", + "tests/test_scorecard_evidence_integrity.py": "a805371c91fc31a6c2b1d5511afc2cfdffe6381370e9f5e4fae62fd695c6d6a1", + "tests/test_scorecard_reporting.py": "04878b10608fd6f702ab2fce8b280a953335b7536823ec45354b63bb8d4f5c1a", + "tests/test_version_decision_reports.py": "7636106f5d37374b876727b92f0b3681ca39c7515532c8d8cc6455fa2787bd1a", + "tests/test_version_decisions.py": "f7051388eb7207fc3642659dd379e70920bd53a744e2224756ecdf645b6a3c47", + "tests/test_version_evidence.py": "cb342258c11208af2af29088cbfdb621a3a6f681954f0b8d7a73d0a216735337" + }, + "status": "BLOCKED", + "type_check": "NOT_CONFIGURED" +} diff --git a/tools/sbom-diff-and-risk/docs/validation/decision-stability/INDEPENDENT-AUDIT.md b/tools/sbom-diff-and-risk/docs/validation/decision-stability/INDEPENDENT-AUDIT.md new file mode 100644 index 0000000..32a47cf --- /dev/null +++ b/tools/sbom-diff-and-risk/docs/validation/decision-stability/INDEPENDENT-AUDIT.md @@ -0,0 +1,30 @@ +# Independent audit disposition + +Status: **AUDIT_PENDING**; overall decision **BLOCKED**. + +Fresh reviewers inspected the C03 repair at semantic tree +`b9931ef405586683b6ce40e1a0d639ad63cb7906`, not the earlier 776-test candidate. + +| Reviewer | Actual execution | Disposition | +| --- | --- | --- | +| Production/C02/C03/C04 | 105 targeted tests passed; synthetic CLI index probe | Found lookup before duplicate/conflicting index rejection | +| Oracle/consumers | 129 targeted tests passed; seven independent corruption probes; N01-N14 reached/detected; source inventory checked | Useful executed evidence, but final review did not finish | +| Linux 3.11 reviewer | Preparation began, no completed test evidence | Infrastructure interruption; no independent execution claim | + +All reviewers again stopped with: "Your workspace is out of credits. Ask your +workspace owner to refill in order to continue." No agreement or unfinished +review is treated as approval. Original private receipts are retained. + +The maintainer reproduced the index finding with two failing regressions and +reused the existing input-index validator before either CLI provider starts. +The final semantic tree is `0760a5543df366f3a8237c60a187cde7f3aae0b0`. +It passed three Windows 817-test rounds and a native Linux 3.11.17 817-test run, +additional targeted tests, goldens and network-denial checks. Those are maintainer +execution results, not independent final acceptance. + +The seven repaired oracle weaknesses have executed independent probes on the +first C03 tree. C04 ASCII-only matching and C02 exception boundaries remain in +the targeted/full suites. Final index-preflight re-review and comprehensive final +sign-off remain pending. A self-contained private package contains the exact +base/candidate source, patch, manifest, tests, RED/GREEN and audit receipts. +No `QUALIFICATION_READY` claim is made. diff --git a/tools/sbom-diff-and-risk/docs/validation/decision-stability/INVARIANT-CATALOG.md b/tools/sbom-diff-and-risk/docs/validation/decision-stability/INVARIANT-CATALOG.md new file mode 100644 index 0000000..3ff635d --- /dev/null +++ b/tools/sbom-diff-and-risk/docs/validation/decision-stability/INVARIANT-CATALOG.md @@ -0,0 +1,49 @@ +# Independent decision invariants + +Status: implemented contract catalog with three executed synthetic rounds. C02 +adds an explicitly approved canonical-name invariant; C03 remains unresolved. +An invariant is asserted only within its documented scope. + +| ID | Independent requirement | Main evidence source / bounded scope | +| --- | --- | --- | +| I01 | Same supported canonical coordinate aligns to one package; namespaces and opaque IDs are preserved | Component identity comparison contract; no universal ecosystem normalization | +| I02 | Duplicate/conflicting same-input coordinates yield explicit invalid-input diagnostics | Identity/diff contract; no first-match reconciliation | +| I03 | Ordinary risk findings remain separate from policy decisions and security verdicts | Risk/report non-claims; SemVer repair is already present | +| I04 | No policy means no policy summary, not an affirmative PASS claim | JSON report contract; consumer needs-review remains consumer-side | +| I05 | Explicit block wins warn, and explicit warn can override a triggered default block | Policy schema and CLI merge contract | +| I06 | Ignore preserves suppressed finding identity, evidence and accounting while removing active enforcement | Policy explainability; inactive severity can be null | +| I07 | Added count equal to the threshold passes; a strictly higher count triggers the configured/default severity | Integer threshold contract | +| I08 | Source host matching is exact; absent/subdomain/different hosts are not authorized by substring inference | Policy source allowlist contract | +| I09 | Unavailable, absent attestation, and unverified publisher states are distinct | Recorded provenance contract; error dominates mixed success | +| I10 | Package missing-attestation exceptions do not waive unavailable or unverified evidence | Narrow PyPI package-name exception scope; C02 equivalence is explicitly approved | +| I11 | Required provenance applies only in its explicitly enabled new-PyPI or suspicious-source context | Capability v2 requirement contract | +| I12 | Scorecard requires an explicit active rule; equality at threshold is not below | Capability v3 contract; missing data alone creates no score violation | +| I13 | YAML version eligibility and CLI rule capability promotion remain separate and deterministic | Policy parser/build contract; v1/v2 files cannot silently accept later-only fields | +| I14 | Accepted Scorecard numbers are numeric and finite; explicit contradictory response identity cannot be rebound | Evidence integrity plus strict machine-consumable JSON; preserve missing/case-only compatibility | +| I15 | Every finding preserves exact component/rule/evidence/reason/severity-source/threshold identity | Public explainability fields; no expected value taken from evaluator output | +| I16 | Full JSON, summary, policy sidecar, Markdown and exit agree under their documented projections | Restricted SARIF map is specified separately; counts alone are insufficient | +| I17 | Repeated identical inputs are byte deterministic; permutations preserve relevant semantic observations | Raw input evidence and format-specific confidence remain visible | +| I18 | No actual network is needed for qualification; arbitrary provider availability is not verification probability | Synthetic byte-stream transport and socket audit, not a live registry | +| I19 | PEP 503-equivalent PyPI names preserve exception eligibility in both policy entries and components | C02 approved clarification; raw evidence and package-name scope are preserved | + +| I20 | Valid explicit/PURL-only release representations are enrichment-equivalent; malformed/conflicting input cannot trigger lookup | C03 owner-approved canonical coordinates; CLI snapshot index validation precedes providers, direct APIs validate identities | + +## Oracle independence + +Expected decisions use literal microcases, explicit finding specifications and +independently calculated count/score boundaries. The expected-value path does not +call the production evaluator, identity resolver, serializer or private matching +helpers. The production adapter calls those functions only to obtain observations +to compare with the frozen expectations. + +Negative controls N01–N14 must first pass the corresponding green observation, then corrupt a reached result and demonstrate detection by the named check. Import errors, unreachable branches, unexecuted controls and unrelated assertion failures do not count as detected semantic mutations. + +## Reversal classes + +- `STABLE`: the chosen transformation preserves the relevant evidence and policy semantics, not merely a raw label. +- `JUSTIFIED_REVERSAL`: a specific authorized rule, threshold, exception, evidence state or added/changed scope changed. +- `SUSPICIOUS_REVERSAL`: a decision changed without a sufficient documented reason. +- `INVALID_INPUT`: the input is contradictory, ambiguous, malformed or outside the supported syntax and must follow its fail-closed contract. +- `UNRESOLVED`: the current contract cannot choose an intended outcome; retain the counterexample rather than blessing implementation behavior. + +More evidence need not reduce enforcement. A newly observed error can make provenance unavailable; a real metadata change can bring a previously unchanged component into the documented policy scope. These are separate from serialization/order transformations. Confidence labels classify evidence sources and are never treated as probabilities. diff --git a/tools/sbom-diff-and-risk/docs/validation/decision-stability/NEGATIVE-CONTROLS.json b/tools/sbom-diff-and-risk/docs/validation/decision-stability/NEGATIVE-CONTROLS.json new file mode 100644 index 0000000..d3181ad --- /dev/null +++ b/tools/sbom-diff-and-risk/docs/validation/decision-stability/NEGATIVE-CONTROLS.json @@ -0,0 +1,760 @@ +{ + "actual_round_executions": 42, + "collection_failures_are_not_detection": true, + "control_results": [ + { + "base_case": "M03", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "corrupted": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "detected_check": "decision", + "green_validated": true, + "id": "N01", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "silently remove blocking violation", + "red_detected": true, + "target_check": "decision" + }, + { + "base_case": "M04", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "WARN", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "warning", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "warn", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "warn_on", + "suppression": null, + "threshold": null + } + ] + }, + "corrupted": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "detected_check": "decision", + "green_validated": true, + "id": "N02", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "upgrade warning to blocking", + "red_detected": true, + "target_check": "decision" + }, + { + "base_case": "M03", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "corrupted": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "WARN", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "warning", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "warn", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "warn_on", + "suppression": null, + "threshold": null + } + ] + }, + "detected_check": "decision", + "green_validated": true, + "id": "N03", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "downgrade blocking to warning", + "red_detected": true, + "target_check": "decision" + }, + { + "base_case": "M06", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 1, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "suppressed", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": "ignored_by_policy", + "threshold": null + } + ] + }, + "corrupted": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "detected_check": "violation-identity", + "green_validated": true, + "id": "N04", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "drop suppression evidence", + "red_detected": true, + "target_check": "violation-identity" + }, + { + "base_case": "M18", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "unavailable", + "policy_rule": "provenance_unavailable", + "reason": "provenance_evidence_unavailable", + "rule": "provenance_unavailable", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "corrupted": { + "assessment": [ + true, + false, + true, + [ + "github actions" + ] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "unavailable", + "policy_rule": "provenance_unavailable", + "reason": "provenance_evidence_unavailable", + "rule": "provenance_unavailable", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "detected_check": "assessment", + "green_validated": true, + "id": "N05", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "unavailable provenance becomes verified", + "red_detected": true, + "target_check": "assessment" + }, + { + "base_case": "M20", + "before": { + "assessment": [ + false, + false, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": false, + "policy_rule": "missing_attestation", + "reason": "attestation_not_published", + "rule": "missing_attestation", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "corrupted": { + "assessment": [ + false, + false, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "unavailable", + "policy_rule": "provenance_unavailable", + "reason": "provenance_evidence_unavailable", + "rule": "provenance_unavailable", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "detected_check": "violation-identity", + "green_validated": true, + "id": "N06", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "missing attestation becomes unavailable", + "red_detected": true, + "target_check": "violation-identity" + }, + { + "base_case": "M22", + "before": { + "assessment": [ + true, + false, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": [], + "policy_rule": "unverified_provenance", + "reason": "provenance_publisher_not_verified", + "rule": "unverified_provenance", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "corrupted": { + "assessment": [ + true, + false, + false, + [] + ], + "decision": "PASS", + "effective_version": 2, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "detected_check": "decision", + "green_validated": true, + "id": "N07", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "accept missing publisher identity", + "red_detected": true, + "target_check": "decision" + }, + { + "base_case": "M42", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 3, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "corrupted": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 3, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "scorecard_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": 6.9, + "policy_rule": "scorecard_below_threshold", + "reason": "scorecard_score_below_threshold", + "rule": "scorecard_below_threshold", + "source": "block_on", + "suppression": null, + "threshold": 7.0 + } + ] + }, + "detected_check": "decision", + "green_validated": true, + "id": "N08", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "enforce advisory-only Scorecard", + "red_detected": true, + "target_check": "decision" + }, + { + "base_case": "M09", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "corrupted": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "group": "blocking", + "key": null, + "observed": 1, + "rule": "max_added_packages", + "threshold": 1 + } + ] + }, + "detected_check": "decision", + "green_validated": true, + "id": "N09", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "use greater-than-or-equal count threshold", + "red_detected": true, + "target_check": "decision" + }, + { + "base_case": "M03", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "corrupted": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "coord:pypi:Qualification_Alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "blocking", + "key": "coord:pypi:Qualification_Alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "detected_check": "identity", + "green_validated": true, + "id": "N10", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "split canonical identity", + "red_detected": true, + "target_check": "identity" + }, + { + "base_case": "identity-conflict", + "before": { + "conflicting_metadata_rejected": true + }, + "corrupted": { + "conflicting_metadata_rejected": false + }, + "detected_check": "identity-conflict-rejection", + "green_validated": true, + "id": "N11", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "silently accept conflicting metadata", + "red_detected": true, + "target_check": "identity-conflict-rejection" + }, + { + "base_case": "M03", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "corrupted": { + "sidecar_observed_value": "corrupted-evidence" + }, + "detected_check": "policy-sidecar", + "green_validated": true, + "id": "N12", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "alter policy sidecar evidence", + "red_detected": true, + "target_check": "policy-sidecar" + }, + { + "base_case": "CLI-warning", + "before": { + "exit_code": 0, + "warning": 1 + }, + "corrupted": { + "exit_code": 1, + "warning": 1 + }, + "detected_check": "exit-blocking", + "green_validated": true, + "id": "N13", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "replace actual CLI exit with blocking exit", + "red_detected": true, + "target_check": "exit-blocking" + }, + { + "base_case": "M03", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "corrupted": { + "package_safety_verdict": "verified_safe" + }, + "detected_check": "no-security-verdict", + "green_validated": true, + "id": "N14", + "mechanism": "reached semantic result replacement; no import or collection failures", + "mutation": "launder heuristic into verified safety verdict", + "red_detected": true, + "target_check": "no-security-verdict" + } + ], + "representative_seed": 103, + "rounds": [ + { + "detected": 14, + "reached": 14, + "seed": 103 + }, + { + "detected": 14, + "reached": 14, + "seed": 117 + }, + { + "detected": 14, + "reached": 14, + "seed": 120 + } + ], + "schema": "local-negative-controls.v1", + "unique_controls": 14 +} diff --git a/tools/sbom-diff-and-risk/docs/validation/decision-stability/PROGRESS.md b/tools/sbom-diff-and-risk/docs/validation/decision-stability/PROGRESS.md new file mode 100644 index 0000000..dcda0da --- /dev/null +++ b/tools/sbom-diff-and-risk/docs/validation/decision-stability/PROGRESS.md @@ -0,0 +1,21 @@ +# Decision stability qualification progress + +Status: **BLOCKED / AUDIT_PENDING**. Both C02 and C03 are approved and repaired. +The local candidate is uncommitted; no public actions occurred. + +- Final semantic tree: `0760a5543df366f3a8237c60a187cde7f3aae0b0`. +- Windows CPython 3.12: 817 passed for each existing seed 103/117/120. +- Native Linux CPython 3.11.17 on ext4: 817 passed, plus targeted rerun. +- 72 literal policy microcases; 55 transitions: 25 STABLE, 23 JUSTIFIED_REVERSAL, + 7 INVALID_INPUT, zero unresolved contract decisions. Seeds do not multiply cases. +- N01-N14 reached/detected every round; golden checks passed; network attempts zero. +- Changed-file Ruff passed. Full Ruff output exactly matches baseline F821. +- Four protected repositories preserved; only candidate branch/worktree additions. +- Fresh independent production/oracle reviewers executed 105/129 tests and probes + on the first C03 repair. The production reviewer found missing index preflight; + it was reproduced and corrected, then the final candidate requalified. +- Reviewer infrastructure failed again. The exact final patch and a self-contained + private evidence package are retained for final review. No final audit approval. + +Earlier 473/776/786-test results remain historical receipts, not final-source proof. +Raw logs, source archives and machine paths remain outside the source checkout. diff --git a/tools/sbom-diff-and-risk/docs/validation/decision-stability/REPRODUCTION.md b/tools/sbom-diff-and-risk/docs/validation/decision-stability/REPRODUCTION.md new file mode 100644 index 0000000..fa88f23 --- /dev/null +++ b/tools/sbom-diff-and-risk/docs/validation/decision-stability/REPRODUCTION.md @@ -0,0 +1,94 @@ +# Reproducing the local decision stability qualification + +The frozen base is `29203f9f14e2a3df2ad77a4e874bf752bf44c599`, root tree +`3e6f45e7968d4ac32f475d6842aaa523860fa4a9`. The candidate is an uncommitted +local change on `stacknil/sbom-evidence-decision-stability`. A base commit alone +does not contain these new qualification files. Use the candidate files and +the implementation inventory in `EXECUTION-MANIFEST.json` to identify the +tested source; no release or remote CI result is implied. + +## Setup and replay + +From `tools/sbom-diff-and-risk`, create a separate environment outside the +entire repository, install the declared dev dependencies, then run: + +```sh +python -m pip install -e '.[dev]' +python -m pip check +python -m pytest -q -ra +python scripts/regenerate-example-artifacts.py --check +python scripts/qualify-decision-stability.py \ + --output-root ../../../decision-stability-evidence \ + --seeds 103 117 120 +``` + +`--output-root` must be a new directory outside the checkout. The runner rejects +existing output directories and retains failures. Do not delete a failed run +to make a later run look successful; use a new sibling directory. The three +seeds change YAML key order, record permutations and JSON serialization; they +are repeated validation of the same semantic catalog, not three independent +discoveries of each property. + +Dependency/tool installation is setup, separate from qualification. The replay +runner installs no packages and makes no evidence-service requests. It installs +a subprocess audit hook that rejects Internet socket and name-resolution +operations before transport. Provider requests inside the tests are routed to +in-memory synthetic byte streams. Production `network_access_performed` fields +can therefore be true for an explicitly exercised provider lookup path while +actual network attempts remain zero. Those meanings must not be conflated. + +Each round records expectations before observations, actual results, synthetic +CLI reports, reached negative controls, full-suite JUnit and stdout/stderr. +The parent records interpreter/package versions, source inventory, artifact +compatibility, bounded timeouts and local consumer-shell checks. No raw output +is written into the source checkout by this runner. Direct pytest commands +above can be given external `--basetemp`, `--junitxml` and +`-o cache_dir=...` paths as done for the retained qualification receipts. + +## Focused checks + +```sh +python -m pytest -q \ + tests/test_policy_validation_qualification.py \ + tests/test_scorecard_evidence_integrity.py +python -m pytest -q tests/test_policy_package_equivalence.py +python -m pytest -q tests/test_decision_stability*.py +``` + +The first group originally had 32 tests: the preserved unchanged-production RED result is +25 failed / 7 passed, and the repaired result was 32 passed. It now has 35 tests, +including three Unicode repository-alias regressions added after independent +review. Those three failed against the initial case-folding repair and passed +after ASCII-only case folding. The C02 exception +group has 29 tests: 17 failed / 12 passed before the approved correction, and +29 passed afterward. Negative controls pass only when their deliberate semantic +corruption raises the specific expected `OracleMismatch` code; a collection, +import or unrelated runtime failure cannot count as detection. + +To reproduce RED without altering the candidate, export the frozen base into +a separate directory, copy only the two first-group regression test files from +the candidate into that export, and install that export in a different dev +environment. To replay the historical 32-case result, exclude the three +Unicode cases with `-k 'not unicode'`. Running all 35 is a different gate and +must not be attributed to that historical receipt. For C02, copy +`test_policy_package_equivalence.py` instead. Do not copy the repaired production +files or use an editable installation pointing back to the candidate. Check +the actual `sbom_diff_risk` import origin before interpreting the result. + +## Deliberate limits + +- Native runtime decisions remain pass/warn/fail when policy is applied; exit + 0 covers both pass and warn. `needs-review` is only a consumer classification. +- SARIF is a subset. Unavailable provenance, staleness and unclassified version + changes are not required to appear there. Missing/unverified provenance + warning results may be intentionally omitted or superseded by a requirement. +- Recorded provenance and publisher metadata are not cryptographic signature, + subject-digest, certificate, transparency-log or tool-release verification. +- No evidence-confidence label is a probability, vulnerability finding or + dependency-safety verdict. +- C03 is approved and repaired. Replay `tests/test_canonical_release_enrichment.py` + for its 31 regression cases. The original counterexample and first divergent + stage are retained in `COUNTEREXAMPLES.md`. Final independent audit is pending. +- Remote CI, release builds/publication and live provider behavior are outside + this campaign. The local environment and actual executed gates are listed + in the manifest; unexecuted platforms are not counted as passing. diff --git a/tools/sbom-diff-and-risk/docs/validation/decision-stability/REVERSAL-MATRIX.json b/tools/sbom-diff-and-risk/docs/validation/decision-stability/REVERSAL-MATRIX.json new file mode 100644 index 0000000..07a216b --- /dev/null +++ b/tools/sbom-diff-and-risk/docs/validation/decision-stability/REVERSAL-MATRIX.json @@ -0,0 +1,4857 @@ +{ + "cases": [ + { + "active_policy": { + "after": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "before": { + "version": 1 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P01", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M02", + "perturbation": "activate block rule", + "perturbed_case_id": "M03", + "previous_decision": "PASS", + "reason": "A local finding is enforced only by an explicit rule.", + "resulting_decision": "FAIL", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "version": 1, + "warn_on": [ + "new_package" + ] + }, + "before": { + "block_on": [ + "new_package" + ], + "version": 1 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "WARN", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "warning", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "warn", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "warn_on", + "suppression": null, + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P02", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M03", + "perturbation": "explicit severity", + "perturbed_case_id": "M04", + "previous_decision": "FAIL", + "reason": "Configured severity changes enforcement.", + "resulting_decision": "WARN", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "new_package" + ], + "version": 1, + "warn_on": [ + "new_package" + ] + }, + "before": { + "version": 1, + "warn_on": [ + "new_package" + ] + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "WARN", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "warning", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "warn", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "warn_on", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P03", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M04", + "perturbation": "overlapping block rule", + "perturbed_case_id": "M05", + "previous_decision": "WARN", + "reason": "Explicit block wins over an existing warn.", + "resulting_decision": "FAIL", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "new_package" + ], + "ignore_rules": [ + "new_package" + ], + "version": 1, + "warn_on": [ + "new_package" + ] + }, + "before": { + "block_on": [ + "new_package" + ], + "version": 1, + "warn_on": [ + "new_package" + ] + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 1, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "suppressed", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": "ignored_by_policy", + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P04", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M05", + "perturbation": "add ignore", + "perturbed_case_id": "M06", + "previous_decision": "FAIL", + "reason": "Ignore takes precedence but retains suppression evidence.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "max_added_packages": 1, + "version": 1 + }, + "before": { + "max_added_packages": 0, + "version": 1 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "blocking", + "key": null, + "level": "block", + "name": null, + "observed": 1, + "policy_rule": "max_added_packages", + "reason": "added_package_count_exceeded_threshold", + "rule": "max_added_packages", + "source": "default_block", + "suppression": null, + "threshold": 0 + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P05", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M08", + "perturbation": "count threshold 0 to 1", + "perturbed_case_id": "M09", + "previous_decision": "FAIL", + "reason": "One added component does not exceed a threshold of one.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "max_added_packages": 2, + "version": 1 + }, + "before": { + "max_added_packages": 1, + "version": 1 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "STABLE", + "id": "P06", + "observed_classification": "STABLE", + "original_case_id": "M09", + "perturbation": "count threshold 1 to 2", + "perturbed_case_id": "M10", + "previous_decision": "PASS", + "reason": "Neither threshold is exceeded.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "max_added_packages": 0, + "version": 1, + "warn_on": [ + "max_added_packages" + ] + }, + "before": { + "max_added_packages": 0, + "version": 1 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "WARN", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "warning", + "key": null, + "level": "warn", + "name": null, + "observed": 1, + "policy_rule": "max_added_packages", + "reason": "added_package_count_exceeded_threshold", + "rule": "max_added_packages", + "source": "warn_on", + "suppression": null, + "threshold": 0 + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "blocking", + "key": null, + "level": "block", + "name": null, + "observed": 1, + "policy_rule": "max_added_packages", + "reason": "added_package_count_exceeded_threshold", + "rule": "max_added_packages", + "source": "default_block", + "suppression": null, + "threshold": 0 + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P07", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M08", + "perturbation": "explicit warning override", + "perturbed_case_id": "M11", + "previous_decision": "FAIL", + "reason": "Warn overrides implicit default block.", + "resulting_decision": "WARN", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "ignore_rules": [ + "max_added_packages" + ], + "max_added_packages": 0, + "version": 1 + }, + "before": { + "max_added_packages": 0, + "version": 1 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 1, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "suppressed", + "key": null, + "level": "block", + "name": null, + "observed": 1, + "policy_rule": "max_added_packages", + "reason": "added_package_count_exceeded_threshold", + "rule": "max_added_packages", + "source": "default_block", + "suppression": "ignored_by_policy", + "threshold": 0 + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "blocking", + "key": null, + "level": "block", + "name": null, + "observed": 1, + "policy_rule": "max_added_packages", + "reason": "added_package_count_exceeded_threshold", + "rule": "max_added_packages", + "source": "default_block", + "suppression": null, + "threshold": 0 + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P08", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M08", + "perturbation": "ignore implicit block", + "perturbed_case_id": "M12", + "previous_decision": "FAIL", + "reason": "The omitted block is accounted for as a suppression.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "allow_sources": [ + "pypi.org" + ], + "version": 1 + }, + "before": { + "allow_sources": [ + "github.com" + ], + "version": 1 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "github.com", + "policy_rule": "allow_sources", + "reason": "source_host_not_allowed", + "rule": "allow_sources", + "source": "default_block", + "suppression": null, + "threshold": [ + "pypi.org" + ] + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 1, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P09", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M13", + "perturbation": "source allowlist membership", + "perturbed_case_id": "M14", + "previous_decision": "PASS", + "reason": "Observed host falls outside the narrowed allowed set.", + "resulting_decision": "FAIL", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "provenance_unavailable" + ], + "version": 2 + }, + "before": { + "block_on": [ + "provenance_unavailable" + ], + "version": 2 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "unavailable", + "policy_rule": "provenance_unavailable", + "reason": "provenance_evidence_unavailable", + "rule": "provenance_unavailable", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "unavailable", + "policy_rule": "provenance_unavailable", + "reason": "provenance_evidence_unavailable", + "rule": "provenance_unavailable", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "STABLE", + "id": "P10", + "observed_classification": "STABLE", + "original_case_id": "M17", + "perturbation": "disabled to failed provenance", + "perturbed_case_id": "M18", + "previous_decision": "FAIL", + "reason": "Both evidence states remain unavailable; confidence describes record presence.", + "resulting_decision": "FAIL", + "source_evidence_changes": { + "provenance": [ + "none", + "error" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "provenance_unavailable" + ], + "version": 2 + }, + "before": { + "block_on": [ + "provenance_unavailable" + ], + "version": 2 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "unavailable", + "policy_rule": "provenance_unavailable", + "reason": "provenance_evidence_unavailable", + "rule": "provenance_unavailable", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "unavailable", + "policy_rule": "provenance_unavailable", + "reason": "provenance_evidence_unavailable", + "rule": "provenance_unavailable", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "STABLE", + "id": "P11", + "observed_classification": "STABLE", + "original_case_id": "M18", + "perturbation": "failed to unsupported provenance", + "perturbed_case_id": "M19", + "previous_decision": "FAIL", + "reason": "Neither state supplies available provenance.", + "resulting_decision": "FAIL", + "source_evidence_changes": { + "provenance": [ + "error", + "unsupported" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "provenance_unavailable" + ], + "version": 2 + }, + "before": { + "block_on": [ + "provenance_unavailable" + ], + "version": 2 + } + }, + "after": { + "assessment": [ + false, + false, + false, + [] + ], + "decision": "PASS", + "effective_version": 2, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "unavailable", + "policy_rule": "provenance_unavailable", + "reason": "provenance_evidence_unavailable", + "rule": "provenance_unavailable", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P12", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M17", + "perturbation": "successful metadata without attestation", + "perturbed_case_id": "M21", + "previous_decision": "FAIL", + "reason": "Unavailable-only policy stops blocking once a lookup succeeds; this is not verified provenance.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "none", + "missing" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "unverified_provenance" + ], + "version": 2 + }, + "before": { + "block_on": [ + "unverified_provenance" + ], + "version": 2 + } + }, + "after": { + "assessment": [ + true, + false, + true, + [ + "manual upload" + ] + ], + "decision": "PASS", + "effective_version": 2, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + true, + false, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": [], + "policy_rule": "unverified_provenance", + "reason": "provenance_publisher_not_verified", + "rule": "unverified_provenance", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P13", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M22", + "perturbation": "recorded publisher supplied", + "perturbed_case_id": "M23", + "previous_decision": "FAIL", + "reason": "A nonempty publisher kind satisfies the current recorded-metadata contract without an allowlist.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "attested", + "attested" + ], + "publishers": [ + [], + [ + "manual upload" + ] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "allow_provenance_publishers": [ + "github actions" + ], + "block_on": [ + "unverified_provenance" + ], + "version": 2 + }, + "before": { + "allow_provenance_publishers": [ + "manual upload" + ], + "block_on": [ + "unverified_provenance" + ], + "version": 2 + } + }, + "after": { + "assessment": [ + true, + false, + true, + [ + "github actions" + ] + ], + "decision": "PASS", + "effective_version": 2, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + true, + false, + false, + [ + "github actions" + ] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": [ + "github actions" + ], + "policy_rule": "unverified_provenance", + "reason": "provenance_publisher_not_verified", + "rule": "unverified_provenance", + "source": "block_on", + "suppression": null, + "threshold": [ + "manual upload" + ] + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P14", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M24", + "perturbation": "publisher allowlist membership", + "perturbed_case_id": "M25", + "previous_decision": "FAIL", + "reason": "The recorded publisher is explicitly authorized.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "attested", + "attested" + ], + "publishers": [ + [ + "github actions" + ], + [ + "github actions" + ] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "allow_provenance_publishers": [ + "github actions" + ], + "block_on": [ + "unverified_provenance" + ], + "version": 2 + }, + "before": { + "allow_provenance_publishers": [ + "github actions" + ], + "block_on": [ + "unverified_provenance" + ], + "version": 2 + } + }, + "after": { + "assessment": [ + true, + false, + true, + [ + "github actions", + "manual upload" + ] + ], + "decision": "PASS", + "effective_version": 2, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + true, + false, + true, + [ + "github actions" + ] + ], + "decision": "PASS", + "effective_version": 2, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "STABLE", + "id": "P15", + "observed_classification": "STABLE", + "original_case_id": "M25", + "perturbation": "additional publisher kind", + "perturbed_case_id": "M26", + "previous_decision": "PASS", + "reason": "The existing allowlist contract requires any matching kind.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "attested", + "attested" + ], + "publishers": [ + [ + "github actions" + ], + [ + "manual upload", + "github actions" + ] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "require_attestations_for_new_packages": true, + "version": 2 + }, + "before": { + "require_attestations_for_new_packages": true, + "version": 2 + } + }, + "after": { + "assessment": [ + false, + false, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "attestation_missing", + "policy_rule": "provenance_required", + "reason": "required_provenance_not_satisfied", + "rule": "provenance_required", + "source": "default_block", + "suppression": null, + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "provenance_unavailable", + "policy_rule": "provenance_required", + "reason": "required_provenance_not_satisfied", + "rule": "provenance_required", + "source": "default_block", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "STABLE", + "id": "P16", + "observed_classification": "STABLE", + "original_case_id": "M27", + "perturbation": "unknown to known missing attestation", + "perturbed_case_id": "M28", + "previous_decision": "FAIL", + "reason": "Required provenance remains unsatisfied; explanation evidence changes.", + "resulting_decision": "FAIL", + "source_evidence_changes": { + "provenance": [ + "none", + "missing" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "require_attestations_for_new_packages": true, + "version": 2 + }, + "before": { + "require_attestations_for_new_packages": true, + "version": 2 + } + }, + "after": { + "assessment": [ + true, + false, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "provenance_unverified", + "policy_rule": "provenance_required", + "reason": "required_provenance_not_satisfied", + "rule": "provenance_required", + "source": "default_block", + "suppression": null, + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + false, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "attestation_missing", + "policy_rule": "provenance_required", + "reason": "required_provenance_not_satisfied", + "rule": "provenance_required", + "source": "default_block", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "STABLE", + "id": "P17", + "observed_classification": "STABLE", + "original_case_id": "M28", + "perturbation": "missing to unverified attestation", + "perturbed_case_id": "M29", + "previous_decision": "FAIL", + "reason": "Presence alone does not satisfy a provenance requirement.", + "resulting_decision": "FAIL", + "source_evidence_changes": { + "provenance": [ + "missing", + "attested" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "require_attestations_for_new_packages": true, + "version": 2 + }, + "before": { + "require_attestations_for_new_packages": true, + "version": 2 + } + }, + "after": { + "assessment": [ + true, + false, + true, + [ + "github actions" + ] + ], + "decision": "PASS", + "effective_version": 2, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + true, + false, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "provenance_unverified", + "policy_rule": "provenance_required", + "reason": "required_provenance_not_satisfied", + "rule": "provenance_required", + "source": "default_block", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P18", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M29", + "perturbation": "recorded publisher identity supplied", + "perturbed_case_id": "M30", + "previous_decision": "FAIL", + "reason": "Recorded publisher metadata satisfies the existing requirement, without a cryptographic claim.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "attested", + "attested" + ], + "publishers": [ + [], + [ + "github actions" + ] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "require_attestations_for_new_packages": true, + "version": 2, + "warn_on": [ + "provenance_required" + ] + }, + "before": { + "require_attestations_for_new_packages": true, + "version": 2 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "WARN", + "effective_version": 2, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "warning", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "warn", + "name": "qualification-alpha", + "observed": "provenance_unavailable", + "policy_rule": "provenance_required", + "reason": "required_provenance_not_satisfied", + "rule": "provenance_required", + "source": "warn_on", + "suppression": null, + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "provenance_unavailable", + "policy_rule": "provenance_required", + "reason": "required_provenance_not_satisfied", + "rule": "provenance_required", + "source": "default_block", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P19", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M27", + "perturbation": "required rule warning override", + "perturbed_case_id": "M31", + "previous_decision": "FAIL", + "reason": "Explicit warn overrides implicit blocking.", + "resulting_decision": "WARN", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "ignore_rules": [ + "provenance_required" + ], + "require_attestations_for_new_packages": true, + "version": 2 + }, + "before": { + "require_attestations_for_new_packages": true, + "version": 2 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 2, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 1, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "suppressed", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "provenance_unavailable", + "policy_rule": "provenance_required", + "reason": "required_provenance_not_satisfied", + "rule": "provenance_required", + "source": "default_block", + "suppression": "ignored_by_policy", + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "provenance_unavailable", + "policy_rule": "provenance_required", + "reason": "required_provenance_not_satisfied", + "rule": "provenance_required", + "source": "default_block", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P20", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M27", + "perturbation": "ignore required rule", + "perturbed_case_id": "M32", + "previous_decision": "FAIL", + "reason": "An explicit suppression is retained.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "allow_unattested_packages": [ + "qualification-alpha" + ], + "require_attestations_for_new_packages": true, + "version": 2 + }, + "before": { + "require_attestations_for_new_packages": true, + "version": 2 + } + }, + "after": { + "assessment": [ + false, + false, + false, + [] + ], + "decision": "PASS", + "effective_version": 2, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + false, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "attestation_missing", + "policy_rule": "provenance_required", + "reason": "required_provenance_not_satisfied", + "rule": "provenance_required", + "source": "default_block", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P21", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M28", + "perturbation": "authorize package exception", + "perturbed_case_id": "M65", + "previous_decision": "FAIL", + "reason": "A name exception may satisfy a requirement caused solely by missing attestations.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "missing", + "missing" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "allow_unattested_packages": [ + "qualification-alpha" + ], + "block_on": [ + "missing_attestation" + ], + "version": 2 + }, + "before": { + "block_on": [ + "missing_attestation" + ], + "version": 2 + } + }, + "after": { + "assessment": [ + false, + false, + false, + [] + ], + "decision": "PASS", + "effective_version": 2, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + false, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": false, + "policy_rule": "missing_attestation", + "reason": "attestation_not_published", + "rule": "missing_attestation", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P22", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M20", + "perturbation": "authorize missing-attestation exception", + "perturbed_case_id": "M66", + "previous_decision": "FAIL", + "reason": "Only the missing-attestation check is waived.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "missing", + "missing" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "allow_unattested_packages": [ + "qualification-alpha" + ], + "require_attestations_for_new_packages": true, + "version": 2 + }, + "before": { + "require_attestations_for_new_packages": true, + "version": 2 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "provenance_unavailable", + "policy_rule": "provenance_required", + "reason": "required_provenance_not_satisfied", + "rule": "provenance_required", + "source": "default_block", + "suppression": null, + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "provenance_unavailable", + "policy_rule": "provenance_required", + "reason": "required_provenance_not_satisfied", + "rule": "provenance_required", + "source": "default_block", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "STABLE", + "id": "P23", + "observed_classification": "STABLE", + "original_case_id": "M27", + "perturbation": "add exception to unavailable evidence", + "perturbed_case_id": "M67", + "previous_decision": "FAIL", + "reason": "A package exception does not waive unavailable provenance.", + "resulting_decision": "FAIL", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "allow_unattested_packages": [ + "qualification-alpha" + ], + "block_on": [ + "unverified_provenance" + ], + "version": 2 + }, + "before": { + "block_on": [ + "unverified_provenance" + ], + "version": 2 + } + }, + "after": { + "assessment": [ + true, + false, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": [], + "policy_rule": "unverified_provenance", + "reason": "provenance_publisher_not_verified", + "rule": "unverified_provenance", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + true, + false, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": [], + "policy_rule": "unverified_provenance", + "reason": "provenance_publisher_not_verified", + "rule": "unverified_provenance", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "STABLE", + "id": "P24", + "observed_classification": "STABLE", + "original_case_id": "M22", + "perturbation": "add exception to unverified evidence", + "perturbed_case_id": "M68", + "previous_decision": "FAIL", + "reason": "A package exception does not waive an unverified publisher.", + "resulting_decision": "FAIL", + "source_evidence_changes": { + "provenance": [ + "attested", + "attested" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "provenance_unavailable" + ], + "require_attestations_for_new_packages": true, + "version": 2 + }, + "before": { + "block_on": [ + "provenance_unavailable" + ], + "require_attestations_for_new_packages": true, + "version": 2 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "provenance_unavailable", + "policy_rule": "provenance_required", + "reason": "required_provenance_not_satisfied", + "rule": "provenance_required", + "source": "default_block", + "suppression": null, + "threshold": null + }, + { + "bucket": null, + "confidence": "provenance_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "unavailable", + "policy_rule": "provenance_unavailable", + "reason": "provenance_evidence_unavailable", + "rule": "provenance_unavailable", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + true, + false, + true, + [ + "github actions" + ] + ], + "decision": "PASS", + "effective_version": 2, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P25", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M69", + "perturbation": "add contradictory enrichment error", + "perturbed_case_id": "M41", + "previous_decision": "PASS", + "reason": "An explicit error dominates attestation availability.", + "resulting_decision": "FAIL", + "source_evidence_changes": { + "provenance": [ + "attested", + "error-and-attested" + ], + "publishers": [ + [ + "github actions" + ], + [ + "github actions" + ] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "scorecard_below_threshold" + ], + "minimum_scorecard_score": 7, + "version": 3 + }, + "before": { + "minimum_scorecard_score": 7, + "version": 3 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 3, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "scorecard_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": 6.9, + "policy_rule": "scorecard_below_threshold", + "reason": "scorecard_score_below_threshold", + "rule": "scorecard_below_threshold", + "source": "block_on", + "suppression": null, + "threshold": 7.0 + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 3, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P26", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M42", + "perturbation": "activate Scorecard enforcement", + "perturbed_case_id": "M43", + "previous_decision": "PASS", + "reason": "A threshold alone is advisory; the block rule activates enforcement.", + "resulting_decision": "FAIL", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + 6.9, + 6.9 + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "scorecard_below_threshold" + ], + "minimum_scorecard_score": 7, + "version": 3 + }, + "before": { + "block_on": [ + "scorecard_below_threshold" + ], + "minimum_scorecard_score": 7, + "version": 3 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 3, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 3, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "scorecard_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": 6.9, + "policy_rule": "scorecard_below_threshold", + "reason": "scorecard_score_below_threshold", + "rule": "scorecard_below_threshold", + "source": "block_on", + "suppression": null, + "threshold": 7.0 + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P27", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M43", + "perturbation": "score reaches threshold", + "perturbed_case_id": "M44", + "previous_decision": "FAIL", + "reason": "Equality is permitted; only a lower score violates.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + 6.9, + 7.0 + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "scorecard_below_threshold" + ], + "minimum_scorecard_score": 7, + "version": 3 + }, + "before": { + "block_on": [ + "scorecard_below_threshold" + ], + "minimum_scorecard_score": 7, + "version": 3 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 3, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 3, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "STABLE", + "id": "P28", + "observed_classification": "STABLE", + "original_case_id": "M44", + "perturbation": "score above threshold", + "perturbed_case_id": "M45", + "previous_decision": "PASS", + "reason": "Both scores meet the minimum.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + 7.0, + 7.1 + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "minimum_scorecard_score": 7, + "version": 3, + "warn_on": [ + "scorecard_below_threshold" + ] + }, + "before": { + "block_on": [ + "scorecard_below_threshold" + ], + "minimum_scorecard_score": 7, + "version": 3 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "WARN", + "effective_version": 3, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "scorecard_recorded", + "group": "warning", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "warn", + "name": "qualification-alpha", + "observed": 6.9, + "policy_rule": "scorecard_below_threshold", + "reason": "scorecard_score_below_threshold", + "rule": "scorecard_below_threshold", + "source": "warn_on", + "suppression": null, + "threshold": 7.0 + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 3, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "scorecard_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": 6.9, + "policy_rule": "scorecard_below_threshold", + "reason": "scorecard_score_below_threshold", + "rule": "scorecard_below_threshold", + "source": "block_on", + "suppression": null, + "threshold": 7.0 + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P29", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M43", + "perturbation": "Scorecard severity", + "perturbed_case_id": "M47", + "previous_decision": "FAIL", + "reason": "Explicit warn changes the policy decision but does not erase evidence.", + "resulting_decision": "WARN", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + 6.9, + 6.9 + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "ignore_rules": [ + "scorecard_below_threshold" + ], + "minimum_scorecard_score": 7, + "version": 3 + }, + "before": { + "minimum_scorecard_score": 7, + "version": 3 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 3, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 1, + "rows": [ + { + "bucket": null, + "confidence": "scorecard_recorded", + "group": "suppressed", + "key": "purl:pkg:pypi/qualification-alpha", + "level": null, + "name": "qualification-alpha", + "observed": 6.9, + "policy_rule": "scorecard_below_threshold", + "reason": "scorecard_score_below_threshold", + "rule": "scorecard_below_threshold", + "source": null, + "suppression": "ignored_by_policy", + "threshold": 7.0 + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 3, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "STABLE", + "id": "P30", + "observed_classification": "STABLE", + "original_case_id": "M42", + "perturbation": "ignore advisory Scorecard", + "perturbed_case_id": "M48", + "previous_decision": "PASS", + "reason": "Decision stays pass while ignored evidence becomes visible.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + 6.9, + 6.9 + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "scorecard_below_threshold" + ], + "minimum_scorecard_score": 7, + "version": 3 + }, + "before": { + "block_on": [ + "scorecard_below_threshold" + ], + "minimum_scorecard_score": 7, + "version": 3 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 3, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 3, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "scorecard_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": 6.9, + "policy_rule": "scorecard_below_threshold", + "reason": "scorecard_score_below_threshold", + "rule": "scorecard_below_threshold", + "source": "block_on", + "suppression": null, + "threshold": 7.0 + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P31", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M43", + "perturbation": "Scorecard availability lost", + "perturbed_case_id": "M49", + "previous_decision": "FAIL", + "reason": "Unknown scores are not fabricated into threshold violations under current policy.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + 6.9, + 6.9 + ], + "score_available": [ + true, + false + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "scorecard_below_threshold" + ], + "minimum_scorecard_score": 6, + "version": 3 + }, + "before": { + "block_on": [ + "scorecard_below_threshold" + ], + "minimum_scorecard_score": 7, + "version": 3 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 3, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 3, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": null, + "confidence": "scorecard_recorded", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": 6.9, + "policy_rule": "scorecard_below_threshold", + "reason": "scorecard_score_below_threshold", + "rule": "scorecard_below_threshold", + "source": "block_on", + "suppression": null, + "threshold": 7.0 + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "P32", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "M43", + "perturbation": "lower Scorecard threshold", + "perturbed_case_id": "M70", + "previous_decision": "FAIL", + "reason": "6.9 meets a configured minimum of 6.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + 6.9, + 6.9 + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "new_package" + ], + "version": 2 + }, + "before": { + "block_on": [ + "new_package" + ], + "version": 1 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 2, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "FAIL", + "effective_version": 1, + "exit_code": 1, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [ + { + "bucket": "new_package", + "confidence": "policy_matched", + "group": "blocking", + "key": "purl:pkg:pypi/qualification-alpha", + "level": "block", + "name": "qualification-alpha", + "observed": "new_package", + "policy_rule": "new_package", + "reason": "risk_finding_matched_policy_rule", + "rule": "new_package", + "source": "block_on", + "suppression": null, + "threshold": null + } + ] + }, + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "STABLE", + "id": "P33", + "observed_classification": "STABLE", + "original_case_id": "M03", + "perturbation": "policy version 1 to 2", + "perturbed_case_id": "M71", + "previous_decision": "FAIL", + "reason": "An existing local rule retains its severity across capability versions.", + "resulting_decision": "FAIL", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "after": { + "block_on": [ + "scorecard_below_threshold" + ], + "version": 3 + }, + "before": { + "block_on": [ + "scorecard_below_threshold" + ], + "version": 2 + } + }, + "after": { + "assessment": [ + false, + true, + false, + [] + ], + "decision": "PASS", + "effective_version": 3, + "exit_code": 0, + "identity": "purl:pkg:pypi/qualification-alpha", + "ignored_checks": 0, + "rows": [] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before": { + "decision": "INVALID_INPUT", + "error_type": "PolicyError", + "exit_code": 2 + }, + "before_canonical_identity": null, + "expected_classification": "INVALID_INPUT", + "id": "P34", + "observed_classification": "INVALID_INPUT", + "original_case_id": "M51", + "perturbation": "policy version 2 to 3", + "perturbed_case_id": "M72", + "previous_decision": "INVALID_INPUT", + "reason": "A Scorecard rule is rejected in v2 and eligible in v3.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "provenance": [ + "none", + "none" + ], + "publishers": [ + [], + [] + ], + "score": [ + null, + null + ], + "score_available": [ + true, + true + ] + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "before_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "expected_classification": "STABLE", + "id": "R01", + "observed_classification": "STABLE", + "original_case_id": "R01-left", + "perturbation": "Supported identity equivalence preserves enforcement; original evidence remains observable.", + "previous_decision": "FAIL", + "reason": "Supported identity equivalence preserves enforcement; original evidence remains observable.", + "record_order": [ + "Qualification_Alpha" + ], + "resulting_decision": "FAIL", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "Qualification_Alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": [ + "purl:pkg:npm/%40scope/qualification" + ], + "before_canonical_identity": [ + "purl:pkg:npm/%40scope/qualification" + ], + "expected_classification": "STABLE", + "id": "R02", + "observed_classification": "STABLE", + "original_case_id": "R02-left", + "perturbation": "Supported identity equivalence preserves enforcement; original evidence remains observable.", + "previous_decision": "FAIL", + "reason": "Supported identity equivalence preserves enforcement; original evidence remains observable.", + "record_order": [ + "qualification" + ], + "resulting_decision": "FAIL", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification", + "purl": "pkg:npm/%40scope/qualification@1.0.0", + "version": "1.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "Qualification", + "purl": "pkg:npm/%40scope/Qualification@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "major_upgrade" + ], + "version": 1 + }, + "after_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "before_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "expected_classification": "STABLE", + "id": "R03", + "observed_classification": "STABLE", + "original_case_id": "R03-left", + "perturbation": "Supported identity equivalence preserves enforcement; original evidence remains observable.", + "previous_decision": "FAIL", + "reason": "Supported identity equivalence preserves enforcement; original evidence remains observable.", + "record_order": [ + "qualification-alpha" + ], + "resulting_decision": "FAIL", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@2.0.0", + "version": "2.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@2.0.0", + "version": null + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "before_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "expected_classification": "STABLE", + "id": "R04", + "observed_classification": "STABLE", + "original_case_id": "R04-left", + "perturbation": "Supported identity equivalence preserves enforcement; original evidence remains observable.", + "previous_decision": "FAIL", + "reason": "Supported identity equivalence preserves enforcement; original evidence remains observable.", + "record_order": [ + "qualification-alpha" + ], + "resulting_decision": "FAIL", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "bom-ref": "opaque:qualification", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": null, + "before_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "expected_classification": "INVALID_INPUT", + "id": "R05", + "observed_classification": "INVALID_INPUT", + "original_case_id": "R05-left", + "perturbation": "Duplicate or conflicting identities must be rejected before report completion.", + "previous_decision": "FAIL", + "reason": "Duplicate or conflicting identities must be rejected before report completion.", + "record_order": [ + "qualification-alpha", + "qualification-alpha" + ], + "resulting_decision": "INVALID_INPUT", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + }, + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": null, + "before_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "expected_classification": "INVALID_INPUT", + "id": "R06", + "observed_classification": "INVALID_INPUT", + "original_case_id": "R06-left", + "perturbation": "Duplicate or conflicting identities must be rejected before report completion.", + "previous_decision": "FAIL", + "reason": "Duplicate or conflicting identities must be rejected before report completion.", + "record_order": [ + "qualification-alpha", + "qualification-alpha" + ], + "resulting_decision": "INVALID_INPUT", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + }, + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "BSD-3-Clause" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": null, + "before_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "expected_classification": "INVALID_INPUT", + "id": "R07", + "observed_classification": "INVALID_INPUT", + "original_case_id": "R07-left", + "perturbation": "An explicit version conflicts with its PURL version.", + "previous_decision": "FAIL", + "reason": "An explicit version conflicts with its PURL version.", + "record_order": [ + "qualification-alpha" + ], + "resulting_decision": "INVALID_INPUT", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "2.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": null, + "before_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "expected_classification": "INVALID_INPUT", + "id": "R08", + "observed_classification": "INVALID_INPUT", + "original_case_id": "R08-left", + "perturbation": "An explicit name conflicts with its PURL name.", + "previous_decision": "FAIL", + "reason": "An explicit name conflicts with its PURL name.", + "record_order": [ + "another-package" + ], + "resulting_decision": "INVALID_INPUT", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "another-package", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": [ + "bom-ref:opaque:x" + ], + "before_canonical_identity": [ + "bom-ref:Opaque:X" + ], + "expected_classification": "STABLE", + "id": "R09", + "observed_classification": "STABLE", + "original_case_id": "R09-left", + "perturbation": "Opaque references remain distinct; identical policy outcomes do not assert identity equivalence.", + "previous_decision": "FAIL", + "reason": "Opaque references remain distinct; identical policy outcomes do not assert identity equivalence.", + "record_order": [ + "qualification-alpha" + ], + "resulting_decision": "FAIL", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "bom-ref": "opaque:x", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "version": "1.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "bom-ref": "Opaque:X", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": [ + "purl:pkg:generic/qualification" + ], + "before_canonical_identity": [ + "purl:pkg:generic/Qualification" + ], + "expected_classification": "STABLE", + "id": "R10", + "observed_classification": "STABLE", + "original_case_id": "R10-left", + "perturbation": "Unsupported ecosystem case equivalence is not invented.", + "previous_decision": "FAIL", + "reason": "Unsupported ecosystem case equivalence is not invented.", + "record_order": [ + "qualification" + ], + "resulting_decision": "FAIL", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification", + "purl": "pkg:generic/qualification@1.0.0", + "version": "1.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "Qualification", + "purl": "pkg:generic/Qualification@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": [ + "purl:pkg:generic/qualification-alpha" + ], + "before_canonical_identity": [ + "purl:pkg:generic/qualification-alpha" + ], + "expected_classification": "STABLE", + "id": "R11", + "observed_classification": "STABLE", + "original_case_id": "R11-left", + "perturbation": "Supported identity equivalence preserves enforcement; original evidence remains observable.", + "previous_decision": "FAIL", + "reason": "Supported identity equivalence preserves enforcement; original evidence remains observable.", + "record_order": [ + "qualification-alpha" + ], + "resulting_decision": "FAIL", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "description": "Irrelevant synthetic description", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:generic/qualification-alpha", + "version": null + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:generic/qualification-alpha", + "version": null + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": null, + "before_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "expected_classification": "INVALID_INPUT", + "id": "R12", + "observed_classification": "INVALID_INPUT", + "original_case_id": "R12-left", + "perturbation": "Malformed identity is rejected rather than silently falling back.", + "previous_decision": "FAIL", + "reason": "Malformed identity is rejected rather than silently falling back.", + "record_order": [ + "qualification-alpha" + ], + "resulting_decision": "INVALID_INPUT", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "not-a-purl", + "version": "1.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "before_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "expected_classification": "STABLE", + "id": "R13", + "observed_classification": "STABLE", + "original_case_id": "R13-left", + "perturbation": "Only the shared component, version, license and source subset is compared.", + "previous_decision": "FAIL", + "reason": "Only the shared component, version, license and source subset is compared.", + "record_order": [], + "resulting_decision": "FAIL", + "source_evidence_changes": { + "after": { + "packages": [ + { + "downloadLocation": "https://github.com/example/qualification", + "externalRefs": [ + { + "referenceLocator": "pkg:pypi/qualification-alpha@1.0.0", + "referenceType": "purl" + } + ], + "licenseConcluded": "MIT", + "name": "qualification-alpha", + "versionInfo": "1.0.0" + } + ], + "spdxVersion": "SPDX-2.3" + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha", + "purl:pkg:pypi/qualification-beta", + "purl:pkg:pypi/qualification-delta", + "purl:pkg:pypi/qualification-gamma" + ], + "before_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha", + "purl:pkg:pypi/qualification-beta", + "purl:pkg:pypi/qualification-delta", + "purl:pkg:pypi/qualification-gamma" + ], + "expected_classification": "STABLE", + "id": "R14", + "observed_classification": "STABLE", + "original_case_id": "R14-left", + "perturbation": "Record ordering is irrelevant; each round records its actual permutation.", + "previous_decision": "FAIL", + "reason": "Record ordering is irrelevant; each round records its actual permutation.", + "record_order": [ + "qualification-beta", + "qualification-alpha", + "qualification-gamma", + "qualification-delta" + ], + "resulting_decision": "FAIL", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-beta", + "purl": "pkg:pypi/qualification-beta@1.0.0", + "version": "1.0.0" + }, + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + }, + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-gamma", + "purl": "pkg:pypi/qualification-gamma@1.0.0", + "version": "1.0.0" + }, + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-delta", + "purl": "pkg:pypi/qualification-delta@1.0.0", + "version": "1.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + }, + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-beta", + "purl": "pkg:pypi/qualification-beta@1.0.0", + "version": "1.0.0" + }, + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-gamma", + "purl": "pkg:pypi/qualification-gamma@1.0.0", + "version": "1.0.0" + }, + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-delta", + "purl": "pkg:pypi/qualification-delta@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "allow_sources": [ + "pypi.org" + ], + "version": 1 + }, + "after_canonical_identity": [], + "before_canonical_identity": [], + "expected_classification": "STABLE", + "id": "R15", + "observed_classification": "STABLE", + "original_case_id": "R15-left", + "perturbation": "Raw ignored metadata does not place an unchanged component into policy scope.", + "previous_decision": "PASS", + "reason": "Raw ignored metadata does not place an unchanged component into policy scope.", + "record_order": [ + "qualification-alpha" + ], + "resulting_decision": "PASS", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "description": "Ignored synthetic annotation", + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "allow_sources": [ + "pypi.org" + ], + "version": 1 + }, + "after_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "before_canonical_identity": [], + "expected_classification": "JUSTIFIED_REVERSAL", + "id": "R16", + "observed_classification": "JUSTIFIED_REVERSAL", + "original_case_id": "R16-left", + "perturbation": "Compared license metadata creates a changed component, so its existing disallowed source enters policy scope.", + "previous_decision": "PASS", + "reason": "Compared license metadata creates a changed component, so its existing disallowed source enters policy scope.", + "record_order": [ + "qualification-alpha" + ], + "resulting_decision": "FAIL", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "BSD-3-Clause" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "before_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "expected_classification": "STABLE", + "id": "R17", + "observed_classification": "STABLE", + "original_case_id": "R17-left", + "perturbation": "Supported field trimming and JSON serialization do not change identity or policy.", + "previous_decision": "FAIL", + "reason": "Supported field trimming and JSON serialization do not change identity or policy.", + "record_order": [ + " qualification-alpha " + ], + "resulting_decision": "FAIL", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": " qualification-alpha ", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": " 1.0.0 " + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "block_on": [ + "new_package" + ], + "version": 1 + }, + "after_canonical_identity": null, + "before_canonical_identity": [ + "purl:pkg:pypi/qualification-alpha" + ], + "expected_classification": "INVALID_INPUT", + "id": "R18", + "observed_classification": "INVALID_INPUT", + "original_case_id": "R18-left", + "perturbation": "A non-string component version is invalid input.", + "previous_decision": "FAIL", + "reason": "A non-string component version is invalid input.", + "record_order": [ + "qualification-alpha" + ], + "resulting_decision": "INVALID_INPUT", + "source_evidence_changes": { + "after": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": 1 + } + ] + }, + "before": { + "bomFormat": "CycloneDX", + "components": [ + { + "externalReferences": [ + { + "type": "vcs", + "url": "https://github.com/example/qualification" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "name": "qualification-alpha", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "version": "1.0.0" + } + ] + } + } + }, + { + "active_policy": { + "version": 1, + "warn_on": [ + "unknown_license" + ] + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "STABLE", + "id": "R19", + "observed_classification": "STABLE", + "original_case_id": "requirements-exact-pin", + "perturbation": "requirements to pyproject exact-pin shared subset", + "previous_decision": "WARN", + "reason": "Exact pins share coordinate identity and unknown-license evidence.", + "resulting_decision": "WARN", + "source_evidence_changes": "Source formats differ; optional groups, markers and unknown constraints are excluded." + }, + { + "active_policy": { + "allow_unattested_packages": [ + "Qualification_Alpha" + ], + "block_on": [ + "missing_attestation" + ], + "version": 2 + }, + "after_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "before_canonical_identity": "purl:pkg:pypi/qualification-alpha", + "expected_classification": "STABLE", + "id": "C02", + "observed_classification": "STABLE", + "original_case_id": "E08", + "perturbation": "PEP 503-equivalent spelling only", + "previous_decision": "PASS", + "reason": "C02 POLICY DECISION APPROVED: canonical PyPI package-name equality; unchanged exception boundary.", + "resulting_decision": "PASS", + "source_evidence_changes": { + "observed_names": [ + "qualification-alpha", + "Qualification_Alpha", + "qualification.alpha" + ] + } + }, + { + "active_policy": { + "block_on": [ + "provenance_unavailable" + ], + "version": 2 + }, + "after_canonical_identity": { + "key": "purl:pkg:pypi/demo-pkg", + "version": "1.0.0" + }, + "before_canonical_identity": { + "key": "purl:pkg:pypi/demo-pkg", + "version": "1.0.0" + }, + "expected_classification": "STABLE", + "id": "C03", + "observed_classification": "STABLE", + "perturbation": "Agreeing explicit versus PURL-only release version", + "previous_decision": "PASS", + "reason": "CONTRACT_CLARIFIED_CANONICAL_RELEASE_VERSION", + "resulting_decision": "PASS", + "source_evidence_changes": { + "enrichment_enabled": true, + "raw_versions": [ + null, + "1.0.0" + ] + } + } + ], + "classification_counts": { + "INVALID_INPUT": 7, + "JUSTIFIED_REVERSAL": 23, + "STABLE": 25, + "SUSPICIOUS_REVERSAL": 0, + "UNRESOLVED": 0 + }, + "distinct_round_seeds": [ + 103, + 117, + 120 + ], + "record_order_permutations": [ + { + "order": [ + "qualification-beta", + "qualification-alpha", + "qualification-gamma", + "qualification-delta" + ], + "seed": 103 + }, + { + "order": [ + "qualification-delta", + "qualification-gamma", + "qualification-alpha", + "qualification-beta" + ], + "seed": 117 + }, + { + "order": [ + "qualification-gamma", + "qualification-delta", + "qualification-alpha", + "qualification-beta" + ], + "seed": 120 + } + ], + "representative_seed": 103, + "round_classifications": [ + { + "counts": { + "INVALID_INPUT": 7, + "JUSTIFIED_REVERSAL": 23, + "STABLE": 25 + }, + "seed": 103 + }, + { + "counts": { + "INVALID_INPUT": 7, + "JUSTIFIED_REVERSAL": 23, + "STABLE": 25 + }, + "seed": 117 + }, + { + "counts": { + "INVALID_INPUT": 7, + "JUSTIFIED_REVERSAL": 23, + "STABLE": 25 + }, + "seed": 120 + } + ], + "rounds_do_not_multiply_independent_cases": true, + "schema": "local-decision-stability-study.v1", + "status": "EXECUTED_AUDIT_PENDING", + "unique_transition_catalog": 55, + "unresolved_observed_reversals": 0 +} diff --git a/tools/sbom-diff-and-risk/docs/validation/decision-stability/TRUST-BOUNDARY-MATRIX.md b/tools/sbom-diff-and-risk/docs/validation/decision-stability/TRUST-BOUNDARY-MATRIX.md new file mode 100644 index 0000000..9c7be1c --- /dev/null +++ b/tools/sbom-diff-and-risk/docs/validation/decision-stability/TRUST-BOUNDARY-MATRIX.md @@ -0,0 +1,60 @@ +# Trust boundary and authority matrix + +This is a qualification contract freeze at base `29203f9f14e2a3df2ad77a4e874bf752bf44c599`, with the explicitly approved C02 clarification. Three synthetic offline rounds have executed; final audit disposition and C03 remain separate gates. + +## Pipeline and authority + +| Stage | Input authority | Conversion/default | Permitted conclusion | Unsupported inference / failure boundary | +| --- | --- | --- | --- | --- | +| Parsers | Untrusted CycloneDX, SPDX, requirements or pyproject bytes | Supported syntax only; retain raw records and conservative parser notes | Observed dependency fields | Parsing is not verification; strict parser notes may exit 2 | +| Canonical identity | Registered ecosystem rules and parsed PURL coordinate | PURL then opaque bom_ref then ecosystem/name; trim versions without resolving precedence | Comparison/alignment key | Contradictions/duplicates fail closed; no universal aliasing | +| Diff | Canonical identity and normalized compared metadata | Added/removed/changed; sorted keys | Observed change scope | Metadata changes can bring an existing package into policy scope | +| Local risk | Diff and local metadata | Heuristic buckets, with unavailable stale evaluation explicitly `not_evaluated` | Local prioritization evidence | No CVE, malware, exploitability or safety verdict | +| PyPI evidence | Explicit opt-in provider record | Requested/supported/lookup/status/file/publisher fields remain distinct; errors dominate mixed success | Recorded release/attestation facts under the existing provider contract | Publisher-kind matching is not newly introduced cryptographic verification | +| Scorecard evidence | Explicit high-confidence source mapping plus requested response | No mapping or unavailable response retains an unknown/error state | Auxiliary repository evidence | Repository identity must not be silently replaced by contradictory response metadata | +| Policy parser | Explicit user policy and CLI rule flags | Schema/capability validation; flags merge, block wins, capability can rise | Authorized local rule configuration | Unknown keys/rules/types must not silently become policy | +| Policy evaluation | Added/changed after components, findings and recorded evidence | Exact rule matching, explicit defaults, suppression evidence, deterministic sort | PASS/WARN/FAIL for an applied policy | Unknown is not verified; exceptions have narrow documented scope | +| Confidence | Recorded evidence source categories | Scorecard, provenance, matched policy, SBOM, manifest precedence | Source label | Not probability, safety or universal evidence strength | +| Reports/exit | One policy evaluation and observed records | Exit 0/1/2; full JSON plus summary/policy/Markdown and restricted SARIF | Auditable local decision and its reason | SARIF is deliberately not equal to full JSON; error exit is not policy FAIL | +| CI consumer | Schema-checked report and actual CLI exit | Disclose warnings and gaps; missing policy is not a PASS finding | Local gate and reviewer routing | Consumer `needs-review` is not a native tool exit/status | +| Tool release provenance | Distribution, workflow and separate attestation authority | A different verification procedure for the tool itself | Tool artifact provenance, when independently verified | Does not establish third-party dependency provenance | + +## Rules, assumptions and failure modes + +Rule matching scope is added and changed after-components; removed/unchanged components are not silently added to policy scope. Risk rules have no default active severity. Threshold/source/explicit provenance requirements default to block when triggered. Explicit warn can override such a default; explicit block wins explicit warn. Ignore suppresses and preserves a decision record, including a match with no active severity. + +| Rule | Evidence / authority | Assumption and supported output | Unsupported inference / expected failure mode | Existing executable anchors | +| --- | --- | --- | --- | --- | +| `new_package` | Canonical added set / local diff | Heuristic exists; policy only when configured | New does not mean vulnerable; missing/spurious violation is a defect | test_diffing, test_policy, test_version_decision_reports | +| `major_upgrade` | Known canonical versions / local syntax heuristic | Strict eligible syntax and increasing major only | No ecosystem precedence or API compatibility proof | test_version_decisions, test_risk | +| `version_change_unclassified` | Known unequal versions / local diff | Explicit uncertainty, independently selectable rule | Not equivalent to a major upgrade or vulnerability | test_risk, test_version_evidence | +| `unknown_license` | Missing/unknown license field | Missing metadata, not an inferred license fact | Do not guess a license or infer compromise | test_risk, test_policy | +| `suspicious_source` | Local URL/scheme/host | Provenance-quality heuristic | Not malware; provenance requirement can supersede its SARIF presentation | test_risk, test_sarif | +| `stale_package` | `stale_package` or `not_evaluated` finding | Offline unanswered state can match an explicit stale policy | Do not turn unavailable evaluation into age evidence | test_policy, test_cli_no_enrichment_regression | +| `max_added_packages` | Added count and integer threshold / explicit policy | Trigger only when count exceeds threshold; default block | Boundary equality must not fail; Boolean is not an integer-policy input | test_policy, test_cli_exit_codes | +| `allow_sources` | Added/changed `source_url` host and policy host list | Exact normalized host membership; default block | No suffix/subdomain or reputation inference; missing host can fail | test_policy, test_cli_policy_json | +| `provenance_unavailable` | Absent, unsupported or failed provider evidence | Distinct unavailable rule, active only by explicit configuration | Not missing attestations and not verified evidence | test_policy_provenance | +| `missing_attestation` | Successful recorded lookup with no attestation | Explicit rule; package exception waives only this absence | Exception must not waive unavailable/unverified evidence | test_policy_provenance | +| `unverified_provenance` | Attestation present but publisher metadata absent/mismatched | Explicit publisher-kind rule | No new cryptographic proof or implicit permission from missing data | test_policy_provenance, test_cli_provenance_policy | +| `provenance_required` | Explicit new-PyPI/suspicious-source requirement | Default block for unmet requirement; explicit warn/ignore respected | The requirement is not activated for every package by merely naming a rule | test_policy_provenance | +| `scorecard_below_threshold` | Available recorded score and finite configured threshold | Below, not equal; only explicit block/warn/ignore activation | Absent/unmapped is not proof of low score; high score is not safety | test_policy_scorecard, test_scorecard_enrichment | + +`allow_unattested_packages`, `allow_provenance_publishers`, version eligibility and CLI merging are policy controls, not additional native risk buckets. Publisher aliases are mutually exclusive keys. Version capability (1/2/3) is distinct from serialized `sbom-diff-risk.policy.v1`; reports remain `sbom-diff-risk.report.v1`. + +## Exceptions and unresolved authority + +A PyPI package exception matches PEP 503-equivalent names in both policy entries +and component names under the explicitly approved C02 decision. It does not +waive unavailable lookup evidence, unverified publisher evidence or unrelated +rules. Authorization remains at package-name scope; no full-PURL, version-specific, +fuzzy or cross-ecosystem matching is introduced, and serialized policy spelling +retains the existing parser behavior. + +C03 is explicitly approved as `CONTRACT_CLARIFIED_CANONICAL_RELEASE_VERSION`. +Opt-in PyPI lookups use canonical ecosystem/name/version coordinates, including +PURL-only versions. Both CLI inputs are indexed before either provider starts; +direct PyPI calls validate identity, and batch calls validate all items first. +No version means unsupported, never latest. Raw component metadata is preserved; +PURL coordinates do not confer authenticity or publisher authorization. + +Qualification calls real parsing/normalization/evaluation/report code with controlled synthetic providers. Provider lookup flags in those records describe the exercised lookup path; a separate socket audit establishes that no actual enrichment network access occurs. diff --git a/tools/sbom-diff-and-risk/scripts/qualify-decision-stability.py b/tools/sbom-diff-and-risk/scripts/qualify-decision-stability.py new file mode 100644 index 0000000..da6e5d6 --- /dev/null +++ b/tools/sbom-diff-and-risk/scripts/qualify-decision-stability.py @@ -0,0 +1,408 @@ +#!/usr/bin/env python3 +"""Bounded offline qualification. All generated evidence goes outside the checkout. + +Run from the tool directory with an installed dev environment. This script never +publishes, installs packages, dispatches workflows or contacts evidence services. +""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import os +import platform +import shutil +import subprocess +import sys +import time +import xml.etree.ElementTree as ET +from collections import Counter +from dataclasses import asdict +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +GUARD = """import json, os, socket, sys +from pathlib import Path +def deny(event, args): + blocked = event in {"socket.getaddrinfo", "socket.gethostbyname", "socket.gethostbyaddr"} + if event in {"socket.connect", "socket.sendto"}: + blocked = bool(args) and getattr(args[0], "family", None) in {socket.AF_INET, socket.AF_INET6} + if blocked: + with Path(os.environ["SCT_QUALIFICATION_NETWORK_LOG"]).open("a", encoding="utf-8") as log: + log.write(json.dumps({"event": event, "action": "blocked-before-transport"}) + "\\n") + raise RuntimeError("Live network access is forbidden during offline qualification") +sys.addaudithook(deny) +os.environ["SCT_QUALIFICATION_OFFLINE_GUARD"] = "1" +""" + + +def write_json(path, value): + path.write_text( + json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + + +def inventory(): + paths = sorted( + [ + *ROOT.glob("src/**/*.py"), + *ROOT.glob("tests/*.py"), + ROOT / "scripts" / "qualify-decision-stability.py", + ROOT / "docs" / "policy-schema.md", + ] + ) + return { + path.relative_to(ROOT).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in paths + } + + +def imports(): + sys.path[:0] = [str(ROOT / "src"), str(ROOT / "tests")] + from test_decision_stability import observe_case, render_bundle + from test_canonical_release_enrichment import exercise_c03_transition + from test_decision_stability_negatives import CONTROLS, exercise_control + from test_decision_stability_oracle import CASES, require, validate_case + from test_decision_stability_pairs import ( + REPRESENTATIONS, + TRANSITIONS, + exercise_manifest_overlap, + exercise_representation, + exercise_transition, + ) + from test_decision_stability_providers import ( + PROVIDERS, + exercise_c02_transition, + exercise_provider, + ) + from test_decision_stability_reports import ( + CLI_CASES, + SARIF_EXPECTED, + cdx_component, + consumer_decision, + run_cli, + validate_bundle, + ) + + return { + "exercise_c03_transition": exercise_c03_transition, + "observe_case": observe_case, + "render_bundle": render_bundle, + "CONTROLS": CONTROLS, + "exercise_control": exercise_control, + "CASES": CASES, + "require": require, + "validate_case": validate_case, + "REPRESENTATIONS": REPRESENTATIONS, + "TRANSITIONS": TRANSITIONS, + "exercise_manifest_overlap": exercise_manifest_overlap, + "exercise_representation": exercise_representation, + "exercise_transition": exercise_transition, + "PROVIDERS": PROVIDERS, + "exercise_c02_transition": exercise_c02_transition, + "exercise_provider": exercise_provider, + "CLI_CASES": CLI_CASES, + "SARIF_EXPECTED": SARIF_EXPECTED, + "cdx_component": cdx_component, + "consumer_decision": consumer_decision, + "run_cli": run_cli, + "validate_bundle": validate_bundle, + } + + +def worker(output, seed): + if os.environ.get("SCT_QUALIFICATION_OFFLINE_GUARD") != "1": + raise RuntimeError("The subprocess offline guard is required") + api = imports() + import sbom_diff_risk + + if ( + Path(sbom_diff_risk.__file__).resolve() + != ROOT / "src" / "sbom_diff_risk" / "__init__.py" + ): + raise RuntimeError("Qualification imported a different source checkout") + # Expectations are persisted before any production observation in this round. + write_json( + output / "preregistration.json", + { + "seed": seed, + "cases": [asdict(case) for case in api["CASES"]], + "transitions": [asdict(item) for item in api["TRANSITIONS"]], + "representations": [*api["REPRESENTATIONS"], "R19"], + "negative_controls": list(api["CONTROLS"]), + "C03": { + "classification": "STABLE", + "decisions": ["PASS", "PASS"], + "contract": "CONTRACT_CLARIFIED_CANONICAL_RELEASE_VERSION", + }, + "providers": [asdict(case) for case in api["PROVIDERS"]], + "expectation_sources": [ + "tests/test_decision_stability_oracle.py", + "tests/test_decision_stability_pairs.py", + "tests/test_decision_stability_providers.py", + ], + }, + ) + microcases = [] + for case in api["CASES"]: + actual, report = api["observe_case"](case, output / case.id, seed) + api["validate_case"](case, actual) + if report is not None: + bundle = api["render_bundle"](report) + api["validate_bundle"]( + bundle, + actual["exit_code"], + sarif_expected=api["SARIF_EXPECTED"].get(case.id), + expected_rows=case.rows, + ) + api["require"]("repeat-render", api["render_bundle"](report), bundle) + microcases.append({"id": case.id, "family": case.family, "observation": actual}) + transitions = [ + api["exercise_transition"](item, output / item.id, seed) + for item in api["TRANSITIONS"] + ] + transitions.extend( + api["exercise_representation"](item, output / item, seed) + for item in api["REPRESENTATIONS"] + ) + transitions.append(api["exercise_manifest_overlap"](output / "R19")) + providers = [ + api["exercise_provider"](case, output / case.id) for case in api["PROVIDERS"] + ] + transitions.append(api["exercise_c02_transition"](output / "C02")) + transitions.append(api["exercise_c03_transition"](output / "C03")) + cli_results = [] + for label, policy, flags, decision, expected_exit in api["CLI_CASES"]: + code, bundle = api["run_cli"]( + output / ("cli-" + label), + {"bomFormat": "CycloneDX", "components": []}, + {"bomFormat": "CycloneDX", "components": [api["cdx_component"]()]}, + policy, + flags, + ) + api["require"]("cli-expected-exit", code, expected_exit) + api["require"]("cli-consumer", api["consumer_decision"](bundle, code), decision) + cli_results.append({"id": label, "decision": decision, "exit_code": code}) + negatives = [ + api["exercise_control"](item, output / item[0], seed) + for item in api["CONTROLS"] + ] + write_json( + output / "results.json", + { + "seed": seed, + "microcases": microcases, + "transitions": transitions, + "providers": providers, + "cli": cli_results, + "negative_controls": negatives, + "import_origin": "src/sbom_diff_risk/__init__.py", + "counts": { + "microcases": len(microcases), + "transitions": len(transitions), + "provider_cases": len(providers), + "cli_cases": len(cli_results), + "negative_controls": len(negatives), + "classifications": dict( + Counter(item["observed_classification"] for item in transitions) + ), + }, + }, + ) + + +def command(output, label, args, environment): + start = time.monotonic() + with ( + (output / (label + ".stdout")).open("w", encoding="utf-8") as stdout, + (output / (label + ".stderr")).open("w", encoding="utf-8") as stderr, + ): + result = subprocess.run( + args, + cwd=ROOT, + env=environment, + stdout=stdout, + stderr=stderr, + timeout=300, + check=False, + ) + receipt = { + "label": label, + "exit_code": result.returncode, + "seconds": round(time.monotonic() - start, 3), + } + write_json(output / (label + ".receipt.json"), receipt) + if result.returncode: + raise RuntimeError( + f"Qualification stopped at {label}, exit {result.returncode}; inspect the external logs" + ) + return receipt + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-root", type=Path, required=True) + parser.add_argument("--seeds", type=int, nargs="+", default=[103, 117, 120]) + parser.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) + args = parser.parse_args() + output = args.output_root.resolve() + if output == ROOT or ROOT in output.parents or ROOT.parents[1] in output.parents: + parser.error("Output must be outside the entire source checkout") + if args.worker: + worker(output, args.seeds[0]) + return + if not 1 <= len(args.seeds) <= 3 or len(set(args.seeds)) != len(args.seeds): + parser.error( + "Use one to three distinct seeds; repeated seeds do not add independent cases" + ) + output.mkdir(parents=True, exist_ok=False) + initial_inventory = inventory() + write_json(output / "source-inventory.json", initial_inventory) + guard_dir = output / "offline-guard" + guard_dir.mkdir() + (guard_dir / "sitecustomize.py").write_text(GUARD, encoding="utf-8") + environment = { + **os.environ, + "PYTHONPATH": str(guard_dir), + "PYTHONDONTWRITEBYTECODE": "1", + "GIT_OPTIONAL_LOCKS": "0", + } + environment["SCT_QUALIFICATION_NETWORK_LOG"] = str( + output / "network-attempts.jsonl" + ) + temporary = output / "tmp" + temporary.mkdir() + environment.update(TMP=str(temporary), TEMP=str(temporary)) + rounds = [] + for seed in args.seeds: + round_root = output / f"round-{seed}" + round_root.mkdir() + command( + round_root, + "qualification", + [ + sys.executable, + str(Path(__file__).resolve()), + "--worker", + "--seeds", + str(seed), + "--output-root", + str(round_root), + ], + environment, + ) + junit = round_root / "full-suite.xml" + command( + round_root, + "full-suite", + [ + sys.executable, + "-m", + "pytest", + "-q", + "-ra", + f"--junitxml={junit}", + f"--basetemp={round_root / 'pytest'}", + "-o", + f"cache_dir={round_root / 'cache'}", + ], + environment, + ) + suite = ET.parse(junit).getroot() + totals = { + key: sum(int(item.get(key, 0)) for item in suite.iter("testsuite")) + for key in ("tests", "failures", "errors", "skipped") + } + if totals["failures"] or totals["errors"] or totals["skipped"]: + raise RuntimeError(f"Non-passing full-suite result: {totals}") + result = json.loads((round_root / "results.json").read_text(encoding="utf-8")) + rounds.append({"seed": seed, "counts": result["counts"], "full_suite": totals}) + print(json.dumps(rounds[-1], sort_keys=True), flush=True) + command( + output, + "example-artifacts", + [sys.executable, "scripts/regenerate-example-artifacts.py", "--check"], + environment, + ) + bash = shutil.which("bash") + consumer = {"status": "NOT_RUN", "reason": "bash unavailable"} + if bash: + records = [] + for expected in (0, 1, 2): + destination = output / f"consumer-{expected}" + destination.mkdir() + # The comparison command is replaced with an actual observed status; + # capture and final exit use the existing consumer example's shell logic. + script = f"""set +e +(exit {expected}) +status=$? +set -e +echo "exit_code=$status" >> "$GITHUB_OUTPUT" +exit "$status" +""" + result = subprocess.run( + [bash, "-c", script], + cwd=destination, + env={**environment, "GITHUB_OUTPUT": "outputs.txt"}, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + captured = (destination / "outputs.txt").read_text(encoding="utf-8").strip() + if result.returncode != expected or captured != f"exit_code={expected}": + raise RuntimeError("Consumer status propagation mismatch") + records.append( + { + "comparison_exit": expected, + "final_exit": result.returncode, + "output": captured, + } + ) + consumer = { + "status": "PASS", + "kind": "local bash status propagation, no workflow execution", + "records": records, + } + write_json(output / "consumer.json", consumer) + if (output / "network-attempts.jsonl").exists(): + raise RuntimeError( + "A network operation was attempted; the guard blocked it and qualification cannot pass" + ) + if inventory() != initial_inventory: + raise RuntimeError( + "Source changed during qualification; results do not qualify the final candidate" + ) + manifest = { + "status": "EXECUTED", + "seeds": args.seeds, + "rounds": rounds, + "python": platform.python_version(), + "python_implementation": platform.python_implementation(), + "os": platform.system(), + "dependencies": { + name: importlib.metadata.version(name) + for name in ("pytest", "packaging", "packageurl-python", "PyYAML") + }, + "source_inventory": initial_inventory, + "actual_network_attempts": 0, + "consumer_shell": consumer, + "parallelism": 1, + "per_command_timeout_seconds": 300, + "independent_cases_are_not_multiplied_by_round_count": True, + "remote_ci": "NOT_RUN", + "live_evidence_enrichment": "FORBIDDEN", + "publication": "NOT_PERFORMED", + } + write_json(output / "execution.json", manifest) + print( + "All requested local rounds executed; final audit and product-policy decisions are separate gates." + ) + + +if __name__ == "__main__": + main() diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/cli.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/cli.py index e2f1ac2..f1df1d7 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/cli.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/cli.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Sequence -from .diffing import diff_components +from .diffing import diff_components, validate_component_identities from .enrichment import DEFAULT_PYPI_TIMEOUT_SECONDS, PyPIProvenanceEnricher, merge_enrichment_metadata from .errors import ParseError, PolicyError from .models import CompareReport, ReportComponents, ReportEnrichmentMetadata, ReportMetadata, ReportSummary @@ -194,6 +194,11 @@ def run_compare(args: argparse.Namespace) -> int: if pyproject_group and before_format != "pyproject-toml" and after_format != "pyproject-toml": raise ValueError("--pyproject-group requires at least one pyproject.toml input.") + if enrich_pypi or enrich_scorecard: + # Validate both inputs before either provider can perform a lookup. + validate_component_identities(before_components, side="before") + validate_component_identities(after_components, side="after") + pypi_enrichment_metadata = None if enrich_pypi: enricher = PyPIProvenanceEnricher(timeout_seconds=pypi_timeout) diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/diffing.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/diffing.py index f7d275c..36a5560 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/diffing.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/diffing.py @@ -63,6 +63,11 @@ def diff_components( return added, removed, changed +def validate_component_identities(components: Iterable[Component], *, side: str) -> None: + """Validate a comparison input before optional network-capable enrichment.""" + _index_components(components, side=side) + + def _index_components(components: Iterable[Component], side: str) -> dict[str, Component]: indexed: dict[str, Component] = {} for component in components: diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/enrichment.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/enrichment.py index ea7f951..448adc9 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/enrichment.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/enrichment.py @@ -3,6 +3,7 @@ from collections import Counter from dataclasses import replace +from .component_identity import canonicalize_component_identity from .models import ( Component, ProvenanceEvidence, @@ -31,9 +32,10 @@ def __init__( self._seen_keys: set[tuple[str, str, str]] = set() def enrich_components(self, components: list[Component]) -> list[Component]: + # Fail the complete batch before a valid earlier item can trigger I/O. + keys = [_component_identity(component) for component in components] enriched: list[Component] = [] - for component in components: - key = _component_identity(component) + for component, key in zip(components, keys): if key not in self._cache: self._seen_keys.add(key) self._cache[key] = normalize_pypi_provenance(component, client=self.client) @@ -116,10 +118,11 @@ def enrichment_metadata_to_dict(metadata: ReportEnrichmentMetadata) -> dict[str, def _component_identity(component: Component) -> tuple[str, str, str]: + identity = canonicalize_component_identity(component) return ( - component.ecosystem.strip().lower(), - component.name.strip().lower(), - (component.version or "").strip().lower(), + identity.ecosystem, + identity.package_name, + identity.version or "", ) diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_evaluator.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_evaluator.py index 598d51c..954d9dd 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_evaluator.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_evaluator.py @@ -3,6 +3,8 @@ from dataclasses import dataclass from urllib.parse import urlparse +from packaging.utils import canonicalize_name + from .diffing import component_key from .models import ( Component, @@ -409,7 +411,12 @@ def _components_for_policy(added: list[Component], changed: list[ComponentChange def _package_is_unattested_allowed(policy: PolicyConfig, component: Component) -> bool: - return component.name.strip().lower() in set(policy.allow_unattested_packages) + if component.ecosystem.strip().lower() != "pypi": + return False + package_name = canonicalize_name(component.name.strip()) + return package_name in { + canonicalize_name(name.strip()) for name in policy.allow_unattested_packages + } def _assess_provenance(component: Component, policy: PolicyConfig) -> ProvenanceAssessment: diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_parser.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_parser.py index 023f03d..86edd96 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_parser.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/policy_parser.py @@ -53,6 +53,11 @@ def load_policy(path: Path) -> PolicyConfig: if not isinstance(payload, dict): raise PolicyError(f"Invalid policy schema in {path}: top-level YAML value must be a mapping.") + if any(not isinstance(key, str) for key in payload): + raise PolicyError( + f"Invalid policy schema in {path}: policy keys must be strings." + ) + unknown_keys = sorted(set(payload) - _SUPPORTED_POLICY_KEYS) if unknown_keys: raise PolicyError(f"Invalid policy schema in {path}: unsupported keys: {', '.join(unknown_keys)}.") @@ -67,7 +72,7 @@ def load_policy(path: Path) -> PolicyConfig: ) version = payload.get("version") - if not isinstance(version, int): + if isinstance(version, bool) or not isinstance(version, int): raise PolicyError(f"Invalid policy schema in {path}: version must be an integer.") if version not in {1, 2, 3}: raise PolicyError(f"Invalid policy schema in {path}: only versions 1, 2, and 3 are supported.") @@ -107,7 +112,11 @@ def load_policy(path: Path) -> PolicyConfig: ) max_added_packages = payload.get("max_added_packages") - if max_added_packages is not None and (not isinstance(max_added_packages, int) or max_added_packages < 0): + if max_added_packages is not None and ( + isinstance(max_added_packages, bool) + or not isinstance(max_added_packages, int) + or max_added_packages < 0 + ): raise PolicyError(f"Invalid policy schema in {path}: max_added_packages must be a non-negative integer.") allow_sources = _parse_string_list(payload.get("allow_sources", []), f"{path}: allow_sources", lower=True) diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/pypi_provenance.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/pypi_provenance.py index e8b2f56..6def8a2 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/pypi_provenance.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/pypi_provenance.py @@ -4,6 +4,7 @@ import binascii import json +from .component_identity import CanonicalComponentIdentity, canonicalize_component_identity from .models import ( Component, ProvenanceEvidence, @@ -27,29 +28,30 @@ def normalize_pypi_provenance(component: Component, *, client: PyPIIntegrityClient) -> ProvenanceEvidence: - if component.ecosystem.strip().lower() != "pypi": - return _unsupported_provenance(component) - if not component.name.strip() or not component.version or not component.version.strip(): - return _unsupported_provenance(component) + identity = canonicalize_component_identity(component) + if identity.ecosystem != "pypi": + return _unsupported_provenance(identity) + if not identity.package_name.strip() or not identity.version or not identity.version.strip(): + return _unsupported_provenance(identity) try: - release = client.fetch_release(component.name, component.version) + release = client.fetch_release(identity.package_name, identity.version) except PyPIClientError as exc: if exc.status_code == 404: - return _unsupported_provenance(component, lookup_performed=True) + return _unsupported_provenance(identity, lookup_performed=True) return ProvenanceEvidence( provider="pypi", requested=True, supported=True, lookup_performed=True, - package_name=component.name, - package_version=component.version, - release_url=_release_url(component.name, component.version), + package_name=identity.package_name, + package_version=identity.version, + release_url=_release_url(identity.package_name, identity.version), statuses=(ProvenanceStatus.ENRICHMENT_ERROR,), error=str(exc), ) - return _normalize_release_provenance(component, release=release, client=client) + return _normalize_release_provenance(identity, release=release, client=client) def normalize_provenance_file( @@ -130,13 +132,13 @@ def provenance_evidence_to_dict(provenance: ProvenanceEvidence | None) -> dict[s def _normalize_release_provenance( - component: Component, + identity: CanonicalComponentIdentity, *, release: PyPIRelease, client: PyPIIntegrityClient, ) -> ProvenanceEvidence: if not release.files: - return _unsupported_provenance(component, lookup_performed=True) + return _unsupported_provenance(identity, lookup_performed=True) component_statuses: set[ProvenanceStatus] = set() file_evidence: list[ProvenanceFileEvidence] = [] @@ -144,7 +146,7 @@ def _normalize_release_provenance( for release_file in release.files: try: - provenance = client.fetch_provenance(component.name, component.version or "", release_file.filename) + provenance = client.fetch_provenance(identity.package_name, identity.version or "", release_file.filename) except PyPIClientError as exc: component_statuses.add(ProvenanceStatus.ENRICHMENT_ERROR) if first_error is None: @@ -173,9 +175,9 @@ def _normalize_release_provenance( requested=True, supported=True, lookup_performed=True, - package_name=component.name, - package_version=component.version, - release_url=release.release_url or _release_url(component.name, component.version or ""), + package_name=identity.package_name, + package_version=identity.version, + release_url=release.release_url or _release_url(identity.package_name, identity.version or ""), statuses=_sorted_statuses(component_statuses or {ProvenanceStatus.ATTESTATION_UNAVAILABLE}), files=tuple(file_evidence), files_evaluated=files_evaluated, @@ -186,7 +188,7 @@ def _normalize_release_provenance( def _unsupported_provenance( - component: Component, + identity: CanonicalComponentIdentity, *, lookup_performed: bool = False, ) -> ProvenanceEvidence: @@ -195,9 +197,9 @@ def _unsupported_provenance( requested=True, supported=False, lookup_performed=lookup_performed, - package_name=component.name, - package_version=component.version, - release_url=_release_url(component.name, component.version or ""), + package_name=identity.package_name, + package_version=identity.version, + release_url=_release_url(identity.package_name, identity.version or ""), statuses=(ProvenanceStatus.UNSUPPORTED_FOR_PACKAGE,), ) diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/scorecard_client.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/scorecard_client.py index de5809f..cffa98d 100644 --- a/tools/sbom-diff-and-risk/src/sbom_diff_risk/scorecard_client.py +++ b/tools/sbom-diff-and-risk/src/sbom_diff_risk/scorecard_client.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import math import socket from dataclasses import dataclass from typing import Any @@ -8,6 +9,8 @@ from .models import ScorecardCheck +_ASCII_LOWER = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") + @dataclass(slots=True, frozen=True) class ScorecardProjectResult: @@ -108,6 +111,13 @@ def parse_project_payload(payload: object, *, expected_canonical_name: str) -> S repo_name = _optional_text(raw_repo.get("name")) or expected_canonical_name repo_commit = _optional_text(raw_repo.get("commit")) + if repo_name.translate(_ASCII_LOWER) != expected_canonical_name.translate( + _ASCII_LOWER + ): + raise ScorecardClientError( + "Scorecard response repository does not match the requested repository." + ) + score = _required_number(payload.get("score"), "Scorecard score") date = _optional_text(payload.get("date")) @@ -171,9 +181,15 @@ def _required_text(value: object, context: str) -> str: def _required_number(value: object, context: str) -> float: - if not isinstance(value, (int, float)): + if isinstance(value, bool) or not isinstance(value, (int, float)): raise ScorecardClientError(f"{context} must be a number.") - return float(value) + try: + number = float(value) + except OverflowError as exc: + raise ScorecardClientError(f"{context} must be finite.") from exc + if not math.isfinite(number): + raise ScorecardClientError(f"{context} must be finite.") + return number def _optional_text(value: object) -> str | None: diff --git a/tools/sbom-diff-and-risk/tests/test_canonical_release_enrichment.py b/tools/sbom-diff-and-risk/tests/test_canonical_release_enrichment.py new file mode 100644 index 0000000..1c25501 --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_canonical_release_enrichment.py @@ -0,0 +1,277 @@ +"""Owner-approved C03 release coordinates and pre-request identity boundary.""" + +from copy import deepcopy +from dataclasses import asdict, replace +from unittest.mock import patch + +import pytest + +from sbom_diff_risk import cli +from sbom_diff_risk.enrichment import PyPIProvenanceEnricher +from sbom_diff_risk.errors import ComponentIdentityError +from sbom_diff_risk.models import Component, ProvenanceStatus +from sbom_diff_risk.pypi_integrity_client import PyPIClientError, PyPIRelease +from sbom_diff_risk.pypi_provenance import normalize_pypi_provenance +from test_pypi_enrichment import FakePyPIClient, _release_file +from test_decision_stability_reports import cdx_component, run_cli + + +def component(version=None, release="1.0.0", name="demo-pkg"): + return Component( + name=name, + version=version, + ecosystem="pypi", + purl=f"pkg:pypi/demo-pkg@{release}", + ) + + +def client_for(*versions, error=None): + releases, provenance, errors = {}, {}, {} + for version in versions: + file = _release_file(filename=f"demo-pkg-{version}.tar.gz") + releases["demo-pkg", version] = PyPIRelease( + project="demo-pkg", + version=version, + release_url=f"https://pypi.org/project/demo-pkg/{version}/", + files=(file,), + ) + provenance["demo-pkg", version, file.filename] = None + if error: + errors["demo-pkg", version] = error + return FakePyPIClient( + releases=releases, provenance=provenance, release_errors=errors + ) + + +@pytest.mark.parametrize("reverse", [False, True]) +def test_equivalent_releases_share_consistent_evidence_without_mutation(reverse): + items = [component(), component("1.0.0", name="Demo_Pkg")] + if reverse: + items.reverse() + observed = deepcopy(items) + client = client_for("1.0.0") + enricher = PyPIProvenanceEnricher(client=client) + results = enricher.enrich_components(items) + assert items == observed + assert client.release_calls == [("demo-pkg", "1.0.0")] + assert results[0].provenance == results[1].provenance + assert results[0].provenance.package_version == "1.0.0" + assert results[0].provenance.package_name == "demo-pkg" + assert results[0].provenance.statuses == (ProvenanceStatus.ATTESTATION_UNAVAILABLE,) + for before, after in zip(items, results): + assert asdict(replace(after, provenance=None)) == asdict(before) + assert enricher.build_report_metadata().candidate_components == 1 + + +def test_distinct_release_cache_isolation(): + client = client_for("1.0.0", "2.0.0") + enricher = PyPIProvenanceEnricher(client=client) + rows = enricher.enrich_components( + [component(), component(release="2.0.0"), component("1.0.0")] + ) + assert client.release_calls == [("demo-pkg", "1.0.0"), ("demo-pkg", "2.0.0")] + assert [r.provenance.package_version for r in rows] == ["1.0.0", "2.0.0", "1.0.0"] + assert rows[0].provenance.files != rows[1].provenance.files + + +INVALID = [ + component("2.0.0"), + replace(component(), purl="not-a-purl"), + replace(component(), ecosystem="npm"), + replace(component(), name="other-package"), +] + + +@pytest.mark.parametrize( + "invalid", INVALID, ids=["version", "purl", "ecosystem", "name"] +) +@pytest.mark.parametrize("batch", [False, True]) +def test_direct_api_rejects_invalid_identity_before_any_lookup(invalid, batch): + client = client_for("1.0.0", "2.0.0") + with pytest.raises(ComponentIdentityError): + if batch: + PyPIProvenanceEnricher(client=client).enrich_components( + [component("1.0.0"), invalid] + ) + else: + normalize_pypi_provenance(invalid, client=client) + assert client.release_calls == client.provenance_calls == [] + + +@pytest.mark.parametrize("purl", [None, "pkg:pypi/demo-pkg"]) +def test_no_version_never_looks_up_latest(purl): + client = client_for("1.0.0") + evidence = normalize_pypi_provenance(replace(component(), purl=purl), client=client) + assert evidence.statuses == (ProvenanceStatus.UNSUPPORTED_FOR_PACKAGE,) + assert evidence.lookup_performed is False + assert client.release_calls == [] + + +@pytest.mark.parametrize("status", [404, 503]) +def test_canonical_release_preserves_failure_states(status): + client = client_for( + "1.0.0", error=PyPIClientError("synthetic failure", status_code=status) + ) + evidence = normalize_pypi_provenance(component(), client=client) + assert evidence.package_version == "1.0.0" + assert evidence.lookup_performed + assert evidence.statuses == ( + (ProvenanceStatus.UNSUPPORTED_FOR_PACKAGE,) + if status == 404 + else (ProvenanceStatus.ENRICHMENT_ERROR,) + ) + + +@pytest.mark.parametrize("flag", ["--enrich-pypi", "--enrich-scorecard"]) +@pytest.mark.parametrize( + "invalid", INVALID, ids=["version", "purl", "ecosystem", "name"] +) +def test_cli_validates_both_inputs_before_enricher_construction( + flag, invalid, tmp_path +): + # Inject normalized metadata to cover contradictions that a format parser + # would otherwise derive or reject; the public CLI ordering is exercised. + with ( + patch.object( + cli, + "normalize_input_with_options", + side_effect=[ + ("cyclonedx-json", [component("1.0.0")], []), + ("cyclonedx-json", [invalid], []), + ], + ), + patch.object( + cli, + "PyPIProvenanceEnricher", + side_effect=AssertionError( + "enricher constructed before identity preflight" + ), + ) as pypi, + patch.object( + cli, + "ScorecardEnricher", + side_effect=AssertionError( + "enricher constructed before identity preflight" + ), + ) as scorecard, + ): + code, bundle = run_cli( + tmp_path, + {"bomFormat": "CycloneDX", "components": []}, + {"bomFormat": "CycloneDX", "components": []}, + flags=[flag], + ) + assert code == 2 and bundle is None + pypi.assert_not_called() + scorecard.assert_not_called() + + +def exercise_c03_transition(tmp_path, enabled=True): + bundles = [] + for label, version in [("purl-only", None), ("explicit", "1.0.0")]: + item = cdx_component(name="demo-pkg", version="1.0.0") + item["version"] = version + client = client_for("1.0.0") + with patch.object( + cli, + "PyPIProvenanceEnricher", + return_value=PyPIProvenanceEnricher(client=client), + ) as factory: + code, bundle = run_cli( + tmp_path / label, + {"bomFormat": "CycloneDX", "components": []}, + {"bomFormat": "CycloneDX", "components": [item]}, + {"version": 2, "block_on": ["provenance_unavailable"]}, + flags=["--enrich-pypi"] if enabled else [], + expected_provider_lookup=enabled, + ) + assert code == (0 if enabled else 1) + if not enabled: + factory.assert_not_called() + assert client.release_calls == ([("demo-pkg", "1.0.0")] if enabled else []) + assert bundle["report"]["components"]["added"][0]["version"] == version + bundles.append(bundle) + for bundle in bundles: + # Invocation-specific policy source paths are not release evidence. + bundle["policy"]["policy_evaluation"].pop("policy_path") + assert bundles[0]["policy"] == bundles[1]["policy"] + assert bundles[0]["summary"] == bundles[1]["summary"] + return { + "id": "C03", + "perturbation": "Agreeing explicit versus PURL-only release version", + "before_canonical_identity": { + "key": "purl:pkg:pypi/demo-pkg", + "version": "1.0.0", + }, + "after_canonical_identity": { + "key": "purl:pkg:pypi/demo-pkg", + "version": "1.0.0", + }, + "previous_decision": "PASS" if enabled else "FAIL", + "resulting_decision": "PASS" if enabled else "FAIL", + "active_policy": {"version": 2, "block_on": ["provenance_unavailable"]}, + "expected_classification": "STABLE", + "observed_classification": "STABLE", + "reason": "CONTRACT_CLARIFIED_CANONICAL_RELEASE_VERSION", + "source_evidence_changes": { + "raw_versions": [None, "1.0.0"], + "enrichment_enabled": enabled, + }, + } + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_original_c03_cli_representation_transition(tmp_path, enabled): + exercise_c03_transition(tmp_path, enabled) + + +def test_direct_normalizer_uses_canonical_release(): + client = client_for("1.0.0") + a = normalize_pypi_provenance(component(), client=client) + b = normalize_pypi_provenance(component("1.0.0", name="Demo_Pkg"), client=client) + assert a == b + assert client.release_calls == [("demo-pkg", "1.0.0")] * 2 + + +@pytest.mark.parametrize("version", ["1.0.0", "2.0.0"]) +def test_cli_rejects_duplicate_or_conflicting_index_before_lookup(version, tmp_path): + client = client_for("1.0.0", "2.0.0") + items = [ + cdx_component(name="demo-pkg", version="1.0.0"), + cdx_component(name="demo-pkg", version=version), + ] + with patch.object( + cli, + "PyPIProvenanceEnricher", + return_value=PyPIProvenanceEnricher(client=client), + ): + code, bundle = run_cli( + tmp_path, + {"bomFormat": "CycloneDX", "components": []}, + {"bomFormat": "CycloneDX", "components": items}, + flags=["--enrich-pypi"], + ) + assert code == 2 and bundle is None + assert client.release_calls == client.provenance_calls == [] + + +@pytest.mark.parametrize( + "change", [{"version": "2.0.0"}, {"purl": "not-a-purl"}, {"name": "other-package"}] +) +def test_actual_cli_input_rejected_before_lookup(change, tmp_path): + item = cdx_component(name="demo-pkg", version="1.0.0") + item.update(change) + client = client_for("1.0.0", "2.0.0") + with patch.object( + cli, + "PyPIProvenanceEnricher", + return_value=PyPIProvenanceEnricher(client=client), + ): + code, bundle = run_cli( + tmp_path, + {"bomFormat": "CycloneDX", "components": []}, + {"bomFormat": "CycloneDX", "components": [item]}, + flags=["--enrich-pypi"], + ) + assert code == 2 and bundle is None + assert client.release_calls == client.provenance_calls == [] diff --git a/tools/sbom-diff-and-risk/tests/test_decision_stability.py b/tools/sbom-diff-and-risk/tests/test_decision_stability.py new file mode 100644 index 0000000..60b23c4 --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_decision_stability.py @@ -0,0 +1,270 @@ +"""Production adapters for the separately authored, bounded decision oracle.""" + +import json +import random +from dataclasses import replace +from pathlib import Path + +import pytest +import yaml + +from sbom_diff_risk.component_identity import canonicalize_component_identity +from sbom_diff_risk.errors import PolicyError +from sbom_diff_risk.models import ( + CompareReport, + Component, + ComponentChange, + ProvenanceEvidence, + ProvenanceFileEvidence, + ProvenanceStatus, + ReportComponents, + ReportMetadata, + ReportSummary, + RepositoryMapping, + RiskBucket, + RiskFinding, + ScorecardEvidence, + ScorecardStatus, +) +from sbom_diff_risk.policy_evaluator import _assess_provenance, evaluate_policy +from sbom_diff_risk.policy_models import PolicyConfig +from sbom_diff_risk.policy_parser import build_policy +from sbom_diff_risk.report_json import ( + render_policy_json, + render_report_json, + render_summary_json, +) +from sbom_diff_risk.report_md import render_report_markdown +from sbom_diff_risk.report_sarif import render_report_sarif +from sbom_diff_risk.risk import summarize_risks + +from test_decision_stability_oracle import CASES, KEY, NAME, require, validate_case + + +def synthetic_component(case): + component = Component( + NAME, + "1.0.0", + "pypi", + purl=f"pkg:pypi/{NAME}@1.0.0", + license_id="MIT", + source_url=case.source_url, + evidence={"source_format": "cyclonedx-json"}, + ) + if case.provenance != "none": + states = { + "error": (ProvenanceStatus.ENRICHMENT_ERROR,), + "unsupported": (ProvenanceStatus.UNSUPPORTED_FOR_PACKAGE,), + "missing": ( + ProvenanceStatus.PROVENANCE_AVAILABLE, + ProvenanceStatus.ATTESTATION_UNAVAILABLE, + ), + "attested": ( + ProvenanceStatus.PROVENANCE_AVAILABLE, + ProvenanceStatus.ATTESTATION_AVAILABLE, + ), + "error-and-attested": ( + ProvenanceStatus.ENRICHMENT_ERROR, + ProvenanceStatus.ATTESTATION_AVAILABLE, + ), + } + statuses = states[case.provenance] + has_attestation = ProvenanceStatus.ATTESTATION_AVAILABLE in statuses + component.provenance = ProvenanceEvidence( + provider="synthetic-pypi", + requested=True, + supported=case.provenance != "unsupported", + lookup_performed=case.provenance != "unsupported", + package_name=NAME, + package_version="1.0.0", + statuses=statuses, + files=( + ProvenanceFileEvidence( + filename="qualification-alpha-1.0.0.whl", + statuses=statuses, + attestation_count=int(has_attestation), + publisher_kinds=case.publishers, + ), + ), + error="Synthetic offline transport error" + if "error" in case.provenance + else None, + ) + if case.score is not None: + component.scorecard = ScorecardEvidence( + provider="synthetic-scorecard", + requested=True, + repository=RepositoryMapping( + "github.com", + "example", + "qualification", + "github.com/example/qualification", + "https://github.com/example/qualification", + "synthetic-explicit-vcs", + ) + if case.score_available + else None, + statuses=(ScorecardStatus.SCORECARD_AVAILABLE,) + if case.score_available + else (ScorecardStatus.REPOSITORY_UNMAPPED,), + score=case.score, + ) + return component + + +def observe_case(case, directory, seed=103): + directory.mkdir(parents=True, exist_ok=True) + policy_path = None + if case.policy is not None: + entries = list(case.policy.items()) + random.Random(seed).shuffle(entries) + policy_path = directory / "policy.yml" + policy_path.write_text( + yaml.safe_dump(dict(entries), sort_keys=False), encoding="utf-8" + ) + try: + policy, _ = build_policy( + policy_path=policy_path, fail_on=case.fail_on, warn_on=case.warn_on + ) + except PolicyError as exc: + return { + "decision": "INVALID_INPUT", + "exit_code": 2, + "error_type": type(exc).__name__, + }, None + component = synthetic_component(case) + added = [component] if case.scope == "added" else [] + changed = ( + [ + ComponentChange( + KEY, + replace(component, license_id="BSD-3-Clause"), + component, + "metadata_changed", + ) + ] + if case.scope == "changed" + else [] + ) + findings = [ + RiskFinding( + RiskBucket(bucket), KEY, component, "Synthetic local heuristic observation." + ) + for bucket in case.findings + ] + random.Random(seed).shuffle(findings) + evaluation = evaluate_policy( + policy, + policy_path="policy.yml" if policy_path else None, + added=added, + changed=changed, + findings=findings, + ) + rows = [] + for group, violations in ( + ("blocking", evaluation.blocking_violations), + ("warning", evaluation.warning_violations), + ("suppressed", evaluation.suppressed_violations), + ): + keys = [ + (item.rule_id, item.component_key or "", item.component_name or "") + for item in violations + ] + require("deterministic-order", keys, sorted(keys)) + for item in violations: + rows.append( + { + "group": group, + "rule": item.rule_id, + "policy_rule": item.policy_rule, + "level": item.level.value if item.level else None, + "reason": item.decision_reason, + "source": item.severity_source, + "observed": item.observed_value, + "threshold": item.matched_threshold, + "key": item.component_key, + "name": item.component_name, + "bucket": item.finding_bucket, + "confidence": item.confidence_level, + "suppression": item.suppression_reason, + } + ) + assessment = _assess_provenance(component, policy or PolicyConfig(version=1)) + decision = ( + "NOT_APPLIED" + if not evaluation.applied + else "FAIL" + if evaluation.blocking_violations + else "WARN" + if evaluation.warning_violations + else "PASS" + ) + observation = { + "decision": decision, + "exit_code": evaluation.exit_code, + "rows": rows, + "identity": canonicalize_component_identity(component).component_key, + "assessment": ( + assessment.attestation_available, + assessment.provenance_unavailable, + assessment.verified, + assessment.publisher_kinds, + ), + "ignored_checks": evaluation.ignored_checks, + "effective_version": policy.version if policy else None, + } + report = CompareReport( + ReportSummary(len(added), 0, len(changed), summarize_risks(findings)), + ReportComponents(added, [], changed), + findings, + ReportMetadata( + "cyclonedx-json", "cyclonedx-json", stub=False, policy_evaluation=evaluation + ), + [ + "This tool uses heuristic risk classification.", + "Synthetic recorded evidence; no cryptographic verification or package safety verdict.", + ], + ) + return observation, report + + +def render_bundle(report): + return { + "report": json.loads(render_report_json(report)), + "summary": json.loads(render_summary_json(report)), + "policy": json.loads(render_policy_json(report)), + "markdown": render_report_markdown(report), + "sarif": json.loads( + render_report_sarif( + report, before_path=Path("before.json"), after_path=Path("after.json") + ) + ), + } + + +@pytest.mark.parametrize("case", CASES, ids=lambda case: case.id) +def test_hand_authored_policy_microcase(case, tmp_path): + actual, report = observe_case(case, tmp_path) + validate_case(case, actual) + if report is not None: + require("renderer-determinism", render_bundle(report), render_bundle(report)) + + +def test_expected_oracle_does_not_import_product_code(): + import ast + import test_decision_stability_oracle as oracle + + tree = ast.parse(Path(oracle.__file__).read_text(encoding="utf-8")) + imports = [ + node.module or "" for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) + ] + imports.extend( + alias.name + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + ) + assert all( + not name.startswith(("sbom_diff_risk", "test_decision_stability.")) + for name in imports + ) diff --git a/tools/sbom-diff-and-risk/tests/test_decision_stability_negatives.py b/tools/sbom-diff-and-risk/tests/test_decision_stability_negatives.py new file mode 100644 index 0000000..7914904 --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_decision_stability_negatives.py @@ -0,0 +1,185 @@ +"""Reached semantic result replacements validate each named oracle boundary.""" + +from copy import deepcopy + +import pytest + +from sbom_diff_risk.diffing import diff_components +from sbom_diff_risk.errors import ComponentIdentityError +from sbom_diff_risk.models import Component + +from test_decision_stability import observe_case, render_bundle +from test_decision_stability_oracle import ( + CASE_BY_ID, + OracleMismatch, + expected_row, + require, + score_row, + validate_case, +) +from test_decision_stability_reports import cdx_component, run_cli, validate_bundle + + +CONTROLS = ( + ("N01", "M03", "silently remove blocking violation", "decision"), + ("N02", "M04", "upgrade warning to blocking", "decision"), + ("N03", "M03", "downgrade blocking to warning", "decision"), + ("N04", "M06", "drop suppression evidence", "violation-identity"), + ("N05", "M18", "unavailable provenance becomes verified", "assessment"), + ("N06", "M20", "missing attestation becomes unavailable", "violation-identity"), + ("N07", "M22", "accept missing publisher identity", "decision"), + ("N08", "M42", "enforce advisory-only Scorecard", "decision"), + ("N09", "M09", "use greater-than-or-equal count threshold", "decision"), + ("N10", "M03", "split canonical identity", "identity"), + ( + "N11", + "identity-conflict", + "silently accept conflicting metadata", + "identity-conflict-rejection", + ), + ("N12", "M03", "alter policy sidecar evidence", "policy-sidecar"), + ( + "N13", + "CLI-warning", + "replace actual CLI exit with blocking exit", + "exit-blocking", + ), + ( + "N14", + "M03", + "launder heuristic into verified safety verdict", + "no-security-verdict", + ), +) + + +def validate_conflict_boundary(diff_function=diff_components): + records = [ + Component( + "qualification-alpha", + "1.0.0", + "pypi", + purl="pkg:pypi/qualification-alpha@1.0.0", + license_id=license_id, + ) + for license_id in ("MIT", "BSD-3-Clause") + ] + rejected = False + try: + diff_function([], records) + except ComponentIdentityError as exc: + require("conflict-diagnostic", exc.code.value, "conflicting_metadata") + rejected = True + require("identity-conflict-rejection", rejected, True) + + +def exercise_control(control, directory, seed=103): + control_id, case_id, mutation, target_check = control + directory.mkdir(parents=True, exist_ok=True) + if control_id == "N11": + validate_conflict_boundary() + checker = lambda: validate_conflict_boundary(lambda before, after: ([], [], [])) + before = {"conflicting_metadata_rejected": True} + after = {"conflicting_metadata_rejected": False} + elif control_id == "N13": + code, bundle = run_cli( + directory, + {"bomFormat": "CycloneDX", "components": []}, + {"bomFormat": "CycloneDX", "components": [cdx_component()]}, + {"version": 1, "warn_on": ["new_package"]}, + ) + validate_bundle(bundle, code) + checker = lambda: validate_bundle(bundle, 1) + before, after = ( + {"exit_code": code, "warning": 1}, + {"exit_code": 1, "warning": 1}, + ) + else: + case = CASE_BY_ID[case_id] + actual, report = observe_case(case, directory, seed) + validate_case(case, actual) + before = deepcopy(actual) + after = deepcopy(actual) + if control_id in {"N12", "N14"}: + bundle = render_bundle(report) + validate_bundle(bundle, actual["exit_code"]) + if control_id == "N12": + bundle["policy"]["blocking_findings"][0]["observed_value"] = ( + "corrupted-evidence" + ) + after = {"sidecar_observed_value": "corrupted-evidence"} + else: + bundle["report"]["package_safety_verdict"] = "verified_safe" + after = {"package_safety_verdict": "verified_safe"} + checker = lambda: validate_bundle(bundle, actual["exit_code"]) + else: + if control_id in {"N01", "N07"}: + after.update(decision="PASS", exit_code=0, rows=[]) + elif control_id == "N02": + after.update(decision="FAIL", exit_code=1) + after["rows"][0].update( + group="blocking", level="block", source="block_on" + ) + elif control_id == "N03": + after.update(decision="WARN", exit_code=0) + after["rows"][0].update(group="warning", level="warn", source="warn_on") + elif control_id == "N04": + after["rows"] = [] + after["ignored_checks"] = 0 + elif control_id == "N05": + after["assessment"] = (True, False, True, ("github actions",)) + elif control_id == "N06": + after["rows"][0].update( + rule="provenance_unavailable", + policy_rule="provenance_unavailable", + reason="provenance_evidence_unavailable", + observed="unavailable", + ) + elif control_id == "N08": + after.update( + decision="FAIL", exit_code=1, rows=[expected_row(score_row())] + ) + elif control_id == "N09": + after.update( + decision="FAIL", + exit_code=1, + rows=[ + { + "group": "blocking", + "rule": "max_added_packages", + "key": None, + "observed": 1, + "threshold": 1, + } + ], + ) + elif control_id == "N10": + after["identity"] = "coord:pypi:Qualification_Alpha" + after["rows"][0]["key"] = "coord:pypi:Qualification_Alpha" + else: + raise AssertionError(f"Unimplemented control {control_id}") + checker = lambda: validate_case(case, after) + detected = None + try: + checker() + except OracleMismatch as exc: + detected = exc.code + require("negative-target-reached", detected, target_check) + return { + "id": control_id, + "base_case": case_id, + "mutation": mutation, + "target_check": target_check, + "green_validated": True, + "red_detected": True, + "detected_check": detected, + "before": before, + "corrupted": after, + "mechanism": "reached semantic result replacement; no import or collection failures", + } + + +@pytest.mark.parametrize("control", CONTROLS, ids=lambda item: item[0]) +def test_negative_control_detects_its_target(control, tmp_path): + result = exercise_control(control, tmp_path) + assert result["green_validated"] and result["red_detected"] diff --git a/tools/sbom-diff-and-risk/tests/test_decision_stability_oracle.py b/tools/sbom-diff-and-risk/tests/test_decision_stability_oracle.py new file mode 100644 index 0000000..6890093 --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_decision_stability_oracle.py @@ -0,0 +1,862 @@ +"""Hand-authored qualification expectations. This module imports no product code. + +Rows are preregistered microcases, not a second general policy implementation. +The small assertion protocol is also used by deliberately corrupted observations. +""" + +from dataclasses import dataclass + + +KEY = "purl:pkg:pypi/qualification-alpha" +NAME = "qualification-alpha" +UNAVAILABLE = (False, True, False, ()) +MISSING = (False, False, False, ()) +UNVERIFIED = (True, False, False, ()) +GITHUB = (True, False, True, ("github actions",)) +MANUAL = (True, False, True, ("manual upload",)) + + +@dataclass(frozen=True) +class Row: + rule: str + level: str | None + reason: str + source: str | None + observed: object + threshold: object = None + key: str | None = KEY + name: str | None = NAME + bucket: str | None = None + confidence: str = "policy_matched" + group: str = "blocking" + suppression: str | None = None + + +def risk( + level="block", source="block_on", group="blocking", rule="new_package", bucket=None +): + return Row( + rule, + level, + "risk_finding_matched_policy_rule", + source, + bucket or rule, + bucket=bucket or rule, + group=group, + suppression="ignored_by_policy" if group == "suppressed" else None, + ) + + +def maximum(level="block", source="default_block", group="blocking"): + return Row( + "max_added_packages", + level, + "added_package_count_exceeded_threshold", + source, + 1, + 0, + key=None, + name=None, + group=group, + suppression="ignored_by_policy" if group == "suppressed" else None, + ) + + +def source_row( + host="github.com", + allowed=None, + level="block", + source="default_block", + group="blocking", +): + return Row( + "allow_sources", + level, + "source_host_not_allowed", + source, + host, + ["pypi.org"] if allowed is None else allowed, + group=group, + ) + + +def unavailable(recorded=False): + return Row( + "provenance_unavailable", + "block", + "provenance_evidence_unavailable", + "block_on", + "unavailable", + confidence="provenance_recorded" if recorded else "policy_matched", + ) + + +def missing(): + return Row( + "missing_attestation", + "block", + "attestation_not_published", + "block_on", + False, + confidence="provenance_recorded", + ) + + +def unverified(publishers=(), allowed=None): + return Row( + "unverified_provenance", + "block", + "provenance_publisher_not_verified", + "block_on", + list(publishers), + allowed, + confidence="provenance_recorded", + ) + + +def required( + observed="provenance_unavailable", + recorded=False, + level="block", + source="default_block", + group="blocking", +): + return Row( + "provenance_required", + level, + "required_provenance_not_satisfied", + source, + observed, + confidence="provenance_recorded" if recorded else "policy_matched", + group=group, + suppression="ignored_by_policy" if group == "suppressed" else None, + ) + + +def score_row(level="block", source="block_on", group="blocking"): + return Row( + "scorecard_below_threshold", + level, + "scorecard_score_below_threshold", + source, + 6.9, + 7.0, + confidence="scorecard_recorded", + group=group, + suppression="ignored_by_policy" if group == "suppressed" else None, + ) + + +@dataclass(frozen=True) +class Case: + id: str + family: str + policy: dict | None + rows: tuple[Row, ...] = () + findings: tuple[str, ...] = () + provenance: str = "none" + publishers: tuple[str, ...] = () + assessment: tuple = UNAVAILABLE + score: float | None = None + score_available: bool = True + source_url: str | None = "https://github.com/example/qualification" + scope: str = "added" + fail_on: str | None = None + warn_on: str | None = None + effective_version: int | None = None + invalid: bool = False + expected_decision: str = "PASS" + note: str = "" + + +CASES = ( + Case( + "M01", + "defaults", + None, + findings=("new_package",), + expected_decision="NOT_APPLIED", + ), + Case("M02", "defaults", {"version": 1}, findings=("new_package",)), + Case( + "M03", + "explicit-severity", + {"version": 1, "block_on": ["new_package"]}, + (risk(),), + ("new_package",), + expected_decision="FAIL", + ), + Case( + "M04", + "explicit-severity", + {"version": 1, "warn_on": ["new_package"]}, + (risk("warn", "warn_on", "warning"),), + ("new_package",), + expected_decision="WARN", + ), + Case( + "M05", + "explicit-severity", + {"version": 1, "block_on": ["new_package"], "warn_on": ["new_package"]}, + (risk(),), + ("new_package",), + expected_decision="FAIL", + ), + Case( + "M06", + "ignore", + { + "version": 1, + "block_on": ["new_package"], + "warn_on": ["new_package"], + "ignore_rules": ["new_package"], + }, + (risk(group="suppressed"),), + ("new_package",), + ), + Case( + "M07", + "ignore", + {"version": 1, "ignore_rules": ["new_package"]}, + (risk(None, None, "suppressed"),), + ("new_package",), + ), + Case( + "M08", + "maximum", + {"version": 1, "max_added_packages": 0}, + (maximum(),), + expected_decision="FAIL", + ), + Case("M09", "maximum", {"version": 1, "max_added_packages": 1}), + Case("M10", "maximum", {"version": 1, "max_added_packages": 2}), + Case( + "M11", + "maximum", + {"version": 1, "max_added_packages": 0, "warn_on": ["max_added_packages"]}, + (maximum("warn", "warn_on", "warning"),), + expected_decision="WARN", + ), + Case( + "M12", + "maximum", + {"version": 1, "max_added_packages": 0, "ignore_rules": ["max_added_packages"]}, + (maximum(group="suppressed"),), + ), + Case("M13", "sources", {"version": 1, "allow_sources": ["github.com"]}), + Case( + "M14", + "sources", + {"version": 1, "allow_sources": ["pypi.org"]}, + (source_row(),), + expected_decision="FAIL", + ), + Case( + "M15", + "sources", + {"version": 1, "allow_sources": ["github.com"], "warn_on": ["allow_sources"]}, + (source_row("missing", ["github.com"], "warn", "warn_on", "warning"),), + source_url=None, + expected_decision="WARN", + ), + Case("M16", "sources", {"version": 1, "allow_sources": []}, source_url=None), + Case( + "M17", + "unavailable", + {"version": 2, "block_on": ["provenance_unavailable"]}, + (unavailable(),), + expected_decision="FAIL", + ), + Case( + "M18", + "unavailable", + {"version": 2, "block_on": ["provenance_unavailable"]}, + (unavailable(True),), + provenance="error", + expected_decision="FAIL", + ), + Case( + "M19", + "unavailable", + {"version": 2, "block_on": ["provenance_unavailable"]}, + (unavailable(True),), + provenance="unsupported", + expected_decision="FAIL", + ), + Case( + "M20", + "missing-attestation", + {"version": 2, "block_on": ["missing_attestation"]}, + (missing(),), + provenance="missing", + assessment=MISSING, + expected_decision="FAIL", + ), + Case( + "M21", + "missing-attestation", + {"version": 2, "block_on": ["provenance_unavailable"]}, + provenance="missing", + assessment=MISSING, + ), + Case( + "M22", + "unverified", + {"version": 2, "block_on": ["unverified_provenance"]}, + (unverified(),), + provenance="attested", + assessment=UNVERIFIED, + expected_decision="FAIL", + ), + Case( + "M23", + "publishers", + {"version": 2, "block_on": ["unverified_provenance"]}, + provenance="attested", + publishers=("manual upload",), + assessment=MANUAL, + note="Recorded publisher kind, not cryptographic verification.", + ), + Case( + "M24", + "publishers", + { + "version": 2, + "block_on": ["unverified_provenance"], + "allow_provenance_publishers": ["manual upload"], + }, + (unverified(("github actions",), ["manual upload"]),), + provenance="attested", + publishers=("github actions",), + assessment=(True, False, False, ("github actions",)), + expected_decision="FAIL", + ), + Case( + "M25", + "publishers", + { + "version": 2, + "block_on": ["unverified_provenance"], + "allow_provenance_publishers": ["github actions"], + }, + provenance="attested", + publishers=("github actions",), + assessment=GITHUB, + ), + Case( + "M26", + "publishers", + { + "version": 2, + "block_on": ["unverified_provenance"], + "allow_provenance_publishers": ["github actions"], + }, + provenance="attested", + publishers=("manual upload", "github actions"), + assessment=(True, False, True, ("github actions", "manual upload")), + ), + Case( + "M27", + "required", + {"version": 2, "require_attestations_for_new_packages": True}, + (required(),), + expected_decision="FAIL", + ), + Case( + "M28", + "required", + {"version": 2, "require_attestations_for_new_packages": True}, + (required("attestation_missing", True),), + provenance="missing", + assessment=MISSING, + expected_decision="FAIL", + ), + Case( + "M29", + "required", + {"version": 2, "require_attestations_for_new_packages": True}, + (required("provenance_unverified", True),), + provenance="attested", + assessment=UNVERIFIED, + expected_decision="FAIL", + ), + Case( + "M30", + "required", + {"version": 2, "require_attestations_for_new_packages": True}, + provenance="attested", + publishers=("github actions",), + assessment=GITHUB, + ), + Case( + "M31", + "required", + { + "version": 2, + "require_attestations_for_new_packages": True, + "warn_on": ["provenance_required"], + }, + (required(level="warn", source="warn_on", group="warning"),), + expected_decision="WARN", + ), + Case( + "M32", + "required", + { + "version": 2, + "require_attestations_for_new_packages": True, + "ignore_rules": ["provenance_required"], + }, + (required(group="suppressed"),), + ), + Case("M33", "required", {"version": 2, "block_on": ["provenance_required"]}), + Case( + "M34", + "package-exception", + { + "version": 2, + "block_on": ["missing_attestation"], + "require_attestations_for_new_packages": True, + "allow_unattested_packages": [NAME], + }, + provenance="missing", + assessment=MISSING, + ), + Case( + "M35", + "package-exception", + { + "version": 2, + "block_on": ["provenance_unavailable"], + "require_attestations_for_new_packages": True, + "allow_unattested_packages": [NAME], + }, + (required(), unavailable()), + expected_decision="FAIL", + ), + Case( + "M36", + "package-exception", + { + "version": 2, + "block_on": ["unverified_provenance"], + "allow_unattested_packages": [NAME], + }, + (unverified(),), + provenance="attested", + assessment=UNVERIFIED, + expected_decision="FAIL", + ), + Case( + "M37", + "required", + {"version": 2, "require_provenance_for_suspicious_sources": True}, + (required("attestation_missing", True),), + findings=("suspicious_source",), + provenance="missing", + assessment=MISSING, + expected_decision="FAIL", + ), + Case( + "M38", + "required", + {"version": 2, "require_provenance_for_suspicious_sources": True}, + ), + Case( + "M39", + "scope", + {"version": 2, "require_attestations_for_new_packages": True}, + scope="changed", + ), + Case( + "M40", + "scope", + {"version": 2, "block_on": ["missing_attestation"]}, + provenance="missing", + assessment=MISSING, + scope="unchanged", + ), + Case( + "M41", + "contradiction", + { + "version": 2, + "block_on": ["provenance_unavailable"], + "require_attestations_for_new_packages": True, + }, + (required(recorded=True), unavailable(True)), + provenance="error-and-attested", + publishers=("github actions",), + expected_decision="FAIL", + ), + Case("M42", "scorecard", {"version": 3, "minimum_scorecard_score": 7}, score=6.9), + Case( + "M43", + "scorecard", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + (score_row(),), + score=6.9, + expected_decision="FAIL", + ), + Case( + "M44", + "scorecard", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + score=7.0, + ), + Case( + "M45", + "scorecard", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + score=7.1, + ), + Case( + "M46", + "scorecard", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + ), + Case( + "M47", + "scorecard", + { + "version": 3, + "minimum_scorecard_score": 7, + "warn_on": ["scorecard_below_threshold"], + }, + (score_row("warn", "warn_on", "warning"),), + score=6.9, + expected_decision="WARN", + ), + Case( + "M48", + "scorecard", + { + "version": 3, + "minimum_scorecard_score": 7, + "ignore_rules": ["scorecard_below_threshold"], + }, + (score_row(None, None, "suppressed"),), + score=6.9, + ), + Case( + "M49", + "scorecard", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + score=6.9, + score_available=False, + ), + Case( + "M50", + "version-eligibility", + {"version": 1, "block_on": ["missing_attestation"]}, + invalid=True, + expected_decision="INVALID_INPUT", + ), + Case( + "M51", + "version-eligibility", + {"version": 2, "block_on": ["scorecard_below_threshold"]}, + invalid=True, + expected_decision="INVALID_INPUT", + ), + Case( + "M52", + "version-eligibility", + {"version": 1, "allow_unattested_packages": [NAME]}, + invalid=True, + expected_decision="INVALID_INPUT", + ), + Case( + "M53", + "version-eligibility", + {"version": 2, "minimum_scorecard_score": 7}, + invalid=True, + expected_decision="INVALID_INPUT", + ), + Case( + "M54", + "cli-merge", + {"version": 1}, + fail_on="scorecard_below_threshold", + effective_version=3, + ), + Case( + "M55", + "cli-merge", + {"version": 1, "warn_on": ["new_package"]}, + (risk(),), + ("new_package",), + fail_on="new_package", + expected_decision="FAIL", + ), + Case( + "M56", + "cli-merge", + {"version": 1, "block_on": ["new_package"]}, + (risk(),), + ("new_package",), + warn_on="new_package", + expected_decision="FAIL", + ), + Case( + "M57", + "cli-merge", + None, + fail_on="provenance_unavailable", + effective_version=2, + rows=(unavailable(),), + expected_decision="FAIL", + ), + Case( + "M58", + "risk-alias", + {"version": 1, "warn_on": ["stale_package"]}, + (risk("warn", "warn_on", "warning", "stale_package", "not_evaluated"),), + ("not_evaluated",), + expected_decision="WARN", + ), + Case( + "M59", + "publisher-alias", + { + "version": 2, + "allow_unattested_publishers": [" GITHUB ACTIONS "], + "block_on": ["unverified_provenance"], + }, + provenance="attested", + publishers=(" GitHub Actions ",), + assessment=GITHUB, + ), + Case( + "M60", + "scope", + {"version": 1, "allow_sources": ["pypi.org"]}, + (source_row(),), + scope="changed", + expected_decision="FAIL", + ), + Case( + "M61", + "sources", + {"version": 1, "allow_sources": ["github.com"]}, + (source_row("sub.github.com", ["github.com"]),), + source_url="https://sub.github.com/example/qualification", + expected_decision="FAIL", + ), + Case( + "M62", + "invalid-types", + {"version": 1.0}, + invalid=True, + expected_decision="INVALID_INPUT", + ), + Case( + "M63", + "invalid-types", + {"version": 3, "minimum_scorecard_score": True}, + invalid=True, + expected_decision="INVALID_INPUT", + ), + Case( + "M64", + "cli-merge", + None, + warn_on="new_package", + effective_version=1, + rows=(risk("warn", "warn_on", "warning"),), + findings=("new_package",), + expected_decision="WARN", + ), + Case( + "M65", + "package-exception", + { + "version": 2, + "require_attestations_for_new_packages": True, + "allow_unattested_packages": [NAME], + }, + provenance="missing", + assessment=MISSING, + ), + Case( + "M66", + "package-exception", + { + "version": 2, + "block_on": ["missing_attestation"], + "allow_unattested_packages": [NAME], + }, + provenance="missing", + assessment=MISSING, + ), + Case( + "M67", + "package-exception", + { + "version": 2, + "require_attestations_for_new_packages": True, + "allow_unattested_packages": [NAME], + }, + (required(),), + expected_decision="FAIL", + ), + Case( + "M68", + "package-exception", + { + "version": 2, + "block_on": ["unverified_provenance"], + "allow_unattested_packages": [NAME], + }, + (unverified(),), + provenance="attested", + assessment=UNVERIFIED, + expected_decision="FAIL", + ), + Case( + "M69", + "contradiction", + { + "version": 2, + "block_on": ["provenance_unavailable"], + "require_attestations_for_new_packages": True, + }, + provenance="attested", + publishers=("github actions",), + assessment=GITHUB, + ), + Case( + "M70", + "scorecard", + { + "version": 3, + "minimum_scorecard_score": 6, + "block_on": ["scorecard_below_threshold"], + }, + score=6.9, + ), + Case( + "M71", + "version-eligibility", + {"version": 2, "block_on": ["new_package"]}, + (risk(),), + ("new_package",), + expected_decision="FAIL", + ), + Case( + "M72", + "version-eligibility", + {"version": 3, "block_on": ["scorecard_below_threshold"]}, + ), +) +CASE_BY_ID = {case.id: case for case in CASES} + + +class OracleMismatch(AssertionError): + def __init__(self, code, actual, expected): + super().__init__(f"{code}: actual={actual!r}, expected={expected!r}") + self.code = code + + +def require(code, actual, expected): + if not evidence_equal(actual, expected): + raise OracleMismatch(code, actual, expected) + + +def evidence_equal(actual, expected): + """Retain numeric equality, but never treat a boolean as numeric evidence.""" + if isinstance(actual, bool) or isinstance(expected, bool): + return type(actual) is type(expected) and actual == expected + if isinstance(actual, dict) and isinstance(expected, dict): + return actual.keys() == expected.keys() and all( + evidence_equal(actual[key], expected[key]) for key in expected + ) + if isinstance(actual, (list, tuple)) and isinstance(expected, type(actual)): + return len(actual) == len(expected) and all( + evidence_equal(left, right) for left, right in zip(actual, expected) + ) + return actual == expected + + +def expected_row(row): + return { + "rule": row.rule, + "level": row.level, + "reason": row.reason, + "source": row.source, + "observed": row.observed, + "threshold": row.threshold, + "key": row.key, + "name": row.name, + "bucket": row.bucket, + "confidence": row.confidence, + "group": row.group, + "suppression": row.suppression, + "policy_rule": row.rule, + } + + +def validate_case(case, actual): + require("decision", actual["decision"], case.expected_decision) + require( + "exit", + actual["exit_code"], + 2 if case.invalid else 1 if case.expected_decision == "FAIL" else 0, + ) + if case.invalid: + require("invalid-type", actual["error_type"], "PolicyError") + return + require("identity", actual["identity"], KEY) + require("assessment", actual["assessment"], case.assessment) + expected = [expected_row(row) for row in case.rows] + sort_key = lambda row: ( + row["group"], + row["rule"], + row["key"] or "", + row["name"] or "", + ) + require( + "violation-identity", + [ + (row["group"], row["rule"], row["key"]) + for row in sorted(actual["rows"], key=sort_key) + ], + [ + (row["group"], row["rule"], row["key"]) + for row in sorted(expected, key=sort_key) + ], + ) + for got, want in zip( + sorted(actual["rows"], key=sort_key), sorted(expected, key=sort_key) + ): + for name in want: + require(f"explanation-{name}", got[name], want[name]) + require( + "suppression-accounting", + actual["ignored_checks"], + sum(row.group == "suppressed" for row in case.rows), + ) + if case.effective_version is not None: + require( + "effective-version", actual["effective_version"], case.effective_version + ) diff --git a/tools/sbom-diff-and-risk/tests/test_decision_stability_pairs.py b/tools/sbom-diff-and-risk/tests/test_decision_stability_pairs.py new file mode 100644 index 0000000..685b567 --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_decision_stability_pairs.py @@ -0,0 +1,641 @@ +"""Preregistered single-dimension transitions, including non-monotonic cases.""" + +import random +from copy import deepcopy +from dataclasses import dataclass + +import pytest + +from sbom_diff_risk.component_identity import canonicalize_component_identity +from sbom_diff_risk.normalize import normalize_input + +from test_decision_stability import observe_case, render_bundle +from test_decision_stability_oracle import CASE_BY_ID, require, validate_case +from test_decision_stability_reports import ( + cdx_component, + consumer_decision, + run_cli, + validate_bundle, +) + + +@dataclass(frozen=True) +class Transition: + id: str + left: str + right: str + dimension: str + expected: str + reason: str + + +TRANSITIONS = tuple( + Transition(f"P{index:02d}", *values) + for index, values in enumerate( + ( + ( + "M02", + "M03", + "activate block rule", + "JUSTIFIED_REVERSAL", + "A local finding is enforced only by an explicit rule.", + ), + ( + "M03", + "M04", + "explicit severity", + "JUSTIFIED_REVERSAL", + "Configured severity changes enforcement.", + ), + ( + "M04", + "M05", + "overlapping block rule", + "JUSTIFIED_REVERSAL", + "Explicit block wins over an existing warn.", + ), + ( + "M05", + "M06", + "add ignore", + "JUSTIFIED_REVERSAL", + "Ignore takes precedence but retains suppression evidence.", + ), + ( + "M08", + "M09", + "count threshold 0 to 1", + "JUSTIFIED_REVERSAL", + "One added component does not exceed a threshold of one.", + ), + ( + "M09", + "M10", + "count threshold 1 to 2", + "STABLE", + "Neither threshold is exceeded.", + ), + ( + "M08", + "M11", + "explicit warning override", + "JUSTIFIED_REVERSAL", + "Warn overrides implicit default block.", + ), + ( + "M08", + "M12", + "ignore implicit block", + "JUSTIFIED_REVERSAL", + "The omitted block is accounted for as a suppression.", + ), + ( + "M13", + "M14", + "source allowlist membership", + "JUSTIFIED_REVERSAL", + "Observed host falls outside the narrowed allowed set.", + ), + ( + "M17", + "M18", + "disabled to failed provenance", + "STABLE", + "Both evidence states remain unavailable; confidence describes record presence.", + ), + ( + "M18", + "M19", + "failed to unsupported provenance", + "STABLE", + "Neither state supplies available provenance.", + ), + ( + "M17", + "M21", + "successful metadata without attestation", + "JUSTIFIED_REVERSAL", + "Unavailable-only policy stops blocking once a lookup succeeds; this is not verified provenance.", + ), + ( + "M22", + "M23", + "recorded publisher supplied", + "JUSTIFIED_REVERSAL", + "A nonempty publisher kind satisfies the current recorded-metadata contract without an allowlist.", + ), + ( + "M24", + "M25", + "publisher allowlist membership", + "JUSTIFIED_REVERSAL", + "The recorded publisher is explicitly authorized.", + ), + ( + "M25", + "M26", + "additional publisher kind", + "STABLE", + "The existing allowlist contract requires any matching kind.", + ), + ( + "M27", + "M28", + "unknown to known missing attestation", + "STABLE", + "Required provenance remains unsatisfied; explanation evidence changes.", + ), + ( + "M28", + "M29", + "missing to unverified attestation", + "STABLE", + "Presence alone does not satisfy a provenance requirement.", + ), + ( + "M29", + "M30", + "recorded publisher identity supplied", + "JUSTIFIED_REVERSAL", + "Recorded publisher metadata satisfies the existing requirement, without a cryptographic claim.", + ), + ( + "M27", + "M31", + "required rule warning override", + "JUSTIFIED_REVERSAL", + "Explicit warn overrides implicit blocking.", + ), + ( + "M27", + "M32", + "ignore required rule", + "JUSTIFIED_REVERSAL", + "An explicit suppression is retained.", + ), + ( + "M28", + "M65", + "authorize package exception", + "JUSTIFIED_REVERSAL", + "A name exception may satisfy a requirement caused solely by missing attestations.", + ), + ( + "M20", + "M66", + "authorize missing-attestation exception", + "JUSTIFIED_REVERSAL", + "Only the missing-attestation check is waived.", + ), + ( + "M27", + "M67", + "add exception to unavailable evidence", + "STABLE", + "A package exception does not waive unavailable provenance.", + ), + ( + "M22", + "M68", + "add exception to unverified evidence", + "STABLE", + "A package exception does not waive an unverified publisher.", + ), + ( + "M69", + "M41", + "add contradictory enrichment error", + "JUSTIFIED_REVERSAL", + "An explicit error dominates attestation availability.", + ), + ( + "M42", + "M43", + "activate Scorecard enforcement", + "JUSTIFIED_REVERSAL", + "A threshold alone is advisory; the block rule activates enforcement.", + ), + ( + "M43", + "M44", + "score reaches threshold", + "JUSTIFIED_REVERSAL", + "Equality is permitted; only a lower score violates.", + ), + ( + "M44", + "M45", + "score above threshold", + "STABLE", + "Both scores meet the minimum.", + ), + ( + "M43", + "M47", + "Scorecard severity", + "JUSTIFIED_REVERSAL", + "Explicit warn changes the policy decision but does not erase evidence.", + ), + ( + "M42", + "M48", + "ignore advisory Scorecard", + "STABLE", + "Decision stays pass while ignored evidence becomes visible.", + ), + ( + "M43", + "M49", + "Scorecard availability lost", + "JUSTIFIED_REVERSAL", + "Unknown scores are not fabricated into threshold violations under current policy.", + ), + ( + "M43", + "M70", + "lower Scorecard threshold", + "JUSTIFIED_REVERSAL", + "6.9 meets a configured minimum of 6.", + ), + ( + "M03", + "M71", + "policy version 1 to 2", + "STABLE", + "An existing local rule retains its severity across capability versions.", + ), + ( + "M51", + "M72", + "policy version 2 to 3", + "INVALID_INPUT", + "A Scorecard rule is rejected in v2 and eligible in v3.", + ), + ), + 1, + ) +) + + +def exercise_transition(transition, directory, seed=103): + left, right = CASE_BY_ID[transition.left], CASE_BY_ID[transition.right] + observations = [] + for label, case in (("left", left), ("right", right)): + observation, report = observe_case(case, directory / label, seed) + validate_case(case, observation) + if report is not None: + validate_bundle( + render_bundle(report), observation["exit_code"], expected_rows=case.rows + ) + observations.append(observation) + a, b = observations + observed = ( + "INVALID_INPUT" + if left.invalid or right.invalid + else "STABLE" + if a["decision"] == b["decision"] + else "JUSTIFIED_REVERSAL" + ) + require("transition-classification", observed, transition.expected) + return { + "id": transition.id, + "original_case_id": left.id, + "perturbed_case_id": right.id, + "perturbation": transition.dimension, + "before_canonical_identity": a.get("identity"), + "after_canonical_identity": b.get("identity"), + "source_evidence_changes": { + "provenance": [left.provenance, right.provenance], + "publishers": [left.publishers, right.publishers], + "score": [left.score, right.score], + "score_available": [left.score_available, right.score_available], + }, + "active_policy": {"before": left.policy, "after": right.policy}, + "previous_decision": a["decision"], + "resulting_decision": b["decision"], + "expected_classification": transition.expected, + "observed_classification": observed, + "reason": transition.reason, + "before": a, + "after": b, + } + + +@pytest.mark.parametrize("transition", TRANSITIONS, ids=lambda item: item.id) +def test_policy_evidence_transition(transition, tmp_path): + exercise_transition(transition, tmp_path) + + +def policy_projection(bundle): + """Raw evidence and presentation paths may differ under representation changes.""" + result = { + key: deepcopy(bundle["report"][key]) + for key in ( + "summary", + "blocking_findings", + "warning_findings", + "suppressed_findings", + ) + } + for section in ("blocking_findings", "warning_findings", "suppressed_findings"): + for row in result[section]: + row.pop("component_name") # Observed spelling is deliberately preserved. + return result + + +def exercise_representation(case_id, directory, seed=103): + base = cdx_component() + empty = {"bomFormat": "CycloneDX", "components": []} + before = {"bomFormat": "CycloneDX", "components": [deepcopy(base)]} + left_after = {"bomFormat": "CycloneDX", "components": [deepcopy(base)]} + right_after = deepcopy(left_after) + policy = {"version": 1, "block_on": ["new_package"]} + expected_decisions = ("FAIL", "FAIL") + expected_ids = ( + ["purl:pkg:pypi/qualification-alpha"], + ["purl:pkg:pypi/qualification-alpha"], + ) + expected_class = "STABLE" + reason = "Supported identity equivalence preserves enforcement; original evidence remains observable." + left_before = right_before = empty + if case_id == "R01": + right_after["components"][0]["name"] = "Qualification_Alpha" + elif case_id == "R02": + for record in (left_after["components"][0], right_after["components"][0]): + record.update( + name="Qualification", purl="pkg:npm/%40scope/Qualification@1.0.0" + ) + right_after["components"][0].update( + name="qualification", purl="pkg:npm/%40scope/qualification@1.0.0" + ) + expected_ids = (["purl:pkg:npm/%40scope/qualification"],) * 2 + elif case_id == "R03": + policy = {"version": 1, "block_on": ["major_upgrade"]} + left_before = right_before = before + left_after["components"][0].update( + version=None, purl="pkg:pypi/qualification-alpha@2.0.0" + ) + right_after["components"][0].update( + version="2.0.0", purl="pkg:pypi/qualification-alpha@2.0.0" + ) + elif case_id == "R04": + left_after["components"][0]["bom-ref"] = "opaque:qualification" + elif case_id in {"R05", "R06"}: + right_after["components"].append(deepcopy(base)) + if case_id == "R06": + right_after["components"][1]["licenses"] = [ + {"license": {"id": "BSD-3-Clause"}} + ] + expected_decisions = ("FAIL", "INVALID_INPUT") + expected_class = "INVALID_INPUT" + reason = "Duplicate or conflicting identities must be rejected before report completion." + expected_ids = (expected_ids[0], None) + elif case_id == "R07": + right_after["components"][0]["version"] = "2.0.0" + expected_decisions, expected_class = ("FAIL", "INVALID_INPUT"), "INVALID_INPUT" + expected_ids = (expected_ids[0], None) + reason = "An explicit version conflicts with its PURL version." + elif case_id == "R08": + right_after["components"][0]["name"] = "another-package" + expected_decisions, expected_class = ("FAIL", "INVALID_INPUT"), "INVALID_INPUT" + expected_ids = (expected_ids[0], None) + reason = "An explicit name conflicts with its PURL name." + elif case_id == "R09": + for record in (left_after["components"][0], right_after["components"][0]): + record.pop("purl") + left_after["components"][0]["bom-ref"] = "Opaque:X" + right_after["components"][0]["bom-ref"] = "opaque:x" + expected_ids = (["bom-ref:Opaque:X"], ["bom-ref:opaque:x"]) + reason = "Opaque references remain distinct; identical policy outcomes do not assert identity equivalence." + elif case_id == "R10": + left_after["components"][0].update( + name="Qualification", purl="pkg:generic/Qualification@1.0.0" + ) + right_after["components"][0].update( + name="qualification", purl="pkg:generic/qualification@1.0.0" + ) + expected_ids = ( + ["purl:pkg:generic/Qualification"], + ["purl:pkg:generic/qualification"], + ) + reason = "Unsupported ecosystem case equivalence is not invented." + elif case_id == "R11": + for record in (left_after["components"][0], right_after["components"][0]): + record.update(version=None, purl="pkg:generic/qualification-alpha") + right_after["components"][0]["description"] = "Irrelevant synthetic description" + expected_ids = (["purl:pkg:generic/qualification-alpha"],) * 2 + elif case_id == "R12": + right_after["components"][0]["purl"] = "not-a-purl" + expected_decisions, expected_class = ("FAIL", "INVALID_INPUT"), "INVALID_INPUT" + expected_ids = (expected_ids[0], None) + reason = "Malformed identity is rejected rather than silently falling back." + elif case_id == "R13": + right_before = {"spdxVersion": "SPDX-2.3", "packages": []} + right_after = { + "spdxVersion": "SPDX-2.3", + "packages": [ + { + "name": "qualification-alpha", + "versionInfo": "1.0.0", + "licenseConcluded": "MIT", + "downloadLocation": "https://github.com/example/qualification", + "externalRefs": [ + { + "referenceType": "purl", + "referenceLocator": "pkg:pypi/qualification-alpha@1.0.0", + } + ], + } + ], + } + reason = ( + "Only the shared component, version, license and source subset is compared." + ) + elif case_id == "R14": + names = [ + "qualification-alpha", + "qualification-beta", + "qualification-gamma", + "qualification-delta", + ] + left_after["components"] = [cdx_component(name) for name in names] + shuffled = list(names) + random.Random(seed).shuffle(shuffled) + right_after["components"] = [cdx_component(name) for name in shuffled] + expected_ids = (sorted(f"purl:pkg:pypi/{name}" for name in names),) * 2 + reason = ( + "Record ordering is irrelevant; each round records its actual permutation." + ) + elif case_id == "R15": + policy = {"version": 1, "allow_sources": ["pypi.org"]} + left_before = right_before = before + right_after["components"][0]["description"] = "Ignored synthetic annotation" + expected_decisions = ("PASS", "PASS") + expected_ids = ([], []) + reason = "Raw ignored metadata does not place an unchanged component into policy scope." + elif case_id == "R16": + policy = {"version": 1, "allow_sources": ["pypi.org"]} + left_before = right_before = before + right_after["components"][0]["licenses"] = [{"license": {"id": "BSD-3-Clause"}}] + expected_decisions, expected_class = ("PASS", "FAIL"), "JUSTIFIED_REVERSAL" + expected_ids = ([], expected_ids[1]) + reason = "Compared license metadata creates a changed component, so its existing disallowed source enters policy scope." + elif case_id == "R17": + right_after["components"][0].update( + name=" qualification-alpha ", version=" 1.0.0 " + ) + reason = "Supported field trimming and JSON serialization do not change identity or policy." + elif case_id == "R18": + right_after["components"][0]["version"] = 1 + expected_decisions, expected_class = ("FAIL", "INVALID_INPUT"), "INVALID_INPUT" + expected_ids = (expected_ids[0], None) + reason = "A non-string component version is invalid input." + else: + raise AssertionError(f"Unregistered representation case {case_id}") + outputs, identities, decisions = [], [], [] + for label, original, changed in ( + ("left", left_before, left_after), + ("right", right_before, right_after), + ): + code, bundle = run_cli( + directory / label, + original, + changed, + policy, + whitespace=(seed % 3 + 1) if label == "right" else 0, + ) + outputs.append(bundle) + decisions.append( + "INVALID_INPUT" if code == 2 else consumer_decision(bundle, code).upper() + ) + if bundle is None: + identities.append(None) + else: + current = [ + *bundle["report"]["components"]["added"], + *[item["after"] for item in bundle["report"]["components"]["changed"]], + ] + from sbom_diff_risk.models import Component + + identities.append( + sorted( + canonicalize_component_identity( + Component( + **{ + key: value + for key, value in record.items() + if key != "evidence" + } + ) + ).component_key + for record in current + ) + ) + require("representation-decisions", tuple(decisions), expected_decisions) + require("representation-identities", tuple(identities), expected_ids) + if expected_class == "STABLE" and case_id not in {"R09", "R10"}: + require( + "representation-policy-evidence", + policy_projection(outputs[0]), + policy_projection(outputs[1]), + ) + observed_class = ( + "INVALID_INPUT" + if "INVALID_INPUT" in decisions + else "STABLE" + if decisions[0] == decisions[1] + else "JUSTIFIED_REVERSAL" + ) + require("representation-classification", observed_class, expected_class) + return { + "id": case_id, + "original_case_id": case_id + "-left", + "perturbation": reason, + "before_canonical_identity": identities[0], + "after_canonical_identity": identities[1], + "source_evidence_changes": {"before": left_after, "after": right_after}, + "active_policy": policy, + "previous_decision": decisions[0], + "resulting_decision": decisions[1], + "expected_classification": expected_class, + "observed_classification": observed_class, + "reason": reason, + "record_order": [ + item.get("name") for item in right_after.get("components", []) + ], + } + + +REPRESENTATIONS = tuple(f"R{index:02d}" for index in range(1, 19)) + + +@pytest.mark.parametrize("case_id", REPRESENTATIONS) +def test_representation_transition(case_id, tmp_path): + exercise_representation(case_id, tmp_path) + + +def exercise_manifest_overlap(directory): + policy = {"version": 1, "warn_on": ["unknown_license"]} + projections = [] + for label, filename, content in ( + ("requirements", "requirements.txt", "qualification-alpha==1.0.0\n"), + ( + "pyproject", + "pyproject.toml", + '[project]\nname = "qualification-input"\nversion = "1.0.0"\ndependencies = ["qualification-alpha==1.0.0"]\n', + ), + ): + target = directory / label + target.mkdir(parents=True, exist_ok=True) + (target / filename).write_text(content, encoding="utf-8") + format_name, components, notes = normalize_input(target / filename) + require("manifest-overlap-count", len(components), 1) + identity = canonicalize_component_identity(components[0]) + require( + "manifest-overlap-key", + identity.component_key, + "purl:pkg:pypi/qualification-alpha", + ) + require("manifest-overlap-version", identity.version, "1.0.0") + from sbom_diff_risk.policy_evaluator import evaluate_policy + from sbom_diff_risk.policy_models import PolicyConfig + from sbom_diff_risk.risk import evaluate_risks + + evaluation = evaluate_policy( + PolicyConfig(version=1, warn_on=("unknown_license",)), + policy_path=None, + added=components, + changed=[], + findings=evaluate_risks(components, [], allowlist=[]), + ) + require( + "manifest-overlap-warning", + [ + (item.rule_id, item.level.value, item.component_key) + for item in evaluation.warning_violations + ], + [("unknown_license", "warn", "purl:pkg:pypi/qualification-alpha")], + ) + require("manifest-overlap-exit", evaluation.exit_code, 0) + projections.append( + (identity.component_key, identity.version, evaluation.exit_code) + ) + require("manifest-overlap", projections[0], projections[1]) + return { + "id": "R19", + "original_case_id": "requirements-exact-pin", + "perturbation": "requirements to pyproject exact-pin shared subset", + "before_canonical_identity": projections[0][0], + "after_canonical_identity": projections[1][0], + "source_evidence_changes": "Source formats differ; optional groups, markers and unknown constraints are excluded.", + "active_policy": policy, + "previous_decision": "WARN", + "resulting_decision": "WARN", + "expected_classification": "STABLE", + "observed_classification": "STABLE", + "reason": "Exact pins share coordinate identity and unknown-license evidence.", + } + + +def test_manifest_shared_subset(tmp_path): + exercise_manifest_overlap(tmp_path) diff --git a/tools/sbom-diff-and-risk/tests/test_decision_stability_providers.py b/tools/sbom-diff-and-risk/tests/test_decision_stability_providers.py new file mode 100644 index 0000000..4095a20 --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_decision_stability_providers.py @@ -0,0 +1,445 @@ +"""Actual CLI and provider parsers with synthetic byte streams, never a live service.""" + +import io +import json +from copy import deepcopy +from dataclasses import dataclass +from unittest.mock import patch +from urllib.error import HTTPError + +import pytest + +from sbom_diff_risk import cli +from sbom_diff_risk.enrichment import PyPIProvenanceEnricher +from sbom_diff_risk.pypi_integrity_client import PyPIIntegrityClient +from sbom_diff_risk.scorecard_client import ScorecardClient +from sbom_diff_risk.scorecard_enrichment import ScorecardEnricher + +from test_decision_stability_oracle import require +from test_decision_stability_reports import cdx_component, consumer_decision, run_cli + + +@dataclass(frozen=True) +class ProviderCase: + id: str + policy: dict + pypi: str | None = None + score: object = None + mapping: str = "vcs" + name: str = "qualification-alpha" + decision: str = "PASS" + rules: tuple[str, ...] = () + pypi_status: str | None = None + score_status: str | None = None + calls: tuple[int, int] = (0, 0) + + +PROVIDERS = ( + ProviderCase( + "E01", + {"version": 2, "block_on": ["missing_attestation"]}, + pypi="missing", + decision="FAIL", + rules=("missing_attestation",), + pypi_status="attestation_unavailable", + calls=(2, 0), + ), + ProviderCase( + "E02", + {"version": 2, "block_on": ["provenance_unavailable"]}, + pypi="error", + decision="FAIL", + rules=("provenance_unavailable",), + pypi_status="enrichment_error", + calls=(1, 0), + ), + ProviderCase( + "E03", + {"version": 2, "require_attestations_for_new_packages": True}, + pypi="missing", + mapping="npm", + pypi_status="unsupported_for_package", + ), + ProviderCase( + "E04", + {"version": 2, "block_on": ["provenance_unavailable"]}, + pypi="missing", + mapping="versionless", + decision="FAIL", + rules=("provenance_unavailable",), + pypi_status="unsupported_for_package", + ), + ProviderCase( + "E05", + {"version": 2, "block_on": ["unverified_provenance"]}, + pypi="unverified", + decision="FAIL", + rules=("unverified_provenance",), + pypi_status="attestation_available", + calls=(2, 0), + ), + ProviderCase( + "E06", + { + "version": 2, + "block_on": ["unverified_provenance"], + "allow_provenance_publishers": ["github actions"], + }, + pypi="github", + pypi_status="attestation_available", + calls=(2, 0), + ), + ProviderCase( + "E07", + { + "version": 2, + "block_on": ["unverified_provenance"], + "allow_provenance_publishers": ["manual upload"], + }, + pypi="github", + decision="FAIL", + rules=("unverified_provenance",), + pypi_status="attestation_available", + calls=(2, 0), + ), + ProviderCase( + "E08", + { + "version": 2, + "block_on": ["missing_attestation"], + "allow_unattested_packages": ["Qualification_Alpha"], + }, + pypi="missing", + pypi_status="attestation_unavailable", + calls=(2, 0), + ), + ProviderCase( + "E09", + { + "version": 2, + "block_on": ["provenance_unavailable"], + "require_attestations_for_new_packages": True, + "allow_unattested_packages": ["Qualification_Alpha"], + }, + pypi="error", + decision="FAIL", + rules=("provenance_required", "provenance_unavailable"), + pypi_status="enrichment_error", + calls=(1, 0), + ), + ProviderCase( + "E10", + { + "version": 2, + "block_on": ["unverified_provenance"], + "require_attestations_for_new_packages": True, + "allow_unattested_packages": ["Qualification_Alpha"], + }, + pypi="unverified", + decision="FAIL", + rules=("provenance_required", "unverified_provenance"), + pypi_status="attestation_available", + calls=(2, 0), + ), + ProviderCase( + "E11", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + score=6.9, + decision="FAIL", + rules=("scorecard_below_threshold",), + score_status="scorecard_available", + calls=(0, 1), + ), + ProviderCase( + "E12", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + score=7.0, + score_status="scorecard_available", + calls=(0, 1), + ), + ProviderCase( + "E13", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + score=7.1, + score_status="scorecard_available", + calls=(0, 1), + ), + ProviderCase( + "E14", + {"version": 3, "minimum_scorecard_score": 7}, + score=6.9, + score_status="scorecard_available", + calls=(0, 1), + ), + ProviderCase( + "E15", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + score=6.9, + mapping="gitlab", + score_status="repository_unmapped", + ), + ProviderCase( + "E16", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + score=6.9, + mapping="ambiguous", + score_status="repository_unmapped", + ), + ProviderCase( + "E17", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + score=6.9, + mapping="website", + score_status="repository_unmapped", + ), + ProviderCase( + "E18", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + score=6.9, + mapping="absent", + score_status="repository_unmapped", + ), + ProviderCase( + "E19", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + score="wrong-repo", + score_status="enrichment_error", + calls=(0, 1), + ), + ProviderCase( + "E20", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + score=True, + score_status="enrichment_error", + calls=(0, 1), + ), + ProviderCase( + "E21", + {"version": 3, "require_attestations_for_new_packages": True}, + pypi="missing", + score=7.0, + decision="FAIL", + rules=("provenance_required",), + pypi_status="attestation_unavailable", + score_status="scorecard_available", + calls=(2, 1), + ), + ProviderCase( + "E22", + { + "version": 3, + "minimum_scorecard_score": 7, + "block_on": ["scorecard_below_threshold"], + }, + score="404", + score_status="scorecard_unavailable", + calls=(0, 1), + ), +) + + +class SyntheticProviderTransport: + def __init__(self, case, provider): + self.case, self.provider, self.calls = case, provider, [] + + def open(self, req, timeout): + url = req.full_url + self.calls.append(url) + if self.provider == "scorecard": + if self.case.score == "404": + raise HTTPError(url, 404, "Synthetic missing score", None, None) + payload = {"score": self.case.score} + if self.case.score == "wrong-repo": + payload = {"score": 9, "repo": {"name": "github.com/other/unrelated"}} + elif "/pypi/" in url: + if self.case.pypi == "error": + raise HTTPError(url, 503, "Synthetic unavailable service", None, None) + payload = { + "info": {}, + "urls": [{"filename": "qualification-alpha-1.0.0.whl"}], + } + elif "/integrity/" in url: + bundles = ( + [] + if self.case.pypi == "missing" + else [{"attestations": [{"envelope": {"statement": None}}]}] + ) + if self.case.pypi == "github": + bundles[0]["publisher"] = {"kind": "github actions"} + payload = {"attestation_bundles": bundles} + else: + raise AssertionError("Unregistered synthetic provider request") + return io.BytesIO(json.dumps(payload).encode("utf-8")) + + +def exercise_provider(case, directory, *, spelling=None): + component = cdx_component( + spelling or case.name, purl="pkg:pypi/qualification-alpha@1.0.0" + ) + if case.mapping == "npm": + component["purl"] = "pkg:npm/qualification-alpha@1.0.0" + elif case.mapping == "versionless": + component.update(purl="pkg:pypi/qualification-alpha", version=None) + elif case.mapping == "gitlab": + component["externalReferences"][0]["url"] = ( + "https://gitlab.com/example/qualification" + ) + elif case.mapping == "ambiguous": + component["externalReferences"].append( + {"type": "vcs", "url": "https://github.com/other/unrelated"} + ) + elif case.mapping == "website": + component["externalReferences"][0]["type"] = "website" + elif case.mapping == "absent": + component.pop("externalReferences") + pypi_transport = SyntheticProviderTransport(case, "pypi") + score_transport = SyntheticProviderTransport(case, "scorecard") + pypi = PyPIProvenanceEnricher(client=PyPIIntegrityClient(opener=pypi_transport)) + scorecard = ScorecardEnricher(client=ScorecardClient(opener=score_transport)) + flags = [ + *(["--enrich-pypi"] if case.pypi else []), + *(["--enrich-scorecard"] if case.score is not None else []), + ] + with ( + patch.object(cli, "PyPIProvenanceEnricher", return_value=pypi), + patch.object(cli, "ScorecardEnricher", return_value=scorecard), + ): + code, bundle = run_cli( + directory, + {"bomFormat": "CycloneDX", "components": []}, + {"bomFormat": "CycloneDX", "components": [component]}, + case.policy, + flags, + expected_provider_lookup=any(case.calls), + ) + require("provider-decision", consumer_decision(bundle, code).upper(), case.decision) + require( + "provider-blocking-identities", + tuple(row["rule_id"] for row in bundle["report"]["blocking_findings"]), + case.rules, + ) + require( + "synthetic-transport-calls", + (len(pypi_transport.calls), len(score_transport.calls)), + case.calls, + ) + require( + "provider-confidence", + bundle["report"]["evidence_confidence"], + "scorecard_recorded" if case.score is not None else "provenance_recorded", + ) + recorded = bundle["report"]["components"]["added"][0] + require("raw-component-name", recorded["name"], spelling or case.name) + require( + "raw-component-version", + recorded["version"], + None if case.mapping == "versionless" else "1.0.0", + ) + for provider, status in ( + ("provenance", case.pypi_status), + ("scorecard", case.score_status), + ): + if status: + require( + "provider-record-status", + status in recorded["evidence"][provider]["statuses"], + True, + ) + return { + "id": case.id, + "decision": case.decision, + "policy": deepcopy(case.policy), + "component_key": bundle["report"]["risks"][0]["component_key"], + "observed_name": recorded["name"], + "rules": case.rules, + "provider_states": { + key: recorded["evidence"].get(key) for key in ("provenance", "scorecard") + }, + "synthetic_requests": { + "pypi": pypi_transport.calls, + "scorecard": score_transport.calls, + }, + "transport_kind": "in-memory byte streams", + "cryptographic_verification": "NOT_PERFORMED", + "tool_release_verification": "OUT_OF_SCOPE", + "actual_network": "forbidden by qualification guard", + } + + +@pytest.mark.parametrize("case", PROVIDERS, ids=lambda case: case.id) +def test_provider_to_cli_decision(case, tmp_path): + exercise_provider(case, tmp_path) + + +def exercise_c02_transition(directory): + case = next(case for case in PROVIDERS if case.id == "E08") + outputs = [ + exercise_provider(case, directory / label, spelling=label) + for label in ( + "qualification-alpha", + "Qualification_Alpha", + "qualification.alpha", + ) + ] + require("c02-stable-decisions", [row["decision"] for row in outputs], ["PASS"] * 3) + require( + "c02-stable-identity", + [row["component_key"] for row in outputs], + ["purl:pkg:pypi/qualification-alpha"] * 3, + ) + return { + "id": "C02", + "original_case_id": "E08", + "perturbation": "PEP 503-equivalent spelling only", + "before_canonical_identity": outputs[0]["component_key"], + "after_canonical_identity": outputs[1]["component_key"], + "source_evidence_changes": { + "observed_names": [row["observed_name"] for row in outputs] + }, + "active_policy": case.policy, + "previous_decision": "PASS", + "resulting_decision": "PASS", + "expected_classification": "STABLE", + "observed_classification": "STABLE", + "reason": "C02 POLICY DECISION APPROVED: canonical PyPI package-name equality; unchanged exception boundary.", + } + + +def test_c02_canonical_spelling_is_stable_through_actual_cli(tmp_path): + exercise_c02_transition(tmp_path) diff --git a/tools/sbom-diff-and-risk/tests/test_decision_stability_reports.py b/tools/sbom-diff-and-risk/tests/test_decision_stability_reports.py new file mode 100644 index 0000000..12fd271 --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_decision_stability_reports.py @@ -0,0 +1,493 @@ +"""Consumer consistency checks; cross-output agreement is not an independent policy oracle.""" + +import json +from copy import deepcopy +from pathlib import Path + +import pytest +import yaml + +from sbom_diff_risk import cli + +from test_decision_stability import observe_case, render_bundle +from test_decision_stability_oracle import ( + CASES, + KEY, + expected_row, + require, + validate_case, +) + + +@pytest.mark.parametrize( + "actual,expected", [(True, 1), (False, 0), ({"value": [True]}, {"value": [1]})] +) +def test_oracle_distinguishes_boolean_from_numeric_evidence(actual, expected): + from test_decision_stability_oracle import OracleMismatch + + with pytest.raises(OracleMismatch): + require("typed-evidence", actual, expected) + + +@pytest.mark.parametrize( + "field", + ["decision_reason", "observed_value", "matched_threshold", "markdown_message"], +) +def test_serialized_microcase_rejects_coherent_corruption(field, tmp_path, monkeypatch): + from test_decision_stability_oracle import CASE_BY_ID, OracleMismatch + + original_render = render_bundle + + def corrupted_render(report): + bundle = original_render(report) + row = bundle["report"]["blocking_findings"][0] + if field == "markdown_message": + assert row["message"] in bundle["markdown"] + bundle["markdown"] = bundle["markdown"].replace( + row["message"], "Incorrect decision explanation" + ) + else: + # Corrupt every mirrored occurrence, so agreement alone cannot detect it. + def corrupt(value): + if isinstance(value, dict): + if field in value: + value[field] = "incorrect evidence" + for child in value.values(): + corrupt(child) + elif isinstance(value, list): + for child in value: + corrupt(child) + + for key in ("report", "policy"): + corrupt(bundle[key]) + return bundle + + monkeypatch.setattr(__import__(__name__), "render_bundle", corrupted_render) + with pytest.raises(OracleMismatch): + test_microcase_reports_agree_with_independent_expectation( + CASE_BY_ID["M08"], tmp_path + ) + + +SARIF_EXPECTED = { + "M01": [], + "M02": [], + "M03": [("sdr.new_package", "error", KEY)], + "M04": [("sdr.new_package", "warning", KEY)], + "M06": [], + "M07": [], + "M08": [("sdr.policy_violation.max_added_packages", "error", None)], + "M11": [("sdr.policy_violation.max_added_packages", "warning", None)], + "M14": [("sdr.policy_violation.allow_sources", "error", KEY)], + "M17": [], + "M18": [], + "M19": [], + "M20": [("sdr.policy_violation.missing_attestation", "error", KEY)], + "M22": [("sdr.policy_violation.unverified_provenance", "error", KEY)], + "M27": [("sdr.policy_violation.provenance_required", "error", KEY)], + "M28": [("sdr.policy_violation.provenance_required", "error", KEY)], + "M31": [("sdr.policy_violation.provenance_required", "warning", KEY)], + "M37": [("sdr.policy_violation.provenance_required", "error", KEY)], + "M41": [("sdr.policy_violation.provenance_required", "error", KEY)], + "M42": [], + "M43": [("sdr.policy_violation.scorecard_below_threshold", "error", KEY)], + "M44": [], + "M47": [("sdr.policy_violation.scorecard_below_threshold", "warning", KEY)], + "M48": [], + "M49": [], + "M58": [], +} + + +def validate_bundle( + bundle, + exit_code, + *, + expected_confidence=None, + sarif_expected=None, + expected_rows=None, +): + full, summary, policy, markdown, sarif = ( + bundle[key] for key in ("report", "summary", "policy", "markdown", "sarif") + ) + require("report-schema", full["report_schema"], "sbom-diff-risk.report.v1") + require("policy-schema", policy["policy_schema"], "sbom-diff-risk.policy.v1") + require("summary-sidecar", summary, full["summary"]) + evaluation = full["policy_evaluation"] + for key in ( + "policy_evaluation", + "blocking_findings", + "warning_findings", + "suppressed_findings", + "rule_catalog", + ): + require("policy-sidecar", policy[key], full[key]) + require("metadata-evaluation", full["metadata"]["policy_evaluation"], evaluation) + require("exit-blocking", exit_code, 1 if full["blocking_findings"] else 0) + require("exit-report", evaluation["exit_code"], exit_code) + counts = {} + serialized_rows = [] + for group, section, title in ( + ("blocking", "blocking_findings", "Blocking violations"), + ("warning", "warning_findings", "Warnings"), + ("suppressed", "suppressed_findings", "Suppressions"), + ): + rows = full[section] + counts[group] = len(rows) + require("evaluation-rows", evaluation[f"{group}_violations"], rows) + for row in rows: + serialized_rows.append( + { + "group": group, + "rule": row["rule_id"], + "level": row["level"], + "reason": row["decision_reason"], + "source": row["severity_source"], + "observed": row["observed_value"], + "threshold": row["matched_threshold"], + "key": row["component_key"], + "name": row["component_name"], + "bucket": row["finding_bucket"], + "confidence": row["confidence_level"], + "suppression": row["suppression_reason"], + "policy_rule": row["policy_rule"], + } + ) + require("matched-rule", row["matched_rule_id"], row["rule_id"]) + require( + "exact-evidence", + row["exact_evidence"], + { + "component_key": row["component_key"], + "finding_bucket": row["finding_bucket"], + "matched_threshold": row["matched_threshold"], + "observed_value": row["observed_value"], + }, + ) + require("decision-reason-present", bool(row["decision_reason"]), True) + section_text = markdown.split(f"## {title}\n", 1)[1].split("\n## ", 1)[0] + level = row["level"] or "n/a" + prefix = f"| {row['rule_id']} | {row['component_name'] or ''} | {level} |" + require("markdown-row", prefix in section_text, True) + message = row["message"].replace("|", "\\|").replace("\n", " ") + tail = ( + f" {row['suppression_reason'] or ''} |" if group == "suppressed" else "" + ) + require( + "markdown-message", f"{prefix}{tail} {message} |" in section_text, True + ) + if group == "suppressed": + require( + "suppression-reason", row["suppression_reason"], "ignored_by_policy" + ) + require( + "markdown-suppression", "ignored_by_policy" in section_text, True + ) + if expected_rows is not None: + sort_key = lambda row: ( + row["group"], + row["rule"], + row["key"] or "", + row["name"] or "", + ) + require( + "serialized-independent-evidence", + sorted(serialized_rows, key=sort_key), + sorted((expected_row(row) for row in expected_rows), key=sort_key), + ) + require( + "totals", + evaluation["totals"], + {**counts, "ignored_checks": counts["suppressed"]}, + ) + if evaluation["applied"]: + expected_summary = { + "status": "fail" + if counts["blocking"] + else "warn" + if counts["warning"] + else "pass", + **counts, + } + require("policy-summary", summary["policy"], expected_summary) + require("sidecar-policy-summary", policy["summary"]["policy"], expected_summary) + else: + require("unapplied-summary", "policy" in summary, False) + require("unapplied-sidecar", "summary" in policy, False) + confidence = full["evidence_confidence"] + require("confidence-summary", summary["evidence_confidence"], confidence) + require("confidence-metadata", full["metadata"]["evidence_confidence"], confidence) + if expected_confidence is not None: + require("confidence-authority", confidence, expected_confidence) + for text in ( + f"- Exit code: {exit_code}", + f"- Blocking findings: {counts['blocking']}", + f"- Warnings: {counts['warning']}", + f"- Suppressed findings: {counts['suppressed']}", + f"- Evidence confidence: {confidence}", + ): + require("markdown-summary", text in markdown, True) + require( + "heuristic-authority", + "This tool uses heuristic risk classification." in full["notes"], + True, + ) + require("no-security-verdict", full.get("package_safety_verdict"), None) + require("sarif-schema", sarif["version"], "2.1.0") + results = sarif["runs"][0]["results"] + if sarif_expected is not None: + require( + "sarif-subset", + sorted( + (row["ruleId"], row["level"], row["properties"].get("component_key")) + for row in results + ), + sorted(sarif_expected), + ) + for row in results: + props = row["properties"] + if "policy_rule_id" in props: + candidates = [ + item + for item in (*full["blocking_findings"], *full["warning_findings"]) + if item["rule_id"] == props["policy_rule_id"] + and item["component_key"] == props["component_key"] + ] + require("sarif-policy-identity", bool(candidates), True) + require( + "sarif-policy-level", + row["level"], + "error" if candidates[0]["level"] == "block" else "warning", + ) + + +def consumer_decision(bundle, exit_code): + """Demonstration consumer classification, not an added native exit status.""" + if exit_code == 2 or not bundle["policy"]["policy_evaluation"]["applied"]: + return "needs-review" + validate_bundle(bundle, exit_code) + return bundle["summary"]["policy"]["status"] + + +@pytest.mark.parametrize( + "case", [case for case in CASES if not case.invalid], ids=lambda case: case.id +) +def test_microcase_reports_agree_with_independent_expectation(case, tmp_path): + actual, report = observe_case(case, tmp_path) + validate_case(case, actual) + confidence = ( + "scorecard_recorded" + if case.score is not None and case.scope != "unchanged" + else "provenance_recorded" + if case.provenance != "none" and case.scope != "unchanged" + else "policy_matched" + if case.rows + else "sbom_present" + ) + bundle = render_bundle(report) + validate_bundle( + bundle, + actual["exit_code"], + expected_confidence=confidence, + sarif_expected=SARIF_EXPECTED.get(case.id), + expected_rows=case.rows, + ) + require( + "consumer-disclosure", + consumer_decision(bundle, actual["exit_code"]), + "needs-review" + if case.expected_decision == "NOT_APPLIED" + else case.expected_decision.lower(), + ) + + +def cdx_component(name="qualification-alpha", version="1.0.0", *, purl=None): + return { + "name": name, + "version": version, + "purl": purl or f"pkg:pypi/{name}@{version}", + "licenses": [{"license": {"id": "MIT"}}], + "externalReferences": [ + {"type": "vcs", "url": "https://github.com/example/qualification"} + ], + } + + +def run_cli( + directory, + before, + after, + policy=None, + flags=(), + *, + whitespace=0, + expected_provider_lookup=False, +): + directory.mkdir(parents=True, exist_ok=True) + for name, value in (("before.json", before), ("after.json", after)): + (directory / name).write_text( + json.dumps(value, indent=whitespace or None, sort_keys=bool(whitespace)), + encoding="utf-8", + ) + argv = [ + "compare", + "--before", + str(directory / "before.json"), + "--after", + str(directory / "after.json"), + ] + if policy is not None: + (directory / "policy.yml").write_text(yaml.safe_dump(policy), encoding="utf-8") + argv.extend(["--policy", str(directory / "policy.yml")]) + argv.extend(flags) + for flag, name in ( + ("--out-json", "report.json"), + ("--summary-json", "summary.json"), + ("--policy-json", "policy.json"), + ("--out-md", "report.md"), + ("--out-sarif", "report.sarif"), + ): + argv.extend([flag, str(directory / name)]) + code = cli.main(argv) + if code == 2: + require( + "invalid-cli-no-report", + any( + (directory / name).exists() + for name in ( + "report.json", + "summary.json", + "policy.json", + "report.md", + "report.sarif", + ) + ), + False, + ) + return code, None + bundle = { + key: json.loads((directory / name).read_text(encoding="utf-8")) + for key, name in ( + ("report", "report.json"), + ("summary", "summary.json"), + ("policy", "policy.json"), + ("sarif", "report.sarif"), + ) + } + bundle["markdown"] = (directory / "report.md").read_text(encoding="utf-8") + validate_bundle(bundle, code) + require( + "provider-lookup-record", + bundle["report"]["enrichment_metadata"]["network_access_performed"], + expected_provider_lookup, + ) + return code, bundle + + +CLI_CASES = ( + ("no-policy", None, (), "needs-review", 0), + ("default", {"version": 1}, (), "pass", 0), + ("warn", {"version": 1, "warn_on": ["new_package"]}, (), "warn", 0), + ("block", {"version": 1, "block_on": ["new_package"]}, (), "fail", 1), + ( + "ignore", + {"version": 1, "block_on": ["new_package"], "ignore_rules": ["new_package"]}, + (), + "pass", + 0, + ), + ( + "required", + {"version": 2, "require_attestations_for_new_packages": True}, + (), + "fail", + 1, + ), + ( + "unavailable", + {"version": 2, "block_on": ["provenance_unavailable"]}, + (), + "fail", + 1, + ), + ("sources", {"version": 1, "allow_sources": ["pypi.org"]}, (), "fail", 1), + ( + "merged", + {"version": 1, "warn_on": ["new_package"]}, + ("--fail-on", "new_package"), + "fail", + 1, + ), + ( + "promoted-provenance", + {"version": 1}, + ("--fail-on", "provenance_unavailable"), + "fail", + 1, + ), + ( + "promoted-scorecard", + {"version": 1}, + ("--warn-on", "scorecard_below_threshold"), + "pass", + 0, + ), +) + + +@pytest.mark.parametrize( + "label,policy,flags,decision,exit_code", + CLI_CASES, + ids=[row[0] for row in CLI_CASES], +) +def test_actual_cli_and_consumer_decisions( + tmp_path, label, policy, flags, decision, exit_code +): + code, bundle = run_cli( + tmp_path, + {"bomFormat": "CycloneDX", "components": []}, + {"bomFormat": "CycloneDX", "components": [cdx_component()]}, + policy, + flags, + ) + require("cli-status", code, exit_code) + require("consumer-status", consumer_decision(bundle, code), decision) + if label == "promoted-provenance": + require( + "cli-promotion-v2", + bundle["policy"]["policy_evaluation"]["effective_policy"]["version"], + 2, + ) + if label == "promoted-scorecard": + require( + "cli-promotion-v3", + bundle["policy"]["policy_evaluation"]["effective_policy"]["version"], + 3, + ) + + +def test_missing_consumer_policy_is_review_not_affirmative_safety(tmp_path): + _, bundle = run_cli( + tmp_path, + {"bomFormat": "CycloneDX", "components": []}, + {"bomFormat": "CycloneDX", "components": [cdx_component()]}, + ) + require("consumer-no-policy", consumer_decision(bundle, 0), "needs-review") + require( + "consumer-runtime-error", consumer_decision(deepcopy(bundle), 2), "needs-review" + ) + for name in ("pass", "warn", "fail", "needs-review"): + example = json.loads( + ( + Path(__file__).resolve().parents[1] + / "examples" + / "policy-decisions" + / f"{name}.json" + ).read_text(encoding="utf-8") + ) + require( + "consumer-example-authority", + "not a dependency safety verdict" + in example["reviewer_interpretation"]["non_claims"], + True, + ) diff --git a/tools/sbom-diff-and-risk/tests/test_decision_stability_runner.py b/tools/sbom-diff-and-risk/tests/test_decision_stability_runner.py new file mode 100644 index 0000000..fbe2ed3 --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_decision_stability_runner.py @@ -0,0 +1,55 @@ +"""The public replay entry point must execute its real offline worker.""" + +import json +import os +import runpy +import subprocess +import sys +from pathlib import Path + + +def test_replay_worker_executes_registered_cases_with_network_guard(tmp_path): + root = Path(__file__).resolve().parents[1] + script = root / "scripts" / "qualify-decision-stability.py" + namespace = runpy.run_path(str(script), run_name="qualification_definition") + guard = tmp_path / "guard" + guard.mkdir() + (guard / "sitecustomize.py").write_text(namespace["GUARD"], encoding="utf-8") + output = tmp_path / "round" + output.mkdir() + attempts = tmp_path / "network-attempts.jsonl" + environment = { + **os.environ, + "PYTHONPATH": str(guard), + "PYTHONDONTWRITEBYTECODE": "1", + "SCT_QUALIFICATION_NETWORK_LOG": str(attempts), + "TMP": str(tmp_path), + "TEMP": str(tmp_path), + } + result = subprocess.run( + [ + sys.executable, + str(script), + "--worker", + "--seeds", + "103", + "--output-root", + str(output), + ], + cwd=root, + env=environment, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + assert result.returncode == 0, result.stderr[-2000:] + data = json.loads((output / "results.json").read_text(encoding="utf-8")) + assert data["counts"]["microcases"] == 72 + assert data["counts"]["negative_controls"] == 14 + assert data["import_origin"] == "src/sbom_diff_risk/__init__.py" + assert all( + item["red_detected"] and item["green_validated"] + for item in data["negative_controls"] + ) + assert not attempts.exists() diff --git a/tools/sbom-diff-and-risk/tests/test_policy_package_equivalence.py b/tools/sbom-diff-and-risk/tests/test_policy_package_equivalence.py new file mode 100644 index 0000000..6948cfb --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_policy_package_equivalence.py @@ -0,0 +1,175 @@ +"""C02: the approved PyPI package-name exception boundary.""" + +import json +from pathlib import Path + +import pytest +import yaml + +from sbom_diff_risk.models import ( + Component, + ProvenanceEvidence, + ProvenanceFileEvidence, + ProvenanceStatus, +) +from sbom_diff_risk.policy_evaluator import ( + _package_is_unattested_allowed, + evaluate_policy, +) +from sbom_diff_risk.policy_models import PolicyConfig +from sbom_diff_risk.policy_parser import load_policy +from sbom_diff_risk.presentation import policy_evaluation_to_dict + + +SPELLINGS = ("demo-pkg", "Demo_Pkg", "demo.pkg") + + +def component(name, state="missing", ecosystem="pypi"): + states = { + "missing": ( + ProvenanceStatus.PROVENANCE_AVAILABLE, + ProvenanceStatus.ATTESTATION_UNAVAILABLE, + ), + "unavailable": (ProvenanceStatus.ENRICHMENT_ERROR,), + "unverified": ( + ProvenanceStatus.PROVENANCE_AVAILABLE, + ProvenanceStatus.ATTESTATION_AVAILABLE, + ), + } + statuses = states[state] + return Component( + name, + "1.0.0", + ecosystem, + purl=f"pkg:{ecosystem}/demo-pkg@1.0.0", + provenance=ProvenanceEvidence( + "synthetic-pypi", + True, + supported=True, + lookup_performed=True, + statuses=statuses, + files=( + ProvenanceFileEvidence( + "synthetic.whl", + statuses=statuses, + attestation_count=int(state == "unverified"), + ), + ), + ), + ) + + +def policy(directory: Path, mode, entry): + values = { + "version": 2, + "block_on": [ + "missing_attestation", + "provenance_unavailable", + "unverified_provenance", + ], + "require_attestations_for_new_packages": True, + "allow_unattested_packages": [entry], + } + if mode == "direct": + return PolicyConfig( + **{ + **values, + "allow_unattested_packages": (entry,), + "block_on": tuple(values["block_on"]), + } + ) + path = directory / "policy.yml" + path.write_text(yaml.safe_dump(values), encoding="utf-8") + return load_policy(path) + + +@pytest.mark.parametrize("mode", ["direct", "yaml"]) +@pytest.mark.parametrize("name", SPELLINGS) +@pytest.mark.parametrize("entry", SPELLINGS) +def test_equivalent_pypi_names_keep_missing_attestation_exception( + tmp_path, mode, name, entry +): + config = policy(tmp_path, mode, entry) + evaluation = evaluate_policy( + config, policy_path=None, added=[component(name)], changed=[], findings=[] + ) + assert evaluation.exit_code == 0 + assert ( + evaluation.blocking_violations + == evaluation.warning_violations + == evaluation.suppressed_violations + == [] + ) + serialized = policy_evaluation_to_dict(evaluation)["effective_policy"][ + "allow_unattested_packages" + ] + assert serialized == [entry if mode == "direct" else entry.lower()] + + +@pytest.mark.parametrize( + "entry", ["demo-pkg-extra", "demopkg", "pkg:pypi/demo-pkg@1.0.0"] +) +def test_non_equivalent_names_do_not_authorize_exception(tmp_path, entry): + evaluation = evaluate_policy( + policy(tmp_path, "direct", entry), + policy_path=None, + added=[component("demo-pkg")], + changed=[], + findings=[], + ) + assert evaluation.exit_code == 1 + assert [item.rule_id for item in evaluation.blocking_violations] == [ + "missing_attestation", + "provenance_required", + ] + + +@pytest.mark.parametrize("mode", ["direct", "yaml"]) +@pytest.mark.parametrize( + "state,expected", + [ + ("unavailable", ["provenance_required", "provenance_unavailable"]), + ("unverified", ["provenance_required", "unverified_provenance"]), + ], +) +def test_name_exception_does_not_waive_unavailable_or_unverified( + tmp_path, mode, state, expected +): + evaluation = evaluate_policy( + policy(tmp_path, mode, "Demo_Pkg"), + policy_path=None, + added=[component("demo.pkg", state)], + changed=[], + findings=[], + ) + assert evaluation.exit_code == 1 + assert [item.rule_id for item in evaluation.blocking_violations] == expected + + +@pytest.mark.parametrize("ecosystem", ["npm", "generic", "maven"]) +def test_package_name_exception_is_pypi_only(ecosystem): + config = PolicyConfig(version=2, allow_unattested_packages=("demo-pkg",)) + assert ( + _package_is_unattested_allowed( + config, component("demo-pkg", ecosystem=ecosystem) + ) + is False + ) + + +def test_original_c02_reproducer(tmp_path): + config = PolicyConfig( + version=2, + block_on=("missing_attestation",), + allow_unattested_packages=("demo-pkg",), + ) + outputs = [] + for name in SPELLINGS: + evaluation = evaluate_policy( + config, policy_path=None, added=[component(name)], changed=[], findings=[] + ) + outputs.append(policy_evaluation_to_dict(evaluation)) + (tmp_path / "c02-observations.json").write_text( + json.dumps(outputs, indent=2), encoding="utf-8" + ) + assert [value["exit_code"] for value in outputs] == [0, 0, 0] diff --git a/tools/sbom-diff-and-risk/tests/test_policy_validation_qualification.py b/tools/sbom-diff-and-risk/tests/test_policy_validation_qualification.py new file mode 100644 index 0000000..219e76a --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_policy_validation_qualification.py @@ -0,0 +1,94 @@ +"""Public YAML/CLI regressions for the documented fail-closed type boundary.""" + +import json + +import pytest + +from sbom_diff_risk import cli +from sbom_diff_risk.errors import PolicyError +from sbom_diff_risk.policy_parser import load_policy + + +@pytest.mark.parametrize( + "text", + [ + "version: true\n", + "version: false\n", + "version: 1\nmax_added_packages: true\n", + "version: 1\nmax_added_packages: false\n", + "version: 1\n0: unsupported\n", + "version: 1\n0: unsupported\nunknown: unsupported\n", + ], + ids=[ + "boolean-version-true", + "boolean-version-false", + "boolean-threshold-true", + "boolean-threshold-false", + "numeric-key", + "mixed-key-types", + ], +) +def test_policy_invalid_integer_fields_and_keys_fail_closed(tmp_path, text): + path = tmp_path / "policy.yml" + path.write_text(text, encoding="utf-8") + with pytest.raises(PolicyError): + load_policy(path) + + +@pytest.mark.parametrize("threshold", [0, 1]) +def test_integer_zero_and_one_remain_valid_thresholds(tmp_path, threshold): + path = tmp_path / "policy.yml" + path.write_text(f"version: 1\nmax_added_packages: {threshold}\n", encoding="utf-8") + policy = load_policy(path) + assert type(policy.version) is int and policy.version == 1 + assert ( + type(policy.max_added_packages) is int + and policy.max_added_packages == threshold + ) + + +@pytest.mark.parametrize( + "policy_text", + [ + "version: true\n", + "version: 1\nmax_added_packages: true\n", + "version: 1\nmax_added_packages: false\n", + "version: 1\n0: unsupported\n", + ], +) +def test_cli_invalid_policy_is_error_before_reports(tmp_path, monkeypatch, policy_text): + monkeypatch.chdir(tmp_path) + before = {"bomFormat": "CycloneDX", "components": []} + after = { + "bomFormat": "CycloneDX", + "components": [ + { + "name": "qualification-alpha", + "version": "1.0.0", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + } + ], + } + for name, payload in [("before.json", before), ("after.json", after)]: + (tmp_path / name).write_text(json.dumps(payload), encoding="utf-8") + (tmp_path / "policy.yml").write_text(policy_text, encoding="utf-8") + assert ( + cli.main( + [ + "compare", + "--before", + "before.json", + "--after", + "after.json", + "--policy", + "policy.yml", + "--out-json", + "report.json", + "--policy-json", + "policy.json", + ] + ) + == 2 + ) + assert not (tmp_path / "report.json").exists() + assert not (tmp_path / "policy.json").exists() diff --git a/tools/sbom-diff-and-risk/tests/test_scorecard_evidence_integrity.py b/tools/sbom-diff-and-risk/tests/test_scorecard_evidence_integrity.py new file mode 100644 index 0000000..9898bb8 --- /dev/null +++ b/tools/sbom-diff-and-risk/tests/test_scorecard_evidence_integrity.py @@ -0,0 +1,169 @@ +"""Synthetic byte-stream evidence must stay bound and strictly serializable.""" + +import io +import json +from unittest.mock import patch + +import pytest + +from sbom_diff_risk import cli +from sbom_diff_risk.scorecard_client import ScorecardClient, ScorecardClientError +from sbom_diff_risk.scorecard_enrichment import ScorecardEnricher + + +class ByteStreamOpener: + def __init__(self, payload): + self.body = json.dumps(payload).encode("utf-8") + self.calls = [] + + def open(self, req, timeout): + self.calls.append(req.full_url) + return io.BytesIO(self.body) + + +@pytest.mark.parametrize( + "value", + [True, False, float("inf"), float("-inf"), float("nan")], + ids=["true", "false", "positive-infinity", "negative-infinity", "nan"], +) +@pytest.mark.parametrize("field", ["overall", "check"]) +def test_invalid_score_numbers_are_controlled_client_errors(value, field): + payload = ( + {"score": value} + if field == "overall" + else {"score": 8, "checks": [{"name": "Synthetic", "score": value}]} + ) + client = ScorecardClient(opener=ByteStreamOpener(payload)) + with pytest.raises(ScorecardClientError): + client.fetch_project("github.com", "example", "qualification") + + +def test_explicit_scorecard_repository_mismatch_is_rejected(): + opener = ByteStreamOpener( + { + "repo": {"name": "github.com/other/unrelated", "commit": "other-commit"}, + "score": 9, + } + ) + with pytest.raises(ScorecardClientError): + ScorecardClient(opener=opener).fetch_project( + "github.com", "example", "qualification" + ) + assert len(opener.calls) == 1 + + +@pytest.mark.parametrize( + "requested,returned", + [("repo-ss", "repo-\u00df"), ("repo-k", "repo-\u212a"), ("repo-ff", "repo-\ufb00")], + ids=["sharp-s-expansion", "kelvin-sign", "ligature-expansion"], +) +def test_scorecard_response_does_not_use_unicode_name_aliases(requested, returned): + payload = {"repo": {"name": f"github.com/example/{returned}"}, "score": 9} + with pytest.raises(ScorecardClientError): + ScorecardClient(opener=ByteStreamOpener(payload)).fetch_project( + "github.com", "example", requested + ) + + +@pytest.mark.parametrize( + "repo", + [ + None, + {"commit": "synthetic-commit"}, + {"name": "github.com/example/qualification"}, + {"name": "GitHub.com/EXAMPLE/QUALIFICATION"}, + ], + ids=["repository-omitted", "name-omitted", "exact", "case-only"], +) +def test_scorecard_identity_compatibility_and_finite_sentinel(repo): + payload = {"score": 7.5, "checks": [{"name": "Synthetic", "score": -1}]} + if repo is not None: + payload["repo"] = repo + result = ScorecardClient(opener=ByteStreamOpener(payload)).fetch_project( + "github.com", "example", "qualification" + ) + assert result.score == 7.5 and result.checks[0].score == -1 + assert result.canonical_name.casefold() == "github.com/example/qualification" + + +def reject_nonfinite(value): + raise ValueError(f"Non-JSON numeric token: {value}") + + +@pytest.mark.parametrize( + "payload", + [ + {"score": True}, + {"score": float("inf")}, + {"score": float("-inf")}, + {"score": float("nan")}, + { + "repo": {"name": "github.com/other/unrelated", "commit": "other-commit"}, + "score": 9, + }, + ], + ids=[ + "boolean", + "positive-infinity", + "negative-infinity", + "nan", + "repository-conflict", + ], +) +def test_cli_bad_scorecard_evidence_is_error_record_not_available_score( + tmp_path, monkeypatch, payload +): + monkeypatch.chdir(tmp_path) + (tmp_path / "before.json").write_text( + json.dumps({"bomFormat": "CycloneDX", "components": []}), encoding="utf-8" + ) + component = { + "name": "qualification-alpha", + "version": "1.0.0", + "purl": "pkg:pypi/qualification-alpha@1.0.0", + "licenses": [{"license": {"id": "MIT"}}], + "externalReferences": [ + {"type": "vcs", "url": "https://github.com/example/qualification"} + ], + } + (tmp_path / "after.json").write_text( + json.dumps({"bomFormat": "CycloneDX", "components": [component]}), + encoding="utf-8", + ) + (tmp_path / "policy.yml").write_text( + "version: 3\nblock_on: [scorecard_below_threshold]\nminimum_scorecard_score: 7\n", + encoding="utf-8", + ) + opener = ByteStreamOpener(payload) + enricher = ScorecardEnricher(client=ScorecardClient(opener=opener)) + with patch.object(cli, "ScorecardEnricher", return_value=enricher): + code = cli.main( + [ + "compare", + "--before", + "before.json", + "--after", + "after.json", + "--policy", + "policy.yml", + "--enrich-scorecard", + "--out-json", + "report.json", + ] + ) + assert code == 0 # Missing Scorecard evidence does not create a new blocking rule. + report = json.loads( + (tmp_path / "report.json").read_text(encoding="utf-8"), + parse_constant=reject_nonfinite, + ) + recorded = report["components"]["added"][0]["evidence"]["scorecard"] + assert recorded["statuses"] == ["enrichment_error"] + assert recorded["score"] is None and recorded["error"] + assert report["summary"]["policy"] == { + "status": "pass", + "blocking": 0, + "warning": 0, + "suppressed": 0, + } + assert report["evidence_confidence"] == "scorecard_recorded" + assert len(opener.calls) == 1