From 55bf0c8a8c468be2ddc090b6d1f22d1067a0e1cd Mon Sep 17 00:00:00 2001 From: stacknil <219124580+stacknil@users.noreply.github.com> Date: Mon, 19 Oct 2026 00:50:57 +0800 Subject: [PATCH] fix(telemetry): clarify empty bounded-correlation evidence Signed-off-by: stacknil <219124580+stacknil@users.noreply.github.com> --- .../artifacts/investigation_hits.json | 2 +- .../artifacts/investigation_report.md | 2 +- .../artifacts/investigation_summary.json | 2 +- docs/evidence-pipeline-contract.md | 4 ++ .../bounded-case-no-evidence-example.json | 2 +- .../pipeline.py | 16 ++++-- .../test_config_change_investigation_demo.py | 50 +++++++++++++++++++ 7 files changed, 70 insertions(+), 8 deletions(-) diff --git a/demos/config-change-investigation-demo/artifacts/investigation_hits.json b/demos/config-change-investigation-demo/artifacts/investigation_hits.json index be38f8e..f28ee78 100644 --- a/demos/config-change-investigation-demo/artifacts/investigation_hits.json +++ b/demos/config-change-investigation-demo/artifacts/investigation_hits.json @@ -131,7 +131,7 @@ }, "trigger_reason": "Break-glass mode was enabled on a sensitive service.", "correlation_window_minutes": 15, - "bounded_correlation_reason": "Attached evidence shares target_system 'vault-gateway' and falls within 15 minutes after the triggering change.", + "bounded_correlation_reason": "No policy denials or follow-on events were attached for target_system 'vault-gateway' within 15 minutes after the triggering change.", "attached_policy_denials": [], "attached_follow_on_events": [], "evidence_counts": { diff --git a/demos/config-change-investigation-demo/artifacts/investigation_report.md b/demos/config-change-investigation-demo/artifacts/investigation_report.md index e37a55a..e82129f 100644 --- a/demos/config-change-investigation-demo/artifacts/investigation_report.md +++ b/demos/config-change-investigation-demo/artifacts/investigation_report.md @@ -53,6 +53,6 @@ Follow-on events: - Trigger reason: Break-glass mode was enabled on a sensitive service. - Attached policy denials: 0 - Attached follow-on events: 0 -- Bounded correlation: Attached evidence shares target_system 'vault-gateway' and falls within 15 minutes after the triggering change. +- Bounded correlation: No policy denials or follow-on events were attached for target_system 'vault-gateway' within 15 minutes after the triggering change. No nearby supporting evidence fell inside the bounded correlation window. diff --git a/demos/config-change-investigation-demo/artifacts/investigation_summary.json b/demos/config-change-investigation-demo/artifacts/investigation_summary.json index c03968d..68ef45c 100644 --- a/demos/config-change-investigation-demo/artifacts/investigation_summary.json +++ b/demos/config-change-investigation-demo/artifacts/investigation_summary.json @@ -33,6 +33,6 @@ "policy_denials": 0, "follow_on_events": 0 }, - "bounded_correlation_reason": "Attached evidence shares target_system 'vault-gateway' and falls within 15 minutes after the triggering change." + "bounded_correlation_reason": "No policy denials or follow-on events were attached for target_system 'vault-gateway' within 15 minutes after the triggering change." } ] diff --git a/docs/evidence-pipeline-contract.md b/docs/evidence-pipeline-contract.md index 73d0404..306be9a 100644 --- a/docs/evidence-pipeline-contract.md +++ b/docs/evidence-pipeline-contract.md @@ -69,6 +69,10 @@ artifact a bounded evidence summary rather than an incident verdict: it does not claim `vault-gateway` was cleared, only that no additional evidence was attached under the demo's bounded correlation rules. +When both attachment arrays are empty, `bounded_correlation_reason` explicitly +states that no policy denials or follow-on events were attached for that target +and window. + ## Verification Run: diff --git a/docs/reviewer-pack-v0.6.0/bounded-case-no-evidence-example.json b/docs/reviewer-pack-v0.6.0/bounded-case-no-evidence-example.json index 41e97c0..58f8ef2 100644 --- a/docs/reviewer-pack-v0.6.0/bounded-case-no-evidence-example.json +++ b/docs/reviewer-pack-v0.6.0/bounded-case-no-evidence-example.json @@ -22,7 +22,7 @@ }, "trigger_reason": "Break-glass mode was enabled on a sensitive service.", "correlation_window_minutes": 15, - "bounded_correlation_reason": "Attached evidence shares target_system 'vault-gateway' and falls within 15 minutes after the triggering change.", + "bounded_correlation_reason": "No policy denials or follow-on events were attached for target_system 'vault-gateway' within 15 minutes after the triggering change.", "evidence_counts": { "policy_denials": 0, "follow_on_events": 0 diff --git a/src/telemetry_lab/config_change_investigation_demo/pipeline.py b/src/telemetry_lab/config_change_investigation_demo/pipeline.py index ea71118..763fffb 100644 --- a/src/telemetry_lab/config_change_investigation_demo/pipeline.py +++ b/src/telemetry_lab/config_change_investigation_demo/pipeline.py @@ -435,6 +435,17 @@ def build_investigations( and change_time <= event["timestamp"] <= window_end ] + if attached_denials or attached_follow_on: + bounded_reason = ( + f"Attached evidence shares target_system '{target_system}' and falls within " + f"{correlation_minutes} minutes after the triggering change." + ) + else: + bounded_reason = ( + "No policy denials or follow-on events were attached for target_system " + f"'{target_system}' within {correlation_minutes} minutes after the triggering change." + ) + investigations.append( { "investigation_id": str(hit["investigation_id"]), @@ -445,10 +456,7 @@ def build_investigations( "triggering_change": dict(change_event), "trigger_reason": str(hit["reason"]), "correlation_window_minutes": correlation_minutes, - "bounded_correlation_reason": ( - f"Attached evidence shares target_system '{target_system}' and falls within " - f"{correlation_minutes} minutes after the triggering change." - ), + "bounded_correlation_reason": bounded_reason, "attached_policy_denials": attached_denials, "attached_follow_on_events": attached_follow_on, "evidence_counts": { diff --git a/tests/test_config_change_investigation_demo.py b/tests/test_config_change_investigation_demo.py index e1480f5..4ef6a5a 100644 --- a/tests/test_config_change_investigation_demo.py +++ b/tests/test_config_change_investigation_demo.py @@ -9,6 +9,8 @@ from telemetry_lab.config_change_investigation_demo import default_demo_root, run_demo from telemetry_lab.config_change_investigation_demo.pipeline import ( + build_investigation_report, + build_investigation_summary, build_investigations, evaluate_risky_config_changes, load_jsonl, @@ -105,6 +107,54 @@ def test_build_investigations_uses_bounded_system_and_time_correlation() -> None ) +def test_bounded_reason_explicitly_reports_zero_evidence() -> None: + demo_root, config, changes, _, _ = _load_demo_inputs() + hits = evaluate_risky_config_changes(changes, config["rules"]) + hit = next(item for item in hits if item["investigation_id"] == "CCI-003") + investigations = build_investigations([hit], [], [], correlation_minutes=15) + investigation = investigations[0] + expected_reason = ( + "No policy denials or follow-on events were attached for target_system " + "'vault-gateway' within 15 minutes after the triggering change." + ) + + assert investigation["evidence_counts"] == {"policy_denials": 0, "follow_on_events": 0} + assert investigation["severity"] == "high" + assert investigation["bounded_correlation_reason"] == expected_reason + summary = build_investigation_summary(investigations, correlation_minutes=15) + assert summary[0]["bounded_correlation_reason"] == expected_reason + report = build_investigation_report(changes, [hit], investigations, correlation_minutes=15) + assert f"- Bounded correlation: {expected_reason}" in report + assert "No nearby supporting evidence fell inside the bounded correlation window." in report + assert "Attached evidence shares" not in report + example = _load_json_file( + demo_root.parents[1] / "docs/reviewer-pack-v0.6.0/bounded-case-no-evidence-example.json" + ) + assert example["investigation"]["bounded_correlation_reason"] == expected_reason + + +@pytest.mark.parametrize( + "with_denials,with_follow_on", [(True, False), (False, True), (True, True)] +) +def test_bounded_reason_preserves_nonempty_wording( + with_denials: bool, with_follow_on: bool +) -> None: + _, config, changes, denials, follow_on = _load_demo_inputs() + hits = evaluate_risky_config_changes(changes, config["rules"]) + hit = next(item for item in hits if item["investigation_id"] == "CCI-001") + investigation = build_investigations( + [hit], + denials if with_denials else [], + follow_on if with_follow_on else [], + correlation_minutes=15, + )[0] + + assert investigation["bounded_correlation_reason"] == ( + "Attached evidence shares target_system 'identity-proxy' and falls within " + "15 minutes after the triggering change." + ) + + def test_build_investigations_includes_evidence_on_window_end_only() -> None: change_time = parse_timestamp("2026-03-20T09:00:00Z") rule_hits = [