diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py index 533b0d40b..fc08f4bb8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py @@ -72,6 +72,7 @@ _V2 = "2" _V2_HEADING_WIDGETS = frozenset({"title", "heading"}) _V2_TAKEAWAY_WIDGET = "takeaway" +_V2_TEXT_WIDGET = "text" _NARRATIVE_EVALUATION_STEPS = [ "The actual output is a document rendered as text: its title, period, and per page the heading, " @@ -208,32 +209,39 @@ def _is_takeaway(node: dict) -> bool: return node.get("type") == _V2_TAKEAWAY_WIDGET +def _is_summary_text(node: dict) -> bool: + return node.get("type") == _V2_TEXT_WIDGET and node.get("id") == _SUMMARY_SLOT + + def _summary_slots(page: dict) -> list[dict]: - """The page's summaries: every v2 ``takeaway`` widget, or the v1 ``summary`` slot as gen-ai reads it. + """The page's summaries, as gen-ai writes them. - A v2 ``text`` widget with id ``summary`` is static text the default page templates ship empty, - not a summary: the copilot writes its summaries into takeaways. + A v2 page's summaries are its ``takeaway`` widgets; a v2 page without one is summarised by its + ``text`` widget with id ``summary``. A v1 page's summary is its ``summary`` slot with a ``paragraph``. """ nodes = list(_nodes(page.get("layout"))) takeaways = [node for node in nodes if _is_takeaway(node)] if takeaways: return takeaways for node in nodes: - if node.get("id") == _SUMMARY_SLOT and "paragraph" in node: + if _is_summary_text(node) or (node.get("id") == _SUMMARY_SLOT and "paragraph" in node): return [node] return [] def _summary_text(slot: dict) -> str | None: - """The slot's written text: a takeaway's markdown, the model's on an AI summary, the string on a static one.""" - if _is_takeaway(slot): + """The slot's written text: the markdown of a takeaway or text summary, the model's on an AI summary, + the string on a static one.""" + if _is_takeaway(slot) or _is_summary_text(slot): return _written(slot.get("markdown")) paragraph = slot.get("paragraph") return _written(paragraph.get("text") if isinstance(paragraph, dict) else paragraph) def _slot_name(slot: dict) -> str: - return "takeaway" if _is_takeaway(slot) else "summary slot" + if _is_takeaway(slot): + return "takeaway" + return "text summary" if _is_summary_text(slot) else "summary slot" def _content_pages(document: dict) -> list[tuple[int, dict]]: @@ -577,7 +585,7 @@ def _score_response( if _summary_text(slot) is None ] if not slots: - missing = "takeaway" if _is_v2(document) else "summary slot" + missing = "takeaway or text summary" if _is_v2(document) else "summary slot" failures.append(f"the document has no {missing}") failures.extend(dict.fromkeys(unwritten)) summaries_present = bool(slots) and not unwritten diff --git a/packages/gooddata-eval/tests/test_agentic_document_skill.py b/packages/gooddata-eval/tests/test_agentic_document_skill.py index 58c2d5f1f..555e18b1a 100644 --- a/packages/gooddata-eval/tests/test_agentic_document_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_document_skill.py @@ -1268,20 +1268,39 @@ def test_a_takeaway_made_only_of_placeholders_is_not_written() -> None: def _with_summary_text(page: dict, markdown: str | None) -> dict: - """``page`` plus the default template's ``text`` widget with id ``summary``, which is not a takeaway.""" + """``page`` plus a page template's ``text`` widget with id ``summary``; ``None`` leaves its markdown out.""" page["layout"]["children"].append( {"type": "text", "id": "summary", **({"markdown": markdown} if markdown is not None else {})} ) return page -def test_a_v2_document_without_a_takeaway_has_no_summaries() -> None: +def test_a_v2_page_without_a_takeaway_is_summarised_by_its_summary_text() -> None: page = _with_summary_text(_v2_content_page("page2", _REVENUE_TREND), "Revenue grew 12%.") part = _v2_document_part([_v2_cover_page(), page]) evaluation = _evaluate(part, expected={"narrative": _NARRATIVE}) + assert evaluation.strict_checks["document_summaries_present"] is True + assert render_document_text(part["document"]).splitlines()[-1] == "Summary: Revenue grew 12%." + + +@pytest.mark.parametrize("markdown", ["", None]) +def test_an_unwritten_summary_text_fails_when_the_page_has_no_takeaway(markdown: str | None) -> None: + pages = [ + _v2_cover_page(), + _v2_content_page("page2", _REVENUE_TREND, takeaway="Revenue grew 12%."), + _with_summary_text(_v2_content_page("page3", _RETURNS_BY_CATEGORY), markdown), + ] + part = _v2_document_part(pages) + evaluation = _evaluate(part, expected={"narrative": _NARRATIVE}, tool=_draft_result(page_count=3)) + assert evaluation.strict_checks["document_summaries_present"] is False + assert evaluation.failures == ["page 'page3' has a text summary with no written text"] + assert render_document_text(part["document"]).count("Summary:") == 1 + + +def test_a_v2_document_without_a_takeaway_or_summary_text_has_no_summaries() -> None: + evaluation = _evaluate(_v2_document_part(), expected={"narrative": _NARRATIVE}) assert evaluation.strict_checks["document_summaries_present"] is False - assert evaluation.failures == ["the document has no takeaway"] - assert "Summary:" not in render_document_text(part["document"]) + assert evaluation.failures == ["the document has no takeaway or text summary"] def test_an_empty_summary_text_beside_a_written_takeaway_is_ignored() -> None: