fix(gooddata-eval): read a v2 text summary on a page with no takeaway

packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py

@@ -72,6 +72,7 @@
 _V2 = "2"
 _V2_HEADING_WIDGETS = frozenset({"title", "heading"})
 _V2_TAKEAWAY_WIDGET = "takeaway"
+_V2_TEXT_WIDGET = "text"
 
 _NARRATIVE_EVALUATION_STEPS = [
     "The actual output is a document rendered as text: its title, period, and per page the heading, "
@@ -208,32 +209,39 @@ def _is_takeaway(node: dict) -> bool:
     return node.get("type") == _V2_TAKEAWAY_WIDGET
 
 
+def _is_summary_text(node: dict) -> bool:
+    return node.get("type") == _V2_TEXT_WIDGET and node.get("id") == _SUMMARY_SLOT
+
+
 def _summary_slots(page: dict) -> list[dict]:
-    """The page's summaries: every v2 ``takeaway`` widget, or the v1 ``summary`` slot as gen-ai reads it.
+    """The page's summaries, as gen-ai writes them.
 
-    A v2 ``text`` widget with id ``summary`` is static text the default page templates ship empty,
-    not a summary: the copilot writes its summaries into takeaways.
+    A v2 page's summaries are its ``takeaway`` widgets; a v2 page without one is summarised by its
+    ``text`` widget with id ``summary``. A v1 page's summary is its ``summary`` slot with a ``paragraph``.
     """
     nodes = list(_nodes(page.get("layout")))
     takeaways = [node for node in nodes if _is_takeaway(node)]
     if takeaways:
         return takeaways
     for node in nodes:
-        if node.get("id") == _SUMMARY_SLOT and "paragraph" in node:
+        if _is_summary_text(node) or (node.get("id") == _SUMMARY_SLOT and "paragraph" in node):
             return [node]
     return []
 
 
 def _summary_text(slot: dict) -> str | None:
-    """The slot's written text: a takeaway's markdown, the model's on an AI summary, the string on a static one."""
-    if _is_takeaway(slot):
+    """The slot's written text: the markdown of a takeaway or text summary, the model's on an AI summary,
+    the string on a static one."""
+    if _is_takeaway(slot) or _is_summary_text(slot):
         return _written(slot.get("markdown"))
     paragraph = slot.get("paragraph")
     return _written(paragraph.get("text") if isinstance(paragraph, dict) else paragraph)
 
 
 def _slot_name(slot: dict) -> str:
-    return "takeaway" if _is_takeaway(slot) else "summary slot"
+    if _is_takeaway(slot):
+        return "takeaway"
+    return "text summary" if _is_summary_text(slot) else "summary slot"
 
 
 def _content_pages(document: dict) -> list[tuple[int, dict]]:
@@ -577,7 +585,7 @@ def _score_response(
             if _summary_text(slot) is None
         ]
         if not slots:
-            missing = "takeaway" if _is_v2(document) else "summary slot"
+            missing = "takeaway or text summary" if _is_v2(document) else "summary slot"
             failures.append(f"the document has no {missing}")
         failures.extend(dict.fromkeys(unwritten))
         summaries_present = bool(slots) and not unwritten

packages/gooddata-eval/tests/test_agentic_document_skill.py

@@ -1268,20 +1268,39 @@ def test_a_takeaway_made_only_of_placeholders_is_not_written() -> None:
 
 
 def _with_summary_text(page: dict, markdown: str | None) -> dict:
-    """``page`` plus the default template's ``text`` widget with id ``summary``, which is not a takeaway."""
+    """``page`` plus a page template's ``text`` widget with id ``summary``; ``None`` leaves its markdown out."""
     page["layout"]["children"].append(
         {"type": "text", "id": "summary", **({"markdown": markdown} if markdown is not None else {})}
     )
     return page
 
 
-def test_a_v2_document_without_a_takeaway_has_no_summaries() -> None:
+def test_a_v2_page_without_a_takeaway_is_summarised_by_its_summary_text() -> None:
     page = _with_summary_text(_v2_content_page("page2", _REVENUE_TREND), "Revenue grew 12%.")
     part = _v2_document_part([_v2_cover_page(), page])
     evaluation = _evaluate(part, expected={"narrative": _NARRATIVE})
+    assert evaluation.strict_checks["document_summaries_present"] is True
+    assert render_document_text(part["document"]).splitlines()[-1] == "Summary: Revenue grew 12%."
+
+
+@pytest.mark.parametrize("markdown", ["", None])
+def test_an_unwritten_summary_text_fails_when_the_page_has_no_takeaway(markdown: str | None) -> None:
+    pages = [
+        _v2_cover_page(),
+        _v2_content_page("page2", _REVENUE_TREND, takeaway="Revenue grew 12%."),
+        _with_summary_text(_v2_content_page("page3", _RETURNS_BY_CATEGORY), markdown),
+    ]
+    part = _v2_document_part(pages)
+    evaluation = _evaluate(part, expected={"narrative": _NARRATIVE}, tool=_draft_result(page_count=3))
+    assert evaluation.strict_checks["document_summaries_present"] is False
+    assert evaluation.failures == ["page 'page3' has a text summary with no written text"]
+    assert render_document_text(part["document"]).count("Summary:") == 1
+
+
+def test_a_v2_document_without_a_takeaway_or_summary_text_has_no_summaries() -> None:
+    evaluation = _evaluate(_v2_document_part(), expected={"narrative": _NARRATIVE})
     assert evaluation.strict_checks["document_summaries_present"] is False
-    assert evaluation.failures == ["the document has no takeaway"]
-    assert "Summary:" not in render_document_text(part["document"])
+    assert evaluation.failures == ["the document has no takeaway or text summary"]
 
 
 def test_an_empty_summary_text_beside_a_written_takeaway_is_ignored() -> None: