fix(chunkers): preserve FAQ prose in docs chunks

apps/sim/lib/chunkers/docs-chunker.test.ts

@@ -0,0 +1,131 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, expect, it, vi } from 'vitest'
+
+vi.mock('@/lib/knowledge/embeddings', () => ({
+  generateEmbeddings: vi.fn(async () => ({ embeddings: [] })),
+  getConfiguredEmbeddingModel: vi.fn(() => 'test-model'),
+}))
+
+import { DocsChunker } from '@/lib/chunkers/docs-chunker'
+
+function cleanContent(content: string): string {
+  const chunker = new DocsChunker()
+  return (chunker as unknown as { cleanContent(content: string): string }).cleanContent.call(
+    chunker,
+    content
+  )
+}
+
+describe('cleanContent FAQ extraction', () => {
+  it('keeps FAQ question/answer prose that the tag and brace strips would otherwise delete', () => {
+    const cleaned = cleanContent(
+      [
+        'Some intro prose.',
+        '',
+        'import { FAQ } from "@/components/ui/faq"',
+        '',
+        '<FAQ items={[',
+        '  { question: "What is the maximum file size for uploads?", answer: "The maximum file size for files processed during a workflow run is 20 MB." },',
+        '  { question: "How are files passed between blocks internally?", answer: "Files are represented as standardized UserFile objects." },',
+        ']} />',
+      ].join('\n')
+    )
+
+    expect(cleaned).toContain('What is the maximum file size for uploads?')
+    expect(cleaned).toContain('20 MB')
+    expect(cleaned).toContain('standardized UserFile objects')
+    expect(cleaned).toContain('Some intro prose.')
+    expect(cleaned).not.toContain('items=')
+    expect(cleaned).not.toContain('question:')
+  })
+
+  it('survives braces and angle-bracket tokens inside answer strings', () => {
+    const cleaned = cleanContent(
+      [
+        '<FAQ items={[',
+        `  { question: "What input formats work?", answer: "Use a data URI with the format 'data:{mime};base64,{data}' or a URL." },`,
+        '  { question: "Do I extract base64 manually?", answer: "No. Pass the entire file reference (e.g., <gmail.attachments[0]>) and the block extracts what it needs." },',
+        ']} />',
+      ].join('\n')
+    )
+
+    // Brace placeholders keep their token text; the wrapper chars are dropped
+    // so the later brace strip cannot punch holes in the sentence.
+    expect(cleaned).toContain("'data:mime;base64,data'")
+    // Angle brackets are dropped so the tag strip cannot re-eat the sentence.
+    expect(cleaned).toContain('(e.g., gmail.attachments[0]) and the block extracts')
+  })
+
+  it('extracts items formatted across multiple lines', () => {
+    const cleaned = cleanContent(
+      [
+        '<FAQ items={[',
+        '  {',
+        '    question: "Is SSO supported?",',
+        '    answer: "Yes, on enterprise plans."',
+        '  },',
+        ']} />',
+      ].join('\n')
+    )
+
+    expect(cleaned).toContain('Is SSO supported?')
+    expect(cleaned).toContain('Yes, on enterprise plans.')
+  })
+
+  it('extracts single-quoted multiline items with trailing commas (session-policies shape)', () => {
+    const cleaned = cleanContent(
+      [
+        '<FAQ',
+        '  items={[',
+        '    {',
+        "      question: 'Do session policies apply to SSO sign-ins?',",
+        '      answer:',
+        "        'Yes. Sessions created through SSO follow the same limits.',",
+        '    },',
+        '    {',
+        '      question: \'Does "Sign out all members" affect API keys?\',',
+        "      answer: 'No. API keys are unaffected.',",
+        '    },',
+        '  ]}',
+        '/>',
+      ].join('\n')
+    )
+
+    expect(cleaned).toContain('Do session policies apply to SSO sign-ins?')
+    expect(cleaned).toContain('Yes. Sessions created through SSO follow the same limits.')
+    expect(cleaned).toContain('Does "Sign out all members" affect API keys?')
+    expect(cleaned).toContain('No. API keys are unaffected.')
+    expect(cleaned).not.toContain('items=')
+  })
+
+  it('unescapes escaped quotes in extracted strings', () => {
+    const cleaned = cleanContent(
+      '<FAQ items={[ { question: "What does \\"draft\\" mean?", answer: "An unsaved workflow." } ]} />'
+    )
+
+    expect(cleaned).toContain('What does "draft" mean?')
+  })
+})
+
+describe('cleanContent scaffolding strips', () => {
+  it('still strips imports, exports, comments, and code-ish brace expressions', () => {
+    const cleaned = cleanContent(
+      [
+        'import { Callout } from "fumadocs-ui/components/callout"',
+        'export const dynamic = "force-static"',
+        '{/* editorial note */}',
+        'Visible prose {props.title} continues here.',
+        '<Callout>Inside text stays</Callout>',
+      ].join('\n')
+    )
+
+    expect(cleaned).not.toContain('import')
+    expect(cleaned).not.toContain('force-static')
+    expect(cleaned).not.toContain('editorial note')
+    expect(cleaned).not.toContain('props.title')
+    expect(cleaned).toContain('Visible prose')
+    expect(cleaned).toContain('Inside text stays')
+  })
+})

apps/sim/lib/chunkers/docs-chunker.ts

@@ -21,6 +21,45 @@ interface Frontmatter {
 
 const logger = createLogger('DocsChunker')
 
+/**
+ * One `{ question: "...", answer: "..." }` FAQ item, in either quote style and
+ * with an optional trailing comma (`session-policies.mdx` uses single-quoted
+ * multiline items). Each captured value keeps its surrounding quotes — the
+ * quoted strings are consumed escape-aware per style, so quotes of the other
+ * style, braces, or escapes inside an answer never end a match early.
+ */
+const FAQ_ITEM_PATTERN =
+  /\{\s*question:\s*("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')\s*,\s*answer:\s*("(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')\s*,?\s*\}/g
+
+/** Strip a captured value's surrounding quotes (either style), then unescape. */
+function unquoteJsxString(value: string): string {
+  return unescapeJsxString(value.slice(1, -1))
+}
+
+function unescapeJsxString(value: string): string {
+  return value.replace(/\\(.)/g, (_, char: string) =>
+    char === 'n' ? '\n' : char === 't' ? '\t' : char
+  )
+}
+
+/**
+ * Emit an FAQ block's question/answer strings as plain prose lines. Must run
+ * BEFORE the tag strip: a `<FAQ items={[` opening tag has no `>` until the
+ * closing `]} />`, so the multiline tag regex would otherwise swallow the
+ * items whole (ending at the first `>` inside an answer). Angle brackets and
+ * braces around inline tokens (`<gmail.attachments[0]>`, `data:{mime}`) are
+ * dropped so the later tag and brace strips cannot re-consume the emitted
+ * text.
+ */
+function extractFaqProse(items: string): string {
+  const lines: string[] = []
+  for (const match of items.matchAll(FAQ_ITEM_PATTERN)) {
+    lines.push(unquoteJsxString(match[1]), unquoteJsxString(match[2]))
+  }
+  if (lines.length === 0) return ' '
+  return `\n${lines.join('\n').replace(/[<>{}]/g, '')}\n`
+}
+
 export class DocsChunker {
   private readonly textChunker: TextChunker
   private readonly baseUrl: string
@@ -216,6 +255,9 @@ export class DocsChunker {
       .replace(/\r/g, '\n')
       .replace(/^import\s+.*$/gm, '')
       .replace(/^export\s+.*$/gm, '')
+      .replace(/<FAQ\s+items=\{\[([\s\S]*?)\]\}\s*\/>/g, (_m, items: string) =>
+        extractFaqProse(items)
+      )
       .replace(/<\/?[a-zA-Z][^>]*>/g, ' ')
       .replace(/\{\/\*[\s\S]*?\*\/\}/g, ' ')
       .replace(/\{[^{}]*\}/g, ' ')