feat(copilot): mount the docs corpus in the VFS with path-scoped search_docs

.github/workflows/test-build.yml

@@ -123,6 +123,9 @@ jobs:
       - name: Repo audits
         run: bun run check:audits
 
+      - name: Verify docs manifest is in sync
+        run: bun run docs-manifest:check
+
       - name: Migration safety (zero-downtime) audit
         run: |
           if [ "${{ github.event_name }}" = "pull_request" ]; then

apps/sim/lib/copilot/chat/process-contents.test.ts

@@ -146,6 +146,23 @@ describe('processContextsServer - skill contexts', () => {
   })
 })
 
+describe('processContextsServer - docs contexts', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  it('resolves a tagged docs context to nothing while @docs tagging is disabled', async () => {
+    const result = await processContextsServer(
+      [{ kind: 'docs', label: 'Docs' } as ChatContext],
+      'user-1',
+      'how do loops work @Docs',
+      'ws-1'
+    )
+
+    expect(result).toEqual([])
+  })
+})
+
 describe('processContextsServer - MCP contexts', () => {
   beforeEach(() => {
     vi.clearAllMocks()

apps/sim/lib/copilot/chat/process-contents.ts

@@ -42,7 +42,6 @@ import { getSkillById } from '@/lib/workflows/skills/operations'
 import { listFolders } from '@/lib/workflows/utils'
 import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils'
 import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check'
-import { escapeRegExp } from '@/executor/constants'
 import type { BrowserTextSelection, ChatContext, TerminalTextSelection } from '@/stores/panel'
 
 type AgentContextType =
@@ -57,7 +56,6 @@ type AgentContextType =
   | 'file'
   | 'file_selection'
   | 'workflow_block'
-  | 'docs'
   | 'folder'
   | 'filefolder'
   | 'active_resource'
@@ -116,7 +114,8 @@ function formatTerminalSelection(selection: TerminalTextSelection): string {
 export async function processContextsServer(
   contexts: ChatContext[] | undefined,
   userId: string,
-  userMessage?: string,
+  /** Retained for call-site compatibility; unused while @docs tagging is disabled. */
+  _userMessage: string | undefined,
   currentWorkspaceId?: string,
   chatId?: string
 ): Promise<AgentContext[]> {
@@ -287,21 +286,9 @@ export async function processContextsServer(
           path: result.path,
         }
       }
-      if (ctx.kind === 'docs') {
-        try {
-          const { searchDocumentationServerTool } = await import(
-            '@/lib/copilot/tools/server/docs/search-documentation'
-          )
-          const rawQuery = (userMessage || '').trim() || ctx.label || 'Sim documentation'
-          const query = sanitizeMessageForDocs(rawQuery, contexts)
-          const res = await searchDocumentationServerTool.execute({ query, topK: 10 })
-          const content = JSON.stringify(res?.results || [])
-          return { type: 'docs', tag: ctx.label ? `@${ctx.label}` : '@', content }
-        } catch (e) {
-          logger.error('Failed to process docs context', e)
-          return null
-        }
-      }
+      // `docs` contexts are intentionally inert: @docs tagging is disabled while
+      // the docs corpus moves to the `docs/` VFS tree. A tagged context resolves
+      // to nothing and is filtered out below.
       return null
     } catch (error) {
       logger.error('Failed processing context (server)', { ctx, error })
@@ -323,53 +310,6 @@ export async function processContextsServer(
   return filtered
 }
 
-function sanitizeMessageForDocs(rawMessage: string, contexts: ChatContext[] | undefined): string {
-  if (!rawMessage) return ''
-  if (!Array.isArray(contexts) || contexts.length === 0) {
-    // No context mapping; conservatively strip all @mentions-like tokens
-    const stripped = rawMessage
-      .replace(/(^|\s)@([^\s]+)/g, ' ')
-      .replace(/\s{2,}/g, ' ')
-      .trim()
-    return stripped
-  }
-
-  // Gather labels by kind
-  const blockLabels = new Set(
-    contexts
-      .filter((c) => c.kind === 'blocks')
-      .map((c) => c.label)
-      .filter((l): l is string => typeof l === 'string' && l.length > 0)
-  )
-  const nonBlockLabels = new Set(
-    contexts
-      .filter((c) => c.kind !== 'blocks')
-      .map((c) => c.label)
-      .filter((l): l is string => typeof l === 'string' && l.length > 0)
-  )
-
-  let result = rawMessage
-
-  // 1) Remove all non-block mentions entirely
-  for (const label of nonBlockLabels) {
-    const pattern = new RegExp(`(^|\\s)@${escapeRegExp(label)}(?!\\S)`, 'g')
-    result = result.replace(pattern, ' ')
-  }
-
-  // 2) For block mentions, strip the '@' but keep the block name
-  for (const label of blockLabels) {
-    const pattern = new RegExp(`@${escapeRegExp(label)}(?!\\S)`, 'g')
-    result = result.replace(pattern, label)
-  }
-
-  // 3) Remove any remaining @mentions (unknown or not in contexts)
-  result = result.replace(/(^|\s)@([^\s]+)/g, ' ')
-
-  // Normalize whitespace
-  result = result.replace(/\s{2,}/g, ' ').trim()
-  return result
-}
-
 async function processSkillFromDb(
   skillId: string,
   workspaceId: string,

apps/sim/lib/copilot/docs/docs-corpus.test.ts

@@ -0,0 +1,225 @@
+/**
+ * @vitest-environment node
+ */
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+vi.mock('@sim/utils/helpers', () => ({
+  sleep: vi.fn(() => Promise.resolve()),
+}))
+
+import {
+  couldMatchDocsScope,
+  DocsCorpusError,
+  globDocs,
+  grepDocs,
+  isDocsPath,
+  readDocsPage,
+} from '@/lib/copilot/docs/docs-corpus'
+import { DOCS_MANIFEST } from '@/lib/copilot/generated/docs-manifest'
+import type { GrepMatch } from '@/lib/copilot/vfs/operations'
+
+const SAMPLE_PAGE = DOCS_MANIFEST.find((path) => path === 'workflows/blocks/agent.mdx')
+
+describe('docs corpus scoping', () => {
+  it('recognizes docs paths', () => {
+    expect(isDocsPath('docs/workflows.mdx')).toBe(true)
+    expect(isDocsPath('docs')).toBe(true)
+    expect(isDocsPath('/docs/workflows.mdx')).toBe(true)
+    expect(isDocsPath('workflows.mdx')).toBe(false)
+    expect(isDocsPath('files/report.pdf')).toBe(false)
+    expect(isDocsPath('docsomething/x')).toBe(false)
+    expect(isDocsPath(undefined)).toBe(false)
+  })
+
+  it('is opt-in: only an explicit docs/ pattern can match', () => {
+    expect(couldMatchDocsScope('docs/**')).toBe(true)
+    expect(couldMatchDocsScope('docs/workflows/**')).toBe(true)
+    expect(couldMatchDocsScope('**')).toBe(false)
+    expect(couldMatchDocsScope('**/*.mdx')).toBe(false)
+    expect(couldMatchDocsScope('*')).toBe(false)
+    expect(couldMatchDocsScope(undefined)).toBe(false)
+  })
+})
+
+describe('globDocs', () => {
+  it('lists the whole corpus under docs/**', () => {
+    const files = globDocs('docs/**')
+    expect(files.length).toBeGreaterThan(DOCS_MANIFEST.length)
+    expect(files).toContain('docs/workflows/blocks/agent.mdx')
+    expect(files).toContain('docs/workflows/blocks')
+  })
+
+  it('scopes to a section', () => {
+    const files = globDocs('docs/integrations/*.mdx')
+    expect(files).toContain('docs/integrations/gmail.mdx')
+    expect(files.every((path) => path.startsWith('docs/integrations/'))).toBe(true)
+  })
+
+  it('excludes academy and api-reference', () => {
+    expect(globDocs('docs/academy/**')).toEqual([])
+    expect(globDocs('docs/api-reference/**')).toEqual([])
+  })
+
+  it('maps section index pages onto their parent URL path', () => {
+    expect(globDocs('docs/workflows.mdx')).toEqual(['docs/workflows.mdx'])
+    expect(globDocs('docs/workflows/index.mdx')).toEqual([])
+  })
+
+  it('treats a trailing-slash pattern like the bare directory instead of matching nothing', () => {
+    expect(globDocs('docs/')).toEqual(['docs'])
+    expect(globDocs('docs/integrations/')).toEqual(['docs/integrations'])
+  })
+})
+
+describe('readDocsPage', () => {
+  const fetchMock = vi.fn()
+
+  beforeEach(() => {
+    fetchMock.mockReset()
+    vi.stubGlobal('fetch', fetchMock)
+  })
+
+  afterEach(() => {
+    vi.unstubAllGlobals()
+  })
+
+  it('fetches the manifest path verbatim from the docs site', async () => {
+    expect(SAMPLE_PAGE).toBeDefined()
+    fetchMock.mockResolvedValue({ ok: true, status: 200, text: async () => '# Agent\n\nbody' })
+
+    const page = await readDocsPage(`docs/${SAMPLE_PAGE}`)
+
+    expect(fetchMock).toHaveBeenCalledOnce()
+    expect(fetchMock.mock.calls[0][0]).toBe(`https://docs.sim.ai/${SAMPLE_PAGE}`)
+    expect(page).toEqual({ content: '# Agent\n\nbody', totalLines: 3 })
+  })
+
+  it('rejects an unknown page without fetching', async () => {
+    await expect(readDocsPage('docs/not-a-real-page.mdx')).rejects.toThrow(DocsCorpusError)
+    expect(fetchMock).not.toHaveBeenCalled()
+  })
+
+  it('points a directory read at glob', async () => {
+    await expect(readDocsPage('docs/workflows/blocks')).rejects.toThrow(/is a directory/)
+    expect(fetchMock).not.toHaveBeenCalled()
+  })
+
+  it('surfaces a docs-site outage as a retryable error after exhausting retries', async () => {
+    fetchMock.mockResolvedValue({ ok: false, status: 502, text: async () => '' })
+    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/could not be reached/)
+    expect(fetchMock).toHaveBeenCalledTimes(3)
+  })
+
+  it('treats a network failure as retryable', async () => {
+    fetchMock.mockRejectedValue(new Error('socket hang up'))
+    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/could not be reached/)
+    expect(fetchMock).toHaveBeenCalledTimes(3)
+  })
+
+  it('recovers when a transient failure clears on retry', async () => {
+    fetchMock
+      .mockRejectedValueOnce(new Error('socket hang up'))
+      .mockResolvedValue({ ok: true, status: 200, text: async () => '# Agent\n\nbody' })
+
+    const page = await readDocsPage(`docs/${SAMPLE_PAGE}`)
+
+    expect(fetchMock).toHaveBeenCalledTimes(2)
+    expect(page).toEqual({ content: '# Agent\n\nbody', totalLines: 3 })
+  })
+
+  it('reports a page the site no longer serves as permanent, without retrying', async () => {
+    fetchMock.mockResolvedValue({ ok: false, status: 404, text: async () => '' })
+    const error = await readDocsPage(`docs/${SAMPLE_PAGE}`).catch((e) => e)
+    expect(error).toBeInstanceOf(DocsCorpusError)
+    expect(error.message).toMatch(/does not serve it/)
+    expect(error.message).toMatch(/retrying will not help/)
+    expect(error.message).not.toMatch(/could not be reached/)
+    expect(fetchMock).toHaveBeenCalledOnce()
+  })
+
+  it('still treats 429 as retryable rather than permanent', async () => {
+    fetchMock.mockResolvedValue({ ok: false, status: 429, text: async () => '' })
+    await expect(readDocsPage(`docs/${SAMPLE_PAGE}`)).rejects.toThrow(/could not be reached/)
+    expect(fetchMock).toHaveBeenCalledTimes(3)
+  })
+})
+
+describe('grepDocs', () => {
+  const fetchMock = vi.fn()
+  const SECTION_DIR = 'docs/workflows/blocks'
+  const SECTION_PAGES = DOCS_MANIFEST.filter((path) => path.startsWith('workflows/blocks/')).map(
+    (path) => `docs/${path}`
+  )
+
+  beforeEach(() => {
+    fetchMock.mockReset()
+    vi.stubGlobal('fetch', fetchMock)
+  })
+
+  afterEach(() => {
+    vi.unstubAllGlobals()
+  })
+
+  it('greps exactly one page for a page path', async () => {
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => 'intro line\nsystemPrompt matters\ntail',
+    })
+
+    const matches = await grepDocs(`docs/${SAMPLE_PAGE}`, 'systemPrompt')
+
+    expect(fetchMock).toHaveBeenCalledOnce()
+    expect(matches).toEqual([
+      { path: `docs/${SAMPLE_PAGE}`, line: 2, content: 'systemPrompt matters' },
+    ])
+  })
+
+  it('greps a directory by fetching every page under it', async () => {
+    fetchMock.mockResolvedValue({
+      ok: true,
+      status: 200,
+      text: async () => 'intro\ncron marker line\ntail',
+    })
+    expect(SECTION_PAGES.length).toBeGreaterThan(1)
+
+    const matches = (await grepDocs(SECTION_DIR, 'cron marker', {
+      maxResults: 10_000,
+    })) as GrepMatch[]
+
+    expect(fetchMock).toHaveBeenCalledTimes(SECTION_PAGES.length)
+    expect(matches.map((match) => match.path)).toEqual(SECTION_PAGES)
+  })
+
+  it('skips pages the site no longer serves instead of failing the directory grep', async () => {
+    const missingUrl = `https://docs.sim.ai/${SECTION_PAGES[0].slice('docs/'.length)}`
+    fetchMock.mockImplementation(async (url: string) =>
+      url === missingUrl
+        ? { ok: false, status: 404, text: async () => '' }
+        : { ok: true, status: 200, text: async () => 'cron marker line' }
+    )
+
+    const matches = (await grepDocs(SECTION_DIR, 'cron marker', {
+      maxResults: 10_000,
+    })) as GrepMatch[]
+
+    expect(matches.map((match) => match.path)).toEqual(SECTION_PAGES.slice(1))
+  })
+
+  it('fails the whole directory grep when a page cannot be reached', async () => {
+    fetchMock.mockImplementation(async (url: string) =>
+      url.endsWith(`/${SAMPLE_PAGE}`)
+        ? { ok: false, status: 502, text: async () => '' }
+        : { ok: true, status: 200, text: async () => 'cron marker line' }
+    )
+
+    await expect(grepDocs(SECTION_DIR, 'cron marker')).rejects.toThrow(/Retry shortly/)
+  })
+
+  it('rejects a path that is neither a page nor a directory without fetching', async () => {
+    await expect(grepDocs('docs/not-a-real-page.mdx', 'cron')).rejects.toThrow(
+      /not a docs page or directory/
+    )
+    expect(fetchMock).not.toHaveBeenCalled()
+  })
+})