From f4b9199470495ec4ada0ee31bac2025655ad42db Mon Sep 17 00:00:00 2001 From: leemour Date: Sat, 31 Oct 2026 13:38:03 +0100 Subject: [PATCH] perf(search): meaning inside the filter and totals, on the owner's store Drive budget date>=30d goes from 7 hits to 20 with cli-core #2236; no hit leaves the filter; totals match separate counts. Scoring a filter's notes one by one costs ~0.5 s for 1,454 documents, which set the 2,000-note threshold. Co-Authored-By: Claude Opus 5.5 --- .../search/notes-meaning-filters/README.md | 72 +++++++++++++++++++ .../measure-in-filter.test.ts | 68 ++++++++++++++++++ .../measure-messages.test.ts | 67 +++++++++++++++++ .../measure-timing.test.ts | 47 ++++++++++++ .../vitest.measure.config.ts | 13 ++++ 5 files changed, 267 insertions(+) create mode 100644 performance/search/notes-meaning-filters/measure-in-filter.test.ts create mode 100644 performance/search/notes-meaning-filters/measure-messages.test.ts create mode 100644 performance/search/notes-meaning-filters/measure-timing.test.ts create mode 100644 performance/search/notes-meaning-filters/vitest.measure.config.ts diff --git a/performance/search/notes-meaning-filters/README.md b/performance/search/notes-meaning-filters/README.md index d77f258..d4193b2 100644 --- a/performance/search/notes-meaning-filters/README.md +++ b/performance/search/notes-meaning-filters/README.md @@ -77,3 +77,75 @@ inside the queried range. - Messages: `search messages --discover` builds its filter by joining every field with AND, a different path that this run did not test. - Note titles, texts and refs: totals only, by design. + +## Meaning inside the filter, and totals (2026-10-31, cli-core #2236) + +### Why this was run + +After #2223 no hit broke a filter, but a narrow filter left few meaning hits: meaning took the 50 notes nearest +overall and dropped those outside the filter. Drive's `budget date>=30d` gave 7 hits while 144 documents match the +date. The owner chose to rank the filter's own notes by meaning (A), and asked for a total of matches "like elastic +does". The questions: does ranking inside the filter fill the page without letting a hit outside it, how much does it +cost, and does the new total equal a count taken another way? The answers set the threshold (`RANK_WITHIN`) in +[cli-core #2236](https://github.com/WireCatLabs/cli-core/pull/2236). + +### Setup + +- The same machine and store as above: 1,454 documents and notes with 23,308 chunks; 1,455 documents chunked. +- Before: cli-core 0.498.0. After: the #2236 branch. Both run from a cli-core worktree through Vitest with + `vitest.measure.config.ts` (the scripts go in `.measure/` of that worktree), calling the services the tools call: + `searchNotes` with drive's one documents account as `folderIds`, or with no scope as memo does; `searchStore` for + messages over the Telegram account that holds the most messages. +- **Read only.** The store opens with `readOnly: true`. Two writes a search makes on its way are stubbed: indexing + notes that wait to be indexed, and topping up the message word index. So the runs search the indexes as they stand. + The store held about 5.5 GB and the disk had 1.5 GB free, so no copy was possible. +- `DBUS_SESSION_BUS_ADDRESS=disabled:` on every run; the default local model `e5-small`. + +```sh +MEASURE_STORE=~/.local/share/WireCat/wirecat.db MEASURE_OUT=out.jsonl MEASURE_LABEL=after \ + DBUS_SESSION_BUS_ADDRESS=disabled: npx vitest run --config .measure/vitest.measure.config.ts +MEASURE_FILE=.measure/measure-timing.test.ts … # scoring cost +MEASURE_FILE=.measure/measure-messages.test.ts … # messages totals +``` + +The query words are generic test words chosen for these runs, not the owner's own searches. Per-query rows are in +cli-private (`reports/2026-10-31-notes-meaning-in-filter.md`); this page holds totals. + +### Results + +| What | Before | After | +| --- | --- | --- | +| Hits outside the query's fields, 12 notes queries (6 each in memo and drive) | 0 | 0 | +| Drive `budget date>=30d`: hits, of them found by meaning | 7, 6 | 20, 19 | +| memo `budget date>=30d`: hits, of them found by meaning | 20, 9 | 20, 13 | +| Totals of the 6 fields-only notes queries equal to the matching notes listed | – | 6 of 6 | +| Drive `date>=30d` total against a separate `sqlite3 -readonly` count | – | 144 `eq`, SQL 144 | +| Messages totals against the grouped count of `search stats`, 9 runs that answered | – | 7 equal; 2 `10000 gte` where the count is 32,180 | + +Cost of scoring a filter's notes one by one, against the vector index's nearest 50 (milliseconds, 3 runs): + +| Notes scored | 150 | 500 | 1,000 | 1,454 | nearest 50 overall | +| --- | --- | --- | --- | --- | --- | +| ms | 14–26 | 35–41 | 132–173 | 478–599 | 165–205 | + +Counting up to 10,000 matching notes took 13–17 ms. + +### What it showed + +- **Ranking inside the filter fills the page.** Drive's `budget date>=30d` went from 7 hits to 20: 50 of the 144 + documents in the date range pass the meaning floor. None of the new hits breaks a field. +- **The cost grows with chunks, not notes.** Long documents come late in the list and cost the most. 2,000 notes + keeps the scoring under about a second at this store's density (16 chunks per document); above it the old path + runs. A chunk budget would scale better on stores of long documents. +- **Totals agree with counts taken another way.** The two capped totals are `gte`, as designed. A notes total with + words also counts the notes found by meaning, so no separate count exists for it. +- **BUG-36 is not reachable.** A `--discover` query with an excluded or ORed field (`встреча -date>=30d`, + `встреча (date>=30d OR …)`) runs strict search, with no meaning half. So its hits cannot break those fields: 0 + dated outside the range. + +### What was not measured + +- Ranking quality inside the filter: whether the 19 meaning hits are the right ones. +- `--discover` with plain words on the real store. Its follow-up step creates a temporary table, which a read-only + connection refuses, so those runs failed here. The unit tests cover their totals. +- The tools' own command lines: the runs called cli-core's services, not `drive` or `memo` binaries. diff --git a/performance/search/notes-meaning-filters/measure-in-filter.test.ts b/performance/search/notes-meaning-filters/measure-in-filter.test.ts new file mode 100644 index 0000000..4dcea96 --- /dev/null +++ b/performance/search/notes-meaning-filters/measure-in-filter.test.ts @@ -0,0 +1,68 @@ +// Read-only: opens the owner's store with readOnly, prints totals per query as JSON lines (no titles, refs or text). +import { appendFileSync } from "node:fs" +import { it, vi } from "vitest" +import { searchNotes, searchNotesQuery } from "../src/services/notes-search.js" +import { openStore } from "../src/store/store.js" + +// The store has notes waiting to be indexed; indexing them is a write, so this run searches the index as it stands. +vi.mock("../src/store/sqlite/note-index.js", async (original) => ({ + ...(await original()), + drainNoteIndex: () => undefined, +})) + +const OUT = process.env.MEASURE_OUT as string +const QUERIES: Record = { + "date:today": "date:today", + "tag:work": "tag:work", + "date>=30d": "date>=30d", + "budget date>=30d": "date>=30d", + "plan tag:work": "tag:work", + "budget source:file": "source:file", +} + +it("measure", async () => { + const store = await openStore({ path: process.env.MEASURE_STORE as string, readOnly: true }) + const label = process.env.MEASURE_LABEL as string + const accounts = await store.notes.documentsAccounts() + const scopes: Record = { memo: {} } + if (accounts.length === 1) scopes.drive = { folderIds: [String(accounts[0]?.id)] } + else appendFileSync(OUT, `${JSON.stringify({ note: "documents accounts", count: accounts.length })}\n`) + for (const [tool, scope] of Object.entries(scopes)) + for (const [query, fields] of Object.entries(QUERIES)) { + const started = performance.now() + const found = await searchNotes(store, query, { + ...scope, + env: process.env, + command: tool, + timezone: "Europe/Madrid", + }) + const ms = Math.round(performance.now() - started) + const exact = await searchNotesQuery(store, { text: fields, limit: 20_000, timezone: "Europe/Madrid", ...scope }) + const inside = new Set(exact.items.map(({ ref }) => ref)) + const words = await searchNotesQuery(store, { + text: query, + limit: 20_000, + timezone: "Europe/Madrid", + ...scope, + }).catch(() => undefined) + appendFileSync( + OUT, + `${JSON.stringify({ + label, + tool, + query, + hits: found.hits.length, + by_meaning_only: found.hits.filter(({ found_by }) => found_by.length === 1 && found_by[0] === "meaning") + .length, + with_meaning: found.hits.filter(({ found_by }) => found_by.includes("meaning")).length, + outside_filter: found.hits.filter(({ ref }) => !inside.has(ref)).length, + field_matches: inside.size, + word_matches: words?.items.length ?? null, + total: (found as { total?: unknown }).total ?? null, + has_more: found.has_more, + ms, + })}\n`, + ) + } + await store.close() +}, 900_000) diff --git a/performance/search/notes-meaning-filters/measure-messages.test.ts b/performance/search/notes-meaning-filters/measure-messages.test.ts new file mode 100644 index 0000000..de7393c --- /dev/null +++ b/performance/search/notes-meaning-filters/measure-messages.test.ts @@ -0,0 +1,67 @@ +import { appendFileSync } from "node:fs" +import { it, vi } from "vitest" +import { searchStore } from "../src/services/messages-archive-search.js" +import { statsLucene } from "../src/services/messages-search.js" +import { openStore } from "../src/store/store.js" + +vi.mock("../src/store/sqlite/note-index.js", async (original) => ({ + ...(await original()), + drainNoteIndex: () => undefined, +})) + +const QUERIES = [ + "встреча", + "встреча date>=30d", + "встреча -date>=30d", + "встреча (date>=30d OR date<=2020-01-01)", + "date>=7d", + "the", +] + +it("messages", async () => { + const store = await openStore({ path: process.env.MEASURE_STORE as string, readOnly: true }) + // Filling the word index is a write; this run searches the index as it stands. + Object.assign(store.searchIndex, { fillSearchIndex: async () => undefined, fillStems: async () => undefined }) + const accounts = (await store.accounts.accounts()).filter(({ provider }) => provider === "telegram") + const out = process.env.MEASURE_OUT as string + appendFileSync(out, `${JSON.stringify({ telegram_accounts: accounts.length })}\n`) + let account = accounts[0] + let most = -1 + for (const one of accounts) { + const held = (await store.chats.chatStats(one)).reduce((sum, { messages }) => sum + messages, 0) + if (held > most) [account, most] = [one, held] + } + if (!account) return + const cutoff = Date.now() - 30 * 86_400_000 + for (const text of QUERIES) + for (const discover of [false, true]) { + const request = { text, limit: 20, timezone: "Europe/Madrid", discover, accounts: [account] } + const found = await searchStore(store, account, request).catch((e: Error) => ({ error: e.message })) + if ("error" in found) { + appendFileSync(out, `${JSON.stringify({ text, discover, error: found.error.slice(0, 120) })}\n`) + continue + } + const stats = await statsLucene(store, account, { ...request, discover: undefined, by: "chat" }).catch( + () => undefined, + ) + const dated = (want: (at: number) => boolean) => + found.items.filter(({ timestamp }) => timestamp && !want(Date.parse(timestamp))).length + appendFileSync( + out, + `${JSON.stringify({ + text, + discover, + ran_discovery: found.query?.discovery !== undefined, + hits: found.items.length, + total: found.total, + direct_count: stats?.total ?? null, + breaks_date: text.includes("-date>=30d") + ? dated((at) => at < cutoff) + : text === "budget date>=30d" + ? dated((at) => at >= cutoff - 86_400_000) + : null, + })}\n`, + ) + } + await store.close() +}, 900_000) diff --git a/performance/search/notes-meaning-filters/measure-timing.test.ts b/performance/search/notes-meaning-filters/measure-timing.test.ts new file mode 100644 index 0000000..a6ea63d --- /dev/null +++ b/performance/search/notes-meaning-filters/measure-timing.test.ts @@ -0,0 +1,47 @@ +import { appendFileSync } from "node:fs" +import { it, vi } from "vitest" +import { DEFAULT_TEXT_MODEL, textModel, vectorModelKey } from "../src/embeddings/models.js" +import { storeQuery } from "../src/services/notes-search.js" +import { openStore } from "../src/store/store.js" + +vi.mock("../src/store/sqlite/note-index.js", async (original) => ({ + ...(await original()), + drainNoteIndex: () => undefined, +})) + +it("timing", async () => { + const store = await openStore({ path: process.env.MEASURE_STORE as string, readOnly: true }) + const key = vectorModelKey(textModel(DEFAULT_TEXT_MODEL)) + const { root } = await storeQuery(store, "in:notes") + const all = await store.notes.matchingRefs({ root, limit: 50_000 }) + const vector = new Float32Array(384).map(() => Math.random() - 0.5) + const norm = Math.hypot(...vector) + for (let i = 0; i < vector.length; i++) vector[i] = (vector[i] as number) / norm + const lines: unknown[] = [{ notes: all.refs.length }] + for (const n of [150, 500, 1000, all.refs.length]) { + const refs = all.refs.slice(0, n) + const runs: number[] = [] + for (let r = 0; r < 3; r++) { + const t = performance.now() + await store.notes.scoreNotes(key, vector, refs) + runs.push(Math.round(performance.now() - t)) + } + lines.push({ notes: n, ms: runs }) + } + const g: number[] = [] + for (let r = 0; r < 3; r++) { + const t = performance.now() + await store.notes.nearest(key, vector, { limit: 50 }) + g.push(Math.round(performance.now() - t)) + } + lines.push({ global_nearest_50_ms: g }) + const c: number[] = [] + for (let r = 0; r < 3; r++) { + const t = performance.now() + await store.notes.matchingRefs({ root, limit: 10_000 }) + c.push(Math.round(performance.now() - t)) + } + lines.push({ count_all_ms: c }) + appendFileSync(process.env.MEASURE_OUT as string, `${lines.map((l) => JSON.stringify(l)).join("\n")}\n`) + await store.close() +}, 900_000) diff --git a/performance/search/notes-meaning-filters/vitest.measure.config.ts b/performance/search/notes-meaning-filters/vitest.measure.config.ts new file mode 100644 index 0000000..15f82b9 --- /dev/null +++ b/performance/search/notes-meaning-filters/vitest.measure.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from "vitest/config" +export default defineConfig({ + tsconfig: "./node_modules/@wirecat/cli-base/config/tsconfig.base.json", + test: { + include: [process.env.MEASURE_FILE ?? ".measure/notes.test.ts"], + testTimeout: 900_000, + deps: { + optimizer: { + ssr: { enabled: true, include: ["drizzle-orm", "drizzle-orm/sqlite-core", "drizzle-orm/node-sqlite"] }, + }, + }, + }, +})