From 222671a7108ea1525583453233ef831dc09d2e37 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 26 Oct 2026 03:09:04 +0000 Subject: [PATCH] Read CSV and TSV as a table on every surface (v0.24.0) Inspired by tabiew. src/csv.ts parses RFC 4180 (quotes, embedded delimiters and newlines, BOM), sniffs comma/tab/semicolon/pipe and the header row, types each column, and sorts, filters (col:text col>n col=v !x #3) and summarises. - terminal reader: CSV/TSV in the tree, drawn with hqtui 0.11's dataGrid: cell cursor, o sort, / row filter, enter row sheet, c column stats, v raw - web editor, desktop app and /p/ paste links: sticky sortable header, filter, row sheet, column stats, Copy/Download view; still editable - pastes: csv is a language, sniffed from unnamed content - --print prints an aligned table; MCP table_query for agents - site, llms.txt, README, desktop file associations Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 2 +- README.md | 41 +++- bun.lock | 4 +- desktop/package.json | 7 +- package.json | 4 +- site/assets/viewer.css | 33 +++ site/build.ts | 7 +- site/content/welcome.md | 3 +- site/table-view.ts | 247 +++++++++++++++++++ site/test/viewer.spec.ts | 76 ++++++ site/viewer.ts | 33 ++- src/cli.ts | 2 +- src/code.ts | 6 +- src/csv.ts | 514 +++++++++++++++++++++++++++++++++++++++ src/mcp.ts | 44 +++- src/print.ts | 5 +- src/tree.ts | 14 +- src/viewer.ts | 322 +++++++++++++++++++++++- test/code.test.ts | 19 ++ test/csv.test.ts | 204 ++++++++++++++++ 20 files changed, 1548 insertions(+), 39 deletions(-) create mode 100644 site/table-view.ts create mode 100644 src/csv.ts create mode 100644 test/csv.test.ts diff --git a/Dockerfile b/Dockerfile index 0a3c598..69942be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ COPY --from=builder --chown=bun:bun /app/server ./server # Only the src modules the server imports at runtime: sync-api needs sync-schema, and # pastes needs code.ts (language detection; its one dependency, highlight.js, is in node_modules), # and billing needs diff.ts (the AI summary is asked about the changed lines). -COPY --from=builder --chown=bun:bun /app/src/sync-schema.ts /app/src/flavors.ts /app/src/emoji.ts /app/src/code.ts /app/src/diff.ts /app/src/sensitive.ts /app/src/sync-crypto.ts /app/src/blob-store.ts ./src/ +COPY --from=builder --chown=bun:bun /app/src/sync-schema.ts /app/src/flavors.ts /app/src/emoji.ts /app/src/code.ts /app/src/csv.ts /app/src/diff.ts /app/src/sensitive.ts /app/src/sync-crypto.ts /app/src/blob-store.ts ./src/ COPY --from=deps --chown=bun:bun /app/node_modules ./node_modules RUN mkdir -p /data && chown bun:bun /data diff --git a/README.md b/README.md index 22dc9d8..84be148 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ npx @profullstack/readm3 readm3 # browse markdown under the current directory readm3 ./docs # browse a directory readm3 CHANGELOG.md # open a file, browse its directory +readm3 sales.csv # a CSV or TSV opens as a sortable, filterable grid readm3 --theme nord # pick a palette ``` @@ -82,6 +83,7 @@ readm3 --print CHANGELOG.md # rendered, colored, to the terminal readm3 --print README.md | less -R # into a pager curl -s https://example.com/x.md | readm3 --print readm3 --print notes.md | grep TODO # no escapes when stdout is a pipe +readm3 --print data.csv # a CSV or TSV prints as an aligned table ``` Color follows the usual rules: on when stdout is a terminal, off when it is not, @@ -89,7 +91,7 @@ and `NO_COLOR` / `FORCE_COLOR` are honored. `--color always|never|auto` override that either way. With `--print`, `--width` sets the render width rather than the sidebar. -The browser only shows markdown. Directories with nothing beneath them are +The browser shows markdown and tables (`.csv`, `.tsv`, `.tab`, `.psv`). Directories with nothing beneath them are pruned, `node_modules` and friends are never walked, and README sorts first. ## Keys @@ -108,6 +110,43 @@ pruned, `node_modules` and friends are never walked, and README sorts first. The mouse works too: the wheel scrolls whichever pane it is over, and a click opens a file. +### Tables + +A CSV or TSV opens as a grid instead of as text, in the spirit of +[tabiew](https://github.com/shshemi/tabiew): a frozen header, row numbers, a cell +cursor, and columns that scroll in from the side. The delimiter (comma, tab, +semicolon, pipe) and the header row are detected, quoted fields with commas and +line breaks are read correctly, and every column is typed (number, date, yes/no, +text) so sorting and comparisons read values the way people do. + +| | | +|---|---| +| `←` `↑` `↓` `→` `h` `j` `k` `l` | move the cell cursor (`←` on the first column returns to the files) | +| `0` `$` | first / last column | +| `o` | sort by the cursor's column: ascending, descending, off | +| `/` | filter the rows | +| `enter` | the row as a sheet, every column on its own line | +| `c` | the column's statistics: rows, empty, distinct, min, max, sum, mean, median, top values | +| `v` | the raw text instead of the grid | +| `e` | edit the text, as with any file | + +The filter takes space-separated terms that must all match: `london` (any cell), +`city:london` (that column contains it), `status=open`, `country!=US`, `age>30`, +`price<=9.99`, `joined>=2024-01-01`, `#3>10` (the third column), and `!draft` to +negate. Quote a value with spaces: `city:"new york"`. + +The same grid is in the web editor, the desktop app and on paste links: click a +column to sort, Σ for its statistics, a row to see it whole, and **Copy view** / +**Download view** for just the rows you kept. A paste with no file name is +recognised as a table from its content. The terminal grid is hqtui's `dataGrid` +widget; the parsing is `src/csv.ts`, shared by every surface. + +Agents get it through the MCP server (`readm3 mcp`) as `table_query`: point it at +CSV text, a local `path`, a paste link or a raw `url`, with an optional `filter`, +`sort`/`descending`, `limit`/`offset` and a `stats` column, and it returns the typed +columns, the match count and just the rows asked for, so a 50,000-row file never +has to go into context whole. It runs locally and needs no account. + ### Editing `e` puts the caret in the file. `esc` takes it out, and the preview re-renders diff --git a/bun.lock b/bun.lock index 5b0911d..102dadf 100644 --- a/bun.lock +++ b/bun.lock @@ -9,7 +9,7 @@ "@profullstack/auth-system": "^0.8.3", "@profullstack/cli": "^0.1.0", "@profullstack/footer": "latest", - "@profullstack/hqtui": "^0.5.0", + "@profullstack/hqtui": "^0.11.0", "@profullstack/synconfig": "^0.1.3", "@profullstack/x402-gateway": "^0.11.0", "highlight.js": "^11.12.0", @@ -150,7 +150,7 @@ "@profullstack/footer": ["@profullstack/footer@0.2.4", "", { "peerDependencies": { "react": ">=18" }, "optionalPeers": ["react"], "bin": { "profullstack-footer": "bin/profullstack-footer.js" } }, "sha512-2ErIzHJCDA2baP8aK6f7rCHohW4EX+wnp7D1mrVMQTUxRcyZwKzmho96hhEFJndXRBGgxT3H/78BHOeBfLxTzg=="], - "@profullstack/hqtui": ["@profullstack/hqtui@0.5.0", "", { "bin": { "hqtui": "bin/hqtui.mjs" } }, "sha512-jDwILBmdQx8pQA0ao/tzzty8u5nFNS4XhvU9xXhZvdIaMeiF3HBJIVtP6cS6kigS4lXmHR7sztbb9QJpaWwhig=="], + "@profullstack/hqtui": ["@profullstack/hqtui@0.11.0", "", { "bin": { "hqtui": "bin/hqtui.mjs" } }, "sha512-kvpruktJsxwcQ8VYxdxDJn6dtPoWOMuOtr87DZbVV0KIrm7BBF5HRf37gA5Am/ReWd/ayEn9rvmHN6rVFTFALA=="], "@profullstack/keys": ["@profullstack/keys@0.1.0", "", {}, "sha512-Vys47LvS5ivhdZu4SYOxh9Le3DEmWhjP0+CtrU45RhXAbY0ZFG/2Dk9V1bIIUGDp8UWMJqctvUD21JrNjfh04w=="], diff --git a/desktop/package.json b/desktop/package.json index 9b9c265..3358ec3 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -31,7 +31,8 @@ "protocols": [{ "name": "readm3", "schemes": ["readm3"] }], "fileAssociations": [ { "ext": ["md", "markdown"], "name": "Markdown", "role": "Editor" }, - { "ext": ["txt", "json", "yaml", "yml", "toml", "js", "ts", "py", "sh", "css", "html"], "name": "Text", "role": "Editor" } + { "ext": ["txt", "json", "yaml", "yml", "toml", "js", "ts", "py", "sh", "css", "html"], "name": "Text", "role": "Editor" }, + { "ext": ["csv", "tsv"], "name": "Table", "role": "Viewer" } ], "mac": { "artifactName": "readm3-mac-${arch}.${ext}", @@ -58,12 +59,12 @@ "category": "Development", "synopsis": "A plain, fast text editor for Markdown and code.", "description": "readm3 is a quiet text editor: line numbers, multiple cursors, find and replace, folding, quick open and a command palette, with Markdown rendered as a document. It opens folders from disk and saves back to them, and works offline.", - "mimeTypes": ["text/plain", "text/markdown", "application/json", "text/x-python", "text/javascript", "text/css", "text/html", "application/xml", "text/x-shellscript", "application/x-yaml", "text/x-c", "text/x-c++", "text/x-go", "text/x-rust", "text/x-java"], + "mimeTypes": ["text/plain", "text/markdown", "application/json", "text/x-python", "text/javascript", "text/css", "text/html", "application/xml", "text/x-shellscript", "application/x-yaml", "text/x-c", "text/x-c++", "text/x-go", "text/x-rust", "text/x-java", "text/csv", "text/tab-separated-values"], "desktop": { "entry": { "Name": "readm3", "GenericName": "Text Editor", - "Keywords": "text;editor;markdown;code;", + "Keywords": "text;editor;markdown;code;csv;tsv;table;", "StartupWMClass": "readm3" } } diff --git a/package.json b/package.json index f82f2ce..304d480 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/readm3", - "version": "0.23.0", + "version": "0.24.0", "description": "Markdown editing and sharing for people and teams. Web/PWA, CLI, API and MCP with version history, view/edit links, and a terminal reader.", "type": "module", "license": "MIT", @@ -46,7 +46,7 @@ "@profullstack/auth-system": "^0.8.3", "@profullstack/cli": "^0.1.0", "@profullstack/footer": "latest", - "@profullstack/hqtui": "^0.5.0", + "@profullstack/hqtui": "^0.11.0", "@profullstack/synconfig": "^0.1.3", "@profullstack/x402-gateway": "^0.11.0", "highlight.js": "^11.12.0", diff --git a/site/assets/viewer.css b/site/assets/viewer.css index 04340ca..0b068b0 100644 --- a/site/assets/viewer.css +++ b/site/assets/viewer.css @@ -235,3 +235,36 @@ html, body.reader-app { height: 100%; overflow: hidden; } .reader-app .summary-listen { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; margin: 4px 0 8px; } .reader-app .summary-listen audio { width: 100%; max-width: 420px; height: 36px; } .reader-app .summary-listen small { color: var(--muted); } + +/* Tables (CSV, TSV): a sticky header that sorts, numbered rows, a filter. Same theme variables as code. */ +.code-view.table-view { width: 100%; min-width: 0; } +.table-view { font: 13px/1.6 var(--mono); color: var(--fg); display: flex; flex-direction: column; gap: 10px; } +.table-tools { display: flex; align-items: center; gap: 8px; flex-wrap: wrap; } +.table-filter { flex: 1 1 260px; min-width: 0; padding: 7px 10px; font: 12px var(--mono); color: var(--fg); background: var(--surface); border: 1px solid var(--border); border-radius: 5px; } +.table-filter:focus { border-color: var(--primary); outline: none; } +.table-count { color: var(--muted); font-size: 11px; white-space: nowrap; } +.reader-app .table-tools button { padding: 5px 9px; font-size: 11px; color: var(--muted); } +.table-scroll { overflow: auto; max-width: 100%; max-height: calc(100dvh - 260px); border: 1px solid var(--border); border-radius: 6px; } +.data-table { border-collapse: separate; border-spacing: 0; width: max-content; min-width: 100%; } +.data-table th, .data-table td { padding: 3px 12px; border-bottom: 1px solid color-mix(in srgb, var(--border) 60%, transparent); border-right: 1px solid color-mix(in srgb, var(--border) 60%, transparent); text-align: left; white-space: pre; max-width: 40ch; overflow: hidden; text-overflow: ellipsis; vertical-align: top; } +.data-table .num { text-align: right; font-variant-numeric: tabular-nums; } +.data-table thead th { position: sticky; top: 0; z-index: 2; background: var(--surface); padding: 0; white-space: nowrap; max-width: none; border-bottom: 1px solid var(--border); } +.data-table .rownum { position: sticky; left: 0; z-index: 1; background: var(--surface); color: var(--muted); text-align: right; font-weight: 400; font-size: 11px; user-select: none; } +.data-table thead .rownum { z-index: 3; padding: 3px 12px; } +.data-table tbody tr { cursor: pointer; } +.data-table tbody tr:nth-child(even) td { background: color-mix(in srgb, var(--fg) 2.5%, transparent); } +.data-table tbody tr:hover td, .data-table tbody tr:focus td { background: color-mix(in srgb, var(--primary) 9%, transparent); } +.reader-app .column-sort { border: 0; border-radius: 0; background: transparent; padding: 5px 4px 5px 12px; font: 600 12px var(--mono); color: var(--fg); } +.reader-app .column-sort:hover { background: transparent; border: 0; color: var(--primary); } +.column-type { margin-left: 4px; color: var(--muted); font-weight: 400; font-size: 10px; } +.reader-app .column-sort[aria-sort="ascending"] .column-type, .reader-app .column-sort[aria-sort="descending"] .column-type { color: var(--primary); font-size: 11px; } +.reader-app .column-stats { border: 0; background: transparent; padding: 5px 10px 5px 2px; font-size: 11px; color: var(--muted); visibility: hidden; } +.data-table thead th:hover .column-stats, .reader-app .column-stats:focus-visible { visibility: visible; } +.reader-app .column-stats:hover { background: transparent; border: 0; color: var(--primary); } +.reader-app .table-more { align-self: flex-start; font-size: 11px; color: var(--muted); } +.sheet-list { display: grid; grid-template-columns: minmax(80px, max-content) 1fr; gap: 8px 20px; margin: 20px 0 0; font: 12px/1.6 var(--mono); } +.sheet-list dt { color: var(--primary); overflow-wrap: anywhere; } +.sheet-list dd { margin: 0; white-space: pre-wrap; overflow-wrap: anywhere; } +.sheet-list dd.empty::before { content: "(empty)"; color: var(--muted); } +@media (hover: none) { .reader-app .column-stats { visibility: visible; } } +@media (max-width: 700px) { .table-scroll { max-height: calc(100dvh - 300px); } .data-table th, .data-table td { max-width: 24ch; } } diff --git a/site/build.ts b/site/build.ts index 47b134c..f4695f8 100644 --- a/site/build.ts +++ b/site/build.ts @@ -250,12 +250,13 @@ function home(): string {

For the files people actually work on: READMEs, project plans, meeting notes, and proposals.

Write, preview, repeat.

Open a file or start from scratch. Edit the Markdown, switch to a clean preview, and download the source whenever you need it.

-

Paste anything, no account.

Text, code, JSON, a PDF or an image behind a secret link that expires. Markdown renders, code is highlighted and folds, and only a hash of the secret is kept. readm3.com/paste, or readm3 paste FILE.

+

Paste anything, no account.

Text, code, JSON, a CSV, a PDF or an image behind a secret link that expires. Markdown renders, code is highlighted and folds, a table is a sortable grid, and only a hash of the secret is kept. readm3.com/paste, or readm3 paste FILE.

“Can view” or “can edit.”

Share a link with a reviewer or invite someone to make changes. You own your files and decide who gets access.

Keep the story of the document.

Every online save creates a version. See who changed it, share a specific version, or restore an earlier draft without losing the history.

A workspace for your team.

Organize people into organizations and teams. Share with a whole group, a few collaborators, or anyone with your link.

A plain text editor.

Line numbers, multiple cursors, find and replace, folding, Ctrl+P to any file and Ctrl+Shift+P for every command. Open a folder from disk and Ctrl+S saves back to it. Open the editor.

Files up to 5 GB, a million of them.

A free account lifts the limits: up to 1,000,000 files (250,000 of them synced), each up to 5 GB, with 5 GB of end-to-end encrypted storage that syncs to every device. Need more? Pay as you go, $0.05 per GB a month, from a CoinPay balance.

+

CSV and TSV as a table.

Open a .csv or .tsv and it reads as a grid, not a wall of commas: click a column to sort, filter rows with city:london age>30 !draft, open a row as a sheet, see a column's min, max, mean and top values, and copy or download just the rows you kept. The same in the terminal (readm3 data.csv), the desktop app, and on a paste link. Try it.

Summarize any document.

Press Summarize in the editor for a short abstract and the key points of the open file, written by Claude. The first 10 documents a week are free on a free account; after that, pay as you go from a CoinPay balance, usually a fraction of a cent each. Try it.

Take the editor with you.

Install readm3 from your browser, or get the desktop app: Linux AppImage or Flatpak, macOS for Apple silicon or Intel, Windows. Then readm3 edit . opens a folder from the terminal, like subl .. Local files work offline; shared documents stay connected.

Use the tools you already use.

The web editor, CLI, API, and MCP work with the same documents and permissions. Your terminal and your agents can join the workflow.

@@ -414,7 +415,8 @@ const llms = `# readm3 ## What it is readm3 opens a directory of markdown files and renders them in the terminal in truecolor. -The browser pane shows only markdown, prunes directories with nothing beneath them, never +CSV and TSV files open as a grid with a cell cursor, sort, row filter, row sheet and column +statistics. The browser pane shows only markdown and tables, prunes directories with nothing beneath them, never walks node_modules, and sorts README first. The reader pane renders headings, emphasis, inline code, links, fenced and indented code blocks, nested and task lists, blockquotes, tables with per-column alignment, rules, images as labelled placeholders, and YAML front @@ -450,6 +452,7 @@ terminal. - Editor: ${SITE}/viewer (installable, offline text editor for Markdown and code: multiple cursors, find and replace, Ctrl+P quick open, command palette, saves back to files on disk) - Desktop app: ${REPO}/releases/latest (the same editor for Linux as AppImage and Flatpak, macOS dmg, Windows installer); \`readm3 edit PATH...\` opens files and folders in it - Summaries: Summarize in ${SITE}/viewer, or POST /api/v1/summaries/document { path, text } signed in (10 free documents a week, then pay as you go) +- Tables: CSV and TSV (also .tab, .psv) open as a sortable, filterable grid in the terminal reader (hqtui data grid), ${SITE}/viewer, the desktop app and paste links; \`readm3 --print data.csv\` prints an aligned table. Filter language: free text, col:text, col=v, col!=v, col>n, col<=n, !negate, #3 for the third column - Paste: ${SITE}/paste (anonymous pastes of any text, PDF or image behind an expiring secret link; CLI, curl, API and MCP examples) - Site: ${SITE} - Source: ${REPO} diff --git a/site/content/welcome.md b/site/content/welcome.md index 765a3db..250d64d 100644 --- a/site/content/welcome.md +++ b/site/content/welcome.md @@ -13,7 +13,8 @@ now in a browser. Pick a file on the left. Make yourself at home. Any text file appears in the explorer: Markdown renders as a document, and JSON, code, config and logs are shown as highlighted, foldable code you can edit. -PDFs and images open in the browser's own viewer. Build output, dependencies, +A CSV or TSV reads as a table: click a column to sort, type in the filter +(`city:london age>30`), click a row to see it whole. PDFs and images open in the browser's own viewer. Build output, dependencies, and hidden directories stay out of the way. README files sort first. ## A little less reaching for the mouse diff --git a/site/table-view.ts b/site/table-view.ts new file mode 100644 index 0000000..65f9e04 --- /dev/null +++ b/site/table-view.ts @@ -0,0 +1,247 @@ +/** + * The table view: a CSV or TSV drawn as a grid instead of as text. Parsing, sorting, + * filtering and statistics come from src/csv.ts, the same code the terminal reader + * uses, so `age>30` filters the same rows in both. This file only owns the DOM. + * + * Rows are added 500 at a time: a 4 MB file is ~50,000 rows, and building them all + * up front would freeze the page for a table nobody scrolls to the end of. Sort and + * filter run over every row, so what is shown first is always the right 500. + */ +import { alignOf, columnStats, delimiterName, displayCell, nextSort, parseTable, statsRows, toDelimited, viewRows, type Sort, type Table } from "../src/csv.ts"; + +export interface TableView { + table: Table; + /** The rows shown, as indices into `table.rows`. */ + shown(): number[]; + /** The shown rows as delimited text, header first. */ + text(): string; +} + +export interface TableActions { + /** Hands text to the clipboard, with a confirmation. */ + copy(text: string): void; + /** Saves text as a file. */ + save(text: string, name: string): void; +} + +const PAGE = 500; +const TYPE_MARK = { number: "123", date: "date", boolean: "y/n", text: "abc" } as const; + +/** Filter and sort per file, so switching away and back keeps them. */ +const remembered = new Map(); + +let dialog: HTMLDialogElement | null = null; +function sheetDialog(): HTMLDialogElement { + if (dialog) return dialog; + dialog = document.createElement("dialog"); + dialog.id = "table-dialog"; + dialog.setAttribute("aria-labelledby", "table-dialog-title"); + dialog.addEventListener("click", (event) => { if (event.target === dialog) dialog!.close(); }); + document.body.append(dialog); + return dialog; +} + +function showPairs(title: string, pairs: { label: string; value: string }[], note?: string) { + const box = sheetDialog(); + const form = document.createElement("form"); + form.method = "dialog"; + const header = document.createElement("div"); + header.className = "dialog-header"; + const heading = document.createElement("h2"); + heading.id = "table-dialog-title"; + heading.textContent = title; + const close = document.createElement("button"); + close.setAttribute("aria-label", "Close"); + close.textContent = "×"; + header.append(heading, close); + const list = document.createElement("dl"); + list.className = "sheet-list"; + for (const { label, value } of pairs) { + const term = document.createElement("dt"); + term.textContent = label; + const detail = document.createElement("dd"); + detail.textContent = value; + if (!value) detail.className = "empty"; + list.append(term, detail); + } + form.append(header, list); + if (note) { + const p = document.createElement("p"); + p.textContent = note; + form.append(p); + } + box.replaceChildren(form); + box.showModal(); +} + +export function renderTable(container: HTMLElement, source: string, path: string, actions: TableActions): TableView { + const table = parseTable(source, { name: path }); + const state = remembered.get(path) ?? { filter: "", sort: null }; + remembered.set(path, state); + let shown = viewRows(table, state); + let drawn = 0; + + container.replaceChildren(); + container.dataset.language = "csv"; + container.classList.add("table-view"); + + const tools = document.createElement("div"); + tools.className = "table-tools"; + const filter = document.createElement("input"); + filter.type = "search"; + filter.className = "table-filter"; + filter.placeholder = "Filter rows: text, city:london, age>30, !draft"; + filter.setAttribute("aria-label", "Filter rows"); + filter.value = state.filter; + const count = document.createElement("span"); + count.className = "table-count"; + count.setAttribute("aria-live", "polite"); + const copyButton = document.createElement("button"); + copyButton.type = "button"; + copyButton.textContent = "Copy view"; + copyButton.title = "Copy the rows shown, sorted and filtered, as CSV"; + const saveButton = document.createElement("button"); + saveButton.type = "button"; + saveButton.textContent = "Download view"; + saveButton.title = "Download the rows shown, sorted and filtered, as a file"; + tools.append(filter, count, copyButton, saveButton); + + const grid = document.createElement("table"); + grid.className = "data-table"; + const head = document.createElement("thead"); + const headRow = document.createElement("tr"); + const corner = document.createElement("th"); + corner.className = "rownum"; + corner.textContent = "#"; + headRow.append(corner); + const sortButtons: HTMLButtonElement[] = []; + table.columns.forEach((column, index) => { + const th = document.createElement("th"); + th.scope = "col"; + th.className = alignOf(column.type) === "right" ? "num" : ""; + const sort = document.createElement("button"); + sort.type = "button"; + sort.className = "column-sort"; + sort.title = `Sort by ${column.name}`; + sort.addEventListener("click", () => { + state.sort = nextSort(state.sort, index); + refresh(); + }); + sortButtons.push(sort); + const stats = document.createElement("button"); + stats.type = "button"; + stats.className = "column-stats"; + stats.textContent = "Σ"; + stats.title = `Statistics for ${column.name}`; + stats.setAttribute("aria-label", `Statistics for ${column.name}`); + stats.addEventListener("click", () => { + showPairs(column.name, statsRows(columnStats(table, index, shown)), state.filter ? "Over the filtered rows." : undefined); + }); + th.append(sort, stats); + headRow.append(th); + }); + head.append(headRow); + const body = document.createElement("tbody"); + grid.append(head, body); + const scroller = document.createElement("div"); + scroller.className = "table-scroll"; + scroller.append(grid); + + const more = document.createElement("button"); + more.type = "button"; + more.className = "table-more"; + more.addEventListener("click", () => drawMore()); + + const note = document.createElement("p"); + note.className = "media-note"; + const notes = [`${table.header ? "First row is the header" : "No header row; columns are lettered"} · ${delimiterName(table.delimiter)}-separated`]; + if (table.ragged) notes.push(`${table.ragged.toLocaleString()} row${table.ragged === 1 ? " has" : "s have"} a different number of fields than the header`); + note.textContent = `${notes.join(" · ")}. Click a row to see it whole; Edit changes the text.`; + + container.append(tools, scroller, more, note); + + body.addEventListener("click", (event) => { + const row = (event.target as HTMLElement).closest("tr"); + if (!row?.dataset.index) return; + openSheet(Number(row.dataset.index)); + }); + body.addEventListener("keydown", (event) => { + const row = (event.target as HTMLElement).closest("tr"); + if (event.key === "Enter" && row?.dataset.index) openSheet(Number(row.dataset.index)); + }); + + function openSheet(index: number) { + const record = table.rows[index]!; + showPairs(`Row ${(index + 1).toLocaleString()}`, table.columns.map((column, c) => ({ label: column.name, value: record[c] ?? "" }))); + } + + let typing: ReturnType | undefined; + filter.addEventListener("input", () => { + clearTimeout(typing); + // A keystroke on a 50,000-row file re-filters everything; wait for a pause. + typing = setTimeout(() => { state.filter = filter.value; refresh(); }, table.rows.length > 20000 ? 200 : 60); + }); + filter.addEventListener("keydown", (event) => { + if (event.key === "Escape" && filter.value) { event.stopPropagation(); filter.value = ""; state.filter = ""; refresh(); } + }); + copyButton.addEventListener("click", () => actions.copy(toDelimited(table, shown))); + saveButton.addEventListener("click", () => { + const base = path.split("/").at(-1)!.replace(/\.[^.]+$/, ""); + const extension = table.delimiter === "\t" ? "tsv" : "csv"; + actions.save(toDelimited(table, shown, table.delimiter === "\t" ? "\t" : ","), `${base}${state.filter || state.sort ? "-view" : ""}.${extension}`); + }); + + function drawMore() { + const fragment = document.createDocumentFragment(); + const end = Math.min(shown.length, drawn + PAGE); + for (let position = drawn; position < end; position++) { + const index = shown[position]!; + const record = table.rows[index]!; + const tr = document.createElement("tr"); + tr.dataset.index = String(index); + tr.tabIndex = -1; + const number = document.createElement("th"); + number.scope = "row"; + number.className = "rownum"; + number.textContent = String(index + 1); + tr.append(number); + table.columns.forEach((column, c) => { + const td = document.createElement("td"); + const value = record[c] ?? ""; + td.textContent = displayCell(value); + if (alignOf(column.type) === "right") td.className = "num"; + if (value.length > 60 || value !== td.textContent) td.title = value; + tr.append(td); + }); + fragment.append(tr); + } + body.append(fragment); + drawn = end; + const left = shown.length - drawn; + more.hidden = left <= 0; + more.textContent = `Show ${Math.min(PAGE, left).toLocaleString()} more (${left.toLocaleString()} left)`; + } + + function refresh() { + shown = viewRows(table, state); + sortButtons.forEach((button, index) => { + const column = table.columns[index]!; + const active = state.sort?.column === index; + button.textContent = `${column.name} `; + const mark = document.createElement("span"); + mark.className = "column-type"; + mark.textContent = active ? (state.sort!.descending ? "▼" : "▲") : TYPE_MARK[column.type]; + button.append(mark); + button.setAttribute("aria-sort", active ? (state.sort!.descending ? "descending" : "ascending") : "none"); + button.parentElement!.setAttribute("aria-sort", active ? (state.sort!.descending ? "descending" : "ascending") : "none"); + }); + const total = table.rows.length; + count.textContent = `${shown.length === total ? "" : `${shown.length.toLocaleString()} of `}${total.toLocaleString()} rows · ${table.columns.length} columns`; + body.replaceChildren(); + drawn = 0; + drawMore(); + } + + refresh(); + return { table, shown: () => shown, text: () => toDelimited(table, shown) }; +} diff --git a/site/test/viewer.spec.ts b/site/test/viewer.spec.ts index 9af5d7b..5c8147d 100644 --- a/site/test/viewer.spec.ts +++ b/site/test/viewer.spec.ts @@ -561,3 +561,79 @@ test("links to the reader from the homepage and serves worker updates uncached", // Railway rejects malformed encodings at its edge before they reach this server. if (!process.env.READM3_TEST_URL) expect((await request.get("/%E0%A4%A")).status()).toBe(404); }); + +test("reads a CSV as a table: sorts, filters, opens a row and a column's stats, and edits the text", async ({ page, context }) => { + await context.grantPermissions(["clipboard-read", "clipboard-write"]); + const csv = 'name,city,age\nAda,London,36\nGrace,"Arlington, VA",85\nLinus,Helsinki,54\nMargaret,Boston,\n'; + await page.locator("#files-input").setInputFiles({ name: "people.csv", mimeType: "text/csv", buffer: Buffer.from(csv) }); + await expect(page.locator("#document-name")).toHaveText("people.csv"); + // A table opens to read, not in the editor the way code does. + await expect(page.locator("#editor")).toBeHidden(); + await expect(page.locator("#file-mark")).toHaveText("CSV"); + await expect(page.locator("#code-tools")).toBeHidden(); + const view = page.locator("#code-content.table-view"); + await expect(view.locator(".table-count")).toHaveText("4 rows · 3 columns"); + await expect(view.locator("thead th")).toHaveCount(4); + const names = () => view.locator("tbody tr td:first-of-type").allTextContents(); + expect(await names()).toEqual(["Ada", "Grace", "Linus", "Margaret"]); + await expect(view.locator("tbody tr").nth(1).locator("td").nth(1)).toHaveText("Arlington, VA"); + await expect(view.locator("tbody td.num").first()).toHaveText("36"); + + const age = view.locator(".column-sort").nth(2); + await age.click(); + await expect(age).toHaveAttribute("aria-sort", "ascending"); + expect(await names()).toEqual(["Ada", "Linus", "Grace", "Margaret"]); + await age.click(); + expect(await names()).toEqual(["Grace", "Linus", "Ada", "Margaret"]); + await age.click(); + await expect(age).toHaveAttribute("aria-sort", "none"); + + // `/` reaches the row filter while a table is open, and the filter language works. + await page.locator("#reader").focus(); + await page.keyboard.press("/"); + await expect(view.locator(".table-filter")).toBeFocused(); + await page.keyboard.type("age>40"); + await expect(view.locator(".table-count")).toHaveText("2 of 4 rows · 3 columns"); + expect(await names()).toEqual(["Grace", "Linus"]); + await view.locator(".table-tools button", { hasText: "Copy view" }).click(); + expect(await page.evaluate(() => navigator.clipboard.readText())).toBe('name,city,age\nGrace,"Arlington, VA",85\nLinus,Helsinki,54\n'); + + await view.locator("tbody tr").first().click(); + const sheet = page.locator("#table-dialog"); + await expect(sheet).toBeVisible(); + await expect(sheet.locator("h2")).toHaveText("Row 2"); + await expect(sheet.locator("dd").nth(1)).toHaveText("Arlington, VA"); + await page.keyboard.press("Escape"); + await expect(sheet).toBeHidden(); + + await view.locator("thead th").nth(3).hover(); + await view.locator(".column-stats").nth(2).click(); + await expect(sheet.locator("h2")).toHaveText("age"); + await expect(sheet).toContainText("Over the filtered rows."); + await expect(sheet.locator("dt", { hasText: "mean" }).locator("+ dd")).toHaveText("69.5"); + await page.keyboard.press("Escape"); + + // Editing is the same text editor as code; reading again shows the change. + await page.locator("#edit-mode").click(); + await expect(page.locator("#editor")).toBeVisible(); + await editorInput(page).press("Control+End"); + await editorInput(page).type("Ken,Tokyo,70\n"); + await page.locator("#read-mode").click(); + await expect(view.locator(".table-count")).toHaveText("3 of 5 rows · 3 columns"); + expect(await names()).toEqual(["Grace", "Linus", "Ken"]); +}); + +test("a pasted table with no name is sniffed as CSV and shown as a table at its private link", async ({ page, request }) => { + const created = await request.post("/api/v1/pastes", { data: { source: "id\tcity\tscore\n1\tOslo\t9.5\n2\tLima\t7\n3\tPune\t8.25\n" } }); + expect(created.status()).toBe(201); + const paste = await created.json(); + expect(paste.language).toBe("csv"); + expect(paste.title).toBe("paste.csv"); + expect(paste.mime).toBe("text/csv"); + await page.goto(paste.url); + const view = page.locator("#code-content.table-view"); + await expect(view.locator(".table-count")).toHaveText("3 rows · 3 columns"); + await expect(view.locator(".media-note")).toContainText("tab-separated"); + await expect(view.locator("tbody tr").nth(2).locator("td").nth(1)).toHaveText("Pune"); + await expect(page.locator("#edit-mode")).toBeDisabled(); +}); diff --git a/site/viewer.ts b/site/viewer.ts index b658039..4a27fe2 100644 --- a/site/viewer.ts +++ b/site/viewer.ts @@ -10,6 +10,8 @@ import type { Flavor } from "../src/flavors.ts"; import type { ReaderSettings } from "../src/sync-schema.ts"; import { toHtml } from "./html.ts"; import { renderCode, type CodeView } from "./code-view.ts"; +import { renderTable } from "./table-view.ts"; +import { TABLE_LANGUAGE } from "../src/csv.ts"; import type { SourceEditor } from "./code-editor.ts"; import * as disk from "./disk.ts"; import * as fileHistory from "./history.ts"; @@ -63,6 +65,8 @@ const kind = () => kindOf(current()); const isCode = () => kind().kind === "code"; const isBinary = () => kind().kind === "binary"; const isMarkdown = () => kind().kind === "markdown"; +/** A CSV or TSV: text like code (edited the same way), but read as a table. */ +const isTable = () => { const active = kind(); return active.kind === "code" && active.language.id === TABLE_LANGUAGE; }; /** * Signed in, the free tier has no file count and files up to 5 GB (src/blob-store.ts); * signed out, the workspace keeps its small local limits. @@ -79,7 +83,7 @@ function applyKind() { const current = kind(); element("flavor-label").hidden = current.kind !== "markdown"; element("spoilers").hidden = current.kind !== "markdown"; - element("code-tools").hidden = current.kind !== "code"; + element("code-tools").hidden = current.kind !== "code" || isTable(); element("copy").hidden = current.kind === "binary" || isLarge(); // Any file can be summarized but text too large to open here (a large PDF or image still can). element("summarize").hidden = isLarge() && current.kind !== "binary"; @@ -391,7 +395,7 @@ function fileRow(path: string, name: string, depth: number, directory: boolean, icon.className = "file-icon"; icon.setAttribute("aria-hidden", "true"); const fileKind = doc ? kindOf(doc) : null; - icon.textContent = directory ? (filter.value || !collapsed.has(path) ? "▾" : "▸") : fileKind!.kind === "markdown" ? "M↓" : fileKind!.kind === "code" ? "{}" : "▣"; + icon.textContent = directory ? (filter.value || !collapsed.has(path) ? "▾" : "▸") : fileKind!.kind === "markdown" ? "M↓" : fileKind!.kind === "code" ? (fileKind!.language.id === TABLE_LANGUAGE ? "▦" : "{}") : "▣"; const label = document.createElement("span"); label.className = "row-name"; label.textContent = name; @@ -533,10 +537,21 @@ function renderDocument() { const active = kind(); if (active.kind === "code") { const key = JSON.stringify(["code", workspace.active, current().source, active.language.id]); - if (key !== previousRender) { + if (key !== previousRender && isTable()) { content.hidden = true; mediaContent.hidden = true; codeContent.hidden = false; + codeView = null; + renderTable(codeContent, current().source, workspace.active, { + copy: (text) => navigator.clipboard.writeText(text).then(() => notice("Copied the rows shown as CSV."), () => notice("The browser refused clipboard access. Use Download view instead.")), + save: (text, name) => saveBlob(new Blob([text], { type: "text/csv;charset=utf-8" }), name), + }); + previousRender = key; + } else if (key !== previousRender) { + content.hidden = true; + mediaContent.hidden = true; + codeContent.hidden = false; + codeContent.classList.remove("table-view"); codeView = renderCode(codeContent, current().source, active.language.id); previousRender = key; if (codeView.formatted) notice("Minified JSON shown formatted. Download gives the file exactly as it is."); @@ -641,8 +656,8 @@ function openDocument(path: string, focus = true) { renderTabs(); applyKind(); documentInfo(); - // Code opens ready to type, as in any text editor; Markdown opens rendered, as in a reader. - setMode(isCode() && canEdit(), focus); + // Code opens ready to type, as in any text editor; Markdown and tables open rendered, as in a reader. + setMode(isCode() && !isTable() && canEdit(), focus); reader.scrollTop = 0; progress(); setFilesOpen(false); @@ -1007,7 +1022,13 @@ function onKey(event: KeyboardEvent) { event.preventDefault(); return; } if (typing) return; - if (event.key === "/") { event.preventDefault(); setFilesOpen(true); filter.focus(); return; } + if (event.key === "/") { + event.preventDefault(); + // Reading a table, `/` filters its rows, as in the terminal reader. + const rows = !editing && isTable() ? codeContent.querySelector(".table-filter") : null; + if (rows) { rows.focus(); return; } + setFilesOpen(true); filter.focus(); return; + } if (event.key === "?") { event.preventDefault(); helpDialog.showModal(); return; } if (fileList.contains(target)) { const rows = [...fileList.querySelectorAll("button")]; diff --git a/src/cli.ts b/src/cli.ts index 39c617e..f5f847b 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -12,7 +12,7 @@ import { run, type ViewerOptions } from "./viewer.ts"; import { isCloudCommand, cloudMain, CLOUD_USAGE } from "./cloud-cli.ts"; import { isDesktopCommand, desktopMain, installedDesktop, prefersDesktop, DESKTOP_USAGE } from "./desktop-cli.ts"; -export const VERSION = "0.23.0"; +export const VERSION = "0.24.0"; /** The help text, exported so a test can hold it to the flags it documents. */ export const USAGE = `readm3 — a plain text editor for Markdown and code diff --git a/src/code.ts b/src/code.ts index ae3d202..38efb24 100644 --- a/src/code.ts +++ b/src/code.ts @@ -51,6 +51,7 @@ import swift from "highlight.js/lib/languages/swift"; import typescript from "highlight.js/lib/languages/typescript"; import xml from "highlight.js/lib/languages/xml"; import yaml from "highlight.js/lib/languages/yaml"; +import { looksLikeTable, TABLE_LANGUAGE } from "./csv.ts"; export interface Language { /** The highlight.js grammar name; also what the API reports as `language`. */ @@ -104,7 +105,9 @@ export const LANGUAGES: Language[] = [ { id: "diff", label: "Diff", extensions: ["diff", "patch"], mime: "text/x-diff" }, { id: "graphql", label: "GraphQL", extensions: ["graphql", "gql"], mime: "application/graphql" }, { id: "protobuf", label: "Protocol Buffers", extensions: ["proto"], mime: "text/plain" }, - { id: "plaintext", label: "Plain text", extensions: ["txt", "text", "log", "csv", "tsv"], mime: "text/plain", auto: false }, + // Tables have no grammar: every surface draws them as a grid (src/csv.ts) instead of highlighting them. + { id: TABLE_LANGUAGE, label: "CSV / TSV", extensions: ["csv", "tsv", "tab", "psv"], mime: "text/csv", auto: false }, + { id: "plaintext", label: "Plain text", extensions: ["txt", "text", "log"], mime: "text/plain", auto: false }, ]; const grammars: Record[1]> = { @@ -284,6 +287,7 @@ export function sniffLanguage(source: string): string { if (/^\s*<\?xml\b/.test(head) || /^\s*= 2 && best.score > (second?.score ?? 0)) return best.language; // The grammar scorer only decides when the signals could not, and only when it is sure. diff --git a/src/csv.ts b/src/csv.ts new file mode 100644 index 0000000..096ba39 --- /dev/null +++ b/src/csv.ts @@ -0,0 +1,514 @@ +/** + * Tables: CSV, TSV and their cousins, read into rows of strings and viewed as a grid. + * + * One module for every surface. The terminal reader draws the result with hqtui's data + * grid, the web viewer builds a from it, `--print` lays it out as text, and the + * paste service uses `looksLikeTable` to recognise a pasted CSV. Nothing here touches a + * DOM, a terminal or a file, so all of it is tested without either. + * + * Inspired by tabiew (github.com/shshemi/tabiew): sort by any column, filter rows, + * open one row as a sheet, and per-column statistics. Querying is a small filter + * language rather than SQL: `city:london age>30 !draft`. + */ + +export const TABLE_LANGUAGE = "csv"; + +export type Delimiter = "," | "\t" | ";" | "|"; +export const DELIMITERS: Delimiter[] = [",", "\t", ";", "|"]; +export type ColumnType = "number" | "date" | "boolean" | "text"; + +export interface Column { + name: string; + type: ColumnType; +} + +export interface Table { + delimiter: Delimiter; + /** Whether the first record was taken as column names. */ + header: boolean; + columns: Column[]; + /** Data records, every one padded or cut to the column count. */ + rows: string[][]; + /** Records with a different field count than the header (or the widest record), before padding. */ + ragged: number; +} + +export interface ParseOptions { + delimiter?: Delimiter | undefined; + /** Force a header row on or off; detected when omitted. */ + header?: boolean | undefined; + /** File name, so `.tsv` means tabs without sniffing. */ + name?: string | undefined; +} + +export const TABLE_EXTENSIONS = ["csv", "tsv", "tab", "psv"]; +const TABLE_NAME = /\.(?:csv|tsv|tab|psv)$/i; + +/** Whether a file name is one the table view opens. */ +export function isTableName(name: string | null | undefined): boolean { + return !!name && TABLE_NAME.test(name); +} + +/** + * RFC 4180 records: quoted fields may hold the delimiter, quotes ("" escapes one) and + * line breaks; CRLF, LF and lone CR all end a record; a leading BOM is dropped. One + * pass, no regex per character, so a 4 MB file parses in milliseconds. + */ +export function parseRecords(source: string, delimiter: Delimiter, limit = Infinity): string[][] { + const text = source.charCodeAt(0) === 0xfeff ? source.slice(1) : source; + const records: string[][] = []; + let record: string[] = []; + let field = ""; + let quoted = false; + let i = 0; + const n = text.length; + while (i < n) { + const ch = text[i]!; + if (quoted) { + if (ch === '"') { + if (text[i + 1] === '"') { field += '"'; i += 2; continue; } + quoted = false; + i++; + continue; + } + field += ch; + i++; + continue; + } + if (ch === '"' && field === "") { quoted = true; i++; continue; } + if (ch === delimiter) { record.push(field); field = ""; i++; continue; } + if (ch === "\n" || ch === "\r") { + record.push(field); + field = ""; + records.push(record); + record = []; + if (records.length >= limit) return records; + i += ch === "\r" && text[i + 1] === "\n" ? 2 : 1; + continue; + } + field += ch; + i++; + } + if (field !== "" || record.length > 0) { record.push(field); records.push(record); } + // Blank lines are not empty records; a trailing newline is not a last record. + return records.filter((r) => r.length > 1 || r[0] !== ""); +} + +/** + * The delimiter a text uses: the candidate that splits the first lines into the same + * number of fields (more than one) most consistently. Ties go to the order of + * DELIMITERS, so a comma wins over a semicolon that merely appears in the data. + */ +export function sniffDelimiter(source: string, name?: string): Delimiter { + if (name && /\.(?:tsv|tab)$/i.test(name)) return "\t"; + if (name && /\.psv$/i.test(name)) return "|"; + return scoreDelimiters(source)[0]?.delimiter ?? ","; +} + +function scoreDelimiters(source: string): { delimiter: Delimiter; fields: number; consistency: number }[] { + const sample = source.slice(0, 64 * 1024); + const scored = DELIMITERS.map((delimiter) => { + const records = parseRecords(sample, delimiter, 50); + // The last sampled record may be cut off by the sample boundary. + const usable = sample.length < source.length && records.length > 2 ? records.slice(0, -1) : records; + const counts = new Map(); + for (const record of usable) counts.set(record.length, (counts.get(record.length) ?? 0) + 1); + let fields = 1; + let most = 0; + for (const [count, times] of counts) if (times > most || (times === most && count > fields)) { fields = count; most = times; } + return { delimiter, fields, consistency: usable.length ? most / usable.length : 0 }; + }); + return scored + .filter((s) => s.fields > 1) + // Ties go to more fields (a TSV whose cells each hold one comma), then to DELIMITERS order. + .sort((a, b) => b.consistency - a.consistency || b.fields - a.fields); +} + +/** + * Whether pasted text with no file name is a table. Strict on purpose, because prose + * and code have commas too: at least three records, every sampled one with the same + * field count of two or more, and no line ending the way code does. + */ +export function looksLikeTable(source: string): boolean { + const lines = source.slice(0, 64 * 1024).split(/\r?\n/).filter((line) => line.trim()).slice(0, 50); + if (lines.length < 3) return false; + if (lines.some((line) => /[{};]\s*$|^\s*(?:\/\/|#|--)/.test(line) && !/\t/.test(line))) return false; + const best = scoreDelimiters(source)[0]; + if (!best || best.consistency < 1) return false; + // Comma-separated prose ("Hello, world") has long cells full of spaces; data does not. + if (best.delimiter === ",") { + const records = parseRecords(source.slice(0, 16 * 1024), ",", 20); + const wordy = records.flat().filter((cell) => cell.trim().split(/\s+/).length > 6).length; + if (wordy > records.length) return false; + // So does a sentence with one comma per line: the text after it starts with a space and a word. + const later = records.flatMap((record) => record.slice(1)); + if (later.length && later.filter((cell) => /^ [A-Za-z]/.test(cell)).length / later.length >= 0.5) return false; + } + return true; +} + +const NUMBER = /^[-+]?(?:\d{1,3}(?:,\d{3})+|\d+)?(?:\.\d+)?(?:[eE][-+]?\d+)?%?$/; +const CURRENCY = /^[-+]?[$€£¥]\s?/; +const DATE = /^\d{4}-\d{2}-\d{2}(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?)?$|^\d{1,2}\/\d{1,2}\/\d{2,4}$/; +const BOOLEAN = /^(?:true|false|yes|no|y|n|t|f)$/i; + +/** A cell as a number, or NaN: thousands commas, a currency sign and a % are allowed. */ +export function toNumber(cell: string): number { + const trimmed = cell.trim().replace(CURRENCY, ""); + if (!trimmed || !/\d/.test(trimmed) || !NUMBER.test(trimmed)) return NaN; + return Number(trimmed.replace(/,/g, "").replace(/%$/, "")); +} + +function typeOfCell(cell: string): ColumnType | null { + const value = cell.trim(); + if (!value) return null; + if (!Number.isNaN(toNumber(value))) return "number"; + if (DATE.test(value) && !Number.isNaN(Date.parse(value))) return "date"; + if (BOOLEAN.test(value)) return "boolean"; + return "text"; +} + +/** A column's type: what at least 95% of its non-empty sampled cells are. */ +export function inferType(cells: readonly string[]): ColumnType { + const counts: Record = { number: 0, date: 0, boolean: 0, text: 0 }; + let seen = 0; + const step = Math.max(1, Math.floor(cells.length / 2000)); + for (let i = 0; i < cells.length; i += step) { + const type = typeOfCell(cells[i] ?? ""); + if (!type) continue; + counts[type]++; + seen++; + } + if (!seen) return "text"; + for (const type of ["number", "date", "boolean"] as const) if (counts[type] / seen >= 0.95) return type; + return "text"; +} + +/** + * Whether the first record names the columns. It does unless it looks like data: a + * value typed the way the rest of its column is (a number over numbers), a blank, or + * a repeated name. + */ +export function detectHeader(records: readonly string[][]): boolean { + const [first, ...rest] = records; + if (!first || rest.length === 0) return true; + if (first.some((cell) => !cell.trim())) return false; + if (new Set(first.map((cell) => cell.trim().toLowerCase())).size !== first.length) return false; + for (let c = 0; c < first.length; c++) { + const below = inferType(rest.slice(0, 200).map((record) => record[c] ?? "")); + if (below !== "text" && typeOfCell(first[c]!) === below) return false; + } + return true; +} + +/** Spreadsheet column names: A … Z, AA, AB … for a file with no header. */ +export function letterName(index: number): string { + let name = ""; + for (let n = index + 1; n > 0; n = Math.floor((n - 1) / 26)) name = String.fromCharCode(65 + ((n - 1) % 26)) + name; + return name; +} + +export function parseTable(source: string, options: ParseOptions = {}): Table { + const delimiter = options.delimiter ?? sniffDelimiter(source, options.name); + const records = parseRecords(source, delimiter); + const header = options.header ?? detectHeader(records); + const head = header ? records[0] ?? [] : []; + const body = header ? records.slice(1) : records; + const width = Math.max(head.length, ...body.slice(0, 5000).map((record) => record.length), 0); + // Ragged means "not the shape the header promised"; with no header, not the widest shape. + const expected = header ? head.length : width; + let ragged = 0; + const rows = body.map((record) => { + if (record.length !== expected) ragged++; + if (record.length === width) return record; + return record.length > width ? record.slice(0, width) : [...record, ...Array(width - record.length).fill("")]; + }); + const columns: Column[] = []; + for (let c = 0; c < width; c++) { + const raw = head[c]?.trim(); + columns.push({ name: raw || letterName(c), type: inferType(rows.map((row) => row[c]!)) }); + } + return { delimiter, header, columns, rows, ragged }; +} + +export interface Sort { + column: number; + descending?: boolean; +} + +const collator = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }); + +/** Compares two cells the way their column's type reads; empty cells always sort last. */ +export function compareCells(a: string, b: string, type: ColumnType): number { + const ea = !a.trim(); + const eb = !b.trim(); + if (ea || eb) return ea === eb ? 0 : ea ? 1 : -1; + if (type === "number") { + const d = toNumber(a) - toNumber(b); + if (!Number.isNaN(d) && d !== 0) return d; + } else if (type === "date") { + const d = Date.parse(a) - Date.parse(b); + if (!Number.isNaN(d) && d !== 0) return d; + } else if (type === "boolean") { + const truthy = (v: string) => (/^(?:true|yes|y|t)$/i.test(v.trim()) ? 1 : 0); + const d = truthy(b) - truthy(a); + if (d) return d; + } + return collator.compare(a, b); +} + +/** One filter term: free text, `col:text`, or `col` with = != > >= < <=. */ +export interface FilterTerm { + column: number | null; + op: ":" | "=" | "!=" | ">" | ">=" | "<" | "<="; + value: string; + negate: boolean; +} + +/** + * Parses a filter. Terms are separated by spaces and must all match. A term is free + * text (any cell contains it), `column:text` (that column contains it), or a + * comparison such as `age>30`, `price<=9.99`, `status=open`, `country!=US`. A leading + * `!` negates. Quotes keep spaces: `city:"new york"`. A column is named, or numbered + * from 1 (`#3>10`). Unknown column names fall back to free text. + */ +export function parseFilter(query: string, columns: readonly Column[]): FilterTerm[] { + const terms: FilterTerm[] = []; + const tokens = query.match(/[^\s"]*"[^"]*"?\S*|\S+/g) ?? []; + const byName = new Map(columns.map((column, index) => [column.name.toLowerCase(), index])); + for (let token of tokens) { + let negate = false; + if (token.startsWith("!") && token.length > 1) { negate = true; token = token.slice(1); } + const match = token.match(/^("[^"]+"|#\d+|[^:=!<>]+)(:|!=|>=|<=|=|>|<)(.*)$/); + const unquote = (s: string) => s.replace(/^"|"$/g, ""); + if (match) { + const name = unquote(match[1]!).toLowerCase(); + const column = /^#\d+$/.test(name) ? Number(name.slice(1)) - 1 : byName.get(name); + if (column !== undefined && column >= 0 && column < columns.length) { + terms.push({ column, op: match[2] as FilterTerm["op"], value: unquote(match[3]!), negate }); + continue; + } + } + terms.push({ column: null, op: ":", value: unquote(token), negate }); + } + return terms.filter((term) => term.value !== "" || term.op !== ":"); +} + +function termMatches(row: readonly string[], term: FilterTerm, columns: readonly Column[]): boolean { + const needle = term.value.toLowerCase(); + if (term.column === null) return row.some((cell) => cell.toLowerCase().includes(needle)); + const cell = row[term.column] ?? ""; + const type = columns[term.column]!.type; + switch (term.op) { + case ":": + return cell.toLowerCase().includes(needle); + case "=": + return type === "number" && !Number.isNaN(toNumber(term.value)) + ? toNumber(cell) === toNumber(term.value) + : cell.trim().toLowerCase() === needle; + case "!=": + return type === "number" && !Number.isNaN(toNumber(term.value)) + ? toNumber(cell) !== toNumber(term.value) + : cell.trim().toLowerCase() !== needle; + default: { + if (!cell.trim()) return false; + // Two numbers compare as numbers whatever the column was typed as. + const numeric = !Number.isNaN(toNumber(cell)) && !Number.isNaN(toNumber(term.value)); + const order = compareCells(cell, term.value, numeric ? "number" : type); + return term.op === ">" ? order > 0 : term.op === ">=" ? order >= 0 : term.op === "<" ? order < 0 : order <= 0; + } + } +} + +/** + * The rows to show, as indices into `table.rows`: filtered, then sorted. The sort is + * stable, so sorting by one column and then another keeps ties in the earlier order. + */ +export function viewRows(table: Table, options: { filter?: string; sort?: Sort | null } = {}): number[] { + const terms = options.filter ? parseFilter(options.filter, table.columns) : []; + const indices: number[] = []; + for (let i = 0; i < table.rows.length; i++) { + const row = table.rows[i]!; + if (terms.every((term) => termMatches(row, term, table.columns) !== term.negate)) indices.push(i); + } + const sort = options.sort; + if (sort && sort.column >= 0 && sort.column < table.columns.length) { + const type = table.columns[sort.column]!.type; + const sign = sort.descending ? -1 : 1; + indices.sort((a, b) => { + const ca = table.rows[a]![sort.column]!; + const cb = table.rows[b]![sort.column]!; + // Empties stay last whichever way the sort runs. + if (!ca.trim() !== !cb.trim()) return ca.trim() ? -1 : 1; + return sign * compareCells(ca, cb, type) || a - b; + }); + } + return indices; +} + +/** The next sort after a click on `column`: ascending, then descending, then none. */ +export function nextSort(current: Sort | null | undefined, column: number): Sort | null { + if (!current || current.column !== column) return { column }; + if (!current.descending) return { column, descending: true }; + return null; +} + +export interface ColumnStats { + name: string; + type: ColumnType; + count: number; + empty: number; + unique: number; + min?: string; + max?: string; + sum?: number; + mean?: number; + median?: number; + /** The most frequent values, most first, at most five. */ + top: { value: string; count: number }[]; +} + +/** Statistics for one column over the given rows (the filtered view, by default all). */ +export function columnStats(table: Table, column: number, indices?: readonly number[]): ColumnStats { + const info = table.columns[column]!; + const rows = indices ?? table.rows.map((_, i) => i); + const counts = new Map(); + const values: string[] = []; + let empty = 0; + for (const index of rows) { + const cell = table.rows[index]![column] ?? ""; + if (!cell.trim()) { empty++; continue; } + values.push(cell); + counts.set(cell, (counts.get(cell) ?? 0) + 1); + } + const top = [...counts].sort((a, b) => b[1] - a[1] || collator.compare(a[0], b[0])).slice(0, 5).map(([value, count]) => ({ value, count })); + const stats: ColumnStats = { name: info.name, type: info.type, count: rows.length, empty, unique: counts.size, top }; + if (!values.length) return stats; + const sorted = [...values].sort((a, b) => compareCells(a, b, info.type)); + stats.min = sorted[0]; + stats.max = sorted.at(-1); + if (info.type === "number") { + const numbers = values.map(toNumber).filter((n) => !Number.isNaN(n)).sort((a, b) => a - b); + if (numbers.length) { + stats.sum = numbers.reduce((total, n) => total + n, 0); + stats.mean = stats.sum / numbers.length; + const mid = numbers.length >> 1; + stats.median = numbers.length % 2 ? numbers[mid] : (numbers[mid - 1]! + numbers[mid]!) / 2; + } + } + return stats; +} + +/** A number as people read it: grouped, at most four decimals. */ +export function formatNumber(n: number): string { + return n.toLocaleString("en-US", { maximumFractionDigits: 4 }); +} + +/** Stats as label/value pairs, for a dialog or a modal. */ +export function statsRows(stats: ColumnStats): { label: string; value: string }[] { + const rows = [ + { label: "type", value: stats.type }, + { label: "rows", value: formatNumber(stats.count) }, + { label: "empty", value: formatNumber(stats.empty) }, + { label: "distinct", value: formatNumber(stats.unique) }, + ]; + if (stats.min !== undefined) rows.push({ label: "min", value: stats.min }); + if (stats.max !== undefined) rows.push({ label: "max", value: stats.max }); + if (stats.sum !== undefined) rows.push({ label: "sum", value: formatNumber(stats.sum) }); + if (stats.mean !== undefined) rows.push({ label: "mean", value: formatNumber(stats.mean) }); + if (stats.median !== undefined) rows.push({ label: "median", value: formatNumber(stats.median) }); + for (const { value, count } of stats.top) rows.push({ label: `× ${formatNumber(count)}`, value }); + return rows; +} + +/** The delimiter as people name it. */ +export function delimiterName(delimiter: Delimiter): string { + return delimiter === "," ? "comma" : delimiter === "\t" ? "tab" : delimiter === ";" ? "semicolon" : "pipe"; +} + +/** A field written back out, quoted only when it has to be. */ +export function quoteField(cell: string, delimiter: Delimiter): string { + return cell.includes(delimiter) || /["\r\n]/.test(cell) || /^\s|\s$/.test(cell) ? `"${cell.replace(/"/g, '""')}"` : cell; +} + +/** The view (filtered, sorted) back as delimited text, header first. */ +export function toDelimited(table: Table, indices: readonly number[], delimiter: Delimiter = table.delimiter): string { + const line = (cells: readonly string[]) => cells.map((cell) => quoteField(cell, delimiter)).join(delimiter); + const out = [line(table.columns.map((column) => column.name)), ...indices.map((index) => line(table.rows[index]!))]; + return `${out.join("\n")}\n`; +} + +/** A cell for one line of a fixed-width layout: first line only, tabs as spaces. */ +export function displayCell(cell: string): string { + const at = cell.search(/[\r\n]/); + return (at < 0 ? cell : `${cell.slice(0, at)}↵`).replace(/\t/g, " "); +} + +/** Numbers and dates read right-aligned, the way a spreadsheet shows them. */ +export function alignOf(type: ColumnType): "left" | "right" { + return type === "number" ? "right" : "left"; +} + +/** + * The table as a GFM pipe table, every cell escaped so nothing in the data turns into + * formatting. `--print` renders a CSV through this, so it shares the Markdown + * renderer's column fitting instead of growing a second one. + */ +export function toMarkdownTable(table: Table, indices: readonly number[] = table.rows.map((_, i) => i)): string { + const escape = (cell: string) => displayCell(cell).replace(/[\\`*_{}[\]()<>#+\-.!|~]/g, "\\$&") || " "; + const line = (cells: readonly string[]) => `| ${cells.map(escape).join(" | ")} |`; + const rule = `|${table.columns.map((column) => (alignOf(column.type) === "right" ? " ---: " : " --- ")).join("|")}|`; + return [line(table.columns.map((column) => column.name)), rule, ...indices.map((index) => line(table.rows[index]!))].join("\n") + "\n"; +} + +export interface TableQuery { + filter?: string | undefined; + /** A column name, or its 1-based number. */ + sort?: string | number | undefined; + descending?: boolean | undefined; + /** Rows to return, at most 1,000. Default 50. */ + limit?: number | undefined; + offset?: number | undefined; + /** A column to summarise, by name or 1-based number. */ + stats?: string | number | undefined; + name?: string | undefined; +} + +/** A column by name (case-insensitive) or 1-based number, or -1. */ +export function columnIndex(table: Table, ref: string | number | undefined): number { + if (ref === undefined || ref === "") return -1; + if (typeof ref === "number" || /^#?\d+$/.test(ref)) { + const n = Number(String(ref).replace("#", "")) - 1; + return n >= 0 && n < table.columns.length ? n : -1; + } + return table.columns.findIndex((column) => column.name.toLowerCase() === ref.toLowerCase()); +} + +/** + * One call's worth of a table for an agent: the typed columns, the rows that match, + * sorted and paged, as objects keyed by column name, and optionally one column's + * statistics over the matching rows. What the MCP `table_query` tool returns. + */ +export function queryTable(source: string, query: TableQuery = {}) { + const table = parseTable(source, { name: query.name }); + const sortColumn = columnIndex(table, query.sort); + if (query.sort !== undefined && sortColumn < 0) throw new Error(`No column ${JSON.stringify(query.sort)}. Columns: ${table.columns.map((c) => c.name).join(", ")}`); + const indices = viewRows(table, { filter: query.filter, sort: sortColumn >= 0 ? { column: sortColumn, descending: query.descending } : null }); + const limit = Math.max(0, Math.min(1000, Math.floor(query.limit ?? 50))); + const offset = Math.max(0, Math.floor(query.offset ?? 0)); + const statsColumn = columnIndex(table, query.stats); + if (query.stats !== undefined && statsColumn < 0) throw new Error(`No column ${JSON.stringify(query.stats)}.`); + return { + delimiter: delimiterName(table.delimiter), + header: table.header, + columns: table.columns, + total: table.rows.length, + matched: indices.length, + offset, + rows: indices.slice(offset, offset + limit).map((index): Record => ({ + row: index + 1, + ...Object.fromEntries(table.columns.map((column, c) => [column.name, table.rows[index]![c] ?? ""])), + })), + ...(statsColumn >= 0 ? { stats: columnStats(table, statsColumn, indices) } : {}), + }; +} diff --git a/src/mcp.ts b/src/mcp.ts index 680e4d3..124c0b7 100644 --- a/src/mcp.ts +++ b/src/mcp.ts @@ -4,11 +4,14 @@ import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js"; +import { readFileSync } from "node:fs"; import { cloudAction, cloudRequest, pasteLocation, sharedLocation } from "./cloud-client.ts"; +import { queryTable } from "./csv.ts"; const string = { type: "string" }; const role = { type: "string", enum: ["view", "edit"] }; const definitions: [string, string, Record, string[]][] = [ ["paste_create", "Create an anonymous paste: any text file behind a secret link, no account needed. Markdown renders as a document; code (JSON, JavaScript, Python, YAML, SQL and 30 more) is highlighted with folding. The language is detected from the title's extension, else from the content; pass language to force one. Expires after 7 days by default (1h, 1d, 7d or 30d), 256 KB at most. Returns the URL and a raw URL; keep them, the server cannot show them again. The result's sensitive field lists any key, token, password, email address, public IP or named home directory in what the link exposes (secrets masked); when its level is \"secret\", tell the user before passing the link on.", { source: string, title: { type: "string", description: "A file name; its extension decides the language." }, language: { type: "string", description: "A highlight.js name such as json, javascript, python, bash, yaml or markdown. Detected when omitted." }, expiresIn: { type: "string", enum: ["1h", "1d", "7d", "30d"] } }, ["source"]], + ["table_query", "Read a CSV or TSV the way the readm3 table view does, without loading it all into context. Give exactly one of source (the text), path (a local file) or url (a readm3 paste link or a raw http(s) file URL). The delimiter, header row and column types (number, date, boolean, text) are detected. filter keeps rows matching every space-separated term: free text, col:text, col=v, col!=v, col>n, col>=n, col, string[]][] = [ ["url", "source", "baseVersion"], ], ]; +/** table_query: the text from wherever it was pointed at, then src/csv.ts does the rest. */ +async function tableQuery(args: Record): Promise { + const given = ["source", "path", "url"].filter((key) => typeof args[key] === "string" && args[key] !== ""); + if (given.length !== 1) throw new Error("Give exactly one of source, path or url."); + let source: string; + let name: string | undefined; + if (typeof args.source === "string" && args.source) source = args.source; + else if (typeof args.path === "string" && args.path) { + source = readFileSync(args.path, "utf8"); + name = args.path; + } else { + const url = String(args.url); + if (/\/p\/[A-Za-z0-9_-]{43}/.test(url)) { + const { token, config } = pasteLocation(url); + const paste = await cloudRequest(`pastes/${token}`, "GET", undefined, config) as { source: string; title: string }; + source = paste.source; + name = paste.title; + } else { + if (!/^https?:\/\//.test(url)) throw new Error("url must be http(s)."); + const response = await fetch(url); + if (!response.ok) throw new Error(`${url} answered ${response.status}.`); + source = await response.text(); + name = new URL(url).pathname; + } + } + return queryTable(source, { + name, + filter: typeof args.filter === "string" ? args.filter : undefined, + sort: typeof args.sort === "string" || typeof args.sort === "number" ? args.sort : undefined, + descending: args.descending === true, + limit: typeof args.limit === "number" ? args.limit : undefined, + offset: typeof args.offset === "number" ? args.offset : undefined, + stats: typeof args.stats === "string" || typeof args.stats === "number" ? args.stats : undefined, + }); +} + export async function runMcp() { const server = new Server( { name: "readm3", version: "0.6.0" }, @@ -276,7 +315,7 @@ export async function runMcp() { additionalProperties: false, }, annotations: { - readOnlyHint: /_(list|get|me|users|revisions)$/.test(name), + readOnlyHint: /_(list|get|me|users|revisions|query)$/.test(name), destructiveHint: /_(delete|remove|revoke|transfer)$/.test(name), openWorldHint: true, }, @@ -289,7 +328,8 @@ export async function runMcp() { if (!definitions.some(([operation]) => operation === name)) throw new Error("Unknown tool."); let result: unknown; - if (name.startsWith("settings_")) { + if (name === "table_query") result = await tableQuery(args); + else if (name.startsWith("settings_")) { if (name === "settings_save" && (!Number.isSafeInteger(args.ifRevision) || Number(args.ifRevision) < 0 || !args.snapshot)) throw new Error("snapshot and a nonnegative ifRevision are required."); result = await cloudRequest(name === "settings_revisions" ? "settings/revisions" : "settings", name === "settings_save" ? "PUT" : "GET", name === "settings_save" ? { snapshot: args.snapshot, ifRevision: args.ifRevision } : undefined); } else if (name.startsWith("paste_")) { diff --git a/src/print.ts b/src/print.ts index b22e126..2186940 100644 --- a/src/print.ts +++ b/src/print.ts @@ -15,6 +15,7 @@ import { type ColorDepth, type Theme, } from "@profullstack/hqtui"; +import { isTableName, parseTable, toMarkdownTable } from "./csv.ts"; import { renderMarkdown, type RenderOptions, type Span } from "./markdown.ts"; import { colorOf } from "./roles.ts"; @@ -108,7 +109,9 @@ export function resolveWidth(explicit?: number): number { } export function printDocument(options: PrintOptions = {}): void { - const source = options.path && options.path !== "-" ? readFileSync(options.path, "utf8") : readStdin(); + const raw = options.path && options.path !== "-" ? readFileSync(options.path, "utf8") : readStdin(); + // A CSV or TSV prints as an aligned table rather than as its text. + const source = isTableName(options.path) ? toMarkdownTable(parseTable(raw, { name: options.path })) : raw; const theme = resolveTheme(options.theme); const text = renderAnsi(source, resolveWidth(options.width), theme, resolveDepth(options.color), { flavor: options.flavor, diff --git a/src/tree.ts b/src/tree.ts index 91ba3e1..7882227 100644 --- a/src/tree.ts +++ b/src/tree.ts @@ -1,8 +1,9 @@ /** - * The file browser's model: a pruned tree of markdown files under a root. + * The file browser's model: a pruned tree of markdown and table (CSV, TSV) files + * under a root. * - * Directories with no markdown anywhere beneath them are dropped, so the pane - * shows a map of the documentation rather than a map of the repository. + * Directories with neither anywhere beneath them are dropped, so the pane shows a + * map of the documentation and data rather than a map of the repository. */ import { readdirSync, statSync } from "node:fs"; import { basename, join, relative, sep } from "node:path"; @@ -35,6 +36,9 @@ export interface ScanOptions { export const MARKDOWN = /\.(?:md|markdown|mdown|mkd|mdx)$/i; +/** What the reader opens: Markdown renders, a table (src/csv.ts) is drawn as a grid. */ +export const READABLE = /\.(?:md|markdown|mdown|mkd|mdx|csv|tsv|tab|psv)$/i; + export const DEFAULT_IGNORE = [ "node_modules", ".git", @@ -97,7 +101,7 @@ export function scan(root: string, options: ScanOptions = {}): Entry[] { if (children.length === 0) continue; budget.left--; entries.push({ name, path, dir: true, children, expanded: depth < 1 }); - } else if (MARKDOWN.test(name)) { + } else if (READABLE.test(name)) { budget.left--; entries.push({ name, path, dir: false, children: [], expanded: false }); } @@ -121,7 +125,7 @@ export function flatten(entries: Entry[], depth = 0, parent = -1, out: FlatEntry return out; } -/** Every markdown file in the tree, in display order. */ +/** Every file in the tree, in display order. */ export function files(entries: Entry[], out: Entry[] = []): Entry[] { for (const entry of entries) { if (entry.dir) files(entry.children, out); diff --git a/src/viewer.ts b/src/viewer.ts index 59c9115..4ce43fa 100644 --- a/src/viewer.ts +++ b/src/viewer.ts @@ -5,6 +5,10 @@ * `markdown.ts`; editing shows the same buffer raw, with a caret. Both read the * same text, so leaving edit mode is all it takes to see the change rendered — * there is no separate preview to keep in step. + * + * A CSV or TSV reads as a grid instead (src/csv.ts parses it, hqtui's data grid + * draws it): a cell cursor, sort by column, a row filter, a row sheet and column + * statistics, the way tabiew does it. Editing it is the same raw editor. */ import { readFileSync, statSync, writeFileSync } from "node:fs"; import { basename, dirname, resolve } from "node:path"; @@ -36,6 +40,19 @@ import { undo, type Editor, } from "./editor.ts"; +import { + alignOf, + columnStats, + delimiterName, + displayCell, + isTableName, + nextSort, + parseTable, + statsRows, + viewRows, + type Sort, + type Table, +} from "./csv.ts"; import { DEFAULT_FLAVOR, type Flavor } from "./flavors.ts"; import { renderMarkdown, type Line } from "./markdown.ts"; import { colorOf } from "./roles.ts"; @@ -111,6 +128,25 @@ export async function run(options: ViewerOptions): Promise { /** Bumped on every edit, so the rendered document is rebuilt exactly then. */ let stamp = 0; + // ---- the table view, for a CSV or TSV + /** The open file parsed, rebuilt when the text changes. */ + let grid: { stamp: number; path: string; table: Table } | null = null; + /** Row indices shown: filtered, then sorted. */ + let gridRows: number[] = []; + let gridKey = ""; + let rowFilter = ""; + let rowFiltering = false; + let sort: Sort | null = null; + let cursor = { row: 0, col: 0 }; + let gridOffset = { row: 0, col: 0 }; + /** Body rows that fit, as the grid last measured them; page keys move by this. */ + let gridPage = 10; + /** Show the table's text rather than the grid. */ + let rawTable = false; + /** The cursor row as a sheet, or the cursor column's statistics. */ + let sheet = false; + let stats = false; + /** A yes/no the next keypress answers. */ let pending: { message: string; run: () => void } | null = null; @@ -148,6 +184,14 @@ export async function run(options: ViewerOptions): Promise { buffer = createEditor(readFileSync(path, "utf8")); openPath = path; doc = null; + grid = null; + gridKey = ""; + rowFilter = ""; + rowFiltering = false; + sort = null; + cursor = { row: 0, col: 0 }; + gridOffset = { row: 0, col: 0 }; + rawTable = false; scroll = 0; editTop = 0; editLeft = 0; @@ -191,6 +235,104 @@ export async function run(options: ViewerOptions): Promise { const maxScroll = (): number => Math.max(0, (doc?.lines.length ?? 0) - viewport.h); + /** Whether the open file is drawn as a grid right now. */ + const tableMode = (): boolean => !!openPath && isTableName(openPath) && !rawTable && mode === "read"; + + const currentTable = (): Table | null => { + if (!openPath || !buffer || !isTableName(openPath)) return null; + if (!grid || grid.stamp !== stamp || grid.path !== openPath) { + grid = { stamp, path: openPath, table: parseTable(bufferText(buffer), { name: openPath }) }; + gridKey = ""; + } + return grid.table; + }; + + /** The rows shown, recomputed only when the text, the filter or the sort changed. */ + const shownRows = (): number[] => { + const table = currentTable(); + if (!table) return []; + const key = JSON.stringify([grid?.stamp, openPath, rowFilter, sort]); + if (key !== gridKey) { + gridRows = viewRows(table, { filter: rowFilter, sort }); + gridKey = key; + } + cursor.row = clamp(cursor.row, 0, Math.max(0, gridRows.length - 1)); + cursor.col = clamp(cursor.col, 0, Math.max(0, table.columns.length - 1)); + return gridRows; + }; + + /** The table's text inside a fence, so `v` shows it raw through the same renderer. */ + const fenced = (text: string): string => { + const longest = Math.max(2, ...(text.match(/`+/g) ?? []).map((run) => run.length)); + const fence = "`".repeat(longest + 1); + return `${fence}csv\n${text}${text.endsWith("\n") ? "" : "\n"}${fence}\n`; + }; + + /** Keys while the grid has focus. */ + const tableKey = (key: string): void => { + const rows = shownRows(); + const table = currentTable(); + if (!table) return; + const last = Math.max(0, rows.length - 1); + const page = Math.max(1, gridPage - 1); + switch (key) { + case "up": + case "k": + cursor.row = clamp(cursor.row - 1, 0, last); + return; + case "down": + case "j": + cursor.row = clamp(cursor.row + 1, 0, last); + return; + case "pageup": + case "b": + cursor.row = clamp(cursor.row - page, 0, last); + return; + case "pagedown": + case "space": + cursor.row = clamp(cursor.row + page, 0, last); + return; + case "home": + case "g": + cursor.row = 0; + return; + case "end": + case "G": + case "shift+g": + cursor.row = last; + return; + case "right": + case "l": + cursor.col = clamp(cursor.col + 1, 0, table.columns.length - 1); + return; + case "left": + case "h": + if (cursor.col === 0) pane = "tree"; + else cursor.col--; + return; + case "0": + case "^": + cursor.col = 0; + return; + case "$": + cursor.col = table.columns.length - 1; + return; + case "o": + // A new order is read from its top, as in a spreadsheet. + sort = nextSort(sort, cursor.col); + cursor.row = 0; + return; + case "enter": + if (rows.length) sheet = true; + return; + case "c": + stats = true; + return; + default: + return; + } + }; + const moveSelection = (delta: number): void => { if (flat.length === 0) return; selected = clamp(selected + delta, 0, flat.length - 1); @@ -348,8 +490,10 @@ export async function run(options: ViewerOptions): Promise { return; } - if (help) { + if (help || sheet || stats) { help = false; + sheet = false; + stats = false; app.invalidate(); return; } @@ -360,6 +504,24 @@ export async function run(options: ViewerOptions): Promise { return; } + if (rowFiltering) { + if (key === "escape") { + rowFiltering = false; + rowFilter = ""; + } else if (name === "enter") { + rowFiltering = false; + } else if (name === "backspace") { + rowFilter = rowFilter.slice(0, -1); + } else if (name === "up" || name === "down") { + tableKey(name); + } else if (char && !event.ctrl && !event.alt) { + rowFilter += char; + cursor.row = 0; + } + app.invalidate(); + return; + } + if (filtering) { if (key === "escape") { filtering = false; @@ -424,8 +586,20 @@ export async function run(options: ViewerOptions): Promise { return; } case "/": - filtering = true; - pane = "tree"; + // Over a grid, `/` filters its rows; anywhere else, the file list. + if (pane === "view" && tableMode()) rowFiltering = true; + else { + filtering = true; + pane = "tree"; + } + app.invalidate(); + return; + case "v": + if (openPath && isTableName(openPath)) { + rawTable = !rawTable; + doc = null; + pane = "view"; + } app.invalidate(); return; case "[": @@ -437,7 +611,9 @@ export async function run(options: ViewerOptions): Promise { app.invalidate(); return; case "escape": - if (filter) { + if (pane === "view" && tableMode() && rowFilter) { + rowFilter = ""; + } else if (filter) { filter = ""; rebuild(openPath); } else { @@ -501,6 +677,12 @@ export async function run(options: ViewerOptions): Promise { return; } + if (tableMode()) { + tableKey(key); + app.invalidate(); + return; + } + const page = Math.max(1, viewport.h - 2); switch (key) { case "up": @@ -544,6 +726,9 @@ export async function run(options: ViewerOptions): Promise { const openRel = openPath ? pathLabel(root, openPath) : ""; const total = doc?.lines.length ?? 0; const percent = total <= viewport.h ? 100 : Math.round((Math.min(scroll + viewport.h, total) / total) * 100); + const gridding = tableMode(); + const table = gridding ? currentTable() : null; + const rows = gridding ? shownRows() : []; ui.column({}, (screen) => { screen.row({ size: "fill" }, (panes) => { @@ -564,7 +749,7 @@ export async function run(options: ViewerOptions): Promise { else if (selected >= treeOffset + capacity) treeOffset = selected - capacity + 1; treeOffset = clamp(treeOffset, 0, Math.max(0, flat.length - capacity)); if (flat.length === 0) { - p.text(filter ? "No match." : "No markdown here.", { fg: theme.muted }); + p.text(filter ? "No match." : "No markdown or CSV here.", { fg: theme.muted }); return; } p.list({ @@ -607,13 +792,23 @@ export async function run(options: ViewerOptions): Promise { { title: openRel ? ` ${dirty() ? "● " : ""}${openRel} ` : " readm3 ", titleColor: editing ? theme.warning : theme.title, - subtitle: openPath ? (editing ? "EDIT" : `${percent}%`) : undefined, + subtitle: openPath + ? editing + ? "EDIT" + : table + ? `${rows.length === table.rows.length ? "" : `${rows.length.toLocaleString()} of `}${table.rows.length.toLocaleString()} rows · ${table.columns.length} cols · ${delimiterName(table.delimiter)}` + : `${percent}%` + : undefined, footer: openPath && editing && buffer ? `${buffer.row + 1}:${buffer.col + 1}` - : openPath && total > 0 - ? `${Math.min(scroll + viewport.h, total)}/${total}` - : undefined, + : table + ? rows.length + ? `${cursor.row + 1}/${rows.length} · ${table.columns[cursor.col]?.name ?? ""}` + : "no rows" + : openPath && total > 0 + ? `${Math.min(scroll + viewport.h, total)}/${total}` + : undefined, borderColor: editing ? theme.warning : pane === "view" ? theme.borderFocused : theme.border, padding: [0, 1], }, @@ -651,13 +846,63 @@ export async function run(options: ViewerOptions): Promise { return; } + if (table) { + if (rowFiltering || rowFilter) { + p.row({ height: 1 }, (r) => { + r.text(`/${rowFilter}${rowFiltering ? "▏" : ""}`, { width: stringWidth(rowFilter) + 2, fg: theme.warning }); + r.text(rowFiltering ? " col:text col>n col=v !term" : " esc clears", { fg: theme.muted }); + }); + } + if (table.columns.length === 0) { + p.text("This file has no rows.", { fg: theme.muted }); + return; + } + p.dataGrid({ + columns: table.columns.map((column) => ({ title: column.name, align: alignOf(column.type) })), + rows: table.rows, + order: rows, + selectedRow: rows.length ? cursor.row : -1, + selectedColumn: cursor.col, + rowOffset: gridOffset.row, + columnOffset: gridOffset.col, + sort, + zebra: true, + scrollbar: true, + onLayout: (frame) => { + gridOffset = { row: frame.rowOffset, col: frame.columnOffset }; + gridPage = Math.max(1, frame.capacity); + }, + onScroll: (delta) => { + cursor.row = clamp(cursor.row + delta * 3, 0, Math.max(0, rows.length - 1)); + pane = "view"; + app.invalidate(); + }, + onSelectCell: (row, column) => { + cursor = { row, col: column >= 0 ? column : cursor.col }; + pane = "view"; + app.invalidate(); + }, + onActivateCell: () => { + sheet = true; + app.invalidate(); + }, + onHeaderClick: (column) => { + cursor.col = column; + tableKey("o"); + pane = "view"; + app.invalidate(); + }, + }); + return; + } + if (!doc || doc.path !== openPath || doc.width !== w || doc.stamp !== stamp || doc.spoilers !== spoilers) { doc = { path: openPath, width: w, stamp, spoilers, - lines: renderMarkdown(bufferText(buffer), w, { flavor, spoilers }), + lines: renderMarkdown(isTableName(openPath) ? fenced(bufferText(buffer)) : bufferText(buffer), w, { flavor, spoilers }), }; } scroll = clamp(scroll, 0, Math.max(0, doc.lines.length - h)); @@ -708,12 +953,30 @@ export async function run(options: ViewerOptions): Promise { { key: "^k", label: "kill" }, { key: "^g", label: "keys" }, ] + : rowFiltering + ? [ + { key: "type", label: "filter rows" }, + { key: "↵", label: "keep" }, + { key: "esc", label: "clear" }, + ] : filtering ? [ { key: "type", label: "filter" }, { key: "↵", label: "open" }, { key: "esc", label: "clear" }, ] + : pane === "view" && gridding + ? [ + { key: "←↑↓→", label: "move" }, + { key: "o", label: "sort" }, + { key: "/", label: "filter" }, + { key: "↵", label: "row" }, + { key: "c", label: "stats" }, + { key: "v", label: "text" }, + { key: "e", label: "edit" }, + { key: "?", label: "keys" }, + { key: "q", label: "quit" }, + ] : pane === "tree" ? [ { key: "↑↓", label: "move" }, @@ -748,10 +1011,47 @@ export async function run(options: ViewerOptions): Promise { m.text("y to continue, any other key to stay", { fg: theme.muted, align: "center" }); m.spacer("fill"); }); + } else if (sheet && table && rows.length) { + const record = table.rows[rows[cursor.row]!] ?? []; + const height = clamp(table.columns.length + 6, 8, Math.max(8, app.height - 2)); + const labelWidth = clamp(Math.max(...table.columns.map((c) => stringWidth(c.name))) + 2, 8, 28); + const valueWidth = Math.max(...record.map((cell) => stringWidth(displayCell(cell))), 12); + // As wide as the row needs, so a short value sits beside its name, not across the screen. + const width = clamp(labelWidth + valueWidth + 6, 40, Math.max(40, app.width - 8)); + screen.modal({ title: ` row ${(rows[cursor.row] ?? 0) + 1} `, width, height }, (m) => { + m.keyValues( + table.columns.map((column, i) => ({ label: column.name, value: displayCell(record[i] ?? "") })), + { labelColor: theme.accent, labelWidth }, + ); + m.spacer("fill"); + m.text("Any key closes.", { fg: theme.muted, align: "center" }); + }); + } else if (stats && table) { + const lines = statsRows(columnStats(table, cursor.col, rows)); + screen.modal({ title: ` ${table.columns[cursor.col]?.name ?? ""} `, width: 56, height: clamp(lines.length + 6, 10, Math.max(10, app.height - 2)) }, (m) => { + m.keyValues(lines.map((line) => ({ label: line.label, value: displayCell(line.value) })), { labelColor: theme.accent, labelWidth: 12 }); + m.spacer("fill"); + m.text(rowFilter ? "Over the filtered rows. Any key closes." : "Any key closes.", { fg: theme.muted, align: "center" }); + }); } else if (help) { screen.modal({ title: " readm3 — keys ", width: 56, height: 24 }, (m) => { m.keyValues( - mode === "edit" + mode !== "edit" && gridding + ? [ + { label: "← ↑ ↓ → h j k l", value: "move the cell cursor" }, + { label: "0 / $", value: "first / last column" }, + { label: "g / G space b", value: "top / bottom, page" }, + { label: "o", value: "sort by column ▲ ▼ off" }, + { label: "/", value: "filter: text col:x col>n !x" }, + { label: "enter", value: "the row as a sheet" }, + { label: "c", value: "column statistics" }, + { label: "v", value: "grid / raw text" }, + { label: "e / i", value: "edit the file" }, + { label: "esc", value: "clear the filter" }, + { label: "tab", value: "switch pane" }, + { label: "q / ctrl+c", value: "quit" }, + ] + : mode === "edit" ? [ { label: "esc", value: "back to the preview" }, { label: "ctrl+g", value: "this help" }, diff --git a/test/code.test.ts b/test/code.test.ts index 080ae5a..1f4a7df 100644 --- a/test/code.test.ts +++ b/test/code.test.ts @@ -137,3 +137,22 @@ describe("folding", () => { assert.deepEqual(foldRegions([]), []); }); }); + +describe("tables", () => { + test("csv, tsv, tab and psv files are tables, not plain text", () => { + for (const name of ["a.csv", "b.TSV", "c.tab", "d.psv"]) assert.equal(languageForName(name)?.id, "csv"); + assert.equal(languageForName("notes.txt")?.id, "plaintext"); + assert.equal(languageOf("csv").mime, "text/csv"); + }); + + test("a pasted table with no name is sniffed as csv; prose and code are not", () => { + assert.equal(detectLanguage(undefined, "id,name,score\n1,Ada,9.5\n2,Grace,8\n3,Linus,7\n"), "csv"); + assert.equal(detectLanguage(undefined, "when\twho\n2026-01-01\tme\n2026-01-02\tyou\n"), "csv"); + assert.equal(sniffLanguage("Hello, world.\nThis is a note, really.\nNothing more, honestly.\n"), "markdown"); + assert.equal(sniffLanguage("const a = [1, 2];\nconst b = f(a, 3);\nconsole.log(a, b);\n"), "javascript"); + }); + + test("a table highlights as plain text when something asks", () => { + assert.deepEqual(highlightLines("a,b\n1,2", "csv"), ["a,b", "1,2"]); + }); +}); diff --git a/test/csv.test.ts b/test/csv.test.ts new file mode 100644 index 0000000..4b001cc --- /dev/null +++ b/test/csv.test.ts @@ -0,0 +1,204 @@ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import { + columnStats, + detectHeader, + isTableName, + letterName, + looksLikeTable, + nextSort, + parseFilter, + parseRecords, + parseTable, + sniffDelimiter, + toDelimited, + toNumber, + viewRows, +} from "../src/csv.ts"; + +const PEOPLE = `name,city,age,joined,active +Ada,London,36,2024-01-05,yes +Grace,"Arlington, VA",85,2023-11-30,no +Linus,Helsinki,54,2025-03-01,yes +Margaret,Boston,,2022-07-14,yes +`; + +describe("parseRecords", () => { + it("handles quotes, escaped quotes, embedded delimiters and line breaks", () => { + const records = parseRecords('a,"b,c","say ""hi""","two\nlines"\r\nx,y,z,w', ","); + assert.deepEqual(records, [["a", "b,c", 'say "hi"', "two\nlines"], ["x", "y", "z", "w"]]); + }); + + it("drops a BOM, blank lines and the trailing newline", () => { + assert.deepEqual(parseRecords("a,b\n\n1,2\n", ","), [["a", "b"], ["1", "2"]]); + }); + + it("keeps empty fields", () => { + assert.deepEqual(parseRecords("a,,c\n,,\n", ","), [["a", "", "c"], ["", "", ""]]); + }); +}); + +describe("delimiters", () => { + it("sniffs comma, tab, semicolon and pipe", () => { + assert.equal(sniffDelimiter(PEOPLE), ","); + assert.equal(sniffDelimiter("a\tb\tc\n1\t2\t3\n"), "\t"); + assert.equal(sniffDelimiter("a;b;c\n1,5;2;3\n4;5;6\n"), ";"); + assert.equal(sniffDelimiter("a|b\n1|2\n"), "|"); + }); + + it("prefers the delimiter giving more fields when two are equally consistent", () => { + assert.equal(sniffDelimiter("name\tnote\tn\nAda\tx, y\t1\nBob\tp, q\t2\n"), "\t"); + }); + + it("trusts a .tsv or .psv name", () => { + assert.equal(sniffDelimiter("a,b\n1,2\n", "data.tsv"), "\t"); + assert.equal(sniffDelimiter("a,b\n1,2\n", "data.psv"), "|"); + }); +}); + +describe("looksLikeTable", () => { + it("recognises data", () => { + assert.equal(looksLikeTable(PEOPLE), true); + assert.equal(looksLikeTable("id\tname\n1\ta\n2\tb\n"), true); + assert.equal(looksLikeTable("x, y, z\n1, 2, 3\n4, 5, 6\n7, 8, 9\n"), true); + }); + + it("does not mistake prose, code or two lines for a table", () => { + assert.equal(looksLikeTable("Hello, world.\nThis is a note, really.\nNothing more, honestly.\n"), false); + assert.equal(looksLikeTable("foo(a, b);\nbar(c, d);\nbaz(e, f);\n"), false); + assert.equal(looksLikeTable("a,b\n1,2\n"), false); + assert.equal(looksLikeTable("one, two\nthree\nfour, five, six\n"), false); + }); +}); + +describe("parseTable", () => { + it("names and types the columns", () => { + const table = parseTable(PEOPLE); + assert.equal(table.header, true); + assert.deepEqual(table.columns.map((c) => `${c.name}:${c.type}`), ["name:text", "city:text", "age:number", "joined:date", "active:boolean"]); + assert.equal(table.rows.length, 4); + assert.equal(table.rows[1]![1], "Arlington, VA"); + }); + + it("detects a file with no header and letters its columns", () => { + const table = parseTable("1,2.5,x\n2,3.5,y\n3,4.5,z\n"); + assert.equal(table.header, false); + assert.deepEqual(table.columns.map((c) => c.name), ["A", "B", "C"]); + assert.equal(table.rows.length, 3); + assert.equal(letterName(27), "AB"); + }); + + it("pads short records, keeps extra fields under letter names, and counts both", () => { + const table = parseTable("a,b,c\n1,2\n1,2,3,4\n1,2,3\n"); + assert.deepEqual(table.columns.map((c) => c.name), ["a", "b", "c", "D"]); + assert.deepEqual(table.rows, [["1", "2", "", ""], ["1", "2", "3", "4"], ["1", "2", "3", ""]]); + assert.equal(table.ragged, 2); + }); + + it("header detection rejects blanks, duplicates and numbers over numbers", () => { + assert.equal(detectHeader([["a", ""], ["1", "2"]]), false); + assert.equal(detectHeader([["a", "a"], ["1", "2"]]), false); + assert.equal(detectHeader([["10", "x"], ["1", "y"]]), false); + assert.equal(detectHeader([["n", "x"], ["1", "y"]]), true); + }); +}); + +describe("numbers", () => { + it("reads grouping, currency, percent and exponents", () => { + assert.equal(toNumber("1,234.5"), 1234.5); + assert.equal(toNumber("$12"), 12); + assert.equal(toNumber("-3e2"), -300); + assert.equal(toNumber("45%"), 45); + assert.ok(Number.isNaN(toNumber("12a"))); + assert.ok(Number.isNaN(toNumber(""))); + }); +}); + +describe("viewRows", () => { + const table = parseTable(PEOPLE); + const names = (indices: number[]) => indices.map((i) => table.rows[i]![0]); + + it("sorts numbers as numbers, with empties last both ways", () => { + assert.deepEqual(names(viewRows(table, { sort: { column: 2 } })), ["Ada", "Linus", "Grace", "Margaret"]); + assert.deepEqual(names(viewRows(table, { sort: { column: 2, descending: true } })), ["Grace", "Linus", "Ada", "Margaret"]); + }); + + it("sorts dates as dates", () => { + assert.deepEqual(names(viewRows(table, { sort: { column: 3 } })), ["Margaret", "Grace", "Ada", "Linus"]); + }); + + it("filters by free text, column text, comparisons and negation", () => { + assert.deepEqual(names(viewRows(table, { filter: "lon" })), ["Ada"]); + assert.deepEqual(names(viewRows(table, { filter: "age>40" })), ["Grace", "Linus"]); + assert.deepEqual(names(viewRows(table, { filter: "age>=36 active=yes" })), ["Ada", "Linus"]); + assert.deepEqual(names(viewRows(table, { filter: "!active=yes" })), ["Grace"]); + assert.deepEqual(names(viewRows(table, { filter: 'city:"arlington, va"' })), ["Grace"]); + assert.deepEqual(names(viewRows(table, { filter: "#1:mar" })), ["Margaret"]); + assert.deepEqual(names(viewRows(table, { filter: "joined<2024-01-01", sort: { column: 0 } })), ["Grace", "Margaret"]); + }); + + it("treats an unknown column as free text", () => { + assert.deepEqual(parseFilter("nope:x", table.columns), [{ column: null, op: ":", value: "nope:x", negate: false }]); + }); + + it("cycles ascending, descending, off", () => { + assert.deepEqual(nextSort(null, 2), { column: 2 }); + assert.deepEqual(nextSort({ column: 2 }, 2), { column: 2, descending: true }); + assert.equal(nextSort({ column: 2, descending: true }, 2), null); + assert.deepEqual(nextSort({ column: 2 }, 1), { column: 1 }); + }); +}); + +describe("columnStats", () => { + it("summarises a number column", () => { + const table = parseTable(PEOPLE); + const stats = columnStats(table, 2); + assert.equal(stats.count, 4); + assert.equal(stats.empty, 1); + assert.equal(stats.min, "36"); + assert.equal(stats.max, "85"); + assert.equal(stats.sum, 175); + assert.equal(stats.median, 54); + }); + + it("ranks the most frequent values", () => { + const stats = columnStats(parseTable(PEOPLE), 4); + assert.deepEqual(stats.top[0], { value: "yes", count: 3 }); + }); +}); + +describe("toDelimited and names", () => { + it("writes the view back, quoting only what needs it", () => { + const table = parseTable(PEOPLE); + const text = toDelimited(table, [1]); + assert.equal(text, 'name,city,age,joined,active\nGrace,"Arlington, VA",85,2023-11-30,no\n'); + assert.deepEqual(parseTable(text).rows, [table.rows[1]]); + }); + + it("knows table file names", () => { + assert.equal(isTableName("x/data.CSV"), true); + assert.equal(isTableName("a.tsv"), true); + assert.equal(isTableName("a.md"), false); + }); +}); + +describe("toMarkdownTable", () => { + it("escapes cells so data never becomes formatting, and right-aligns numbers", async () => { + const { toMarkdownTable } = await import("../src/csv.ts"); + const text = toMarkdownTable(parseTable("name,n\n*bold*,1\na|b,2\n")); + assert.equal(text, "| name | n |\n| --- | ---: |\n| \\*bold\\* | 1 |\n| a\\|b | 2 |\n"); + }); +}); + +describe("queryTable", () => { + it("filters, sorts, pages and summarises for an agent", async () => { + const { queryTable } = await import("../src/csv.ts"); + const result = queryTable(PEOPLE, { filter: "active=yes", sort: "age", descending: true, limit: 1, stats: "age" }); + assert.equal(result.total, 4); + assert.equal(result.matched, 3); + assert.deepEqual(result.rows, [{ row: 3, name: "Linus", city: "Helsinki", age: "54", joined: "2025-03-01", active: "yes" }]); + assert.equal(result.stats?.mean, 45); + assert.throws(() => queryTable(PEOPLE, { sort: "nope" }), /No column "nope"/); + assert.equal(queryTable(PEOPLE, { sort: 3 }).rows[0]!.name, "Ada"); + }); +});