From 9a0f8778320dffdcbc89ea2463222b8a06619b6d Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 23 Oct 2026 08:04:32 +0000 Subject: [PATCH 1/2] Raise the file cap from 5,000 to 1,000,000; sync up to 250,000; fold big folders (v0.23.0) A signed-in workspace (and the desktop folder walk) now holds up to 1,000,000 files. Sync carries up to 250,000: each file reference is about 100 bytes sealed (measured), so that many fit the 32 MB snapshot; past it sync pauses with a message and nothing is dropped from the account copy. The file list no longer searches the workspace for every row (quadratic at scale), and a workspace over 2,000 files opens with every folder below the top level folded, except the way to the open file. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ++-- desktop/main.cjs | 2 +- package.json | 2 +- site/autosync.ts | 6 +++++- site/build.ts | 2 +- site/test/sync.spec.ts | 24 +++++++++++++++++++++++- site/usage.ts | 4 ++-- site/viewer.ts | 35 +++++++++++++++++++++++++++-------- src/blob-store.ts | 7 ++++++- src/cli.ts | 2 +- src/sync-schema.ts | 4 ++-- test/blob-store.test.ts | 8 ++++---- 12 files changed, 75 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 9c9d145..22dc9d8 100644 --- a/README.md +++ b/README.md @@ -376,7 +376,7 @@ directly with no credentials or server proxy. Local files are uploaded only when | | Without an account | Free account | Pay as you go | | --- | --- | --- | --- | | Largest file | 4 MB | 5 GB | 5 GB | -| Number of files | 1,000 (20 MB in all) | 5,000 | 5,000 | +| Number of files | 1,000 (20 MB in all) | 1,000,000 (250,000 synced) | 1,000,000 (250,000 synced) | | Synced storage | none | 5 GB | $0.05 per GB a month over 5 GB | The meter beside Sync in the editor's header shows storage and files against these @@ -777,7 +777,7 @@ explicitly overrides a conflict. Replaced local files receive numbered `.bak-NNN backups. Browser replacements receive backups in IndexedDB, accessible through **Sync → Local backups** for download or restoration. Clearing browser site data also removes those local backups. The CLI syncs a readable workspace of up to 4 MB -per file and 20 MB in total; the browser and desktop app sync up to 5,000 files of +per file and 20 MB in total; the browser and desktop app sync up to 250,000 files (a workspace holds up to 1,000,000) of up to 5 GB each (see File limits and storage). The authenticated API exposes: diff --git a/desktop/main.cjs b/desktop/main.cjs index 5ba3e99..6d67392 100644 --- a/desktop/main.cjs +++ b/desktop/main.cjs @@ -28,7 +28,7 @@ const LOCAL = "app://readm3"; const MAX_FILE_BYTES = 5 * 1024 ** 3; const MAX_RANGE = 16 * 1024 * 1024; /** A signed-in workspace's file count (src/blob-store.ts); the walk stops there. */ -const MAX_FILES = 5000; +const MAX_FILES = 1_000_000; const IGNORED = new Set(["node_modules", "dist", "build", "out", "target", "vendor", "coverage", "venv", "__pycache__", ".git"]); const NEVER = /\.(?:zip|gz|tgz|bz2|xz|7z|rar|tar|exe|dll|so|dylib|o|a|class|jar|war|wasm|bin|dat|db|sqlite|sqlite3|mp3|mp4|m4a|mov|avi|mkv|webm|ogg|wav|flac|woff2?|ttf|otf|eot|psd|ai|doc|docx|xls|xlsx|ppt|pptx|iso|dmg|pkg|deb|rpm|apk|lock)$/i; /** The bundled editor: inside the package, or the repo's site build when run with `electron .`. */ diff --git a/package.json b/package.json index 30bfaa9..f82f2ce 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/readm3", - "version": "0.22.0", + "version": "0.23.0", "description": "Markdown editing and sharing for people and teams. Web/PWA, CLI, API and MCP with version history, view/edit links, and a terminal reader.", "type": "module", "license": "MIT", diff --git a/site/autosync.ts b/site/autosync.ts index abb8ef8..eea4ddf 100644 --- a/site/autosync.ts +++ b/site/autosync.ts @@ -20,7 +20,7 @@ import { digestDocument, digests, mergeWorkspaces } from "../src/sync-merge.ts"; import { isSealed, validateSyncFiles, type ReaderSettings, type SyncedWorkspace } from "../src/sync-schema.ts"; import { store, stored } from "./sync.ts"; import * as blobs from "./blobs.ts"; -import { formatStorage, newBlobRef, SNAPSHOT_INLINE_BYTES, type BlobRef } from "../src/blob-store.ts"; +import { formatStorage, MAX_SYNC_FILES, newBlobRef, SNAPSHOT_INLINE_BYTES, type BlobRef } from "../src/blob-store.ts"; const OFF = "readm3:autosync"; export const autoSyncEnabled = (): boolean => { try { return localStorage.getItem(OFF) !== "off"; } catch { return true; } }; @@ -264,6 +264,10 @@ async function pass(user: CloudUser) { const accountOnly = remoteWorkspace?.documents.filter((doc) => keptOnDevice(doc, limit)) ?? []; const local = { ...all, documents: all.documents.filter((doc) => !keptOnDevice(doc, limit)) }; const remoteSynced = remoteWorkspace && { ...remoteWorkspace, documents: remoteWorkspace.documents.filter((doc) => !keptOnDevice(doc, limit)) }; + // Past what one sealed snapshot carries, sync waits rather than drop files from the account copy. + if (local.documents.length + accountOnly.length > MAX_SYNC_FILES) { + throw new Paused(`Sync paused · ${local.documents.length.toLocaleString()} files; sync carries up to ${MAX_SYNC_FILES.toLocaleString()}. Your files are safe on this device.`); + } const merge = mergeWorkspaces({ local, remote: remoteSynced, base: state?.base, starter: await starterDigests(), conflictLabel: `${h.device} ${new Date().toISOString().slice(0, 10)}`, diff --git a/site/build.ts b/site/build.ts index abcf6f0..47b134c 100644 --- a/site/build.ts +++ b/site/build.ts @@ -255,7 +255,7 @@ function home(): string {

Keep the story of the document.

Every online save creates a version. See who changed it, share a specific version, or restore an earlier draft without losing the history.

A workspace for your team.

Organize people into organizations and teams. Share with a whole group, a few collaborators, or anyone with your link.

A plain text editor.

Line numbers, multiple cursors, find and replace, folding, Ctrl+P to any file and Ctrl+Shift+P for every command. Open a folder from disk and Ctrl+S saves back to it. Open the editor.

-

Files up to 5 GB, 5,000 of them.

A free account lifts the limits: up to 5,000 files, each up to 5 GB, with 5 GB of end-to-end encrypted storage that syncs to every device. Need more? Pay as you go, $0.05 per GB a month, from a CoinPay balance.

+

Files up to 5 GB, a million of them.

A free account lifts the limits: up to 1,000,000 files (250,000 of them synced), each up to 5 GB, with 5 GB of end-to-end encrypted storage that syncs to every device. Need more? Pay as you go, $0.05 per GB a month, from a CoinPay balance.

Summarize any document.

Press Summarize in the editor for a short abstract and the key points of the open file, written by Claude. The first 10 documents a week are free on a free account; after that, pay as you go from a CoinPay balance, usually a fraction of a cent each. Try it.

Take the editor with you.

Install readm3 from your browser, or get the desktop app: Linux AppImage or Flatpak, macOS for Apple silicon or Intel, Windows. Then readm3 edit . opens a folder from the terminal, like subl .. Local files work offline; shared documents stay connected.

Use the tools you already use.

The web editor, CLI, API, and MCP work with the same documents and permissions. Your terminal and your agents can join the workflow.

diff --git a/site/test/sync.spec.ts b/site/test/sync.spec.ts index a1e1ac9..07a15f9 100644 --- a/site/test/sync.spec.ts +++ b/site/test/sync.spec.ts @@ -180,6 +180,28 @@ test("files over the sync size limit stay on this device until the limit is turn expect(await page.evaluate(() => localStorage.getItem("readm3:sync-limit-mb"))).toBe("0"); }); +test("signed in, a folder past the old 5,000-file cap opens whole with its subfolders folded", async ({ page, context }) => { + await register(context, username()); + await page.addInitScript(() => { + const file = (name: string) => ({ kind: "file", name, getFile: async () => new File([`# ${name}\n`], name) }); + const directory = (name: string, entries: unknown[]) => ({ kind: "directory", name, values: async function* () { yield* entries; } }); + const sections = Array.from({ length: 60 }, (_, s) => directory(`section-${s}`, Array.from({ length: 100 }, (_, i) => file(`page-${i}.md`)))); + Object.assign(window, { showDirectoryPicker: async () => directory("site", [file("README.md"), ...sections]) }); + }); + await page.goto("/viewer"); + await setPassphrase(page); + await page.locator("#open-folder").click(); + await expect(page.locator("#workspace-name")).toHaveText("site"); + await expect(page.locator("#document-name")).toHaveText("site/README.md"); + // 6,001 files plus the starter documents, all kept. + expect(Number(await page.locator("#file-count").textContent())).toBeGreaterThan(6000); + // Folded: the site folder, README and the 60 section folders, plus the starter rows. + expect(await page.locator("#file-list button").count()).toBeLessThan(100); + await page.locator('#file-list .folder-row[title="site/section-7"]').click(); + await expect(page.locator('#file-list button[title="site/section-7/page-42.md"]')).toBeVisible(); + await expect(page.locator("#sync-note")).toContainText("Synced", { timeout: 30_000 }); +}); + test("the header shows usage against the plan's limits, signed out and signed in", async ({ page, context }) => { await page.goto("/viewer"); const usage = page.locator("#usage"); @@ -194,7 +216,7 @@ test("the header shows usage against the plan's limits, signed out and signed in await register(context, username()); await page.reload(); await setPassphrase(page); - await expect(usage).toHaveAttribute("title", /\/ 5\.0 GB · \d+ \/ 5,000 files/); + await expect(usage).toHaveAttribute("title", /\/ 5\.0 GB · \d+ \/ 1,000,000 files/); await usage.click(); await expect(dialog).toContainText("Free account"); await expect(dialog).toContainText("$0.05 per GB a month"); diff --git a/site/usage.ts b/site/usage.ts index b4bea41..f611c7c 100644 --- a/site/usage.ts +++ b/site/usage.ts @@ -1,7 +1,7 @@ /** * The usage meter in the editor's header: files and storage against the plan's limits. * Signed out it measures this device's workspace (1,000 files, 20 MB); signed in it - * measures the account (5,000 files, 5 GB stored free, then pay as you go). A click + * measures the account (1,000,000 files, 5 GB stored free, then pay as you go). A click * opens the limits, the price, the balance and a way to add funds or sign in. */ import { modal } from "./cloud.ts"; @@ -88,7 +88,7 @@ export function showUsage() { const note = document.createElement("p"); note.textContent = last.signedIn ? "Files over 4 MB are stored end-to-end encrypted on your account. Storage over the free allowance is charged hourly from your balance; with no balance, new uploads past it wait, and nothing stored is deleted." - : "A free account raises the limits to 5,000 files of up to 5 GB each, with 5 GB of encrypted storage that syncs to every device."; + : "A free account raises the limits to 1,000,000 files of up to 5 GB each, with 5 GB of encrypted storage that syncs to every device."; const actions = document.createElement("p"); if (last.signedIn) { const add = document.createElement("button"); diff --git a/site/viewer.ts b/site/viewer.ts index bb799fd..b658039 100644 --- a/site/viewer.ts +++ b/site/viewer.ts @@ -68,7 +68,7 @@ const isMarkdown = () => kind().kind === "markdown"; * signed out, the workspace keeps its small local limits. */ let signedIn = false; -const ACCOUNT_LIMITS = "Sign in (free) for files up to 5 GB and up to 5,000 files."; +const ACCOUNT_LIMITS = "Sign in (free) for files up to 5 GB and up to 1,000,000 files."; /** A stored file too large for the workspace (not a spilled small one): shown read-only. */ const isLarge = (doc = current()) => !!doc.blob && !doc.blob.digest; const PREVIEW_BYTES = 1024 * 1024; @@ -167,6 +167,23 @@ let noticeTimer: ReturnType; let importBusy = false; let saveFailed = false; const collapsed = new Set(); +/** + * A workspace of thousands of files (a whole ~/Desktop) shows its top folders with every + * folder below them folded, except the way to the open file; a small one shows everything. + */ +function foldLarge(open: string) { + collapsed.clear(); + if (workspace.documents.length <= 2000) return; + const parts = open.split("/"); + const keep = new Set(parts.slice(0, -1).map((_, depth) => parts.slice(0, depth + 1).join("/"))); + for (const doc of workspace.documents) { + const folders = doc.path.split("/"); + for (let depth = 1; depth < folders.length - 1; depth++) { + const path = folders.slice(0, depth + 1).join("/"); + if (!keep.has(path)) collapsed.add(path); + } + } +} function notice(message: string) { element("notice").textContent = message; @@ -211,7 +228,7 @@ async function applySynced(incoming: Workspace | undefined, settings: ReaderSett await persist(incoming); workspace = incoming; filter.value = ""; - collapsed.clear(); + foldLarge(workspace.active); } if (settings?.theme && [...theme.options].some((option) => option.value === settings.theme)) theme.value = settings.theme; if (settings?.flavor) flavor.value = settings.flavor; @@ -349,7 +366,7 @@ function renderFiles(focusPath?: string) { } if (!query && collapsed.has(path)) { hidden = true; break; } } - if (!hidden) fileList.append(fileRow(doc.path, parts.at(-1)!, parts.length - 1, false)); + if (!hidden) fileList.append(fileRow(doc.path, parts.at(-1)!, parts.length - 1, false, doc)); } if (!docs.length) { const empty = document.createElement("p"); @@ -362,18 +379,18 @@ function renderFiles(focusPath?: string) { if (focusPath) Array.from(fileList.querySelectorAll("button")).find((button) => button.dataset.path === focusPath)?.focus(); } -function fileRow(path: string, name: string, depth: number, directory: boolean) { +function fileRow(path: string, name: string, depth: number, directory: boolean, doc?: Document) { const row = document.createElement("button"); row.className = `file-row${directory ? " folder-row" : ""}`; row.dataset.path = path; row.style.paddingLeft = `${10 + depth * 15}px`; row.title = path; if (!directory && path === workspace.active) row.setAttribute("aria-current", "page"); - if (!directory && unsavedOnDisk(workspace.documents.find((doc) => doc.path === path)!)) row.classList.add("unsaved"); + if (doc && unsavedOnDisk(doc)) row.classList.add("unsaved"); const icon = document.createElement("span"); icon.className = "file-icon"; icon.setAttribute("aria-hidden", "true"); - const fileKind = directory ? null : kindOf(workspace.documents.find((doc) => doc.path === path)!); + const fileKind = doc ? kindOf(doc) : null; icon.textContent = directory ? (filter.value || !collapsed.has(path) ? "▾" : "▸") : fileKind!.kind === "markdown" ? "M↓" : fileKind!.kind === "code" ? "{}" : "▣"; const label = document.createElement("span"); label.className = "row-name"; @@ -669,8 +686,8 @@ function addDocuments(incoming: Document[], name?: string): Document[] { workspace.documents = [...map.values()].sort((a, b) => comparePaths(a.path, b.path)); if (name) workspace.name = name; filter.value = ""; - collapsed.clear(); const first = added.find((doc) => /^readme\b/i.test(doc.path.split("/").at(-1)!)) ?? added[0]; + foldLarge(first.path); openDocument(first.path); return added; } @@ -686,6 +703,7 @@ async function importFiles(files: disk.Picked[], name = "Your workspace") { let needAccount = 0; let overCount = 0; let unreadable = 0; + const existing = new Map(workspace.documents.map((doc) => [doc.path, doc])); for (const { file, path, handle, size = file.size, onDisk } of files) { if (!accepts(path)) continue; // Past the account's file count nothing more is read. @@ -700,7 +718,8 @@ async function importFiles(files: disk.Picked[], name = "Your workspace") { if (size > MAX_BLOB_BYTES) { skipped++; continue; } if (!binaryType(path) && looksBinary(await file.slice(0, 8192).text())) { unreadable++; continue; } // Opening the folder again keeps a file's stored reference, so sync does not upload it again. - const known = readFromDisk ? workspace.documents.find((doc) => doc.path === path && doc.blob?.size === size && !doc.blob.digest)?.blob : undefined; + const before = readFromDisk ? existing.get(path)?.blob : undefined; + const known = before && before.size === size && !before.digest ? before : undefined; const doc: Document = { path, source: "", blob: known ?? newBlobRef(size) }; if (!binaryType(path) && !languageForName(path)) { const language = detectLanguage(path, await file.slice(0, 65536).text()); diff --git a/src/blob-store.ts b/src/blob-store.ts index 38479c8..42b38c0 100644 --- a/src/blob-store.ts +++ b/src/blob-store.ts @@ -21,7 +21,12 @@ export const GiB = 1024 ** 3; /** The largest single file an account may keep. */ export const MAX_BLOB_BYTES = 5 * GiB; /** Files in one signed-in workspace (signed out, the workspace keeps 1,000). */ -export const MAX_ACCOUNT_FILES = 5000; +export const MAX_ACCOUNT_FILES = 1_000_000; +/** + * Files one synced workspace carries. Every file's reference is about 100 bytes sealed + * (its own random key does not compress), so this many fit the 32 MB snapshot with room. + */ +export const MAX_SYNC_FILES = 250_000; /** Storage every signed-in account gets free; more is pay as you go from the prepaid balance. */ export const FREE_STORAGE_BYTES = 5 * GiB; /** Pay as you go: micro-dollars per GiB stored per month, over the free allowance. */ diff --git a/src/cli.ts b/src/cli.ts index 2aea740..39c617e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -12,7 +12,7 @@ import { run, type ViewerOptions } from "./viewer.ts"; import { isCloudCommand, cloudMain, CLOUD_USAGE } from "./cloud-cli.ts"; import { isDesktopCommand, desktopMain, installedDesktop, prefersDesktop, DESKTOP_USAGE } from "./desktop-cli.ts"; -export const VERSION = "0.22.0"; +export const VERSION = "0.23.0"; /** The help text, exported so a test can hold it to the flags it documents. */ export const USAGE = `readm3 — a plain text editor for Markdown and code diff --git a/src/sync-schema.ts b/src/sync-schema.ts index 3be5095..7abc347 100644 --- a/src/sync-schema.ts +++ b/src/sync-schema.ts @@ -1,7 +1,7 @@ /** Browser-safe account data contract. Only these two files may be synced. */ import { isFlavor, type Flavor } from "./flavors.ts"; import { envelopeProblem, SEALED_FILE } from "./sync-crypto.ts"; -import { blobRefProblem, INLINE_MAX_BYTES, MAX_ACCOUNT_FILES, SNAPSHOT_INLINE_BYTES, type BlobRef } from "./blob-store.ts"; +import { blobRefProblem, INLINE_MAX_BYTES, MAX_SYNC_FILES, SNAPSHOT_INLINE_BYTES, type BlobRef } from "./blob-store.ts"; /** * A workspace file. `source` is the text, or base64 bytes when the path names a PDF or @@ -33,7 +33,7 @@ export function validateWorkspace(value: unknown): SyncedWorkspace { if (!value || typeof value !== "object") throw new Error("Workspace must be an object."); const ws = value as SyncedWorkspace; if (Object.keys(ws).some((key) => !["name", "active", "documents"].includes(key))) throw new Error("Workspace can only contain name, active and documents."); - if (typeof ws.name !== "string" || ws.name.length > 200 || !Array.isArray(ws.documents) || !ws.documents.length || ws.documents.length > MAX_ACCOUNT_FILES) throw new Error("Workspace needs a name and 1–5,000 files."); + if (typeof ws.name !== "string" || ws.name.length > 200 || !Array.isArray(ws.documents) || !ws.documents.length || ws.documents.length > MAX_SYNC_FILES) throw new Error(`Workspace needs a name and 1–${MAX_SYNC_FILES.toLocaleString("en-US")} files.`); const paths = new Set(); let total = 0; const encoder = new TextEncoder(); diff --git a/test/blob-store.test.ts b/test/blob-store.test.ts index 66725b7..559e452 100644 --- a/test/blob-store.test.ts +++ b/test/blob-store.test.ts @@ -28,12 +28,12 @@ test("chunk arithmetic covers a 5 GB file exactly", () => { assert.match(blobRefProblem({ ...newBlobRef(1), id: "../../etc" })!, /invalid id/); }); -test("a synced workspace holds up to 5,000 files and carries stored files by reference", () => { - const documents = Array.from({ length: 4999 }, (_, i) => ({ path: `notes/${i}.md`, source: `# ${i}` })); +test("a synced workspace holds up to 250,000 files and carries stored files by reference", () => { + const documents = Array.from({ length: 249_999 }, (_, i) => ({ path: `notes/${i}.md`, source: `# ${i}` })); documents.push({ path: "big/dump.json", source: "", blob: newBlobRef(3 * 1024 ** 3) } as never); - assert.equal(validateWorkspace({ name: "Many", active: "notes/0.md", documents }).documents.length, 5000); + assert.equal(validateWorkspace({ name: "Many", active: "notes/0.md", documents }).documents.length, 250_000); documents.push({ path: "one/more.md", source: "" }); - assert.throws(() => validateWorkspace({ name: "Many", active: "notes/0.md", documents }), /1–5,000 files/); + assert.throws(() => validateWorkspace({ name: "Many", active: "notes/0.md", documents }), /1–250,000 files/); assert.throws(() => validateWorkspace({ name: "x", active: "a.md", documents: [{ path: "a.md", source: "text", blob: newBlobRef(1) }] }), /source must be empty/); }); From 060fb196411ad76f9cca5e1e2b5a3f753e1cbf94 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Fri, 23 Oct 2026 08:09:36 +0000 Subject: [PATCH 2/2] Give the 6,001-file test time on slow CI runners Co-Authored-By: Claude Opus 5.5 --- site/test/sync.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/site/test/sync.spec.ts b/site/test/sync.spec.ts index 07a15f9..39dea4a 100644 --- a/site/test/sync.spec.ts +++ b/site/test/sync.spec.ts @@ -191,7 +191,8 @@ test("signed in, a folder past the old 5,000-file cap opens whole with its subfo await page.goto("/viewer"); await setPassphrase(page); await page.locator("#open-folder").click(); - await expect(page.locator("#workspace-name")).toHaveText("site"); + // Reading 6,001 files takes a few seconds on a slow CI runner. + await expect(page.locator("#workspace-name")).toHaveText("site", { timeout: 60_000 }); await expect(page.locator("#document-name")).toHaveText("site/README.md"); // 6,001 files plus the starter documents, all kept. expect(Number(await page.locator("#file-count").textContent())).toBeGreaterThan(6000);