From d83e0cfe8b0c8e9581169ea9a00e546485573836 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 26 Oct 2026 11:04:11 +0000 Subject: [PATCH] bookdl: yt-dlp for DRM-free ebooks (0.72.0) bookdl , info, search, humble [list|get], check. Sources: Project Gutenberg (gutendex metadata, fallback URLs), Standard Ebooks (?source=download, the bare link is a thank-you page), Internet Archive open items (lending-library items and LCP/ACS copies refused), Humble Bundle library via the _simpleauth_sess cookie (HUMBLE_SESSION in the credential store, md5 checked), and any direct DRM-free link. Library: "Author - Title.ext" in ~/Books ($BOOKDL_DIR, --to), a yt-dlp style download archive so reruns fetch only what is new, -f preference order, --all-formats, --calibre. Every file is checked for DRM after download (ADEPT, LCP, Mobipocket, KFX, .acsm, PDF EBX/APS; EPUB font obfuscation passes) and refused with exit 3, never removed. Kindle/Apple/Google Play/Kobo store links get what that store does allow instead. Co-Authored-By: Claude Opus 5.5 --- README.md | 53 +++++ bin/bookdl.ts | 376 +++++++++++++++++++++++++++++++ bin/cli-tools.ts | 1 + package.json | 2 +- src/bookdl.ts | 537 ++++++++++++++++++++++++++++++++++++++++++++ src/credentials.ts | 3 + src/registry.ts | 1 + test/bookdl.test.ts | 263 ++++++++++++++++++++++ 8 files changed, 1235 insertions(+), 1 deletion(-) create mode 100755 bin/bookdl.ts create mode 100644 src/bookdl.ts create mode 100644 test/bookdl.test.ts diff --git a/README.md b/README.md index 7def8f7..98203d2 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,8 @@ Two things here are not `PATH` commands and do not need Node: best single stream and says so - **[`yt-dlp`](https://github.com/yt-dlp/yt-dlp)** — `dl` only (`moshcode install yt-dlp`, `pipx install yt-dlp`, `brew install yt-dlp`) +- **[Calibre](https://calibre-ebook.com)** (`calibredb`) — `bookdl --calibre` + only; `bookdl` itself needs nothing beyond Node - **`apt`, and `sudo` unless you are root** — `sysupdate` only, and the one command here that is not portable: it updates Debian and Ubuntu boxes and refuses anything else. `snap` is optional; without snapd that step is skipped @@ -1775,6 +1777,57 @@ merge. `dl audio` is `-x`, which *is* ffmpeg, so there it is a hard requirement. A URL that fails does not stop the ones after it; the exit code still reports the failure. +### `bookdl` + +yt-dlp for the ebooks you are allowed to keep as files: public-domain +libraries, your Humble Bundle library, and any DRM-free download link a store +gives you. + +```sh +bookdl search dracula # Project Gutenberg +bookdl https://www.gutenberg.org/ebooks/345 # → ~/Books/Bram Stoker - Dracula.epub +bookdl https://standardebooks.org/ebooks/jane-austen/emma +bookdl -f pdf https://archive.org/details/prideprejudice00aust +bookdl info https://archive.org/details/prideprejudice00aust # formats, nothing downloaded +bookdl 'https://leanpub.com/…/download.epub' # any direct DRM-free link +bookdl humble # your Humble ebooks +bookdl humble get --all-formats --calibre # all of them, into Calibre too +bookdl check ~/Books/*.epub # DRM-free or not +``` + +| Source | How | +|---|---| +| Project Gutenberg | `gutenberg.org/ebooks/`, metadata from gutendex.com; `bookdl search` | +| Standard Ebooks | `standardebooks.org/ebooks//`; epub, azw3, kepub | +| Internet Archive | `archive.org/details/<id>`, open items only | +| Humble Bundle | `bookdl humble`, or one order's `humblebundle.com/downloads?key=…` | +| Direct links | Leanpub, Gumroad, itch.io, Payhip, publisher stores, a Google Play "Export" link, a Kobo DRM-free download | + +**Humble login.** Humble has no buyer API token, so `bookdl` uses your browser +session: in a logged-in humblebundle.com tab, copy the `_simpleauth_sess` +cookie (DevTools > Application > Cookies) and `cli-tools config set humble`. +A 401 means it expired; copy it again. The download links themselves are +signed CDN URLs and each file is checked against Humble's md5. + +**A library, not a pile.** Files are named `Author - Title.ext` in `~/Books` +(`--to` or `$BOOKDL_DIR` to change it) and recorded in +`~/.local/share/cli-tools/bookdl/archive.txt`, like yt-dlp's +`--download-archive`: a second `bookdl humble get` fetches only what is new. +`-f epub,pdf` is a preference order (first one the book has); `--all-formats` +takes each of them. + +**No DRM removal, ever.** Every file is checked after it lands (Adobe ADEPT, +Readium LCP, Mobipocket, Kindle KFX, an `.acsm` ticket, PDF EBX/APS handlers; +EPUB font obfuscation is not DRM and passes). A locked file is not saved, and +the exit code is 3. Removing DRM is circumvention under DMCA §1201, so this +tool does not do it. A Kindle, Apple Books, Google Play or Kobo store link gets +an answer instead: what that store does let you export (Kindle highlights at +`read.amazon.com/notebook`, Google Play's Export link, Kobo's DRM-free titles). +archive.org lending-library books are borrowed, not owned, and are refused the +same way. + +Exit codes: 0 ok, 1 a download failed, 3 refused (DRM or lending library). + ### `torrent` Turn a directory into a torrent, and get it seeded: diff --git a/bin/bookdl.ts b/bin/bookdl.ts new file mode 100755 index 0000000..ed9f3f1 --- /dev/null +++ b/bin/bookdl.ts @@ -0,0 +1,376 @@ +#!/usr/bin/env node +/** + * bookdl -- yt-dlp for the ebooks you are allowed to have as files. + * + * Public-domain libraries (Project Gutenberg, Standard Ebooks, the Internet + * Archive's open items), your Humble Bundle library, and any direct DRM-free + * link a store gives you (Leanpub, Gumroad, itch.io, Kobo/Google Play exports, + * publishers). Every file is checked for DRM after it lands and refused if it + * has any; nothing here removes DRM. Kindle and other locked stores get an + * answer about what they do allow instead of a download. + */ + +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; + +import { UsageError, parseArgs } from '../src/args.ts'; +import { + type Book, + type BookFile, + LockedError, + archiveId, + archiveKey, + archivePath, + chooseFiles, + defaultLibrary, + detectDrm, + detectSource, + directName, + extensionOf, + fileName, + gutenbergFallback, + gutenbergId, + humbleKey, + lockedStore, + looksLikeHtml, + looksLikeUrl, + parseArchive, + parseArchiveFile, + parseFormats, + parseGutendex, + parseGutendexSearch, + parseHumbleOrder, + safeName, + standardEbooks, + EBOOK_EXTENSIONS, + type HumbleOrder, +} from '../src/bookdl.ts'; +import { resolveCredentials } from '../src/credentials.ts'; +import { isMain } from '../src/is-main.ts'; + +const USAGE = `Usage: + bookdl <url>... download the book(s) at these URLs + bookdl info <url>... title, authors and formats; nothing downloaded + bookdl search <query> search Project Gutenberg + bookdl humble [list] every ebook in your Humble Bundle library + bookdl humble get [--all] download them (just new ones unless --force) + bookdl check <file>... does this file carry DRM? + +Sources: + gutenberg.org/ebooks/<id> standardebooks.org/ebooks/<author>/<title> + archive.org/details/<id> (open items; lending-library books are refused) + humblebundle.com/downloads?key=<key> (needs \`cli-tools config set humble\`) + any direct .epub/.pdf/.mobi/.azw3/.cbz link (Leanpub, Gumroad, itch.io, + Kobo and Google Play DRM-free exports, publisher stores) + +Options: + -f, --format LIST preference order (default: epub,pdf,azw3,mobi,cbz,txt) + --all-formats every format in LIST the book has, not just the first + --to DIR library directory (default: ~/Books, or $BOOKDL_DIR) + --calibre add each new file to Calibre with calibredb + --force download again even if the archive says you have it + --json info/search/humble list as JSON + --help show this help + +Files are named "Author - Title.ext" and recorded in +~/.local/share/cli-tools/bookdl/archive.txt, so a second run fetches only what +is new. A file that turns out to have DRM (Adobe ADEPT, Readium LCP, Mobipocket, +KFX, or an .acsm ticket) is deleted and reported; bookdl never removes DRM. +`; + +const UA = 'bookdl (+https://github.com/profullstack/cli-tools)'; + +async function get(url: string, headers: Record<string, string> = {}): Promise<Response> { + const res = await fetch(url, { headers: { 'user-agent': UA, ...headers }, redirect: 'follow' }); + if (!res.ok) throw new Error(`${res.status} ${res.statusText} from ${new URL(url).host}`); + return res; +} + +async function getJson<T>(url: string, headers: Record<string, string> = {}): Promise<T> { + return (await (await get(url, { accept: 'application/json', ...headers })).json()) as T; +} + +function humbleHeaders(): Record<string, string> { + const session = resolveCredentials(process.env).HUMBLE_SESSION; + if (!session) { + throw new Error( + 'Humble needs your session: in a logged-in humblebundle.com tab, copy the _simpleauth_sess cookie (DevTools > Application > Cookies), then `cli-tools config set humble`.', + ); + } + const value = session.replace(/^_simpleauth_sess=/, '').replace(/^"|"$/g, ''); + return { cookie: `_simpleauth_sess="${value}"`, 'x-requested-by': 'hb_android_app' }; +} + +async function humbleOrder(key: string): Promise<HumbleOrder> { + return getJson<HumbleOrder>(`https://www.humblebundle.com/api/v1/order/${encodeURIComponent(key)}?all_tpkds=true`, humbleHeaders()); +} + +async function humbleLibrary(): Promise<Book[]> { + let keys: { gamekey: string }[]; + try { + keys = await getJson<{ gamekey: string }[]>('https://www.humblebundle.com/api/v1/user/order?ajax=true', humbleHeaders()); + } catch (err) { + if (/^401/.test((err as Error).message)) { + throw new Error('Humble said 401: the saved session cookie has expired. Copy a fresh _simpleauth_sess and `cli-tools config set humble`.'); + } + throw err; + } + const books: Book[] = []; + const seen = new Set<string>(); + for (const { gamekey } of keys) { + for (const book of parseHumbleOrder(await humbleOrder(gamekey))) { + if (seen.has(book.id)) continue; + seen.add(book.id); + books.push(book); + } + } + return books; +} + +/** Resolve a URL to a Book. Direct links have no metadata beyond their file name. */ +async function resolve(url: string): Promise<Book[]> { + const locked = lockedStore(url); + if (locked && detectSource(url) === 'direct' && !/\.(epub|pdf|mobi|azw3|cbz)(\?|$)/i.test(url)) { + throw new LockedError(locked); + } + switch (detectSource(url)) { + case 'gutenberg': { + const id = gutenbergId(url); + try { + return [parseGutendex(await getJson(`https://gutendex.com/books/${id}`))]; + } catch { + return [gutenbergFallback(id)]; + } + } + case 'standardebooks': { + let title: string | undefined; + try { + const html = await (await get(url)).text(); + title = html.match(/<title>([^<]*)<\/title>/)?.[1]; + } catch { + // The slugs are enough to name and fetch it. + } + return [standardEbooks(url, title)]; + } + case 'archive': { + const id = archiveId(url); + return [parseArchive(id, await getJson(`https://archive.org/metadata/${encodeURIComponent(id)}`))]; + } + case 'humble': + return parseHumbleOrder(await humbleOrder(humbleKey(url))); + case 'direct': { + const name = directName(url); + const format = extensionOf(name); + const stem = name.replace(/\.kepub\.epub$|\.[^.]+$/i, ''); + return [{ source: 'direct', id: url, title: stem || 'book', authors: [], files: [{ format, url }] }]; + } + } +} + +interface Options { + formats: string[]; + all: boolean; + dir: string; + calibre: boolean; + force: boolean; +} + +function loadArchive(): Set<string> { + try { + return parseArchiveFile(readFileSync(archivePath(), 'utf8')); + } catch { + return new Set(); + } +} + +function record(key: string): void { + const path = archivePath(); + mkdirSync(dirname(path), { recursive: true }); + appendFileSync(path, `${key}\n`); +} + +/** Fetch one file to its place in the library. Returns the path, or null when skipped. */ +async function fetchFile(book: Book, file: BookFile, opts: Options, archive: Set<string>): Promise<string | null> { + const key = archiveKey(book, file.format); + if (!opts.force && archive.has(key)) { + process.stderr.write(`skip ${book.title} (${file.format}): already downloaded\n`); + return null; + } + + // Humble's download links are signed CDN URLs: no cookie needed or wanted. + const res = await get(file.url); + const buf = Buffer.from(await res.arrayBuffer()); + + // A direct link that answered with a page is a login wall, an interstitial + // or a store page: say so instead of saving HTML as "book.epub". + if (looksLikeHtml(buf)) { + throw new Error(`${file.url} returned a web page, not a book file (a login page or a store page?). Copy the file's own download link.`); + } + + let format = file.format; + let title = book.title; + if (book.source === 'direct') { + const name = directName(res.url || file.url, res.headers.get('content-disposition')); + format = extensionOf(name) || format; + title = name.replace(/\.kepub\.epub$|\.[^.]+$/i, '') || title; + if (!(EBOOK_EXTENSIONS as readonly string[]).includes(format) && format !== 'kepub') { + const sniff = detectDrm(buf); + if (sniff.drm) throw new LockedError(`${file.url}: ${sniff.scheme}. bookdl does not remove DRM.`); + throw new Error(`${file.url}: "${name}" is not an ebook format bookdl knows (${EBOOK_EXTENSIONS.join(', ')}).`); + } + } + + const verdict = detectDrm(buf); + if (verdict.drm) { + throw new LockedError(`${title} (${format}) has DRM: ${verdict.scheme}. Not saved; bookdl does not remove DRM.`); + } + if (file.md5) { + const got = createHash('md5').update(buf).digest('hex'); + if (got !== file.md5) throw new Error(`${title} (${format}): md5 ${got} does not match the store's ${file.md5}`); + } + + mkdirSync(opts.dir, { recursive: true }); + const target = join( + opts.dir, + book.source === 'direct' ? `${safeName(title)}.${format === 'kepub' ? 'kepub.epub' : format}` : fileName(book, format), + ); + if (existsSync(target) && !opts.force) { + process.stderr.write(`skip ${target}: exists\n`); + record(key); + return null; + } + const part = `${target}.part`; + writeFileSync(part, buf); + renameSync(part, target); + record(key); + archive.add(key); + process.stdout.write(`saved ${target} (${(buf.length / 1048576).toFixed(1)} MB)\n`); + + if (opts.calibre) { + const r = spawnSync('calibredb', ['add', '--duplicates', target], { stdio: 'inherit' }); + if (r.error) process.stderr.write('bookdl: --calibre: calibredb is not on PATH (install Calibre)\n'); + } + return target; +} + +async function download(books: Book[], opts: Options): Promise<number> { + const archive = loadArchive(); + let failed = 0; + for (const book of books) { + const files = chooseFiles(book.files, opts.formats, opts.all); + if (files.length === 0) { + process.stderr.write(`bookdl: ${book.title}: none of ${opts.formats.join(',')} (has ${book.files.map((f) => f.format).join(',') || 'nothing'})\n`); + failed += 1; + continue; + } + for (const file of files) { + try { + await fetchFile(book, file, opts, archive); + } catch (err) { + // One bad file does not stop a library run; the exit code reports it. + process.stderr.write(`bookdl: ${(err as Error).message}\n`); + failed += 1; + } + } + } + return failed === 0 ? 0 : 1; +} + +function describe(books: Book[], json: boolean): void { + if (json) { + process.stdout.write(`${JSON.stringify(books, null, 2)}\n`); + return; + } + for (const b of books) { + const who = b.authors.length ? ` -- ${b.authors.join(', ')}` : ''; + process.stdout.write(`${b.title}${who} [${b.source} ${b.id}]\n ${b.files.map((f) => f.format).join(', ') || '(no files)'}\n`); + } +} + +if (isMain(import.meta.url)) { + try { + const { flags, values, positional } = parseArgs(process.argv.slice(2), { + boolean: ['--help', '--all-formats', '--all', '--calibre', '--force', '--json'], + string: ['-f', '--format', '--to'], + }); + if (flags.has('--help') || positional.length === 0) { + process.stdout.write(USAGE); + process.exit(positional.length === 0 ? 1 : 0); + } + + const opts: Options = { + formats: parseFormats(values.get('-f') ?? values.get('--format')), + all: flags.has('--all-formats'), + dir: values.get('--to') ?? defaultLibrary(), + calibre: flags.has('--calibre'), + force: flags.has('--force'), + }; + const json = flags.has('--json'); + const [verb, ...rest] = positional as [string, ...string[]]; + + if (verb === 'check') { + if (rest.length === 0) throw new UsageError('check needs a file'); + let locked = 0; + for (const path of rest) { + const v = detectDrm(readFileSync(path)); + if (v.drm) locked += 1; + process.stdout.write(`${v.drm ? `DRM (${v.scheme})` : 'DRM-free'} ${path}\n`); + } + process.exit(locked === 0 ? 0 : 3); + } + + if (verb === 'search') { + if (rest.length === 0) throw new UsageError('search needs a query'); + const found = parseGutendexSearch(await getJson(`https://gutendex.com/books?search=${encodeURIComponent(rest.join(' '))}`)); + if (json) describe(found, true); + else for (const b of found) process.stdout.write(`https://www.gutenberg.org/ebooks/${b.id} ${b.title}${b.authors[0] ? ` -- ${b.authors[0]}` : ''}\n`); + process.exit(found.length === 0 ? 1 : 0); + } + + if (verb === 'humble') { + const sub = rest[0] ?? 'list'; + const books = await humbleLibrary(); + if (sub === 'list') { + describe(books, json); + process.exit(0); + } + if (sub === 'get') process.exit(await download(books, { ...opts, all: opts.all || flags.has('--all') })); + throw new UsageError(`unknown humble command: ${sub}`); + } + + const info = verb === 'info'; + const urls = info ? rest : positional; + if (urls.length === 0) throw new UsageError('info needs a URL'); + const books: Book[] = []; + let failed = 0; + let locked = 0; + for (const url of urls) { + if (!looksLikeUrl(url)) throw new UsageError(`not a URL: ${url}`); + try { + books.push(...(await resolve(url))); + } catch (err) { + process.stderr.write(`bookdl: ${(err as Error).message}\n`); + if (err instanceof LockedError) locked += 1; + else failed += 1; + } + } + // 3 = refused because of DRM or a lending library, so a script can tell + // "this book is locked" from "this download broke". + const refusal = failed === 0 && locked > 0 ? 3 : failed > 0 ? 1 : 0; + if (info) { + describe(books, json); + process.exit(refusal); + } + const code = await download(books, opts); + process.exit(refusal || code); + } catch (err) { + if (err instanceof UsageError) { + process.stderr.write(`bookdl: ${err.message}\n\n${USAGE}`); + process.exit(1); + } + process.stderr.write(`bookdl: ${(err as Error).message}\n`); + process.exit(err instanceof LockedError ? 3 : 1); + } +} diff --git a/bin/cli-tools.ts b/bin/cli-tools.ts index 4020d05..daf2859 100755 --- a/bin/cli-tools.ts +++ b/bin/cli-tools.ts @@ -105,6 +105,7 @@ Keys (config set <key>): hproxy HPROXY_API_KEY proxy (status, and generated lines); hproxy_proxy_user/_password: traffic ngc NGC_API_KEY ngc, nim (fallback); ngc_org: NGC_ORG nvidia NVIDIA_API_KEY nim + humble HUMBLE_SESSION bookdl humble (the _simpleauth_sess cookie) Options: --force link: take over a symlink owned by another checkout diff --git a/package.json b/package.json index 89bfe2f..4bca6c6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@profullstack/cli-tools", - "version": "0.71.0", + "version": "0.72.0", "private": true, "description": "Local command-line tools, in TypeScript, exposed on PATH.", "type": "module", diff --git a/src/bookdl.ts b/src/bookdl.ts new file mode 100644 index 0000000..2662494 --- /dev/null +++ b/src/bookdl.ts @@ -0,0 +1,537 @@ +import { inflateRawSync } from 'node:zlib'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; + +/** + * bookdl: the yt-dlp shape for ebooks you are allowed to have as files. + * + * Every extractor here reads a store or library that hands out DRM-free files + * (public domain, or a store that sells without DRM), and every file is checked + * for DRM after it lands. A file that turns out to carry Adobe ADEPT, Readium + * LCP, Mobipocket or KFX protection is deleted and reported, never "fixed": + * removing DRM is circumvention under DMCA 1201, and this tool does not do it. + * Kindle, Apple Books and the DRM side of Kobo and Google Play are answered + * with what they do allow instead (see `lockedStore`). + */ + +export type Source = 'gutenberg' | 'standardebooks' | 'archive' | 'humble' | 'direct'; + +export interface BookFile { + /** Lowercase extension without the dot: epub, pdf, azw3, mobi, txt... */ + format: string; + url: string; + size?: number; + md5?: string; +} + +export interface Book { + source: Source; + /** Stable per source: gutenberg id, archive identifier, SE slug, humble machine name. */ + id: string; + title: string; + authors: string[]; + files: BookFile[]; +} + +export const EBOOK_EXTENSIONS = ['epub', 'pdf', 'azw3', 'mobi', 'cbz', 'cbr', 'fb2', 'djvu', 'txt'] as const; + +export const DEFAULT_FORMATS = ['epub', 'pdf', 'azw3', 'mobi', 'cbz', 'txt']; + +export function defaultLibrary(env: NodeJS.ProcessEnv = process.env): string { + return env.BOOKDL_DIR || join(homedir(), 'Books'); +} + +export function archivePath(env: NodeJS.ProcessEnv = process.env): string { + const data = env.XDG_DATA_HOME || join(homedir(), '.local', 'share'); + return join(data, 'cli-tools', 'bookdl', 'archive.txt'); +} + +/** + * Stores whose books are DRM-locked by default, and what to do instead. + * + * Matched before any extractor so a pasted Kindle link gets an answer rather + * than a download of the reader's HTML. + */ +export function lockedStore(url: string): string | null { + let host: string; + try { + host = new URL(url).hostname.replace(/^www\./, ''); + } catch { + return null; + } + if (/(^|\.)amazon\.[a-z.]+$/.test(host) || host === 'read.amazon.com' || /(^|\.)kindle\./.test(host)) { + return 'Kindle books are DRM-protected and bookdl does not remove DRM. Your highlights and notes export from https://read.amazon.com/notebook; a DRM-free edition, if the publisher sells one, will be on their own site, Humble, Leanpub or Gumroad.'; + } + if (host === 'books.apple.com') { + return 'Apple Books files are FairPlay-protected and bookdl does not remove DRM.'; + } + if (host === 'play.google.com') { + return 'Google Play Books: open the book in your library on the web, and if the publisher allowed it, "Export" offers a DRM-free EPUB or PDF link. Pass that link to bookdl. An .acsm file means the book is DRM-locked.'; + } + if (host.endsWith('kobo.com')) { + return 'Kobo: DRM-free titles show "DRM-Free" on the book page and download as a plain EPUB from My Books; pass that download link to bookdl. Others come as .acsm (Adobe DRM), which bookdl refuses.'; + } + return null; +} + +export function detectSource(url: string): Source { + const u = new URL(url); + const host = u.hostname.replace(/^www\./, ''); + if (host === 'gutenberg.org' && /^\/ebooks\/\d+/.test(u.pathname)) return 'gutenberg'; + if (host === 'standardebooks.org' && /^\/ebooks\/[^/]+\/[^/]+/.test(u.pathname)) return 'standardebooks'; + if (host === 'archive.org' && /^\/details\/[^/]+/.test(u.pathname)) return 'archive'; + if (host === 'humblebundle.com' && u.searchParams.has('key')) return 'humble'; + return 'direct'; +} + +export function looksLikeUrl(value: string): boolean { + try { + const u = new URL(value); + return u.protocol === 'https:' || u.protocol === 'http:'; + } catch { + return false; + } +} + +/** "Austen, Jane" -> "Jane Austen"; anything without exactly one comma is left alone. */ +export function naturalName(name: string): string { + const parts = name.split(',').map((p) => p.trim()); + return parts.length === 2 && parts[0] && parts[1] ? `${parts[1]} ${parts[0]}` : name.trim(); +} + +const UNSAFE = /[\\/:*?"<>|\u0000-\u001f]/g; + +export function safeName(value: string, max = 120): string { + const cleaned = value.replace(UNSAFE, ' ').replace(/\s+/g, ' ').trim().replace(/[. ]+$/, ''); + return cleaned.length > max ? cleaned.slice(0, max).trim() : cleaned || 'untitled'; +} + +/** "Author - Title.ext", the shape Calibre and most readers sort well. */ +export function fileName(book: Pick<Book, 'title' | 'authors'>, format: string): string { + // First author only: archive.org "creators" include illustrators and editors. + const author = book.authors[0] ? safeName(book.authors[0], 60) : ''; + const title = safeName(book.title); + const ext = format === 'kepub' ? 'kepub.epub' : format; + return `${author ? `${author} - ` : ''}${title}.${ext}`; +} + +/** + * Pick the files to fetch: the first format in `wanted` the book has, or every + * wanted format when `all` is set. + */ +export function chooseFiles(files: readonly BookFile[], wanted: readonly string[], all = false): BookFile[] { + const order = wanted.map((w) => w.toLowerCase()); + const ranked = files + .filter((f) => order.includes(f.format)) + .sort((a, b) => order.indexOf(a.format) - order.indexOf(b.format)); + if (all) { + const seen = new Set<string>(); + return ranked.filter((f) => !seen.has(f.format) && seen.add(f.format)); + } + return ranked.slice(0, 1); +} + +export function parseFormats(value: string | undefined): string[] { + if (!value) return DEFAULT_FORMATS; + return value + .split(',') + .map((v) => v.trim().toLowerCase().replace(/^\./, '')) + .filter(Boolean); +} + +/* ------------------------------------------------------------------ */ +/* Extractors: pure parsers over what each site's API returns. */ +/* ------------------------------------------------------------------ */ + +interface GutendexBook { + id: number; + title: string; + authors: { name: string }[]; + formats: Record<string, string>; + copyright?: boolean | null; +} + +const GUTENBERG_TYPES: [string, string][] = [ + ['application/epub+zip', 'epub'], + ['application/x-mobipocket-ebook', 'azw3'], + ['text/plain; charset=utf-8', 'txt'], + ['text/plain; charset=us-ascii', 'txt'], + ['application/pdf', 'pdf'], +]; + +export function gutenbergId(url: string): string { + const m = new URL(url).pathname.match(/^\/ebooks\/(\d+)/); + if (!m) throw new Error(`not a Project Gutenberg book URL: ${url}`); + return m[1]!; +} + +export function parseGutendex(json: GutendexBook): Book { + const files: BookFile[] = []; + for (const [mime, format] of GUTENBERG_TYPES) { + const url = json.formats[mime]; + if (url && !files.some((f) => f.format === format)) files.push({ format, url }); + } + return { + source: 'gutenberg', + id: String(json.id), + title: json.title, + authors: json.authors.map((a) => naturalName(a.name)), + files, + }; +} + +/** When gutendex is down: the URLs gutenberg.org itself redirects from. */ +export function gutenbergFallback(id: string): Book { + return { + source: 'gutenberg', + id, + title: `Project Gutenberg ${id}`, + authors: [], + files: [ + { format: 'epub', url: `https://www.gutenberg.org/ebooks/${id}.epub3.images` }, + { format: 'azw3', url: `https://www.gutenberg.org/ebooks/${id}.kf8.images` }, + { format: 'txt', url: `https://www.gutenberg.org/ebooks/${id}.txt.utf-8` }, + ], + }; +} + +export function parseGutendexSearch(json: { results: GutendexBook[] }): Book[] { + return json.results.map(parseGutendex); +} + +function titleCase(slug: string): string { + return slug + .split('-') + .map((w) => (w ? w[0]!.toUpperCase() + w.slice(1) : w)) + .join(' '); +} + +/** + * Standard Ebooks: /ebooks/<author>/<title>[/<translator>...]. The download + * file is the path segments joined with "_", and a bare link answers with a + * thank-you page: `?source=download` is what returns the file itself. + */ +export function standardEbooks(url: string, pageTitle?: string): Book { + const u = new URL(url); + const segments = u.pathname + .replace(/^\/ebooks\//, '') + .replace(/\/(downloads|text)(\/.*)?$/, '') + .split('/') + .filter(Boolean); + if (segments.length < 2) throw new Error(`not a Standard Ebooks book URL: ${url}`); + const base = `https://standardebooks.org/ebooks/${segments.join('/')}/downloads/${segments.join('_')}`; + const q = '?source=download'; + let title = titleCase(segments[1]!); + let authors = [titleCase(segments[0]!)]; + const m = pageTitle?.match(/^(.*?), by (.*?) - Free ebook download/); + if (m) { + title = m[1]!.trim(); + authors = m[2]!.split(/,\s*|\s+and\s+/).map((a) => a.trim()).filter(Boolean); + } + return { + source: 'standardebooks', + id: segments.join('/'), + title, + authors, + files: [ + { format: 'epub', url: `${base}.epub${q}` }, + { format: 'azw3', url: `${base}.azw3${q}` }, + { format: 'kepub', url: `${base}.kepub.epub${q}` }, + ], + }; +} + +interface ArchiveFile { + name: string; + format?: string; + size?: string; + md5?: string; + private?: string | boolean; +} + +interface ArchiveMetadata { + metadata?: { + identifier?: string; + title?: string | string[]; + creator?: string | string[]; + 'access-restricted-item'?: string | boolean; + }; + files?: ArchiveFile[]; +} + +export class LockedError extends Error { + constructor(message: string) { + super(message); + this.name = 'LockedError'; + } +} + +const ARCHIVE_FORMATS: Record<string, string> = { + EPUB: 'epub', + 'Text PDF': 'pdf', + 'Additional Text PDF': 'pdf', + 'Image Container PDF': 'pdf', + 'Grayscale PDF': 'pdf', + 'Comic Book ZIP': 'cbz', + 'Comic Book RAR': 'cbr', + DjVu: 'djvu', + DjVuTXT: 'txt', + Mobi: 'mobi', + 'Kindle (AZW3)': 'azw3', +}; + +export function archiveId(url: string): string { + const m = new URL(url).pathname.match(/^\/details\/([^/]+)/); + if (!m) throw new Error(`not an Internet Archive item URL: ${url}`); + return decodeURIComponent(m[1]!); +} + +/** + * An archive.org item. Lending-library books (access-restricted, or only + * "LCP/ACS Encrypted" copies) are borrowed, not owned, and refused. + */ +export function parseArchive(id: string, json: ArchiveMetadata): Book { + const meta = json.metadata; + if (!meta || !json.files) throw new Error(`archive.org has no item ${id}`); + const restricted = String(meta['access-restricted-item'] ?? '') === 'true'; + const files: BookFile[] = []; + for (const f of json.files) { + const format = f.format ? ARCHIVE_FORMATS[f.format] : undefined; + if (!format || String(f.private ?? '') === 'true' || /encrypted/i.test(f.format ?? '')) continue; + files.push({ + format, + url: `https://archive.org/download/${encodeURIComponent(id)}/${f.name.split('/').map(encodeURIComponent).join('/')}`, + ...(f.size ? { size: Number(f.size) } : {}), + ...(f.md5 ? { md5: f.md5 } : {}), + }); + } + if (restricted || files.length === 0) { + throw new LockedError( + `archive.org item ${id} is a lending-library book (borrow it in the browser; its files are LCP/ACS encrypted), or has no ebook files.`, + ); + } + const first = (v: string | string[] | undefined) => (Array.isArray(v) ? v : v ? [v] : []); + return { + source: 'archive', + id, + title: first(meta.title)[0] ?? id, + authors: first(meta.creator).map((c) => naturalName(c.replace(/,\s*\d{4}-(\d{4})?.*$/, ''))), + files, + }; +} + +interface HumbleDownloadStruct { + name?: string; + url?: { web?: string }; + md5?: string; + file_size?: number; +} + +interface HumbleSubproduct { + machine_name: string; + human_name: string; + payee?: { human_name?: string }; + downloads?: { platform: string; download_struct?: HumbleDownloadStruct[] }[]; +} + +export interface HumbleOrder { + gamekey: string; + product?: { human_name?: string }; + subproducts?: HumbleSubproduct[]; +} + +/** Humble's "name" is the button label: EPUB, PDF, MOBI, "PDF (HQ)", "CBZ"... */ +function humbleFormat(name: string | undefined, url: string): string | null { + const fromUrl = new URL(url).pathname.split('.').pop()?.toLowerCase(); + if (fromUrl && (EBOOK_EXTENSIONS as readonly string[]).includes(fromUrl)) return fromUrl; + const label = (name ?? '').toLowerCase(); + return (EBOOK_EXTENSIONS as readonly string[]).find((ext) => label.includes(ext)) ?? null; +} + +/** The ebooks in one Humble order. Games, audio and video are skipped. */ +export function parseHumbleOrder(order: HumbleOrder): Book[] { + const books: Book[] = []; + for (const sub of order.subproducts ?? []) { + const files: BookFile[] = []; + for (const d of sub.downloads ?? []) { + if (d.platform !== 'ebook') continue; + for (const s of d.download_struct ?? []) { + const url = s.url?.web; + if (!url) continue; + const format = humbleFormat(s.name, url); + if (!format) continue; + files.push({ + format, + url, + ...(s.md5 ? { md5: s.md5 } : {}), + ...(s.file_size ? { size: s.file_size } : {}), + }); + } + } + if (files.length === 0) continue; + books.push({ + source: 'humble', + id: sub.machine_name, + title: sub.human_name, + authors: sub.payee?.human_name ? [sub.payee.human_name] : [], + files, + }); + } + return books; +} + +export function humbleKey(url: string): string { + const key = new URL(url).searchParams.get('key'); + if (!key) throw new Error(`no ?key= in Humble URL: ${url}`); + return key; +} + +/** A direct file link: the name comes from Content-Disposition, else the path. */ +export function directName(url: string, disposition?: string | null): string { + const star = disposition?.match(/filename\*\s*=\s*[^']*''([^;]+)/i); + const plain = disposition?.match(/filename\s*=\s*"?([^";]+)"?/i); + const raw = star ? decodeURIComponent(star[1]!) : plain ? plain[1]! : decodeURIComponent(new URL(url).pathname.split('/').pop() || ''); + return raw.trim(); +} + +export function extensionOf(name: string): string { + const lower = name.toLowerCase(); + if (lower.endsWith('.kepub.epub')) return 'kepub'; + return lower.split('.').pop() ?? ''; +} + +/* ------------------------------------------------------------------ */ +/* Download archive: one line per fetched file, like yt-dlp's. */ +/* ------------------------------------------------------------------ */ + +export function archiveKey(book: Pick<Book, 'source' | 'id'>, format: string): string { + return `${book.source} ${book.id} ${format}`; +} + +export function parseArchiveFile(text: string): Set<string> { + return new Set( + text + .split('\n') + .map((l) => l.trim()) + .filter(Boolean), + ); +} + +/* ------------------------------------------------------------------ */ +/* DRM detection. Detect and refuse; never remove. */ +/* ------------------------------------------------------------------ */ + +export type DrmVerdict = { drm: false } | { drm: true; scheme: string }; + +/** Font obfuscation, which is in every other EPUB and is not DRM. */ +const FONT_OBFUSCATION = ['http://www.idpf.org/2008/embedding', 'http://ns.adobe.com/pdf/enc#RC']; + +interface ZipEntry { + name: string; + method: number; + compressedSize: number; + localOffset: number; +} + +/** Central directory of a zip, enough to find and read META-INF files. */ +export function zipEntries(buf: Buffer): ZipEntry[] { + // End of central directory: last 22 bytes plus up to 64 KiB of comment. + const min = Math.max(0, buf.length - 22 - 0xffff); + let eocd = -1; + for (let i = buf.length - 22; i >= min; i -= 1) { + if (buf.readUInt32LE(i) === 0x06054b50) { + eocd = i; + break; + } + } + if (eocd < 0) throw new Error('not a zip file'); + const count = buf.readUInt16LE(eocd + 10); + let p = buf.readUInt32LE(eocd + 16); + const entries: ZipEntry[] = []; + for (let n = 0; n < count && p + 46 <= buf.length; n += 1) { + if (buf.readUInt32LE(p) !== 0x02014b50) break; + const method = buf.readUInt16LE(p + 10); + const compressedSize = buf.readUInt32LE(p + 20); + const nameLen = buf.readUInt16LE(p + 28); + const extraLen = buf.readUInt16LE(p + 30); + const commentLen = buf.readUInt16LE(p + 32); + const localOffset = buf.readUInt32LE(p + 42); + const name = buf.toString('utf8', p + 46, p + 46 + nameLen); + entries.push({ name, method, compressedSize, localOffset }); + p += 46 + nameLen + extraLen + commentLen; + } + return entries; +} + +export function zipRead(buf: Buffer, entry: ZipEntry): Buffer { + const p = entry.localOffset; + const start = p + 30 + buf.readUInt16LE(p + 26) + buf.readUInt16LE(p + 28); + const data = buf.subarray(start, start + entry.compressedSize); + if (entry.method === 0) return data; + if (entry.method === 8) return inflateRawSync(data); + throw new Error(`unsupported zip method ${entry.method}`); +} + +export function epubDrm(buf: Buffer): DrmVerdict { + const entries = zipEntries(buf); + const names = new Set(entries.map((e) => e.name)); + if (names.has('META-INF/rights.xml')) return { drm: true, scheme: 'Adobe ADEPT' }; + if (names.has('META-INF/license.lcpl')) return { drm: true, scheme: 'Readium LCP' }; + const enc = entries.find((e) => e.name === 'META-INF/encryption.xml'); + if (enc) { + const xml = zipRead(buf, enc).toString('utf8'); + const algorithms = [...xml.matchAll(/Algorithm\s*=\s*"([^"]+)"/g)].map((m) => m[1]!); + // An EncryptionMethod that is not font obfuscation encrypts content. + const real = algorithms.filter( + (a) => !FONT_OBFUSCATION.includes(a) && !/xmldsig|#sha|#rsa|keyinfo/i.test(a), + ); + if (real.length > 0) return { drm: true, scheme: `encrypted content (${real[0]})` }; + } + return { drm: false }; +} + +export function pdfDrm(buf: Buffer): DrmVerdict { + // Look at the head and tail, where the trailer and /Encrypt dictionary live, + // plus a cheap scan of the whole file for the handler names. + const text = buf.toString('latin1'); + if (/\/Filter\s*\/(EBX_HANDLER|Adobe\.APS|FOPN_\w+)/.test(text)) { + const m = text.match(/\/Filter\s*\/(EBX_HANDLER|Adobe\.APS|FOPN_\w+)/)!; + return { drm: true, scheme: m[1] === 'EBX_HANDLER' ? 'Adobe ADEPT' : `PDF DRM (${m[1]})` }; + } + return { drm: false }; +} + +/** Mobipocket / AZW: the encryption type in the MOBI header of record 0. */ +export function mobiDrm(buf: Buffer): DrmVerdict { + if (buf.length < 86) return { drm: false }; + const type = buf.toString('latin1', 60, 68); + if (type !== 'BOOKMOBI' && type !== 'TEXtREAd') return { drm: false }; + const rec0 = buf.readUInt32BE(78); + if (rec0 + 14 > buf.length) return { drm: false }; + const encryption = buf.readUInt16BE(rec0 + 12); + return encryption === 0 ? { drm: false } : { drm: true, scheme: 'Mobipocket DRM' }; +} + +/** Sniff the container and check it. Unknown types pass: there is nothing to say about them. */ +export function detectDrm(buf: Buffer): DrmVerdict { + const head = buf.subarray(0, 512).toString('latin1'); + if (head.includes('DRMION')) return { drm: true, scheme: 'Kindle KFX' }; + if (/<fulfillmentToken|adept:fulfillmentToken/.test(head) || head.includes('ns.adobe.com/adept')) { + return { drm: true, scheme: 'Adobe ACSM (a DRM download ticket, not a book)' }; + } + if (buf.length >= 4 && buf.readUInt32LE(0) === 0x04034b50) { + try { + return epubDrm(buf); + } catch { + return { drm: false }; + } + } + if (head.startsWith('%PDF')) return pdfDrm(buf); + return mobiDrm(buf); +} + +/** Bytes that are an HTML page, not a book: a login wall or an interstitial. */ +export function looksLikeHtml(buf: Buffer): boolean { + const head = buf.subarray(0, 512).toString('utf8').trimStart().toLowerCase(); + return head.startsWith('<!doctype html') || head.startsWith('<html'); +} diff --git a/src/credentials.ts b/src/credentials.ts index 9269bc6..e759b4b 100644 --- a/src/credentials.ts +++ b/src/credentials.ts @@ -81,6 +81,9 @@ export const KNOWN_KEYS: Record<string, string> = { ngc_org: 'NGC_ORG', // Read by `nim`: a build.nvidia.com key for the hosted inference API. nvidia: 'NVIDIA_API_KEY', + // Read by `bookdl humble`: the value of the _simpleauth_sess cookie from a + // logged-in humblebundle.com tab. Humble has no API token for buyers. + humble: 'HUMBLE_SESSION', }; export type Source = 'env' | 'file' | 'unset'; diff --git a/src/registry.ts b/src/registry.ts index 5bde5fd..c6781e8 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -37,6 +37,7 @@ const SUMMARIES: Record<string, string> = { 'cli-tools': 'This dispatcher: list, update and wire up the others', codeburn: 'See where your AI spend goes, by task, tool, model and project', cronjob: 'Install, list and remove cron jobs without ever installing one twice', + bookdl: 'Download DRM-free ebooks (Gutenberg, Standard Ebooks, archive.org, Humble, direct links)', dl: 'Download a video, or just its audio, through yt-dlp', domainfree: 'Which of these domains can you actually register', domainjson: 'whois-style, JSON-first name lookup', diff --git a/test/bookdl.test.ts b/test/bookdl.test.ts new file mode 100644 index 0000000..d14d80f --- /dev/null +++ b/test/bookdl.test.ts @@ -0,0 +1,263 @@ +import { deflateRawSync } from 'node:zlib'; +import { describe, expect, it } from 'vitest'; + +import { + LockedError, + archiveKey, + chooseFiles, + detectDrm, + detectSource, + directName, + extensionOf, + fileName, + gutenbergFallback, + lockedStore, + looksLikeHtml, + naturalName, + parseArchive, + parseArchiveFile, + parseFormats, + parseGutendex, + parseHumbleOrder, + safeName, + standardEbooks, +} from '../src/bookdl.ts'; + +/** A minimal zip: deflated entries, a central directory, an end record. */ +function zip(entries: Record<string, string>): Buffer { + const locals: Buffer[] = []; + const centrals: Buffer[] = []; + let offset = 0; + for (const [name, text] of Object.entries(entries)) { + const nameBuf = Buffer.from(name); + const data = deflateRawSync(Buffer.from(text)); + const local = Buffer.alloc(30); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(8, 8); + local.writeUInt32LE(data.length, 18); + local.writeUInt32LE(text.length, 22); + local.writeUInt16LE(nameBuf.length, 26); + const central = Buffer.alloc(46); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(8, 10); + central.writeUInt32LE(data.length, 20); + central.writeUInt32LE(text.length, 24); + central.writeUInt16LE(nameBuf.length, 28); + central.writeUInt32LE(offset, 42); + locals.push(local, nameBuf, data); + centrals.push(central, nameBuf); + offset += 30 + nameBuf.length + data.length; + } + const cd = Buffer.concat(centrals); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(Object.keys(entries).length, 8); + end.writeUInt16LE(Object.keys(entries).length, 10); + end.writeUInt32LE(cd.length, 12); + end.writeUInt32LE(offset, 16); + return Buffer.concat([...locals, cd, end]); +} + +const EPUB = { mimetype: 'application/epub+zip', 'META-INF/container.xml': '<container/>' }; + +function encryption(algorithm: string): string { + return `<encryption xmlns="urn:oasis:names:tc:opendocument:xmlns:container"><EncryptedData xmlns="http://www.w3.org/2001/04/xmlenc#"><EncryptionMethod Algorithm="${algorithm}"/></EncryptedData></encryption>`; +} + +function mobi(encryptionType: number): Buffer { + const buf = Buffer.alloc(200); + buf.write('BOOKMOBI', 60, 'latin1'); + buf.writeUInt32BE(100, 78); + buf.writeUInt16BE(encryptionType, 112); + return buf; +} + +describe('detectDrm', () => { + it('passes a plain EPUB', () => { + expect(detectDrm(zip(EPUB))).toEqual({ drm: false }); + }); + + /* + * Font obfuscation lives in encryption.xml in a large share of retail and + * Standard Ebooks EPUBs. Treating it as DRM would refuse half of every + * DRM-free library. + */ + it('passes EPUB font obfuscation, which is not DRM', () => { + const idpf = zip({ ...EPUB, 'META-INF/encryption.xml': encryption('http://www.idpf.org/2008/embedding') }); + const adobe = zip({ ...EPUB, 'META-INF/encryption.xml': encryption('http://ns.adobe.com/pdf/enc#RC') }); + expect(detectDrm(idpf).drm).toBe(false); + expect(detectDrm(adobe).drm).toBe(false); + }); + + it('flags Adobe ADEPT, Readium LCP and encrypted content', () => { + expect(detectDrm(zip({ ...EPUB, 'META-INF/rights.xml': '<rights/>' }))).toEqual({ drm: true, scheme: 'Adobe ADEPT' }); + expect(detectDrm(zip({ ...EPUB, 'META-INF/license.lcpl': '{}' }))).toEqual({ drm: true, scheme: 'Readium LCP' }); + const aes = zip({ ...EPUB, 'META-INF/encryption.xml': encryption('http://www.w3.org/2001/04/xmlenc#aes128-cbc') }); + expect(detectDrm(aes).drm).toBe(true); + }); + + it('reads the PDF security handler', () => { + expect(detectDrm(Buffer.from('%PDF-1.7\n1 0 obj<</Title(x)>>endobj')).drm).toBe(false); + // A password-only /Standard handler is permissions, not a store's DRM. + expect(detectDrm(Buffer.from('%PDF-1.7\n<</Filter /Standard /V 2>>')).drm).toBe(false); + expect(detectDrm(Buffer.from('%PDF-1.7\n<</Filter /EBX_HANDLER /V 4>>'))).toEqual({ drm: true, scheme: 'Adobe ADEPT' }); + }); + + it('reads the MOBI encryption field', () => { + expect(detectDrm(mobi(0)).drm).toBe(false); + expect(detectDrm(mobi(2))).toEqual({ drm: true, scheme: 'Mobipocket DRM' }); + }); + + it('refuses KFX and an .acsm ticket', () => { + expect(detectDrm(Buffer.from('êDRMIONî....', 'latin1')).drm).toBe(true); + const acsm = Buffer.from('<?xml version="1.0"?><fulfillmentToken xmlns="http://ns.adobe.com/adept">'); + expect(detectDrm(acsm).drm).toBe(true); + }); +}); + +describe('lockedStore', () => { + it('answers Kindle, Apple, Google Play and Kobo links', () => { + expect(lockedStore('https://read.amazon.com/?asin=B0GY3BMFKK')).toMatch(/Kindle/); + expect(lockedStore('https://www.amazon.co.uk/dp/B0X')).toMatch(/Kindle/); + expect(lockedStore('https://books.apple.com/us/book/x/id1')).toMatch(/FairPlay/); + expect(lockedStore('https://play.google.com/store/books/details?id=x')).toMatch(/Export/); + expect(lockedStore('https://www.kobo.com/us/en/ebook/x')).toMatch(/DRM-Free/); + expect(lockedStore('https://www.gutenberg.org/ebooks/1')).toBeNull(); + }); +}); + +describe('detectSource', () => { + it('routes each library and falls back to a direct link', () => { + expect(detectSource('https://www.gutenberg.org/ebooks/345')).toBe('gutenberg'); + expect(detectSource('https://standardebooks.org/ebooks/jane-austen/emma')).toBe('standardebooks'); + expect(detectSource('https://archive.org/details/prideprejudice00aust')).toBe('archive'); + expect(detectSource('https://www.humblebundle.com/downloads?key=abc')).toBe('humble'); + expect(detectSource('https://leanpub.com/s/x.epub')).toBe('direct'); + }); +}); + +describe('names', () => { + it('turns catalogue names around and keeps file names safe', () => { + expect(naturalName('Austen, Jane')).toBe('Jane Austen'); + expect(naturalName('Plato')).toBe('Plato'); + expect(safeName('a/b: c?')).toBe('a b c'); + expect(fileName({ title: 'Dracula', authors: ['Bram Stoker', 'Editor'] }, 'epub')).toBe('Bram Stoker - Dracula.epub'); + expect(fileName({ title: 'Emma', authors: [] }, 'kepub')).toBe('Emma.kepub.epub'); + }); + + it('reads a direct link name from Content-Disposition first', () => { + expect(directName('https://x.test/dl/123', 'attachment; filename="My Book.epub"')).toBe('My Book.epub'); + expect(directName('https://x.test/dl/123', "attachment; filename*=UTF-8''Caf%C3%A9.pdf")).toBe('Café.pdf'); + expect(directName('https://x.test/files/Book%20One.pdf?sig=1')).toBe('Book One.pdf'); + expect(extensionOf('x.kepub.epub')).toBe('kepub'); + }); +}); + +describe('formats', () => { + const files = [ + { format: 'pdf', url: 'p' }, + { format: 'epub', url: 'e' }, + { format: 'pdf', url: 'p2' }, + ]; + + it('takes the first preferred format, or each one once with --all-formats', () => { + expect(chooseFiles(files, ['epub', 'pdf']).map((f) => f.url)).toEqual(['e']); + expect(chooseFiles(files, ['epub', 'pdf'], true).map((f) => f.url)).toEqual(['e', 'p']); + expect(chooseFiles(files, ['mobi'])).toEqual([]); + expect(parseFormats('.EPUB, pdf')).toEqual(['epub', 'pdf']); + }); +}); + +describe('extractors', () => { + it('parses gutendex, and has a fallback when it is down', () => { + const book = parseGutendex({ + id: 345, + title: 'Dracula', + authors: [{ name: 'Stoker, Bram' }], + formats: { + 'application/epub+zip': 'https://www.gutenberg.org/ebooks/345.epub3.images', + 'text/plain; charset=utf-8': 'https://www.gutenberg.org/ebooks/345.txt.utf-8', + 'image/jpeg': 'cover.jpg', + }, + }); + expect(book.authors).toEqual(['Bram Stoker']); + expect(book.files.map((f) => f.format)).toEqual(['epub', 'txt']); + expect(gutenbergFallback('1').files[0]!.url).toBe('https://www.gutenberg.org/ebooks/1.epub3.images'); + }); + + /* + * A bare Standard Ebooks .epub link answers with a thank-you page; only + * `?source=download` returns the file. Every URL we build must carry it. + */ + it('builds Standard Ebooks download links that return the file', () => { + const book = standardEbooks( + 'https://standardebooks.org/ebooks/jane-austen/pride-and-prejudice/downloads/jane-austen_pride-and-prejudice.epub', + 'Pride and Prejudice, by Jane Austen - Free ebook download - Standard Ebooks: ...', + ); + expect(book.id).toBe('jane-austen/pride-and-prejudice'); + expect(book.title).toBe('Pride and Prejudice'); + expect(book.files[0]!.url).toBe( + 'https://standardebooks.org/ebooks/jane-austen/pride-and-prejudice/downloads/jane-austen_pride-and-prejudice.epub?source=download', + ); + }); + + it('takes open archive.org files and refuses lending-library items', () => { + const open = parseArchive('pp', { + metadata: { title: 'Pride and prejudice', creator: ['Austen, Jane, 1775-1817'] }, + files: [ + { name: 'pp.epub', format: 'EPUB', size: '10' }, + { name: 'pp_lcp.epub', format: 'LCP Encrypted EPUB' }, + { name: 'pp.jpg', format: 'JPEG' }, + ], + }); + expect(open.authors).toEqual(['Jane Austen']); + expect(open.files.map((f) => f.url)).toEqual(['https://archive.org/download/pp/pp.epub']); + + const lending = { + metadata: { title: 'x', 'access-restricted-item': 'true' }, + files: [{ name: 'x.epub', format: 'EPUB', private: 'true' }], + }; + expect(() => parseArchive('x', lending)).toThrow(LockedError); + }); + + it('keeps only the ebooks from a Humble order', () => { + const books = parseHumbleOrder({ + gamekey: 'k', + subproducts: [ + { + machine_name: 'pythonbook', + human_name: 'Python Book', + payee: { human_name: 'No Starch Press' }, + downloads: [ + { + platform: 'ebook', + download_struct: [ + { name: 'EPUB', url: { web: 'https://dl.humble.com/pythonbook.epub?gamekey=k&ttl=1' }, md5: 'abc' }, + { name: 'PDF', url: { web: 'https://dl.humble.com/pythonbook.pdf?gamekey=k' } }, + { name: 'Supplement', url: { web: 'https://dl.humble.com/code.zip' } }, + ], + }, + ], + }, + { machine_name: 'game', human_name: 'A Game', downloads: [{ platform: 'windows', download_struct: [] }] }, + ], + }); + expect(books).toHaveLength(1); + expect(books[0]!.files.map((f) => f.format)).toEqual(['epub', 'pdf']); + expect(books[0]!.files[0]!.md5).toBe('abc'); + }); +}); + +describe('archive file', () => { + it('keys by source, id and format', () => { + const key = archiveKey({ source: 'gutenberg', id: '345' }, 'epub'); + expect(parseArchiveFile(`${key}\n\nother 1 pdf\n`).has('gutenberg 345 epub')).toBe(true); + }); +}); + +describe('looksLikeHtml', () => { + it('spots a login wall saved as a book', () => { + expect(looksLikeHtml(Buffer.from(' <!DOCTYPE html><html>'))).toBe(true); + expect(looksLikeHtml(Buffer.from('%PDF-1.7'))).toBe(false); + }); +});