diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f8e8ae..8808ea0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to ManagedCode.FileContext are documented here. +## 1.0.19 + +- Apply file, result, page, concurrency, and timeout limits only when explicitly configured or requested. Default text and document reads return the remaining content. +- Inspect CSV records without materializing non-header values. Preserve quoted multiline fields, blank lines, and trailing tab-delimited columns. +- Preserve DOCX tabs, line breaks, and continuation offsets. Return XLSX base cell text without phonetic annotations. +- Reduce allocations during file search and XLSX inspection. Reuse report text styles and index table highlights once per table. +- Paginate long PDF headings and KPI rows. Check configured output budgets before uploading generated files. + ## 1.0.17 - Update deterministic agent integration dependencies and APIs to `ManagedCode.LlmTestServer` 1.0.0. diff --git a/Directory.Build.props b/Directory.Build.props index 93780d7..d3f91d0 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -12,7 +12,7 @@ Recommended true $(NoWarn);CS1591;MAAI001 - 1.0.18 + 1.0.19 $(Version) diff --git a/README.md b/README.md index 001c7f8..998d21b 100644 --- a/README.md +++ b/README.md @@ -197,9 +197,9 @@ an existing host directory. Disposal, cancellation and staging failures remove t `IFileContextPdf.ReadPdfTextAsync(path)` returns bounded text, `PageCount`, and one-based `PagesWithoutText`. It does not perform OCR. A scanned page can instead be rendered with `RenderPdfPageAsync(path, pageNumber)`, which returns PNG `DataContent`. Use `CountPdfPageImagesAsync` and `ExtractPdfImageAsync` when the original embedded pictures are needed rather than the complete page. The four read-only `file_context_pdf_*` tools expose the same operations from scoped storage. -For an authenticated PDF already held as bytes, `FileContextPdfTextExtractor.Extract`, `FileContextPdfImages.RenderPagePng`, and `FileContextPdfImages.ExtractPageImagesPng` work without storing it. PDF source reads default to 100 MiB and accept `FileContextOptions` for a different limit; page rasterization also uses configured pixel and PNG limits. `FileContextImageContent` creates model-visible `DataContent` from PNG bytes or base64 and `UriContent` from an HTTPS URL. A URL reference is not fetched by FileContext, so the model provider must be able to access it. A host must pass image content to its model as image content. A generic OpenAI Chat function result serializes it as text, so hosts must explicitly bridge image tool results into a multimodal model message. +For an authenticated PDF already held as bytes, `FileContextPdfTextExtractor.Extract`, `FileContextPdfImages.RenderPagePng`, and `FileContextPdfImages.ExtractPageImagesPng` work without storing it. Hosts can set PDF source, pixel and PNG limits through `FileContextOptions`. `FileContextImageContent` creates model-visible `DataContent` from PNG bytes or base64 and `UriContent` from an HTTPS URL. A URL reference is not fetched by FileContext, so the model provider must be able to access it. A host must pass image content to its model as image content. A generic OpenAI Chat function result serializes it as text, so hosts must explicitly bridge image tool results into a multimodal model message. -`file_context_docx_text(path, startParagraph?, startCharacter?, paragraphCount?)` reads ordinary paragraph and table text from a scoped DOCX package. The result contains numbered paragraph segments and `nextParagraph`/`nextCharacter`; use that cursor to continue a long document. Reads are limited to 50 paragraphs and 20,000 characters per call, with a configurable 25 MiB source limit (`MaximumDocxReadBytes`). It does not OCR embedded images. DOCX and XLSX packages are excluded from generic text reads and grep. +`file_context_docx_text(path, startParagraph?, startCharacter?, paragraphCount?)` reads ordinary paragraph and table text from a scoped DOCX package. The result contains numbered paragraph segments and `nextParagraph`/`nextCharacter`; use that cursor to continue a long document. The caller can request a paragraph count. Hosts can also configure `MaximumDocxTextCharacters` and `MaximumDocxReadBytes`. It does not OCR embedded images. DOCX and XLSX packages are excluded from generic text reads and grep. ## Explore Markdown as a graph @@ -253,32 +253,20 @@ var tools = workspaces.GetRequiredKeyedService("research"); Paths are logical, relative, and `/`-separated. `RootPrefix` scopes storage access. Path validation rejects traversal and unsafe path forms before storage calls. File contents remain untrusted tool data. +Size, count, PDF concurrency, and deadline limits apply when configured by the host or requested by the caller. Range, DOCX and PDF text operations return the complete remaining content when the caller omits the window size and the host has left the relevant options at their defaults. + | Option | Default | -| --- | ---: | -| `MaximumPdfReadBytes` | 100 MiB | -| `MaximumImageBytes` | 8 MiB | -| `MaximumRenderedPagePixels` / `MaximumImagesPerPdfPage` | 4,000,000 / 20 | -| `DefaultPdfPageScale` / `MinimumPdfPageScale` / `MaximumPdfPageScale` | 1.5 / 0.5 / 3 | -| `PdfPngQuality` | 100 | -| `MaximumFullReadBytes` | 1 MiB | -| `MaximumRangeReadBytes` | 256 KiB | -| `MaximumDirectoryEntries` | 1,000 direct children | -| `MaximumOpenXmlPartCharacters` | 32 Mi characters per decompressed XML part | -| `MaximumGeneratedFileBytes` | 64 MiB per generated file or complete report | -| `MaximumReportInputBytes` / `MaximumReportPages` | 10,000,000 bytes / 60 pages | -| `DefaultRangeLineCount` / `MaximumRangeLineCount` | 200 / 1,000 | -| `MaximumSearchFiles` | 500 | -| `MaximumSearchFileBytes` | 4 MiB | -| `MaximumSearchResults` / `MaximumMatchesPerFile` | 100 / 20 | -| `RegexTimeout` | 2 seconds per regex match | -| `OperationTimeout` | `null` (reports default to 60 seconds) | +| --- | --- | +| Numeric `Maximum*` capacity options | No FileContext limit. `int.MaxValue` or `long.MaxValue` represents the unset capacity | +| `MaximumOpenXmlPartCharacters` | `0`, which delegates to Open XML's unlimited setting | +| `DefaultRangeLineCount` | Remaining lines within any configured maximum | +| `DefaultPdfPageScale` / `PdfPngQuality` | 1.5 / 100 | +| `MinimumPdfPageScale` / `MaximumPdfPageScale` | Positive scales representable by the raster engine | +| `RegexTimeout` | `Regex.InfiniteMatchTimeout` | +| `OperationTimeout` | `null` | | `MarkdownGlob` | `**/*.md` | -| `MaximumMarkdownFiles` | 100 | -| `MaximumMarkdownSourceBytes` | 1 MiB per file | -| `MaximumGraphResults` | 20 | -| `MaximumGraphExportCharacters` | 200,000 | -These settings are available through `FileContextOptions`; their named defaults are exposed by `FileContextDefaults`. Credentials, container lifecycle, authorization, and session persistence remain host responsibilities. When saving conversations, preserve tool-call/result pairs and handle interrupted turns before replaying history. +These settings are available through `FileContextOptions`. Credentials, container lifecycle, authorization, and session persistence remain host responsibilities. When saving conversations, preserve tool-call/result pairs and handle interrupted turns before replaying history. ### Configure limits and timeouts @@ -305,7 +293,7 @@ services.AddManagedCodeFileContext(storage, options => }); ``` -`RegexTimeout` limits **one regex match on one line**, protecting the process from excessive backtracking in a model-supplied pattern. It is not a two-second deadline for the whole search. Choose a finite positive value supported by .NET Regex. Size/count limits must be positive, and the default line count cannot exceed the maximum. +`RegexTimeout` limits **one regex match on one line**, protecting the process from excessive backtracking in a model-supplied pattern. Set a finite positive value supported by .NET Regex to enable it. Configured size/count limits must be positive, except that zero disables the Open XML part limit. An explicitly configured default line count must fit its maximum. In a host that uses Microsoft configuration binding, the same options can come from `appsettings.json`, environment variables, or another configuration source: @@ -421,7 +409,7 @@ await using var bytes = await store.OpenReadAsync(report.Path); `FileContextProvider` exposes `file_context_create_text`, `file_context_create_csv`, `file_context_create_workbook`, and `file_context_create_pdf` when writes are enabled. `FileContextDocumentTools.Create(store, options)` returns those same native functions for host-managed tool registries, including sandboxed JavaScript bridges. -CSV preserves literal values and escapes quotes, delimiters and newlines. XLSX supports multiple sheets and text/number/boolean/formula cells; Excel calculates explicit formulas when opened. PDF supports paginated paragraphs and a title with embedded Noto Sans for Latin/Cyrillic text. The Noto Sans font is distributed under the bundled SIL Open Font License. The default generated-file budget is 64 MiB, configurable through `MaximumGeneratedFileBytes`; cancellation and `OperationTimeout` apply to generation and persistence. +CSV preserves literal values and escapes quotes, delimiters and newlines. XLSX supports multiple sheets and text/number/boolean/formula cells; Excel calculates explicit formulas when opened. PDF supports paginated paragraphs and a title with embedded Noto Sans for Latin/Cyrillic text. The Noto Sans font is distributed under the bundled SIL Open Font License. Hosts can set `MaximumGeneratedFileBytes` explicitly. Cancellation and any configured `OperationTimeout` apply to generation and persistence. Document generation uses DocumentFormat.OpenXml (MIT), PdfPig (Apache-2.0), and bundled Noto Sans (OFL-1.1). CSV writing uses the .NET runtime. These components do not require a paid commercial license. See [dependency licenses](docs/Development/dependency-licenses.md) for the audited package boundary and font notice. @@ -450,6 +438,7 @@ The read-only agent tool is `file_context_tables_info`; it follows read approval XLSX returns each named table (excluding its header and totals), or a worksheet summary when no named tables exist. Blank and duplicate headers retain column positions. CSV detects comma, semicolon, tab or pipe, or accepts an explicit single-character delimiter. +Inspection materializes the header and scans data records for width and count. Quoted line endings and blank lines remain part of the field. Quoted multiline fields form one record; physically empty lines are skipped. The optional one-based headerRow selects a CSV logical record or worksheet row; otherwise the first nonempty record/row is the header. Named Excel tables own their headers. diff --git a/docs/Architecture.md b/docs/Architecture.md index 56d194a..a00c318 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -80,13 +80,13 @@ flowchart TD ## Operational limits PDF callers share `FileContextPdfProcessor`, acquiring a permit before reading or downloading the -source and retaining it through parsing/rendering. `MaximumConcurrentPdfOperations` defaults to one. +source and retaining it through parsing/rendering when the host sets `MaximumConcurrentPdfOperations`. `FileContextPdfRenderDocument` exposes page count and sequential page rendering from one parsed PDF; dispose it after the batch. The low-level synchronous image helpers remain caller-scheduled APIs. PDF text, page rendering and embedded-image APIs accept bounded seekable streams. Storage-backed PDF tools retain seekable local `FileStream`/`MemoryStream` inputs but asynchronously stage other streams, including seekable cloud streams, to an automatically deleted temporary file. PdfPig's synchronous byte reads and seeks then remain local, without blocking on repeated network ranges. `PdfSourceStagingMode.TemporaryFile` also stages local inputs. `PdfSourceBufferBytes` bounds every copy read and `PdfTemporaryDirectory` optionally selects an existing host directory. No whole-document managed array is created. Native raster decoding has a separate per-page source-image pixel budget; lowering output scale does not reduce source bitmap allocation. -All potentially large operations are controlled by `IOptions`: PDF source/page/image budgets, full-read bytes, range bytes, files scanned, bytes per searched file, matches per file, total search results, graph documents, graph source bytes, and exported graph characters. Non-seekable cloud streams are supported by sequential streaming. +Hosts can configure bounds for large operations through `IOptions`: PDF source/page/image budgets, full-read bytes, range bytes, files scanned, bytes per searched file, matches per file, total search results, graph documents, graph source bytes, and exported graph characters. Non-seekable cloud streams are supported by sequential streaming. ## Start here @@ -112,7 +112,7 @@ flowchart LR ## Native workbook reads -`FileContextDocumentService` opens XLSX packages read-only from the scoped storage adapter. +`FileContextDocumentService` opens XLSX packages read-only from the scoped storage adapter. Local seekable streams can be read directly when the host has left the source budget unset. Other streams and reads with a configured source budget are staged before parsing. `FileContextWorkbookReader` maps worksheets and cell rectangles into package-owned result records. It preserves coordinates and stored types without evaluating formulas or accessing external links. The same service opens DOCX packages read-only and streams bounded paragraph text through diff --git a/docs/Features/ReportRendering.md b/docs/Features/ReportRendering.md index 10221cd..c9a1ef0 100644 --- a/docs/Features/ReportRendering.md +++ b/docs/Features/ReportRendering.md @@ -17,13 +17,13 @@ Block content is JSON: - `heading`: `title`, optional `subtitle`. - `text`: `text` with bold spans (`**bold**`), dash bullets, and HTTPS Markdown links. -- `kpi-row`: `items` with 2-6 entries containing `label`, `value`, optional `change`, and optional `indicator` (`up` or `down`). +- `kpi-row`: `items` with entries containing `label`, `value`, optional `change`, and optional `indicator` (`up` or `down`). - `chart`: `title`, `chartType` (`line`, `bar`, `stacked-bar`, `pie`, `donut`), `labels`, `series` entries with `name` and numeric `values`, optional `xAxis` and `yAxis`. - `table`: `columns` with `key`, `label`, optional .NET numeric `format`; `rows` as objects; optional `totals` object and `highlightCells` entries with zero-based `row` and column `key`. - `image`: base64 PNG/JPEG bytes in `base64`, optional `caption`. - `callout`: `text`. -- `page-break`: empty content. +- `page-break`: omit `content` or pass an empty object. -All requested outputs render in memory before FileContext stores any file. The combined size of the outputs must fit `MaximumGeneratedFileBytes`. `MaximumReportInputBytes` and `MaximumReportPages` default to 10 MB and 60 pages. `OperationTimeout` covers rendering and upload, with a 60-second report default. Image blocks are capped at 5 MB. PDF chart and table rows are kept intact across pages, table headers repeat after a page break, and each page has a title, NYWD mark or theme logo, UTC date, and page number. Failures name the block index and type where possible. +All requested outputs render in memory before FileContext stores any file. When configured, `MaximumGeneratedFileBytes` applies to the combined size of the outputs. Hosts can also set `MaximumReportInputBytes`, `MaximumReportPages`, and an `OperationTimeout` for rendering and upload. PDF headings can continue onto additional pages, and KPI cards flow onto additional rows and pages as needed. PDF chart and table rows are kept intact across pages, table headers repeat after a page break, and each page has a title, NYWD mark or theme logo, UTC date, and page number. Failures name the block index and type where possible. Named template and theme persistence, network allowlists, operator authorization, and authenticated download links belong to the host. `AIAgents.App` implements that boundary in its `AgentFiles/Reports` module. diff --git a/docs/Features/file-context.md b/docs/Features/file-context.md index 2661dae..7484b7a 100644 --- a/docs/Features/file-context.md +++ b/docs/Features/file-context.md @@ -16,7 +16,7 @@ In scope: standard file access, bounded line navigation, metadata, native PDF/DO 6. Full reads fail before allocation when metadata exceeds `MaximumFullReadBytes`. Decoded UTF-8 limits retain encoder state across buffer boundaries, including split surrogate pairs. 7. Range reads are 1-based by line, stream sequentially, return continuation metadata, and stop at configured line/byte limits. 8. Content search is case-insensitive regular expression search with a timeout, file/match/result limits, and optional standard glob filtering. -9. Directory listing returns direct child directories before direct child files and keeps the storage prefix private. A listing with more than `MaximumDirectoryEntries` distinct children fails with a limit error. The default is 1,000. +9. Directory listing returns direct child directories before direct child files and keeps the storage prefix private. When a host sets `MaximumDirectoryEntries`, a listing that exceeds it fails with a limit error. 10. Markdown graph builds include only `.md` and `.markdown` files that match the configured glob and limits. The default glob selects `.md` files. Builds use `ManagedCode.MarkdownLd.Kb`. 11. Graph search returns package-owned match records; graph export supports JSON-LD, Turtle, Mermaid, and DOT with an output limit. 12. Each graph operation builds from the current selected storage contents, so graph results cannot become stale between calls. @@ -98,7 +98,7 @@ Independent writes and range reads on eight different files are tested concurren ## Generated documents -The shared store and `IFileContext.Documents` create unique text/CSV/XLSX/PDF outputs. Standard listing, metadata, range reads and host streaming access use their returned paths. The document tool factory and context provider share the same implementations; hosts may register a selected subset in a native or JavaScript tool catalog. Writes must be explicitly enabled on shared options and human approval is enabled by default. +The shared store and `IFileContext.Documents` create unique text/CSV/XLSX/PDF outputs. When configured, CSV generation checks the output byte budget as it writes UTF-8, including the BOM and escaped fields. Standard listing, metadata, range reads and host streaming access use their returned paths. The document tool factory and context provider share the same implementations; hosts may register a selected subset in a native or JavaScript tool catalog. Writes must be explicitly enabled on shared options and human approval is enabled by default. ```mermaid sequenceDiagram @@ -125,10 +125,11 @@ delete any staged file. Source ownership continues through document rendering an DOCX reading uses the native `file_context_docx_text` tool. It reads ordinary paragraph and table text from the scoped `.docx` package in bounded windows. Each result includes the next paragraph and character offset when more text remains, so an agent can continue without loading a long -document into one model call. The tool does not execute macros, fetch external links, or perform -OCR on embedded images. Generic text reads refuse DOCX, and text search skips it. -XLSX and DOCX reads also enforce `MaximumOpenXmlPartCharacters` per decompressed XML part. -The default is 32 Mi characters. +document into one model call. Tabs and explicit line breaks are included in paragraph text and +cursor offsets. Text nested in an inline text box belongs to the containing paragraph and keeps the +same paragraph numbering across windows. The tool does not execute macros, fetch external links, +or perform OCR on embedded images. Generic text reads refuse DOCX, and text search skips it. +Hosts can set `MaximumOpenXmlPartCharacters` to bound individual decompressed XLSX and DOCX XML parts. `file_context_pdf_text` reports a bounded text-layer prefix, total page count, and one-based pages with almost no text. It performs no OCR. `file_context_pdf_page_image` renders a complete page as PNG. `file_context_pdf_images_info` counts embedded image objects, and `file_context_pdf_image` returns one object as PNG. The direct `IFileContextPdf` methods and public byte-oriented PDF APIs support the same operations. Storage-scoped PDF reads enforce a byte cap; page rendering enforces pixel and image-byte caps. Image tools return `DataContent`; host chat pipelines must forward it as image content rather than stringify a function result. diff --git a/docs/Security.md b/docs/Security.md index eb564f2..9febac3 100644 --- a/docs/Security.md +++ b/docs/Security.md @@ -12,7 +12,7 @@ - Path normalization rejects rooted, traversal, backslash, null-byte, and ambiguous segment inputs. - `RootPrefix` confines every operation to a configured storage subtree. - Write tools are disabled by default. If enabled, Agent Framework approval is still required by default. -- File reads, individual range lines, searches, graph sources, graph exports, regex execution, and result counts are bounded while streaming; even a giant line is rejected before proportional buffering. +- Hosts can set explicit bounds for file reads, range lines, searches, graph sources, exports, result counts, and regex execution. The library applies the configured byte bounds while streaming. - The provider advertises tools and safe usage instructions; it does not promote file contents into system instructions. - Secrets stay inside the configured storage provider. Tool results expose logical paths and content only, never provider connection details. diff --git a/src/ManagedCode.FileContext/Documents/CsvRecordReader.cs b/src/ManagedCode.FileContext/Documents/CsvRecordReader.cs new file mode 100644 index 0000000..b151e8d --- /dev/null +++ b/src/ManagedCode.FileContext/Documents/CsvRecordReader.cs @@ -0,0 +1,171 @@ +using System.Text; +using Microsoft.VisualBasic.FileIO; + +namespace ManagedCode.FileContext; + +/// Scans logical CSV records and materializes field values only for a requested header. +internal sealed class CsvRecordReader(TextReader reader, char delimiter, CancellationToken token) +{ + private const char Quote = '"'; + private const int EndOfStream = -1; + private readonly char[] _buffer = new char[4_096]; + private int _position; + private int _length; + private long _line = 1; + private bool _previousCarriageReturn; + + public bool TryRead(bool captureFields, out Record record) + { + var fields = captureFields ? new List() : null; + var field = captureFields ? new StringBuilder() : null; + while (Peek() != EndOfStream) + { + token.ThrowIfCancellationRequested(); + fields?.Clear(); + record = ReadRecord(fields, field, out var blank); + if (!blank) { return true; } + } + + record = default; + return false; + } + + private Record ReadRecord(List? fields, StringBuilder? field, out bool blank) + { + var startLine = _line; + var count = 0; + var hasValue = false; + var meaningful = false; + while (true) + { + field?.Clear(); + hasValue |= ReadField(field, startLine, out var hasSyntax); + meaningful |= hasSyntax; + count++; + fields?.Add(field!.ToString()); + + if (Peek() != delimiter) { break; } + meaningful = true; + _position++; + } + + ConsumeRecordEnding(); + blank = !meaningful; + return new Record(count, hasValue, fields); + } + + private bool ReadField(StringBuilder? field, long startLine, out bool meaningful) + { + var hasPadding = false; + meaningful = false; + while (Peek() is var next && IsPadding(next)) + { + _position++; + hasPadding = true; + meaningful |= !char.IsWhiteSpace((char)next); + field?.Append((char)next); + } + + if (Peek() == Quote) + { + _position++; + field?.Clear(); + meaningful = true; + return ReadQuotedField(field, startLine); + } + + var hasText = ReadUnquotedField(field); + meaningful |= hasText; + return hasPadding || hasText; + } + + private bool ReadUnquotedField(StringBuilder? field) + { + var hasValue = false; + while (EnsureBuffered()) + { + var remaining = _buffer.AsSpan(_position, _length - _position); + var ending = remaining.IndexOfAny(delimiter, '\r', '\n'); + var value = ending < 0 ? remaining : remaining[..ending]; + field?.Append(value); + hasValue |= !value.IsEmpty; + _position += value.Length; + if (ending >= 0) { break; } + } + return hasValue; + } + + private bool ReadQuotedField(StringBuilder? field, long startLine) + { + var hasValue = false; + while (EnsureBuffered()) + { + var remaining = _buffer.AsSpan(_position, _length - _position); + var quote = remaining.IndexOf(Quote); + var value = quote < 0 ? remaining : remaining[..quote]; + CountLineEndings(value); + field?.Append(value); + hasValue |= !value.IsEmpty; + _position += value.Length; + if (quote < 0) { continue; } + + _position++; + _previousCarriageReturn = false; + if (Peek() != Quote) + { + while (IsPadding(Peek())) { _position++; } + if (Peek() is var next && next != delimiter && next is not (EndOfStream or '\r' or '\n')) + { throw Malformed(startLine); } + return hasValue; + } + + _position++; + field?.Append(Quote); + hasValue = true; + } + + throw Malformed(startLine); + } + + private void CountLineEndings(ReadOnlySpan value) + { + if (value.IsEmpty) { return; } + + var firstEnding = value.IndexOfAny('\r', '\n'); + if (firstEnding != 0) { _previousCarriageReturn = false; } + if (firstEnding < 0) { return; } + + foreach (var character in value[firstEnding..]) + { + if (character == '\r' || (character == '\n' && !_previousCarriageReturn)) { _line++; } + _previousCarriageReturn = character == '\r'; + } + } + + private void ConsumeRecordEnding() + { + var ending = Peek(); + if (ending == EndOfStream) { return; } + _position++; + if (ending == '\r' && Peek() == '\n') { _position++; } + _line++; + } + + private bool IsPadding(int character) => character >= 0 && character != delimiter && character is not ('\r' or '\n') && + (char.IsWhiteSpace((char)character) || character is '\u200b' or '\ufeff'); + + private int Peek() => EnsureBuffered() ? _buffer[_position] : EndOfStream; + + private bool EnsureBuffered() + { + if (_position < _length) { return true; } + token.ThrowIfCancellationRequested(); + _length = reader.Read(_buffer.AsSpan()); + _position = 0; + return _length > 0; + } + + private static MalformedLineException Malformed(long line) => new($"CSV record starting on line {line} is malformed.", line); + + public readonly record struct Record(int ColumnCount, bool HasValue, IReadOnlyList? Fields); +} diff --git a/src/ManagedCode.FileContext/Documents/FileContextCsvTables.cs b/src/ManagedCode.FileContext/Documents/FileContextCsvTables.cs index 406bd97..395cdf6 100644 --- a/src/ManagedCode.FileContext/Documents/FileContextCsvTables.cs +++ b/src/ManagedCode.FileContext/Documents/FileContextCsvTables.cs @@ -12,42 +12,36 @@ public static FileContextTablesInfo Read(Stream source, string path, int? header if (delimiter.Length != 1 || delimiter[0] is '"' or '\r' or '\n' or '\0') { throw new ArgumentException("CSV delimiter must be one character other than quote, newline or NUL.", nameof(delimiter)); } source.Position = 0; - using var parser = Create(source, delimiter); - string[]? headers = null; + using var reader = CreateReader(source); + var parser = new CsvRecordReader(reader, delimiter[0], token); + IReadOnlyList? headers = null; var record = 0; int? foundHeader = null; long rows = 0; var width = 0; - while (!parser.EndOfData) + while (parser.TryRead(headers is null && record >= (headerRow ?? 1) - 1, out var fields)) { - token.ThrowIfCancellationRequested(); - var fields = parser.ReadFields()!; record++; if (headerRow.HasValue && record < headerRow.Value) { continue; } if (headers is null) { - if (!headerRow.HasValue && fields.All(string.IsNullOrEmpty)) { continue; } - headers = fields; + if (!headerRow.HasValue && !fields.HasValue) { continue; } + headers = fields.Fields!; foundHeader = record; - width = fields.Length; + width = fields.ColumnCount; continue; } - width = Math.Max(width, fields.Length); - if (fields.Any(static field => field.Length > 0)) { rows++; } + width = Math.Max(width, fields.ColumnCount); + if (fields.HasValue) { rows++; } } if (headerRow.HasValue && headers is null) { throw new ArgumentException("The requested CSV header record does not exist.", nameof(headerRow)); } - var names = Enumerable.Range(0, width).Select(index => index < (headers?.Length ?? 0) ? headers![index] : string.Empty).ToArray(); + var names = Enumerable.Range(0, width).Select(index => index < (headers?.Count ?? 0) ? headers![index] : string.Empty).ToArray(); return new(path, "csv", delimiter, [new(Path.GetFileName(path), null, foundHeader, 1, names, rows)]); } - private static TextFieldParser Create(Stream source, string delimiter) - { - var parser = new TextFieldParser(source, new UTF8Encoding(false, true), true, true) - { TextFieldType = FieldType.Delimited, HasFieldsEnclosedInQuotes = true, TrimWhiteSpace = false }; - parser.SetDelimiters(delimiter); - return parser; - } + private static StreamReader CreateReader(Stream source) => + new(source, new UTF8Encoding(false, true), detectEncodingFromByteOrderMarks: true, leaveOpen: true); private static string DetectDelimiter(Stream source, int? headerRow, CancellationToken token) { @@ -57,17 +51,16 @@ private static string DetectDelimiter(Stream source, int? headerRow, Cancellatio { token.ThrowIfCancellationRequested(); source.Position = 0; - using var parser = Create(source, candidate); + using var reader = CreateReader(source); + var parser = new CsvRecordReader(reader, candidate[0], token); try { var record = 0; - while (!parser.EndOfData) + while (parser.TryRead(captureFields: false, out var fields)) { - token.ThrowIfCancellationRequested(); - var fields = parser.ReadFields()!; record++; if (record < (headerRow ?? 1)) { continue; } - if (fields.Length > columns) { best = candidate; columns = fields.Length; } + if (fields.ColumnCount > columns) { best = candidate; columns = fields.ColumnCount; } break; } } diff --git a/src/ManagedCode.FileContext/Documents/FileContextDocumentService.Docx.cs b/src/ManagedCode.FileContext/Documents/FileContextDocumentService.Docx.cs index 656cd1b..37f33c4 100644 --- a/src/ManagedCode.FileContext/Documents/FileContextDocumentService.Docx.cs +++ b/src/ManagedCode.FileContext/Documents/FileContextDocumentService.Docx.cs @@ -5,21 +5,19 @@ namespace ManagedCode.FileContext; public sealed partial class FileContextDocumentService { public Task ReadDocxTextAsync(string path, int startParagraph = 1, - int startCharacter = 0, int paragraphCount = 20, CancellationToken cancellationToken = default) + int startCharacter = 0, int paragraphCount = FileContextDefaults.MaximumDocxParagraphsPerRead, CancellationToken cancellationToken = default) { RequireExtension(path, ".docx"); ArgumentOutOfRangeException.ThrowIfLessThan(startParagraph, 1); ArgumentOutOfRangeException.ThrowIfNegative(startCharacter); ArgumentOutOfRangeException.ThrowIfLessThan(paragraphCount, 1); - ArgumentOutOfRangeException.ThrowIfGreaterThan(paragraphCount, - FileContextDefaults.MaximumDocxParagraphsPerRead); return ReadSourceAsync(path, (buffer, token) => { using var document = WordprocessingDocument.Open(buffer, false, new OpenSettings { MaxCharactersInPart = options.MaximumOpenXmlPartCharacters }); return FileContextDocxReader.Read(document, path, startParagraph, startCharacter, - paragraphCount, token); - }, options.MaximumDocxReadBytes, cancellationToken); + paragraphCount, options.MaximumDocxTextCharacters, token); + }, options.DocxSourceReadBudget, cancellationToken); } } diff --git a/src/ManagedCode.FileContext/Documents/FileContextDocumentService.Reading.cs b/src/ManagedCode.FileContext/Documents/FileContextDocumentService.Reading.cs index cf3289c..238fec3 100644 --- a/src/ManagedCode.FileContext/Documents/FileContextDocumentService.Reading.cs +++ b/src/ManagedCode.FileContext/Documents/FileContextDocumentService.Reading.cs @@ -27,24 +27,42 @@ private Task ReadWorkbookAsync(string path, Func ReadSourceAsync(string path, Func read, + private Task ReadSourceAsync(string path, Func read, long maximumBytes, CancellationToken cancellationToken) => FileContextOperation.RunAsync(options.OperationTimeout, async token => { + var source = await OpenScopedSourceAsync(path, maximumBytes, token).ConfigureAwait(false); + await using var lifetime = source.ConfigureAwait(false); + if (maximumBytes == long.MaxValue && source is FileStream or MemoryStream && source.CanSeek && source.Position == 0) + { + token.ThrowIfCancellationRequested(); + return read(source, token); + } + using var buffer = new MemoryStream(); - await CopyScopedSourceAsync(path, buffer, maximumBytes, token).ConfigureAwait(false); + await CopySourceAsync(source, buffer, maximumBytes, token).ConfigureAwait(false); token.ThrowIfCancellationRequested(); return read(buffer, token); }, cancellationToken); private async Task CopyScopedSourceAsync(string path, MemoryStream buffer, long maximumBytes, CancellationToken token) + { + var source = await OpenScopedSourceAsync(path, maximumBytes, token).ConfigureAwait(false); + await using var lifetime = source.ConfigureAwait(false); + await CopySourceAsync(source, buffer, maximumBytes, token).ConfigureAwait(false); + } + + private async Task OpenScopedSourceAsync(string path, long maximumBytes, CancellationToken token) { var metadata = await store.GetMetadataAsync(path, token).ConfigureAwait(false) ?? throw new FileNotFoundException("The document was not found in this file context.", path); if (metadata.Length > (ulong)maximumBytes) { throw new IOException("Document exceeds the configured source read budget."); } - var source = await store.OpenReadInOperationAsync(path, token).ConfigureAwait(false); - await using var lifetime = source.ConfigureAwait(false); + return await store.OpenReadInOperationAsync(path, token).ConfigureAwait(false); + } + + private static async Task CopySourceAsync(Stream source, MemoryStream buffer, long maximumBytes, CancellationToken token) + { using var bounded = new StorageReadBudgetStream(source, maximumBytes); await bounded.CopyToAsync(buffer, StorageReadBudgetStream.DefaultCopyBufferSize, token).ConfigureAwait(false); buffer.Position = 0; diff --git a/src/ManagedCode.FileContext/Documents/FileContextDocumentService.cs b/src/ManagedCode.FileContext/Documents/FileContextDocumentService.cs index fb53c68..c2fc972 100644 --- a/src/ManagedCode.FileContext/Documents/FileContextDocumentService.cs +++ b/src/ManagedCode.FileContext/Documents/FileContextDocumentService.cs @@ -8,16 +8,19 @@ public sealed partial class FileContextDocumentService(ManagedCodeStorageFileSto public Task CreateTextAsync(string fileName, string text, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(text); - return CreateAsync(fileName, "text/plain; charset=utf-8", _ => + return CreateAsync(fileName, "text/plain; charset=utf-8", (limit, _) => { - StorageFileCreator.EnsureWithinLimit(Encoding.UTF8.GetByteCount(text), options.MaximumGeneratedFileBytes); - return Encoding.UTF8.GetBytes(text); + if (limit != long.MaxValue) + { + StorageFileCreator.EnsureWithinLimit(Encoding.UTF8.GetByteCount(text), limit); + } + return new MemoryStream(Encoding.UTF8.GetBytes(text), writable: false); }, cancellationToken); } public Task CreateCsvAsync(string fileName, FileContextCsvDocument document, CancellationToken cancellationToken = default) => CreateAsync(RequireExtension(fileName, ".csv"), "text/csv; charset=utf-8", - token => FileContextCsvWriter.Create(document, token), cancellationToken); + (limit, token) => FileContextCsvWriter.Create(document, limit, token), cancellationToken); public async Task CreateWorkbookAsync(string fileName, FileContextWorkbook workbook, CancellationToken cancellationToken = default) { @@ -41,19 +44,19 @@ public async Task CreateWorkbookAsync(string fileName, F public Task CreatePdfAsync(string fileName, FileContextPdfDocument document, CancellationToken cancellationToken = default) => CreateAsync(RequireExtension(fileName, ".pdf"), "application/pdf", - token => FileContextPdfWriter.Create(document, token), cancellationToken); + (_, token) => new MemoryStream(FileContextPdfWriter.Create(document, token), writable: false), cancellationToken); private Task CreateAsync(string fileName, string mediaType, - Func generate, CancellationToken cancellationToken) + Func generate, CancellationToken cancellationToken) { options.Validate(); if (!options.EnableWriteTools) { throw new InvalidOperationException("Document creation requires EnableWriteTools."); } var normalized = StoragePathScope.Normalize(fileName); return FileContextOperation.RunAsync(options.OperationTimeout, async token => { - var bytes = generate(token); - using var content = new MemoryStream(bytes, writable: false); - return await store.CreatePreparedFileAsync(normalized, content, mediaType, options.MaximumGeneratedFileBytes, token).ConfigureAwait(false); + var limit = store.GetGeneratedFileLimit(options.MaximumGeneratedFileBytes); + using var content = generate(limit, token); + return await store.CreatePreparedFileAsync(normalized, content, mediaType, limit, token).ConfigureAwait(false); }, cancellationToken); } @@ -73,8 +76,7 @@ public async Task CreateReportAsync(FileContextReport r options.Validate(); if (!options.EnableWriteTools) { throw new InvalidOperationException("Document creation requires EnableWriteTools."); } var limit = store.GetGeneratedFileLimit(options.MaximumGeneratedFileBytes); - var timeout = options.OperationTimeout ?? TimeSpan.FromSeconds(FileContextDefaults.ReportTimeoutSeconds); - return await FileContextOperation.RunAsync(timeout, async token => + return await FileContextOperation.RunAsync(options.OperationTimeout, async token => { var outputs = FileContextReportRenderer.Render(report, options, limit, token); var files = new List(outputs.Count); diff --git a/src/ManagedCode.FileContext/Documents/FileContextDocxReader.cs b/src/ManagedCode.FileContext/Documents/FileContextDocxReader.cs index 3443f05..ae98018 100644 --- a/src/ManagedCode.FileContext/Documents/FileContextDocxReader.cs +++ b/src/ManagedCode.FileContext/Documents/FileContextDocxReader.cs @@ -1,3 +1,4 @@ +using System.Text; using DocumentFormat.OpenXml; using DocumentFormat.OpenXml.Packaging; using DocumentFormat.OpenXml.Wordprocessing; @@ -7,25 +8,22 @@ namespace ManagedCode.FileContext; internal static class FileContextDocxReader { public static FileContextDocxText Read(WordprocessingDocument document, string path, - int startParagraph, int startCharacter, int paragraphCount, CancellationToken cancellationToken) + int startParagraph, int startCharacter, int paragraphCount, int maximumCharacters, CancellationToken cancellationToken) { var main = document.MainDocumentPart ?? throw new InvalidDataException("The DOCX has no main document part."); using var reader = OpenXmlReader.Create(main); var paragraphs = new List(); var number = 0; - var remaining = FileContextDefaults.MaximumDocxTextCharacters; - while (reader.Read()) + var remaining = maximumCharacters == int.MaxValue ? long.MaxValue : maximumCharacters; + reader.Read(); + while (MoveToParagraph(reader, cancellationToken)) { - cancellationToken.ThrowIfCancellationRequested(); - if (!reader.IsStartElement || reader.ElementType != typeof(Paragraph)) - { - continue; - } - number++; if (number < startParagraph) { + // Read and skipped paragraphs must consume the same subtree, including text boxes. + reader.Skip(); continue; } if (paragraphs.Count == paragraphCount || remaining == 0) @@ -35,14 +33,14 @@ public static FileContextDocxText Read(WordprocessingDocument document, string p var paragraph = reader.LoadCurrentElement() as Paragraph ?? throw new InvalidDataException("A DOCX paragraph could not be read."); - var text = paragraph.InnerText; + var text = ReadParagraphText(paragraph, cancellationToken); var offset = number == startParagraph ? startCharacter : 0; if (offset > text.Length || SplitsSurrogatePair(text, offset)) { throw new ArgumentOutOfRangeException(nameof(startCharacter), "The character offset must be within this paragraph and at a Unicode character boundary."); } - var length = Math.Min(text.Length - offset, remaining); + var length = (int)Math.Min(text.Length - offset, remaining); if (SplitsSurrogatePair(text, offset + length)) { length--; } if (length > 0 || offset == text.Length) { @@ -53,11 +51,42 @@ public static FileContextDocxText Read(WordprocessingDocument document, string p { return new FileContextDocxText(path, paragraphs, number, offset + length); } + reader.Read(); } return new FileContextDocxText(path, paragraphs, null, 0); } + private static string ReadParagraphText(Paragraph paragraph, CancellationToken token) + { + var text = new StringBuilder(); + foreach (var element in paragraph.Descendants()) + { + token.ThrowIfCancellationRequested(); + switch (element) + { + case TabChar or PositionalTab: text.Append('\t'); break; + case Break or CarriageReturn: text.Append('\n'); break; + case { HasChildren: false }: text.Append(element.InnerText); break; + } + } + return text.ToString(); + } + + private static bool MoveToParagraph(OpenXmlReader reader, CancellationToken token) + { + while (!reader.EOF) + { + token.ThrowIfCancellationRequested(); + if (reader.IsStartElement && reader.ElementType == typeof(Paragraph)) + { + return true; + } + reader.Read(); + } + return false; + } + private static bool SplitsSurrogatePair(string text, int index) => index > 0 && index < text.Length && char.IsSurrogatePair(text, index - 1); } diff --git a/src/ManagedCode.FileContext/Documents/FileContextTableReader.cs b/src/ManagedCode.FileContext/Documents/FileContextTableReader.cs index adbec81..862a335 100644 --- a/src/ManagedCode.FileContext/Documents/FileContextTableReader.cs +++ b/src/ManagedCode.FileContext/Documents/FileContextTableReader.cs @@ -26,7 +26,8 @@ public static FileContextTablesInfo Read(Stream source, string fileName, int? he result = new(fileName, "xlsx", null, FileContextWorkbookTables.Read(workbook, headerRow, cancellationToken)); } else { result = FileContextCsvTables.Read(source, fileName, headerRow, delimiter, cancellationToken); } - if (Encoding.UTF8.GetByteCount(JsonSerializer.Serialize(result)) > options.MaximumRangeReadBytes) + if (options.MaximumRangeReadBytes != long.MaxValue && + JsonSerializer.SerializeToUtf8Bytes(result).LongLength > options.MaximumRangeReadBytes) { throw new IOException("Table metadata exceeds the configured output read budget."); } return result; } diff --git a/src/ManagedCode.FileContext/Documents/FileContextWorkbookReader.cs b/src/ManagedCode.FileContext/Documents/FileContextWorkbookReader.cs index 0187f18..8a2ae1a 100644 --- a/src/ManagedCode.FileContext/Documents/FileContextWorkbookReader.cs +++ b/src/ManagedCode.FileContext/Documents/FileContextWorkbookReader.cs @@ -55,8 +55,7 @@ public static FileContextWorkbookRange Read(SpreadsheetDocument document, string ?? throw new ArgumentException("The requested worksheet does not exist.", nameof(sheetName)); var worksheet = workbook.GetPartById(sheet.Id!) as WorksheetPart ?? throw new InvalidDataException("The requested sheet is not a worksheet."); - var strings = workbook.SharedStringTablePart?.SharedStringTable?.Elements() - .Select(static item => string.Concat(item.Descendants().Select(text => text.Text))).ToArray() ?? []; + var strings = ReadSharedStrings(workbook); var cells = ReadCells(worksheet, strings, startRow, rowCount, startColumn, columnCount, byteLimit, token); var range = $"{ColumnName(startColumn)}{startRow}:{ColumnName(startColumn + columnCount - 1)}{startRow + rowCount - 1}"; return new(path, sheetName, range, cells); @@ -129,18 +128,10 @@ private static int ReadRowNumber(OpenXmlReader reader, int previous) return previous + 1; } - internal static FileContextWorkbookCell ReadCell(Cell cell, string[] strings, string address) + private static FileContextWorkbookCell ReadCell(Cell cell, string[] strings, string address) { var type = cell.DataType?.Value; - var value = cell.CellValue?.Text ?? string.Empty; - if (type == CellValues.SharedString) - { - if (!int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var index) || index < 0 || index >= strings.Length) - { throw new InvalidDataException("Workbook cell contains an invalid shared-string reference."); } - value = strings[index]; - } - if (type == CellValues.InlineString) - { value = string.Concat(cell.InlineString?.Descendants().Select(static text => text.Text) ?? []); } + var value = ReadCellValue(cell, strings, type); var kind = "number"; if (type == CellValues.SharedString || type == CellValues.InlineString || type == CellValues.String) { kind = "text"; } else if (type == CellValues.Boolean) { kind = "boolean"; } @@ -150,6 +141,27 @@ internal static FileContextWorkbookCell ReadCell(Cell cell, string[] strings, st return new(address, kind, value, cell.CellFormula?.Text); } + internal static string ReadCellValue(Cell cell, string[] strings, CellValues? type) + { + if (type == CellValues.InlineString) { return ReadBaseText(cell.InlineString); } + var value = cell.CellValue?.Text ?? string.Empty; + if (type != CellValues.SharedString) { return value; } + if (!int.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out var index) || index < 0 || index >= strings.Length) + { throw new InvalidDataException("Workbook cell contains an invalid shared-string reference."); } + return strings[index]; + } + + internal static string[] ReadSharedStrings(WorkbookPart workbook) => + workbook.SharedStringTablePart?.SharedStringTable?.Elements().Select(ReadBaseText).ToArray() ?? []; + + private static string ReadBaseText(RstType? value) + { + if (value is null) { return string.Empty; } + if (value.FirstChild is Text text && ReferenceEquals(text, value.LastChild)) { return text.Text; } + return string.Concat(value.Descendants() + .Where(static text => text.Parent is not PhoneticRun).Select(static text => text.Text)); + } + private static WorkbookPart RequireWorkbook(SpreadsheetDocument document) => document.WorkbookPart ?? throw new InvalidDataException("The package has no Excel workbook."); diff --git a/src/ManagedCode.FileContext/Documents/FileContextWorkbookTables.cs b/src/ManagedCode.FileContext/Documents/FileContextWorkbookTables.cs index eef1b89..ea92e48 100644 --- a/src/ManagedCode.FileContext/Documents/FileContextWorkbookTables.cs +++ b/src/ManagedCode.FileContext/Documents/FileContextWorkbookTables.cs @@ -15,8 +15,7 @@ public static IReadOnlyList Read(SpreadsheetDocument docum { if (headerRow > MaximumRows) { throw new ArgumentOutOfRangeException(nameof(headerRow)); } var workbook = document.WorkbookPart ?? throw new InvalidDataException("The package has no Excel workbook."); - var strings = workbook.SharedStringTablePart?.SharedStringTable?.Elements() - .Select(static item => string.Concat(item.Descendants().Select(text => text.Text))).ToArray() ?? []; + var strings = FileContextWorkbookReader.ReadSharedStrings(workbook); var result = new List(); foreach (var sheet in workbook.Workbook?.Sheets?.Elements() ?? []) { @@ -51,6 +50,7 @@ private static FileContextTableInfo Inspect(WorksheetPart part, string[] strings { var accumulator = new TableAccumulator(bounds); using var reader = OpenXmlReader.Create(part); + var headerCells = new Dictionary(); var rowNumber = 0; while (reader.Read()) { @@ -59,20 +59,24 @@ private static FileContextTableInfo Inspect(WorksheetPart part, string[] strings var row = (Row)reader.LoadCurrentElement()!; rowNumber = row.RowIndex is null ? rowNumber + 1 : checked((int)row.RowIndex.Value); if (rowNumber < bounds.FirstRow || rowNumber > bounds.LastRow) { continue; } - var cells = new Dictionary(); + var captureHeader = accumulator.ShouldCaptureHeader(rowNumber); var hasData = false; var column = 0; + var lastColumn = 0; foreach (var cell in row.Elements()) { token.ThrowIfCancellationRequested(); column = cell.CellReference is null ? column + 1 : FileContextWorkbookReader.ReadColumn(cell.CellReference.Value!); if (column < bounds.StartColumn || column > bounds.EndColumn) { continue; } - var value = FileContextWorkbookReader.ReadCell(cell, strings, string.Empty); - hasData |= value.Value.Length > 0 || value.Formula is not null; - if (value.Value.Length > 0 || value.Formula is not null || cell.InlineString is not null || cell.CellValue is not null) - { cells[column] = value.Value; } + var value = FileContextWorkbookReader.ReadCellValue(cell, strings, cell.DataType?.Value); + var containsData = value.Length > 0 || cell.CellFormula?.Text is not null; + hasData |= containsData; + if (!containsData && cell.InlineString is null && cell.CellValue is null) { continue; } + lastColumn = Math.Max(lastColumn, column); + if (captureHeader) { headerCells[column] = value; } } - accumulator.Add(rowNumber, cells, hasData); + accumulator.Add(rowNumber, headerCells, lastColumn, hasData); + headerCells.Clear(); } return accumulator.Result(); } @@ -91,7 +95,9 @@ private sealed class TableAccumulator(TableBounds bounds) private int lastColumn = bounds.Names is null ? bounds.StartColumn - 1 : bounds.EndColumn; private long rows; - public void Add(int row, Dictionary cells, bool hasData) + public bool ShouldCaptureHeader(int row) => row == headerRow || (headerRow is null && bounds.Names is null); + + public void Add(int row, Dictionary cells, int rowLastColumn, bool hasData) { if (headerRow.HasValue && row < headerRow) { return; } if (bounds.Names is null && !headerRow.HasValue && hasData) { headerRow = row; } @@ -101,7 +107,7 @@ public void Add(int row, Dictionary cells, bool hasData) foreach (var (column, value) in cells) { headers[column] = value; } } else if ((headerRow.HasValue || bounds.Names is not null) && hasData) { rows++; } - if (cells.Count > 0 && (hasData || row == headerRow)) { lastColumn = Math.Max(lastColumn, cells.Keys.Max()); } + if (hasData || row == headerRow) { lastColumn = Math.Max(lastColumn, rowLastColumn); } } public FileContextTableInfo Result() diff --git a/src/ManagedCode.FileContext/Documents/Generation/FileContextCsvWriter.cs b/src/ManagedCode.FileContext/Documents/Generation/FileContextCsvWriter.cs index e987119..5ff6905 100644 --- a/src/ManagedCode.FileContext/Documents/Generation/FileContextCsvWriter.cs +++ b/src/ManagedCode.FileContext/Documents/Generation/FileContextCsvWriter.cs @@ -1,10 +1,14 @@ +using System.Buffers; using System.Text; namespace ManagedCode.FileContext; internal static class FileContextCsvWriter { - public static byte[] Create(FileContextCsvDocument document, CancellationToken cancellationToken) + private const string RowEnding = "\r\n"; + private const string EscapedQuote = "\"\""; + + public static MemoryStream Create(FileContextCsvDocument document, long maximumBytes, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(document); if (document.Rows is null) { throw new ArgumentException("Document members cannot be null.", nameof(document)); } @@ -12,19 +16,71 @@ public static byte[] Create(FileContextCsvDocument document, CancellationToken c { throw new ArgumentException("CSV delimiter must be comma, semicolon or tab.", nameof(document)); } - var text = new StringBuilder(); + var output = new MemoryStream(); + try + { + WriteRows(output, document, maximumBytes, cancellationToken); + return output; + } + catch + { + output.Dispose(); + throw; + } + } + + private static void WriteRows(Stream output, FileContextCsvDocument document, long maximumBytes, CancellationToken token) + { + var bytes = (long)Encoding.UTF8.Preamble.Length; + StorageFileCreator.EnsureWithinLimit(bytes, maximumBytes); + using var writer = new StreamWriter(output, Encoding.UTF8, leaveOpen: true); + var delimiter = document.Delimiter[0]; + var hasLimit = maximumBytes != long.MaxValue; + var quoteTriggers = SearchValues.Create(stackalloc char[] { delimiter, '"', '\r', '\n' }); foreach (var row in document.Rows) { - cancellationToken.ThrowIfCancellationRequested(); + token.ThrowIfCancellationRequested(); if (row is null) { throw new ArgumentException("Document members cannot be null.", nameof(document)); } - text.AppendJoin(document.Delimiter, row.Select(value => Escape(value ?? string.Empty, document.Delimiter))); - text.Append("\r\n"); + if (hasLimit) + { + bytes += RowEnding.Length; + StorageFileCreator.EnsureWithinLimit(bytes, maximumBytes); + } + var first = true; + foreach (var field in row) + { + token.ThrowIfCancellationRequested(); + var value = field.AsSpan(); + var quote = value.ContainsAny(quoteTriggers); + if (hasLimit) + { + bytes += (long)Encoding.UTF8.GetByteCount(value) + (quote ? EscapedQuote.Length + (long)value.Count('"') : 0) + (first ? 0 : 1); + StorageFileCreator.EnsureWithinLimit(bytes, maximumBytes); + } + if (!first) { writer.Write(delimiter); } + WriteField(writer, value, quote); + first = false; + } + writer.Write(RowEnding); } - return Encoding.UTF8.GetPreamble().Concat(Encoding.UTF8.GetBytes(text.ToString())).ToArray(); } - private static string Escape(string value, string delimiter) => - value.Contains(delimiter, StringComparison.Ordinal) || value.IndexOfAny(['"', '\r', '\n']) >= 0 - ? $"\"{value.Replace("\"", "\"\"", StringComparison.Ordinal)}\"" - : value; + private static void WriteField(StreamWriter writer, ReadOnlySpan value, bool quote) + { + if (!quote) + { + writer.Write(value); + return; + } + + writer.Write('"'); + while (value.IndexOf('"') is var index && index >= 0) + { + writer.Write(value[..index]); + writer.Write(EscapedQuote); + value = value[(index + 1)..]; + } + writer.Write(value); + writer.Write('"'); + } } diff --git a/src/ManagedCode.FileContext/Documents/Reports/FileContextReportBlock.cs b/src/ManagedCode.FileContext/Documents/Reports/FileContextReportBlock.cs index 2f2d253..3a39a84 100644 --- a/src/ManagedCode.FileContext/Documents/Reports/FileContextReportBlock.cs +++ b/src/ManagedCode.FileContext/Documents/Reports/FileContextReportBlock.cs @@ -1,4 +1,8 @@ using System.Text.Json; +using System.Text.Json.Serialization; + namespace ManagedCode.FileContext; + /// A report block. Content follows the block JSON schema. -public sealed record FileContextReportBlock(string Type, JsonElement Content); +public sealed record FileContextReportBlock(string Type, + [property: JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] JsonElement Content = default); diff --git a/src/ManagedCode.FileContext/Documents/Reports/FileContextReportHtml.cs b/src/ManagedCode.FileContext/Documents/Reports/FileContextReportHtml.cs index 3e553be..675c988 100644 --- a/src/ManagedCode.FileContext/Documents/Reports/FileContextReportHtml.cs +++ b/src/ManagedCode.FileContext/Documents/Reports/FileContextReportHtml.cs @@ -9,11 +9,10 @@ namespace ManagedCode.FileContext; internal static partial class FileContextReportHtml { - private const int MinimumKpis = 2; - private const int MaximumKpis = 6; private const int ListMarkerLength = 2; - private static readonly Regex Bold = new(@"\*\*(?.+?)\*\*", RegexOptions.Compiled, TimeSpan.FromSeconds(1)); - private static readonly Regex Link = new(@"\[(?[^\]]+)\]\((?https://[^\s)]+)\)", RegexOptions.Compiled, TimeSpan.FromSeconds(1)); + private const RegexOptions InlineRegexOptions = RegexOptions.ExplicitCapture | RegexOptions.NonBacktracking; + private static readonly Regex Bold = new(@"\*\*(?.+?)\*\*", InlineRegexOptions, Regex.InfiniteMatchTimeout); + private static readonly Regex Link = new(@"\[(?[^\]]+)\]\((?https://[^\s)]+)\)", InlineRegexOptions, Regex.InfiniteMatchTimeout); public static string Render(FileContextReport report, FileContextReportTheme theme, CancellationToken token) { @@ -82,7 +81,6 @@ private static void Block(StringBuilder html, FileContextReportBlock block, File private static void Kpis(StringBuilder html, JsonElement content) { var items = ReportJson.Array(content, "items"); - if (items.Count < MinimumKpis || items.Count > MaximumKpis) { throw new ArgumentException("kpi-row.items requires 2 to 6 cards."); } html.Append("
"); foreach (var item in items) { @@ -101,6 +99,7 @@ private static void Table(StringBuilder html, JsonElement content) { var columns = ReportJson.Array(content, "columns"); if (columns.Count == 0) { throw new ArgumentException("table.columns is required."); } + var highlights = ReportJson.ReadHighlights(content); html.Append(""); foreach (var column in columns) { html.Append(""); } @@ -112,7 +111,7 @@ private static void Table(StringBuilder html, JsonElement content) for (var columnIndex = 0; columnIndex < columns.Count; columnIndex++) { var column = columns[columnIndex]; - var highlighted = ReportJson.IsHighlighted(content, rowIndex, ReportJson.Text(column, "key")); + var highlighted = highlights.Contains((rowIndex, ReportJson.Text(column, "key"))); html.Append(highlighted ? ""); } diff --git a/src/ManagedCode.FileContext/Documents/Reports/FileContextReportImage.cs b/src/ManagedCode.FileContext/Documents/Reports/FileContextReportImage.cs index 10eeb90..fb901e6 100644 --- a/src/ManagedCode.FileContext/Documents/Reports/FileContextReportImage.cs +++ b/src/ManagedCode.FileContext/Documents/Reports/FileContextReportImage.cs @@ -2,15 +2,14 @@ namespace ManagedCode.FileContext; internal static class FileContextReportImage { - private const int MaximumImageBytes = 5_000_000; private const int MinimumImageBytes = 8; private static readonly byte[] PngSignature = [137, 80, 78, 71]; private static readonly byte[] JpegSignature = [255, 216]; public static string Validate(byte[] bytes) { - if (bytes.Length is < MinimumImageBytes or > MaximumImageBytes) - { throw new ArgumentException("image must be a PNG or JPEG under 5 MB.", nameof(bytes)); } + if (bytes.Length < MinimumImageBytes) + { throw new ArgumentException("image must be a PNG or JPEG.", nameof(bytes)); } if (bytes.AsSpan().StartsWith(PngSignature)) { return "image/png"; } if (bytes.AsSpan().StartsWith(JpegSignature)) diff --git a/src/ManagedCode.FileContext/Documents/Reports/FileContextReportPdf.cs b/src/ManagedCode.FileContext/Documents/Reports/FileContextReportPdf.cs index 93d51ef..7e31e0d 100644 --- a/src/ManagedCode.FileContext/Documents/Reports/FileContextReportPdf.cs +++ b/src/ManagedCode.FileContext/Documents/Reports/FileContextReportPdf.cs @@ -2,6 +2,7 @@ #pragma warning disable S109, MA0015 using System.Globalization; using System.Text.Json; +using System.Text.RegularExpressions; using SkiaSharp; namespace ManagedCode.FileContext; @@ -114,29 +115,36 @@ private void DrawHeading(JsonElement content) var titleLines = ReportCanvas.Wrap(ReportJson.Text(content, "title"), title, pageWidth - Margin * 2); var subtitleLines = ReportCanvas.Wrap(ReportJson.Text(content, "subtitle"), subtitle, pageWidth - Margin * 2); var height = titleLines.Count * 29 + subtitleLines.Count * 17 + 18; - Ensure(height); + if (height <= Bottom - ContentTop) { Ensure(height); } foreach (var line in titleLines) { + token.ThrowIfCancellationRequested(); + Ensure(29); ReportCanvas.Text(canvas, line, Margin, Y + 22, title); Y += 29; } foreach (var line in subtitleLines) { + token.ThrowIfCancellationRequested(); + Ensure(17); ReportCanvas.Text(canvas, line, Margin, Y + 12, subtitle); Y += 17; } Y += 18; } + private static readonly Regex ReportLinks = new(@"\[(?"); + } + + [Fact] + public async Task Html_preserves_long_unmatched_inline_markup() + { + await using var storage = await TestStorageScope.CreateAsync(); + var options = new FileContextOptions { EnableWriteTools = true }; + var store = new ManagedCodeStorageFileStore(storage.Storage, options); + var text = new string('[', 1_000_000) + " **Tail**"; + var report = new FileContextReport("Inline markup", "inline", [Block("text", new { text })], ["html"]); + + var output = await new FileContextService(store, options).Documents.CreateReportAsync(report); + var html = await store.ReadAsync(output.Files.Single().Path); + + html!.ShouldContain(new string('[', 1_000_000) + " Tail"); } [Fact] @@ -173,6 +190,45 @@ public async Task Explicit_page_break_starts_a_new_branded_page() pdf.GetPage(2).Text.ShouldContain("Quarterly report"); } + [Fact] + public void Pdf_wrap_preserves_exact_fit_overflow_empty_lines_and_unicode() + { + using var paint = ReportCanvas.Paint(SkiaSharp.SKColors.Black, 10); + var width = ReportCanvas.MeasureText("WW", paint); + + var lines = ReportCanvas.Wrap("WW\r\nWWW\n\nЇ😀", paint, width); + + lines.ShouldBe(["WW", "WW", "W", "", "Ї😀"]); + } + + [Fact] + public async Task Long_pdf_heading_flows_across_pages_and_honors_an_explicit_page_budget() + { + await using var storage = await TestStorageScope.CreateAsync(); + var options = new FileContextOptions { EnableWriteTools = true }; + var store = new ManagedCodeStorageFileStore(storage.Storage, options); + var context = new FileContextService(store, options); + var title = string.Join('\n', Enumerable.Range(1, 30).Select(index => $"Section {index}")); + var report = new FileContextReport("Overview", "overview", [ + Block("heading", new { title, subtitle = "Final subtitle" }), + Block("text", new { text = "Body follows" }) + ], ["pdf"]); + + var output = (await context.Documents.CreateReportAsync(report)).Files.Single(); + await using var stream = await store.OpenReadAsync(output.Path); + using var document = PdfDocument.Open(stream); + document.NumberOfPages.ShouldBe(2); + document.GetPage(1).Text.ShouldContain("Section 1"); + document.GetPage(2).Text.ShouldContain("Section 30"); + document.GetPage(2).Text.ShouldContain("Final subtitle"); + document.GetPage(2).Text.ShouldContain("Body follows"); + + options.MaximumReportPages = 1; + var error = await Should.ThrowAsync(() => context.Documents.CreateReportAsync(report)); + error.Message.ShouldContain("exceeds 1 pages"); + (await context.ListFilesAsync()).Select(static file => file.Path).ShouldBe([output.Path]); + } + [Fact] public async Task Unsupported_image_fails_before_any_requested_output_is_written() { diff --git a/tests/ManagedCode.FileContext.Tests/ReportInputValidationTests.cs b/tests/ManagedCode.FileContext.Tests/ReportInputValidationTests.cs new file mode 100644 index 0000000..9cb24be --- /dev/null +++ b/tests/ManagedCode.FileContext.Tests/ReportInputValidationTests.cs @@ -0,0 +1,63 @@ +using System.Text.Json; +using UglyToad.PdfPig; + +namespace ManagedCode.FileContext.Tests; + +public sealed class ReportInputValidationTests +{ + private static readonly JsonSerializerOptions StrictJson = new() { RespectRequiredConstructorParameters = true }; + + [Fact] + public async Task Page_break_can_omit_content() + { + await using var storage = await TestStorageScope.CreateAsync(); + var options = new FileContextOptions { EnableWriteTools = true }; + var store = new ManagedCodeStorageFileStore(storage.Storage, options); + var context = new FileContextService(store, options); + var pageBreak = JsonSerializer.Deserialize("""{"Type":"page-break"}""", StrictJson)!; + var report = new FileContextReport("Report", "pages.pdf", + [pageBreak, new("text", JsonSerializer.SerializeToElement(new { text = "Second page" }))], ["pdf"]); + + var result = await context.Documents.CreateReportAsync(report); + + await using var stream = await store.OpenReadAsync(result.Files.Single().Path); + using var pdf = PdfDocument.Open(stream); + pdf.NumberOfPages.ShouldBe(2); + pdf.GetPage(2).Text.ShouldContain("Second page"); + } + + [Fact] + public async Task Null_block_reports_its_index_without_persisting_any_files() + { + await using var storage = await TestStorageScope.CreateAsync(); + var options = new FileContextOptions { EnableWriteTools = true }; + var context = new FileContextService(new ManagedCodeStorageFileStore(storage.Storage, options), options); + var report = new FileContextReport("Report", "bad.html", [null!], ["html"]); + + var error = await Should.ThrowAsync(() => context.Documents.CreateReportAsync(report)); + + error.ParamName.ShouldBe("report"); + error.Message.ShouldContain("blocks[0]"); + (await context.ListFilesAsync()).ShouldBeEmpty(); + } + + [Fact] + public async Task Input_budget_counts_the_serialized_utf8_bytes_at_the_exact_boundary() + { + await using var storage = await TestStorageScope.CreateAsync(); + var report = new FileContextReport("Звіт 😀", "budget.html", [], ["html"]); + var options = new FileContextOptions + { + EnableWriteTools = true, + MaximumReportInputBytes = JsonSerializer.SerializeToUtf8Bytes(report).Length, + }; + var context = new FileContextService(new ManagedCodeStorageFileStore(storage.Storage, options), options); + + var first = await context.Documents.CreateReportAsync(report); + options.MaximumReportInputBytes--; + var error = await Should.ThrowAsync(() => context.Documents.CreateReportAsync(report)); + + error.Message.ShouldContain("Report input exceeds"); + (await context.ListFilesAsync()).Select(static file => file.Path).ShouldBe([first.Files.Single().Path]); + } +} diff --git a/tests/ManagedCode.FileContext.Tests/SearchStreamingTests.cs b/tests/ManagedCode.FileContext.Tests/SearchStreamingTests.cs index 1cee586..af9f809 100644 --- a/tests/ManagedCode.FileContext.Tests/SearchStreamingTests.cs +++ b/tests/ManagedCode.FileContext.Tests/SearchStreamingTests.cs @@ -29,6 +29,19 @@ public async Task Search_preserves_matching_lines_across_buffers_and_mixed_newli matches.Select(static match => (match.LineNumber, match.Line)).ShouldBe([(2, longLine), (3, "target last")]); } + [Fact] + public async Task Search_reuses_long_line_storage_without_retaining_stale_matches() + { + var first = new string('a', 17_000) + "target"; + var nonmatch = new string('é', 8_500); + var last = new string('z', 9_500) + "target"; + using var stream = new MemoryStream(Encoding.UTF8.GetBytes($"{first}\r\n{nonmatch}\n{last}")); + + var matches = await StorageFileSearcher.SearchLinesAsync(stream, Target(), new(), CancellationToken.None); + + matches.Select(static match => (match.LineNumber, match.Line)).ShouldBe([(1, first), (3, last)]); + } + [GeneratedRegex("target", RegexOptions.CultureInvariant, 2_000)] private static partial Regex Target(); } diff --git a/tests/ManagedCode.FileContext.Tests/TableInspectionTests.cs b/tests/ManagedCode.FileContext.Tests/TableInspectionTests.cs index b1550c5..c0c2faf 100644 --- a/tests/ManagedCode.FileContext.Tests/TableInspectionTests.cs +++ b/tests/ManagedCode.FileContext.Tests/TableInspectionTests.cs @@ -64,6 +64,23 @@ public async Task Workbook_uses_actual_rows_not_declared_dimension_and_retains_b await Should.ThrowAsync(() => context.Documents.GetTablesInfoAsync(file.Path, headerRow: 100)); } + [Fact] + public async Task Workbook_explicit_header_counts_formula_only_rows_and_keeps_trailing_empty_cells() + { + await using var scope = await TestStorageScope.CreateAsync(); + var options = new FileContextOptions { EnableWriteTools = true }; + var context = new FileContextService(new ManagedCodeStorageFileStore(scope.Storage, options), options); + var file = await context.Documents.CreateWorkbookAsync("headers.xlsx", new([new("Data", [ + [new(Text: "Preface"), new(), new(), new(Text: "Outside the table")], [], [new(Text: "Label")], + [new(), new(Formula: "1+1")], [new(), new(), new(), new(Text: "")], [new(Text: "Data"), new(), new(Text: "")]])])); + + var table = (await context.Documents.GetTablesInfoAsync(file.Path, headerRow: 3)).Tables.Single(); + + table.HeaderRow.ShouldBe(3); + table.Headers.ShouldBe(["Label", "", ""]); + table.RowCount.ShouldBe(2); + } + [Fact] public async Task Named_excel_tables_exclude_totals_and_keep_offsets() { @@ -117,4 +134,77 @@ public void Malformed_csv_cancellation_and_budgets_fail_without_partial_metadata Should.Throw(() => FileContextTableReader.Read(source, "file.csv", options: new() { MaximumFullReadBytes = 1 })); Should.Throw(() => FileContextTableReader.Read(source, "file.csv", options: new() { MaximumRangeReadBytes = 1 })); } + + [Fact] + public void Csv_counts_a_quoted_record_beyond_the_former_parser_ceiling() + { + using var source = new MemoryStream(); + var chunk = new string('x', 5_100_000); + using (var writer = new StreamWriter(source, leaveOpen: true)) + { + writer.Write("Header,Second\n\""); + writer.Write(chunk); + writer.Write('\n'); + writer.Write(chunk); + writer.Write("\",1\nlast,2"); + } + + var table = FileContextTableReader.Read(source, "file.csv", delimiter: ",").Tables.Single(); + + table.Headers.ShouldBe(["Header", "Second"]); + table.RowCount.ShouldBe(2); + } + + [Fact] + public void Csv_preserves_blank_lines_and_crlf_inside_quoted_headers() + { + using var source = new MemoryStream(Encoding.UTF8.GetBytes(" \"line 1\r\n\r\nline 3\" ,second\r\n,,\r\nlast,\"\"")); + + var table = FileContextTableReader.Read(source, "file.csv").Tables.Single(); + + table.Headers.ShouldBe(["line 1\r\n\r\nline 3", "second", ""]); + table.RowCount.ShouldBe(1); + } + + [Fact] + public async Task Csv_round_trip_preserves_columns_in_tab_only_records() + { + await using var scope = await TestStorageScope.CreateAsync(); + var options = new FileContextOptions { EnableWriteTools = true }; + var context = new FileContextService(new ManagedCodeStorageFileStore(scope.Storage, options), options); + var csv = await context.Documents.CreateCsvAsync("ragged.csv", new([["A", "B"], ["1", "2"], ["", "", "", ""]], "\t")); + + var table = (await context.Documents.GetTablesInfoAsync(csv.Path)).Tables.Single(); + + table.Headers.ShouldBe(["A", "B", "", ""]); + table.RowCount.ShouldBe(1); + } + + [Fact] + public void Csv_handles_escaped_quotes_across_buffers_and_reports_the_malformed_physical_line() + { + var prefix = new string('x', 4_094); + var header = "\"" + prefix + "\"\"tail\",second\r\n"; + using var source = new MemoryStream(Encoding.UTF8.GetBytes(header + "value,1")); + var table = FileContextTableReader.Read(source, "file.csv", delimiter: ",").Tables.Single(); + table.Headers.ShouldBe([prefix + "\"tail", "second"]); + table.RowCount.ShouldBe(1); + + using var malformed = new MemoryStream(Encoding.UTF8.GetBytes(header + "\"value\"junk,1")); + var error = Should.Throw(() => FileContextTableReader.Read(malformed, "file.csv", delimiter: ",")); + error.LineNumber.ShouldBe(2); + } + + [Fact] + public void Csv_counts_crlf_split_across_buffers_and_standalone_line_endings() + { + const string prefix = "key,value\r\nkey,\""; + var padding = new string('x', 4_095 - prefix.Length); + var record = prefix + padding + "\r\nfirst\nsecond\rthird\"\r\n"; + using var source = new MemoryStream(Encoding.UTF8.GetBytes(record + "\"unterminated")); + + var error = Should.Throw(() => FileContextTableReader.Read(source, "file.csv", delimiter: ",")); + + error.LineNumber.ShouldBe(6); + } } diff --git a/tests/ManagedCode.FileContext.Tests/WorkbookReadingTests.cs b/tests/ManagedCode.FileContext.Tests/WorkbookReadingTests.cs index 07e7550..3c2c6f8 100644 --- a/tests/ManagedCode.FileContext.Tests/WorkbookReadingTests.cs +++ b/tests/ManagedCode.FileContext.Tests/WorkbookReadingTests.cs @@ -111,6 +111,34 @@ public async Task Workbook_info_preserves_chart_sheet_names_without_reading_them info.Sheets[1].Dimension.ShouldBeNull(); } + [Fact] + public async Task Workbook_ranges_and_headers_keep_base_text_separate_from_phonetic_hints() + { + await using var scope = await TestStorageScope.CreateAsync(); + var options = new FileContextOptions { EnableWriteTools = true }; + var context = new FileContextService(new ManagedCodeStorageFileStore(scope.Storage, options), options); + var created = await context.Documents.CreateWorkbookAsync("phonetic.xlsx", new([ + new("Data", [[new(Text: "東京"), new(Text: "京都")], [new(Number: 1), new(Number: 2)]])])); + using (var document = SpreadsheetDocument.Open(Path.Combine(scope.Directory, created.Path), true)) + { + var workbook = document.WorkbookPart!; + var strings = workbook.AddNewPart(); + strings.SharedStringTable = new SharedStringTable(); + var item = strings.SharedStringTable.AppendChild(new SharedStringItem()); + item.Append(new Text("東京"), new PhoneticRun { BaseTextStartIndex = 0, EndingBaseIndex = 2, Text = new Text("とうきょう") }); + var cells = workbook.WorksheetParts.Single().Worksheet!.Descendants().First().Elements().ToArray(); + cells[0].InlineString = null; + cells[0].DataType = CellValues.SharedString; + cells[0].CellValue = new CellValue("0"); + cells[1].InlineString!.AppendChild(new PhoneticRun { BaseTextStartIndex = 0, EndingBaseIndex = 2, Text = new Text("きょうと") }); + } + + var range = await context.Documents.ReadWorkbookRangeAsync(created.Path, "Data", 1, 1, 1, 2); + range.Cells.Select(static cell => cell.Value).ShouldBe(["東京", "京都"]); + var table = (await context.Documents.GetTablesInfoAsync(created.Path)).Tables.Single(); + table.Headers.ShouldBe(["東京", "京都"]); + } + [Fact] public async Task Workbook_range_and_table_inspection_enforce_decompressed_part_limit() {
").Append(E(ReportJson.Text(column, "label", ReportJson.Text(column, "key")))).Append("" : "") .Append(E(ReportJson.FormatCell(row, column))).Append("Search12.5