From b5e04953bd3194ef80ea637a65be5091bf102542 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Fri, 23 Oct 2026 21:51:20 -0400 Subject: [PATCH 1/2] Decompress into one growing buffer in _GzipReader.readall() readall() decompressed into chunks of four times the input buffer size and joined them at the end. When the output didn't fit in the first chunk, the join copied all of the output into new memory, so readall() needed about twice the output in memory. Every file opened with igzip.open() got a first chunk of 2 MiB. When the amount of compressed data left is known, for bytes-like input, regular files, and streams that fit in the input buffer, readall() now decompresses into one buffer. When the buffer is full, it grows to the size that the compression ratio so far predicts, plus an eighth, by at least a quarter and to at most eight times its size. Longer streams of unknown length, such as pipes, still use chunks, because on Windows, doubling one buffer for regular files took up to 1.3 times as long as joining chunks. The first buffer is four times the compressed bytes in the input buffer, and at least 16 KiB. readall() first fills an empty input buffer outside a deflate block, so a small file is read to the end and its size is known without fstat(). It doesn't refill the buffer inside a deflate block, because the end of a member needs the bytes just before current_pos. --- CHANGELOG.rst | 9 ++ src/isal/isal_zlibmodule.c | 233 +++++++++++++++++++++++++++++-------- tests/test_readall.py | 208 +++++++++++++++++++++++++++++++++ 3 files changed, 399 insertions(+), 51 deletions(-) create mode 100644 tests/test_readall.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1fb79cb4..439f756b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -31,6 +31,15 @@ version 1.8.1-dev + Speed up each ``write()`` call to binary ``igzip`` files on CPython. On an AMD Ryzen 7 9800X3D with Python 3.14.8, writing FASTQ data or Python source code one line at a time is 1.1 times faster. ++ Speed up ``igzip.decompress`` and reading a whole ``igzip`` file with + ``read()``. When the size of the compressed data is known, as for bytes and + regular files, they decompress into one buffer that grows to the expected + size, instead of into chunks that are joined at the end. On Linux, reading + 32 MiB of output with ``read()`` needs about 1.14 times the output in memory + instead of 2 times. On an AMD Ryzen 7 9800X3D, ``igzip.decompress`` of 4 to + 32 MiB of JSON is 1.8 to 2.0 times faster in a Linux container and 1.5 to + 1.8 times faster on Windows. Reading a small file no longer allocates a + 2 MiB buffer. On PyPy, large outputs can be up to 10% slower. + Fix zlib streams compressed with a preset dictionary so they can be decoded by standard zlib, including builds linked against older ISA-L releases. + Fix backward seeking in igzip files by discarding buffered compressed data diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 7686d0a7..1c731125 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -25,6 +25,8 @@ Changes compared to CPython: #include "isal_shared.h" +#include + #include #include "crc32_combine.h" @@ -1612,6 +1614,9 @@ typedef struct _GzipReaderStruct { char all_bytes_read; char closed; uint32_t _last_mtime; + /* Compressed bytes read from fp so far. readall() uses it to find the + compression ratio. */ + Py_ssize_t compressed_read; PyThread_type_lock lock; struct inflate_state state; } GzipReader; @@ -1710,6 +1715,7 @@ GzipReader__new__(PyTypeObject *type, PyObject *args, PyObject *kwargs) self->stream_phase = GzipReader_HEADER; self->closed = 0; self->_last_mtime = 0; + self->compressed_read = 0; self->lock = PyThread_allocate_lock(); if (self->lock == NULL) { Py_DECREF(self); @@ -1765,6 +1771,7 @@ GzipReader_read_from_file(GzipReader *self) self->all_bytes_read = 1; } buffer_end += new_size; + self->compressed_read += new_size; self->current_pos = current_pos; self->buffer_end = buffer_end; return 0; @@ -2127,76 +2134,200 @@ GzipReader_seek(GzipReader *self, PyObject *args, PyObject *kwargs) return PyLong_FromLongLong(self->_pos); } +/* Return the number of compressed bytes that are not decompressed yet, or -1 + if that is unknown. It is known for bytes-like objects, for regular files, + and for streams that were read to the end. readall() uses it only to choose + how much memory to allocate. */ +static Py_ssize_t +GzipReader_compressed_left(GzipReader *self) +{ + Py_ssize_t left = self->buffer_end - self->current_pos; + if (self->all_bytes_read) { + return left; + } + int fd = PyObject_AsFileDescriptor(self->fp); + if (fd == -1) { + PyErr_Clear(); + return -1; + } +#ifdef MS_WINDOWS + struct _stat64 st; + if (_fstat64(fd, &st) != 0 || (st.st_mode & _S_IFMT) != _S_IFREG) { + return -1; + } +#else + struct stat st; + if (fstat(fd, &st) != 0 || !S_ISREG(st.st_mode)) { + return -1; + } +#endif + PyObject *pos_obj = PyObject_CallMethod(self->fp, "tell", NULL); + if (pos_obj == NULL) { + PyErr_Clear(); + return -1; + } + long long pos = PyLong_AsLongLong(pos_obj); + Py_DECREF(pos_obj); + if (pos == -1 && PyErr_Occurred()) { + PyErr_Clear(); + return -1; + } + if (pos < 0 || pos > (long long)st.st_size) { + return left; + } + long long file_left = (long long)st.st_size - pos; + if (file_left > PY_SSIZE_T_MAX - left) { + return -1; + } + return left + (Py_ssize_t)file_left; +} + +/* Return the size for the readall() buffer when it is full: the output so + far scaled by the input it came from, plus an eighth. The buffer grows by + at least a quarter, so a low estimate still converges, and to at most 8 + times its size, which limits the allocation when the rest of the data + compresses less well. */ +static Py_ssize_t +readall_new_size(Py_ssize_t written, Py_ssize_t consumed, Py_ssize_t left) +{ + double size = (double)written; + if (consumed > 0) { + size += size * (double)left / (double)consumed; + } + size += size / 8; + double min_size = (double)written + (double)(written / 4); + double max_size = (double)written * 8; + if (size < min_size) { + size = min_size; + } + if (size > max_size) { + size = max_size; + } + if (size >= (double)PY_SSIZE_T_MAX) { + return PY_SSIZE_T_MAX; + } + return (Py_ssize_t)size; +} + static PyObject * GzipReader_readall(GzipReader *self, PyObject *Py_UNUSED(ignore)) { - /* Try to consume the entire buffer without too much overallocation */ - Py_ssize_t chunk_size = self->buffer_size * 4; - /* Rather than immediately creating a list, read one chunk first and - only create a list when more read operations are necessary. */ - PyObject *first_chunk = PyBytes_FromStringAndSize(NULL, chunk_size); - if (first_chunk == NULL) { - return NULL; - } + PyObject *result = NULL; + PyObject *chunks = NULL; + Py_ssize_t written = 0; + ENTER_ZLIB(self); - Py_ssize_t written_size = GzipReader_read_into_buffer( - self, (uint8_t *)PyBytes_AS_STRING(first_chunk), chunk_size); - LEAVE_ZLIB(self); - if (written_size < 0) { - Py_DECREF(first_chunk); - return NULL; - } - if (written_size < chunk_size) { - if (_PyBytes_Resize(&first_chunk, written_size) < 0) { - return NULL; + /* Fill the input buffer first, so that a small file gets a buffer for + its own size rather than for the size of the input buffer, and its size + is known without calling fstat(). Only do this with an empty buffer + outside a deflate block: the end of a member can still need the bytes + just before current_pos, which reading would overwrite. */ + if (self->current_pos == self->buffer_end && + self->stream_phase != GzipReader_DEFLATE_BLOCK) { + while (!self->all_bytes_read && (size_t)(self->buffer_end - + self->input_buffer) < + self->buffer_size) { + if (GzipReader_read_from_file(self) < 0) { + goto error; + } } - return first_chunk; } - - PyObject *chunk_list = PyList_New(1); - if (chunk_list == NULL) { - return NULL; + Py_ssize_t available = self->buffer_end - self->current_pos; + Py_ssize_t read_at_start = self->compressed_read; + int size_known = 1; + Py_ssize_t size = DEF_BUF_SIZE; + if (available > PY_SSIZE_T_MAX / 4) { + size = PY_SSIZE_T_MAX; + } else if (available * 4 > size) { + size = available * 4; + } + result = PyBytes_FromStringAndSize(NULL, size); + if (result == NULL) { + goto error; } - PyList_SET_ITEM(chunk_list, 0, first_chunk); while (1) { - PyObject *chunk = PyBytes_FromStringAndSize(NULL, chunk_size); - if (chunk == NULL) { - Py_DECREF(chunk_list); - return NULL; - } - ENTER_ZLIB(self); - written_size = GzipReader_read_into_buffer( - self, (uint8_t *)PyBytes_AS_STRING(chunk), chunk_size); - LEAVE_ZLIB(self); - if (written_size < 0) { - Py_DECREF(chunk); - Py_DECREF(chunk_list); - return NULL; + Py_ssize_t written_now = GzipReader_read_into_buffer( + self, (uint8_t *)PyBytes_AS_STRING(result) + written, + size - written); + if (written_now < 0) { + goto error; } - if (written_size == 0) { - Py_DECREF(chunk); + written += written_now; + if (written < size) { + // The end of the stream. break; } - if (_PyBytes_Resize(&chunk, written_size) < 0) { - Py_DECREF(chunk_list); - return NULL; + Py_ssize_t left = -1; + if (size_known) { + left = GzipReader_compressed_left(self); + size_known = left >= 0; } - int ret = PyList_Append(chunk_list, chunk); - Py_DECREF(chunk); - if (ret < 0) { - Py_DECREF(chunk_list); - return NULL; + if (left >= 0) { + /* Grow the buffer to the expected size. Unlike joining chunks, + this does not copy all of the output into new memory at the + end. On Linux, realloc() usually moves large blocks without + copying them. */ + Py_ssize_t consumed = self->compressed_read - read_at_start + + available - (self->buffer_end - + self->current_pos); + size = readall_new_size(written, consumed, left); + if (_PyBytes_Resize(&result, size) < 0) { + goto error; + } + continue; + } + /* With an unknown amount of input, for example from a pipe, collect + chunks and join them at the end. Doubling one buffer was slower on + Windows. */ + if (chunks == NULL) { + chunks = PyList_New(0); + if (chunks == NULL) { + goto error; + } + } + if (PyList_Append(chunks, result) < 0) { + goto error; } + Py_ssize_t chunk_size = (Py_ssize_t)self->buffer_size * 4; + if (chunk_size > size) { + size = chunk_size; + } + Py_SETREF(result, PyBytes_FromStringAndSize(NULL, size)); + if (result == NULL) { + goto error; + } + written = 0; + } + LEAVE_ZLIB(self); + + if (_PyBytes_Resize(&result, written) < 0) { + Py_XDECREF(chunks); + return NULL; + } + if (chunks == NULL) { + return result; + } + int ret = PyList_Append(chunks, result); + Py_DECREF(result); + if (ret < 0) { + Py_DECREF(chunks); + return NULL; } PyObject *empty_bytes = PyBytes_FromStringAndSize(NULL, 0); if (empty_bytes == NULL) { - Py_DECREF(chunk_list); + Py_DECREF(chunks); return NULL; } - PyObject *ret = PyObject_CallMethod(empty_bytes, "join", "O", chunk_list); + result = PyObject_CallMethod(empty_bytes, "join", "O", chunks); Py_DECREF(empty_bytes); - Py_DECREF(chunk_list); - return ret; + Py_DECREF(chunks); + return result; + +error: + LEAVE_ZLIB(self); + Py_XDECREF(result); + Py_XDECREF(chunks); + return NULL; } static PyObject * diff --git a/tests/test_readall.py b/tests/test_readall.py new file mode 100644 index 00000000..538c8f4d --- /dev/null +++ b/tests/test_readall.py @@ -0,0 +1,208 @@ +"""Tests for reading all of a gzip stream at once, as igzip.decompress() +and IGzipFile.read() do with _GzipReader.readall().""" +import io +import random + +from isal import igzip + +import pytest + + +def compressible(size): + """JSON-like lines that compress well.""" + rng = random.Random(size) + lines = [] + length = 0 + while length < size: + line = (b'{"id": %d, "type": "event", "entity": "sensor.temp_%d"}\n' + % (rng.randrange(1000), rng.randrange(50))) + lines.append(line) + length += len(line) + return b"".join(lines)[:size] + + +def incompressible(size): + return random.Random(size).randbytes(size) + + +def zeros(size): + return bytes(size) + + +KINDS = [compressible, incompressible, zeros] +SIZES = [0, 1, 1000, 100_000, 3_000_000] + + +class UnknownSize(io.RawIOBase): + """A stream without fileno(), so its size is unknown.""" + def __init__(self, data, read_size=65536): + self._data = io.BytesIO(data) + self._read_size = read_size + + def readable(self): + return True + + def readinto(self, b): + # Return less than asked for, as pipes do. + return self._data.readinto(memoryview(b)[:self._read_size]) + + +@pytest.mark.parametrize("kind", KINDS) +@pytest.mark.parametrize("size", SIZES) +def test_decompress(kind, size): + data = kind(size) + assert igzip.decompress(igzip.compress(data)) == data + + +@pytest.mark.parametrize("kind", KINDS) +@pytest.mark.parametrize("size", SIZES) +def test_read_file(tmp_path, kind, size): + data = kind(size) + path = tmp_path / "test.gz" + path.write_bytes(igzip.compress(data)) + with igzip.open(path) as f: + assert f.read() == data + with open(path, "rb") as f: + assert igzip._GzipReader(f).readall() == data + + +@pytest.mark.parametrize("kind", KINDS) +def test_read_unknown_size(kind): + data = kind(5_000_000) + with igzip.IGzipFile(fileobj=UnknownSize(igzip.compress(data))) as f: + assert f.read() == data + with igzip.IGzipFile(fileobj=io.BytesIO(igzip.compress(data))) as f: + assert f.read() == data + # readall() reads a stream that fits in the input buffer to the end + # first. With a small input buffer, it must join chunks. + compressed = igzip.compress(data) + assert len(compressed) > 4096 + reader = igzip._GzipReader(UnknownSize(compressed), 4096) + assert reader.readall() == data + + +def members(): + """Members that compress very differently, so that the size that + readall() expects is wrong in both directions.""" + parts = [zeros(4_000_000), incompressible(1_000_000), + compressible(2_000_000), zeros(8_000_000), b"end"] + return b"".join(parts), b"".join(igzip.compress(p) for p in parts) + + +def test_decompress_members(): + data, compressed = members() + assert igzip.decompress(compressed) == data + + +def test_read_file_members(tmp_path): + data, compressed = members() + path = tmp_path / "test.gz" + path.write_bytes(compressed) + with igzip.open(path) as f: + assert f.read() == data + + +@pytest.mark.parametrize("buffersize", [4096, 32 * 1024, 512 * 1024]) +def test_read_many_members(tmp_path, buffersize): + # Members end at many places relative to the input and output buffers. + rng = random.Random(buffersize) + parts = [rng.choice(KINDS)(rng.randrange(1, 50_000)) for _ in range(300)] + data = b"".join(parts) + compressed = b"".join(igzip.compress(p, 1) for p in parts) + path = tmp_path / "test.gz" + path.write_bytes(compressed) + with open(path, "rb") as f: + assert igzip._GzipReader(f, buffersize).readall() == data + reader = igzip._GzipReader(UnknownSize(compressed), buffersize) + assert reader.readall() == data + assert igzip._GzipReader(compressed).readall() == data + + +@pytest.mark.parametrize("buffersize", [4096, 32 * 1024]) +@pytest.mark.parametrize("kind", [compressible, zeros]) +def test_readall_after_read(tmp_path, kind, buffersize): + # read() stops at many places in a member and in the input buffer. + data = kind(100_000) + path = tmp_path / "test.gz" + path.write_bytes(igzip.compress(data) + igzip.compress(data[:5000])) + expected = data + data[:5000] + for size in sorted({1, 100, 4095, 4096, 16384, 65536, 99_000, 99_992, + 99_999, 100_000, 100_001, *range(30_000, 100_000, + 7919)}): + with open(path, "rb") as f: + reader = igzip._GzipReader(f, buffersize) + start = reader.read(size) + assert start + reader.readall() == expected, size + + +def test_readall_after_read_inside_deflate_block(): + # Some of these read() calls stop inside a deflate block with all input + # in the buffer used. If readall() refilled the input buffer there, the + # end of the member would fail with "CRC check failed". + data = b"".join(b"line %d of some text %d\n" % (i, i % 97) + for i in range(8000)) + compressed = igzip.compress(data, 0) + igzip.compress(b"tail", 1) + expected = data + b"tail" + for size in range(201_600, 201_800): + reader = igzip._GzipReader(UnknownSize(compressed, 1000), 32 * 1024) + start = reader.read(size) + assert start + reader.readall() == expected, size + + +def test_read_file_after_partial_read(tmp_path): + data = compressible(3_000_000) + path = tmp_path / "test.gz" + path.write_bytes(igzip.compress(data)) + with igzip.open(path) as f: + assert f.read(10) == data[:10] + assert f.read() == data[10:] + with open(path, "rb") as f: + reader = igzip._GzipReader(f) + assert reader.read(100_000) == data[:100_000] + assert reader.readall() == data[100_000:] + + +class BadTell(io.FileIO): + def tell(self): + raise OSError("tell() does not work") + + +class TellPastEnd(io.FileIO): + def tell(self): + return 2**40 + + +@pytest.mark.parametrize("file_class", [BadTell, TellPastEnd]) +def test_read_file_with_unusable_tell(tmp_path, file_class): + # The size that tell() and fileno() suggest only guides the allocation. + data = compressible(3_000_000) + path = tmp_path / "test.gz" + path.write_bytes(igzip.compress(data)) + with file_class(path, "rb") as f: + assert igzip._GzipReader(f).readall() == data + + +@pytest.mark.parametrize("kind", [compressible, zeros]) +@pytest.mark.parametrize("source", ["bytes", "file"]) +def test_peak_memory(tmp_path, kind, source): + # Joining chunks needs memory for the chunks and the joined result, so + # about twice the output. One buffer that grows needs about the output. + # PyPy has no tracemalloc. + tracemalloc = pytest.importorskip("tracemalloc") + data = kind(8_000_000) + compressed = igzip.compress(data) + path = tmp_path / "test.gz" + path.write_bytes(compressed) + tracemalloc.start() + try: + if source == "bytes": + result = igzip.decompress(compressed) + else: + with igzip.open(path) as f: + result = f.read() + peak = tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + assert result == data + # One MiB allows for the 512 KiB input buffer of a file. + assert peak < 1.5 * len(data) + 1024 * 1024 From 1b9e4e1a78d32ebf70b3a271571567be677a8534 Mon Sep 17 00:00:00 2001 From: ptweezy Date: Sat, 24 Oct 2026 14:22:33 -0400 Subject: [PATCH 2/2] Make the CHANGELOG entry for readall() match the measurements --- CHANGELOG.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 439f756b..9649b987 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -32,14 +32,15 @@ version 1.8.1-dev AMD Ryzen 7 9800X3D with Python 3.14.8, writing FASTQ data or Python source code one line at a time is 1.1 times faster. + Speed up ``igzip.decompress`` and reading a whole ``igzip`` file with - ``read()``. When the size of the compressed data is known, as for bytes and - regular files, they decompress into one buffer that grows to the expected - size, instead of into chunks that are joined at the end. On Linux, reading - 32 MiB of output with ``read()`` needs about 1.14 times the output in memory - instead of 2 times. On an AMD Ryzen 7 9800X3D, ``igzip.decompress`` of 4 to - 32 MiB of JSON is 1.8 to 2.0 times faster in a Linux container and 1.5 to - 1.8 times faster on Windows. Reading a small file no longer allocates a - 2 MiB buffer. On PyPy, large outputs can be up to 10% slower. + ``read()``, and cut their peak memory to about half for most large outputs. + When the size of the compressed data is known, as for bytes and regular + files, they decompress into one buffer that grows to the expected size, + instead of into chunks that are joined at the end. On Linux, + ``igzip.decompress`` of data that compresses better than 4 to 1, and + ``read()`` of files with more than 2 MiB of output, are 1.1 to 3.1 times + faster. The gains are smaller on macOS and Windows, and on Windows, + ``read()`` of a file with 1 to 2 MiB of output can take up to 1.4 times as + long. + Fix zlib streams compressed with a preset dictionary so they can be decoded by standard zlib, including builds linked against older ISA-L releases. + Fix backward seeking in igzip files by discarding buffered compressed data