diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 997ce4a..394f878 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,9 @@ version 1.0.1-dev (`gh-134745 `__) ``PyThread_release_lock`` to use ``PyMutex`` internally, which now raises a fatal error when releasing a lock that was never acquired. ++ Fix ``gzip_ng_threaded`` writers at compression level 1, which raised + ``OverflowError`` for data that does not compress and consists mostly of + bytes 144 to 255. version 1.0.0 ----------------- diff --git a/src/zlib_ng/gzip_ng_threaded.py b/src/zlib_ng/gzip_ng_threaded.py index dee8be7..172c20e 100644 --- a/src/zlib_ng/gzip_ng_threaded.py +++ b/src/zlib_ng/gzip_ng_threaded.py @@ -226,9 +226,9 @@ def __init__(self, self.exception: Optional[Exception] = None self.level = level self.previous_block = b"" - # Deflating random data results in an output a little larger than the - # input. Making the output buffer 10% larger is sufficient overkill. - compress_buffer_size = block_size + max(block_size // 10, 500) + # Level 1 never falls back to stored blocks and codes bytes 144 to 255 + # with 9 bits each, so its output can be an eighth larger. + compress_buffer_size = block_size + block_size // 8 + 500 self.block_size = block_size self.compressors: List[zlib_ng._ParallelCompress] = [ zlib_ng._ParallelCompress(buffersize=compress_buffer_size, diff --git a/tests/test_gzip_ng_threaded.py b/tests/test_gzip_ng_threaded.py index a587193..6aaa1b2 100644 --- a/tests/test_gzip_ng_threaded.py +++ b/tests/test_gzip_ng_threaded.py @@ -98,6 +98,24 @@ def test_threaded_write_oversized_block_no_error(threads): assert data == decompressed +@pytest.mark.timeout(5) +@pytest.mark.parametrize(["threads", "block_size"], + list(itertools.product([1, 3], + [8 * 1024, 1024 * 1024]))) +def test_threaded_write_high_bytes_level_1(tmp_path, threads, block_size): + # Level 1 codes all data with the fixed Huffman codes and never falls back + # to stored blocks. Bytes 144 to 255 take 9 bits in these codes, so data + # made of them grows by 12.5%. Map random bytes onto 144 to 255. + data = os.urandom(3 * block_size).translate( + bytes(144 + i % 112 for i in range(256))) + test_file = tmp_path / "output.gz" + with gzip_ng_threaded.open(test_file, "wb", compresslevel=1, + threads=threads, block_size=block_size) as f: + f.write(data) + with gzip.open(test_file, "rb") as f: + assert f.read() == data + + @pytest.mark.timeout(5) @pytest.mark.parametrize("threads", [1, 3]) def test_threaded_write_error(threads):