From 33c483f2df0be8de9df847c7cdfa9f9b32afc7aa Mon Sep 17 00:00:00 2001 From: "Bernhard M. Wiedemann" Date: Fri, 21 Feb 2025 20:59:18 +0100 Subject: [PATCH 1/3] Use zstd-1.5.7 this changed compression results, so we have to adapt tests --- c-ext/backend_c.c | 2 +- tests/test_compressor_compress.py | 2 +- tests/test_compressor_compressobj.py | 2 +- tests/test_compressor_copy_stream.py | 2 +- tests/test_compressor_stream_writer.py | 2 +- tests/test_module_attributes.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/c-ext/backend_c.c b/c-ext/backend_c.c index aabe30b..4b09d1f 100644 --- a/c-ext/backend_c.c +++ b/c-ext/backend_c.c @@ -152,7 +152,7 @@ void zstd_module_init(PyObject *m) { PyObject *features = NULL; PyObject *feature = NULL; unsigned zstd_ver_no = ZSTD_versionNumber(); - unsigned our_hardcoded_version = 10506; + unsigned our_hardcoded_version = 10507; if (ZSTD_VERSION_NUMBER != our_hardcoded_version || zstd_ver_no != our_hardcoded_version) { PyErr_Format( diff --git a/tests/test_compressor_compress.py b/tests/test_compressor_compress.py index d96d476..c978c19 100644 --- a/tests/test_compressor_compress.py +++ b/tests/test_compressor_compress.py @@ -52,7 +52,7 @@ class TestCompressor_compress(unittest.TestCase): cctx = zstd.ZstdCompressor(level=3, write_content_size=False) result = cctx.compress(b"".join(chunks)) - self.assertEqual(len(result), 999) + self.assertEqual(len(result), 1029) self.assertEqual(result[0:4], b"\x28\xb5\x2f\xfd") # This matches the test for read_to_iter() below. diff --git a/tests/test_compressor_compressobj.py b/tests/test_compressor_compressobj.py index f429b32..b89ae20 100644 --- a/tests/test_compressor_compressobj.py +++ b/tests/test_compressor_compressobj.py @@ -39,7 +39,7 @@ class TestCompressor_compressobj(unittest.TestCase): cobj = cctx.compressobj() result = cobj.compress(b"".join(chunks)) + cobj.flush() - self.assertEqual(len(result), 999) + self.assertEqual(len(result), 1029) self.assertEqual(result[0:4], b"\x28\xb5\x2f\xfd") params = zstd.get_frame_parameters(result) diff --git a/tests/test_compressor_copy_stream.py b/tests/test_compressor_copy_stream.py index 82c7ce7..685660f 100644 --- a/tests/test_compressor_copy_stream.py +++ b/tests/test_compressor_copy_stream.py @@ -50,7 +50,7 @@ class TestCompressor_copy_stream(unittest.TestCase): r, w = cctx.copy_stream(source, dest) self.assertEqual(r, 255 * 16384) - self.assertEqual(w, 999) + self.assertEqual(w, 1029) params = zstd.get_frame_parameters(dest.getvalue()) self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN) diff --git a/tests/test_compressor_stream_writer.py b/tests/test_compressor_stream_writer.py index cfa198b..3baa788 100644 --- a/tests/test_compressor_stream_writer.py +++ b/tests/test_compressor_stream_writer.py @@ -301,7 +301,7 @@ class TestCompressor_stream_writer(unittest.TestCase): d = zstd.train_dictionary(8192, samples) h = hashlib.sha1(d.as_bytes()).hexdigest() - self.assertEqual(h, "a46d2f7a3bc3357c9d717d3dadf9a26fde23e93d") + self.assertEqual(h, "f32ddfbe0878bbd428afc00b17810387c6752191") buffer = io.BytesIO() cctx = zstd.ZstdCompressor(level=9, dict_data=d) diff --git a/tests/test_module_attributes.py b/tests/test_module_attributes.py index a99ed2e..3e3b482 100644 --- a/tests/test_module_attributes.py +++ b/tests/test_module_attributes.py @@ -5,7 +5,7 @@ import zstandard as zstd class TestModuleAttributes(unittest.TestCase): def test_version(self): - self.assertEqual(zstd.ZSTD_VERSION, (1, 5, 6)) + self.assertEqual(zstd.ZSTD_VERSION, (1, 5, 7)) self.assertEqual(zstd.__version__, "0.23.0") From f598b951f36b6259fd7871892dd1e136f96c58ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 24 Feb 2025 11:30:29 +0100 Subject: [PATCH 2/3] Update make_cffi.py to skip type alias macros in zstd 1.5.7 --- make_cffi.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/make_cffi.py b/make_cffi.py index 170d1e5..763abb2 100644 --- a/make_cffi.py +++ b/make_cffi.py @@ -179,7 +179,7 @@ ffi.set_source( include_dirs=INCLUDE_DIRS, ) -DEFINE = re.compile(b"^\\#define ([a-zA-Z0-9_]+) ") +DEFINE = re.compile(rb"^#define\s+([a-zA-Z0-9_]+)\s+(\S+)") sources = [] @@ -204,9 +204,14 @@ for header in HEADERS: if m.group(1) in (b"ZSTD_LIB_VERSION", b"ZSTD_VERSION_STRING"): continue + # These defines create aliases from old (camelCase) type names + # to the new PascalCase names, which breaks CFFI. + if m.group(1).lower() == m.group(2).lower(): + continue + # The ... is magic syntax by the cdef parser to resolve the # value at compile time. - sources.append(m.group(0) + b" ...") + sources.append(b"#define " + m.group(1) + b" ...") cdeflines = b"\n".join(sources).splitlines() cdeflines = [l for l in cdeflines if l.strip()] From fb56d2b5ab38132bff998b15a0d3de4e91569167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Mon, 24 Feb 2025 11:30:46 +0100 Subject: [PATCH 3/3] Update backend_cffi for new type names in zstd 1.5.7 --- zstandard/backend_cffi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zstandard/backend_cffi.py b/zstandard/backend_cffi.py index 80d6fc8..38427f4 100644 --- a/zstandard/backend_cffi.py +++ b/zstandard/backend_cffi.py @@ -2574,7 +2574,7 @@ def get_frame_parameters(data): :return: :py:class:`FrameParameters` """ - params = ffi.new("ZSTD_frameHeader *") + params = ffi.new("ZSTD_FrameHeader *") data_buffer = ffi.from_buffer(data) zresult = lib.ZSTD_getFrameHeader(params, data_buffer, len(data_buffer)) @@ -4289,7 +4289,7 @@ class ZstdDecompressor(object): # All chunks should be zstd frames and should have content size set. chunk_buffer = ffi.from_buffer(chunk) - params = ffi.new("ZSTD_frameHeader *") + params = ffi.new("ZSTD_FrameHeader *") zresult = lib.ZSTD_getFrameHeader( params, chunk_buffer, len(chunk_buffer) )