diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 359e88d0b8bff7..ce245af5569b29 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -1864,3 +1864,33 @@ Build Changes subclass of a built-in type (for example an :class:`int` or :class:`datetime.datetime` subclass) returned :data:`NotImplemented` instead of raising :exc:`TypeError`. (Backport of :issue:`42825`.) + + +Notable changes in 3.9.25.4 +=========================== + +This is an ActiveState release of the 3.9 line, on top of 3.9.25.3. + +tarfile +------- + +* Opening an archive no longer pre-allocates the size a GNU long name/link + or pax extended header claims. The header is read in bounded chunks, so a + small crafted archive cannot force a multi-gigabyte allocation before any + extraction filter runs. (:gh:`151497`.) + +Build Changes +------------- + +* :mod:`ctypes` builds on Windows against libffi 3.5 and later. Those + versions always declare libffi's ``ffi_type_*`` objects as DLL imports + under MSVC, which broke the static initializers in + :file:`Modules/_ctypes/cfield.c`. The format table is now filled in at + run time, and the Windows project no longer defines ``FFI_BUILDING``. + (Backport of :issue:`45898` and :gh:`92356`.) +* :file:`Modules/expat/pyexpatns.h` now also renames the three functions + Expat 2.9.0 added (``XML_SetHashSalt16Bytes``, + ``XML_SetAllocTrackerActivationThreshold`` and + ``XML_SetAllocTrackerMaximumAmplification``), so every function of the + bundled Expat carries the ``PyExpat_`` prefix and cannot clash with + another copy of Expat loaded in the same process. diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 9db78558a0ceac..30ca6b6064889a 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.9.25.3" +#define PY_VERSION "3.9.25.4" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 4a688bc4e96b7c..26305866b32975 100755 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -260,6 +260,32 @@ def copyfileobj(src, dst, length=None, exception=OSError, bufsize=None): dst.write(buf) return +# Maximum number of bytes read in a single call when reading a member's +# extended header (a GNU long name/link or a pax header). The size of such +# a header is taken from the archive and is not trustworthy, so it is read in +# bounded chunks to avoid a huge up-front allocation when a crafted or +# truncated archive claims far more data than the file actually contains +# (gh-151497). +_EXTHEADER_READ_CHUNK = 1024 * 1024 # 1 MiB + +def _safe_read(fileobj, size): + """Read up to *size* bytes from *fileobj* in bounded chunks. + + Returns the same bytes as ``fileobj.read(size)`` would (including a short + result at end of file), but limits pre-allocation, so an + oversized size field in a crafted header cannot force a huge allocation. + """ + if size <= _EXTHEADER_READ_CHUNK: + return fileobj.read(size) + chunks = [] + while size > 0: + chunk = fileobj.read(min(size, _EXTHEADER_READ_CHUNK)) + if not chunk: + break + chunks.append(chunk) + size -= len(chunk) + return b"".join(chunks) + def _safe_print(s): encoding = getattr(sys.stdout, 'encoding', None) if encoding is not None: @@ -1395,7 +1421,7 @@ def _proc_gnulong(self, tarfile): """Process the blocks that hold a GNU longname or longlink member. """ - buf = tarfile.fileobj.read(self._block(self.size)) + buf = _safe_read(tarfile.fileobj, self._block(self.size)) # Fetch the next header and process it. try: @@ -1446,7 +1472,7 @@ def _proc_pax(self, tarfile): POSIX.1-2008. """ # Read the header information. - buf = tarfile.fileobj.read(self._block(self.size)) + buf = _safe_read(tarfile.fileobj, self._block(self.size)) # A pax header stores supplemental information for either # the following file (extended) or all following files diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 2d5bf9f34b3af5..9fd6e3173f5a71 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -466,6 +466,54 @@ def test_length_zero_header(self): with tarfile.open(support.findfile('recursion.tar')) as tar: pass + +class ReadSizeRecorder(io.BytesIO): + # Records the largest size ever passed to read(), so a test can check + # that tarfile does not request far more data than the archive holds + # (which on a real file would pre-allocate it). + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.max_read_size = 0 + + def read(self, size=-1): + if size is not None and size >= 0: + self.max_read_size = max(self.max_read_size, size) + return super().read(size) + + +@support.cpython_only +class ExtendedHeaderMemoryTest(unittest.TestCase): + # gh-151497: the size of a GNU long name/link or a pax extended header is + # read from the archive and is untrusted. A crafted header can claim a + # size far larger than the file actually contains; opening such an archive + # must not try to read (and so pre-allocate) the claimed size in one go. + + def crafted_archive(self, hdrtype): + tarinfo = tarfile.TarInfo("A") + tarinfo.type = hdrtype + tarinfo.size = 0xFFFFFFFF # ~4 GiB claimed in a 512-byte header + return tarinfo.tobuf(format=tarfile.GNU_FORMAT) + + def check(self, hdrtype): + fobj = ReadSizeRecorder(self.crafted_archive(hdrtype)) + try: + with tarfile.open(fileobj=fobj, mode="r:") as tar: + tar.getmembers() + except tarfile.ReadError: + pass # a truncated header is fine; we only check the allocation + # The bogus ~4 GiB size must never reach a single read() call. + self.assertLessEqual(fobj.max_read_size, tarfile._EXTHEADER_READ_CHUNK) + + def test_gnu_longname_oversized_size(self): + self.check(tarfile.GNUTYPE_LONGNAME) + + def test_gnu_longlink_oversized_size(self): + self.check(tarfile.GNUTYPE_LONGLINK) + + def test_pax_header_oversized_size(self): + self.check(tarfile.XHDTYPE) + + class MiscReadTestBase(CommonReadTest): def requires_name_attribute(self): pass diff --git a/Misc/NEWS.d/next/Library/2021-11-26-10-46-09.bpo-45898.UIfhsb.rst b/Misc/NEWS.d/next/Library/2021-11-26-10-46-09.bpo-45898.UIfhsb.rst new file mode 100644 index 00000000000000..9da5c258ab7fd8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-11-26-10-46-09.bpo-45898.UIfhsb.rst @@ -0,0 +1,2 @@ +:mod:`ctypes` no longer defines ``ffi_type_*`` symbols in ``cfield.c``. The +symbols have been provided by libffi for over a decade. diff --git a/Misc/NEWS.d/next/Library/2022-05-05-19-25-09.gh-issue-92356.uvxWdu.rst b/Misc/NEWS.d/next/Library/2022-05-05-19-25-09.gh-issue-92356.uvxWdu.rst new file mode 100644 index 00000000000000..9c9566e5b9d719 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-05-05-19-25-09.gh-issue-92356.uvxWdu.rst @@ -0,0 +1 @@ +Fixed a performance regression in ctypes function calls. diff --git a/Misc/NEWS.d/next/Library/2026-06-15-15-32-36.gh-issue-151497.1cfmSV.rst b/Misc/NEWS.d/next/Library/2026-06-15-15-32-36.gh-issue-151497.1cfmSV.rst new file mode 100644 index 00000000000000..a4c03c9d71d761 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-15-15-32-36.gh-issue-151497.1cfmSV.rst @@ -0,0 +1,4 @@ +Opening a :mod:`tarfile` archive no longer attempts to pre-allocate a huge +buffer when a crafted or truncated member claims an oversized extended header +(a GNU long name/link or a pax header). The extended header is now read in +bounded chunks, so its size field can no longer trigger memory exhaustion. diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 8582af9c3a209a..7e962c632452c0 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -1505,57 +1505,42 @@ P_get(void *ptr, Py_ssize_t size) } static struct fielddesc formattable[] = { - { 's', s_set, s_get, &ffi_type_pointer}, - { 'b', b_set, b_get, &ffi_type_schar}, - { 'B', B_set, B_get, &ffi_type_uchar}, - { 'c', c_set, c_get, &ffi_type_schar}, - { 'd', d_set, d_get, &ffi_type_double, d_set_sw, d_get_sw}, - { 'g', g_set, g_get, &ffi_type_longdouble}, - { 'f', f_set, f_get, &ffi_type_float, f_set_sw, f_get_sw}, - { 'h', h_set, h_get, &ffi_type_sshort, h_set_sw, h_get_sw}, - { 'H', H_set, H_get, &ffi_type_ushort, H_set_sw, H_get_sw}, - { 'i', i_set, i_get, &ffi_type_sint, i_set_sw, i_get_sw}, - { 'I', I_set, I_get, &ffi_type_uint, I_set_sw, I_get_sw}, -/* XXX Hm, sizeof(int) == sizeof(long) doesn't hold on every platform */ -/* As soon as we can get rid of the type codes, this is no longer a problem */ -#if SIZEOF_LONG == 4 - { 'l', l_set, l_get, &ffi_type_sint32, l_set_sw, l_get_sw}, - { 'L', L_set, L_get, &ffi_type_uint32, L_set_sw, L_get_sw}, -#elif SIZEOF_LONG == 8 - { 'l', l_set, l_get, &ffi_type_sint64, l_set_sw, l_get_sw}, - { 'L', L_set, L_get, &ffi_type_uint64, L_set_sw, L_get_sw}, -#else -# error -#endif -#if SIZEOF_LONG_LONG == 8 - { 'q', q_set, q_get, &ffi_type_sint64, q_set_sw, q_get_sw}, - { 'Q', Q_set, Q_get, &ffi_type_uint64, Q_set_sw, Q_get_sw}, -#else -# error -#endif - { 'P', P_set, P_get, &ffi_type_pointer}, - { 'z', z_set, z_get, &ffi_type_pointer}, + { 's', s_set, s_get, NULL}, + { 'b', b_set, b_get, NULL}, + { 'B', B_set, B_get, NULL}, + { 'c', c_set, c_get, NULL}, + { 'd', d_set, d_get, NULL, d_set_sw, d_get_sw}, + { 'g', g_set, g_get, NULL}, + { 'f', f_set, f_get, NULL, f_set_sw, f_get_sw}, + { 'h', h_set, h_get, NULL, h_set_sw, h_get_sw}, + { 'H', H_set, H_get, NULL, H_set_sw, H_get_sw}, + { 'i', i_set, i_get, NULL, i_set_sw, i_get_sw}, + { 'I', I_set, I_get, NULL, I_set_sw, I_get_sw}, + { 'l', l_set, l_get, NULL, l_set_sw, l_get_sw}, + { 'L', L_set, L_get, NULL, L_set_sw, L_get_sw}, + { 'q', q_set, q_get, NULL, q_set_sw, q_get_sw}, + { 'Q', Q_set, Q_get, NULL, Q_set_sw, Q_get_sw}, + { 'P', P_set, P_get, NULL}, + { 'z', z_set, z_get, NULL}, #ifdef CTYPES_UNICODE - { 'u', u_set, u_get, NULL}, /* ffi_type set later */ - { 'U', U_set, U_get, &ffi_type_pointer}, - { 'Z', Z_set, Z_get, &ffi_type_pointer}, + { 'u', u_set, u_get, NULL}, + { 'U', U_set, U_get, NULL}, + { 'Z', Z_set, Z_get, NULL}, #endif #ifdef MS_WIN32 - { 'X', BSTR_set, BSTR_get, &ffi_type_pointer}, + { 'X', BSTR_set, BSTR_get, NULL}, #endif - { 'v', vBOOL_set, vBOOL_get, &ffi_type_sshort}, -#if SIZEOF__BOOL == 1 - { '?', bool_set, bool_get, &ffi_type_uchar}, /* Also fallback for no native _Bool support */ -#elif SIZEOF__BOOL == SIZEOF_SHORT - { '?', bool_set, bool_get, &ffi_type_ushort}, -#elif SIZEOF__BOOL == SIZEOF_INT - { '?', bool_set, bool_get, &ffi_type_uint, I_set_sw, I_get_sw}, + { 'v', vBOOL_set, vBOOL_get, NULL}, +#if SIZEOF__BOOL == SIZEOF_INT + { '?', bool_set, bool_get, NULL, I_set_sw, I_get_sw}, #elif SIZEOF__BOOL == SIZEOF_LONG - { '?', bool_set, bool_get, &ffi_type_ulong, L_set_sw, L_get_sw}, + { '?', bool_set, bool_get, NULL, L_set_sw, L_get_sw}, #elif SIZEOF__BOOL == SIZEOF_LONG_LONG - { '?', bool_set, bool_get, &ffi_type_ulong, Q_set_sw, Q_get_sw}, + { '?', bool_set, bool_get, NULL, Q_set_sw, Q_get_sw}, +#else + { '?', bool_set, bool_get, NULL}, #endif /* SIZEOF__BOOL */ - { 'O', O_set, O_get, &ffi_type_pointer}, + { 'O', O_set, O_get, NULL}, { 0, NULL, NULL, NULL}, }; @@ -1564,6 +1549,81 @@ static struct fielddesc formattable[] = { Use '?' as code for BOOL. */ +/* Delayed initialization. Windows cannot statically reference dynamically + loaded addresses from DLLs. */ +void +_ctypes_init_fielddesc(void) +{ + struct fielddesc *fd = formattable; + for (; fd->code; ++fd) { + switch (fd->code) { + case 's': fd->pffi_type = &ffi_type_pointer; break; + case 'b': fd->pffi_type = &ffi_type_schar; break; + case 'B': fd->pffi_type = &ffi_type_uchar; break; + case 'c': fd->pffi_type = &ffi_type_schar; break; + case 'd': fd->pffi_type = &ffi_type_double; break; + case 'g': fd->pffi_type = &ffi_type_longdouble; break; + case 'f': fd->pffi_type = &ffi_type_float; break; + case 'h': fd->pffi_type = &ffi_type_sshort; break; + case 'H': fd->pffi_type = &ffi_type_ushort; break; + case 'i': fd->pffi_type = &ffi_type_sint; break; + case 'I': fd->pffi_type = &ffi_type_uint; break; + /* XXX Hm, sizeof(int) == sizeof(long) doesn't hold on every platform */ + /* As soon as we can get rid of the type codes, this is no longer a problem */ + #if SIZEOF_LONG == 4 + case 'l': fd->pffi_type = &ffi_type_sint32; break; + case 'L': fd->pffi_type = &ffi_type_uint32; break; + #elif SIZEOF_LONG == 8 + case 'l': fd->pffi_type = &ffi_type_sint64; break; + case 'L': fd->pffi_type = &ffi_type_uint64; break; + #else + #error + #endif + #if SIZEOF_LONG_LONG == 8 + case 'q': fd->pffi_type = &ffi_type_sint64; break; + case 'Q': fd->pffi_type = &ffi_type_uint64; break; + #else + #error + #endif + case 'P': fd->pffi_type = &ffi_type_pointer; break; + case 'z': fd->pffi_type = &ffi_type_pointer; break; + #ifdef CTYPES_UNICODE + case 'u': + if (sizeof(wchar_t) == sizeof(short)) + fd->pffi_type = &ffi_type_sshort; + else if (sizeof(wchar_t) == sizeof(int)) + fd->pffi_type = &ffi_type_sint; + else if (sizeof(wchar_t) == sizeof(long)) + fd->pffi_type = &ffi_type_slong; + else + Py_UNREACHABLE(); + break; + case 'U': fd->pffi_type = &ffi_type_pointer; break; + case 'Z': fd->pffi_type = &ffi_type_pointer; break; + #endif + #ifdef MS_WIN32 + case 'X': fd->pffi_type = &ffi_type_pointer; break; + #endif + case 'v': fd->pffi_type = &ffi_type_sshort; break; + #if SIZEOF__BOOL == 1 + case '?': fd->pffi_type = &ffi_type_uchar; break; /* Also fallback for no native _Bool support */ + #elif SIZEOF__BOOL == SIZEOF_SHORT + case '?': fd->pffi_type = &ffi_type_ushort; break; + #elif SIZEOF__BOOL == SIZEOF_INT + case '?': fd->pffi_type = &ffi_type_uint; break; + #elif SIZEOF__BOOL == SIZEOF_LONG + case '?': fd->pffi_type = &ffi_type_ulong; break; + #elif SIZEOF__BOOL == SIZEOF_LONG_LONG + case '?': fd->pffi_type = &ffi_type_ulong; break; + #endif /* SIZEOF__BOOL */ + case 'O': fd->pffi_type = &ffi_type_pointer; break; + default: + Py_UNREACHABLE(); + } + } + +} + struct fielddesc * _ctypes_get_fielddesc(const char *fmt) { @@ -1572,14 +1632,7 @@ _ctypes_get_fielddesc(const char *fmt) if (!initialized) { initialized = 1; -#ifdef CTYPES_UNICODE - if (sizeof(wchar_t) == sizeof(short)) - _ctypes_get_fielddesc("u")->pffi_type = &ffi_type_sshort; - else if (sizeof(wchar_t) == sizeof(int)) - _ctypes_get_fielddesc("u")->pffi_type = &ffi_type_sint; - else if (sizeof(wchar_t) == sizeof(long)) - _ctypes_get_fielddesc("u")->pffi_type = &ffi_type_slong; -#endif + _ctypes_init_fielddesc(); } for (; table->code; ++table) { @@ -1589,77 +1642,4 @@ _ctypes_get_fielddesc(const char *fmt) return NULL; } -typedef struct { char c; char x; } s_char; -typedef struct { char c; short x; } s_short; -typedef struct { char c; int x; } s_int; -typedef struct { char c; long x; } s_long; -typedef struct { char c; float x; } s_float; -typedef struct { char c; double x; } s_double; -typedef struct { char c; long double x; } s_long_double; -typedef struct { char c; char *x; } s_char_p; -typedef struct { char c; void *x; } s_void_p; - -/* -#define CHAR_ALIGN (sizeof(s_char) - sizeof(char)) -#define SHORT_ALIGN (sizeof(s_short) - sizeof(short)) -#define LONG_ALIGN (sizeof(s_long) - sizeof(long)) -*/ -#define INT_ALIGN (sizeof(s_int) - sizeof(int)) -#define FLOAT_ALIGN (sizeof(s_float) - sizeof(float)) -#define DOUBLE_ALIGN (sizeof(s_double) - sizeof(double)) -#define LONGDOUBLE_ALIGN (sizeof(s_long_double) - sizeof(long double)) - -/* #define CHAR_P_ALIGN (sizeof(s_char_p) - sizeof(char*)) */ -#define VOID_P_ALIGN (sizeof(s_void_p) - sizeof(void*)) - -/* -#ifdef HAVE_USABLE_WCHAR_T -typedef struct { char c; wchar_t x; } s_wchar; -typedef struct { char c; wchar_t *x; } s_wchar_p; - -#define WCHAR_ALIGN (sizeof(s_wchar) - sizeof(wchar_t)) -#define WCHAR_P_ALIGN (sizeof(s_wchar_p) - sizeof(wchar_t*)) -#endif -*/ - -typedef struct { char c; long long x; } s_long_long; -#define LONG_LONG_ALIGN (sizeof(s_long_long) - sizeof(long long)) - -/* from ffi.h: -typedef struct _ffi_type -{ - size_t size; - unsigned short alignment; - unsigned short type; - struct _ffi_type **elements; -} ffi_type; -*/ - -/* align and size are bogus for void, but they must not be zero */ -ffi_type ffi_type_void = { 1, 1, FFI_TYPE_VOID }; - -ffi_type ffi_type_uint8 = { 1, 1, FFI_TYPE_UINT8 }; -ffi_type ffi_type_sint8 = { 1, 1, FFI_TYPE_SINT8 }; - -ffi_type ffi_type_uint16 = { 2, 2, FFI_TYPE_UINT16 }; -ffi_type ffi_type_sint16 = { 2, 2, FFI_TYPE_SINT16 }; - -ffi_type ffi_type_uint32 = { 4, INT_ALIGN, FFI_TYPE_UINT32 }; -ffi_type ffi_type_sint32 = { 4, INT_ALIGN, FFI_TYPE_SINT32 }; - -ffi_type ffi_type_uint64 = { 8, LONG_LONG_ALIGN, FFI_TYPE_UINT64 }; -ffi_type ffi_type_sint64 = { 8, LONG_LONG_ALIGN, FFI_TYPE_SINT64 }; - -ffi_type ffi_type_float = { sizeof(float), FLOAT_ALIGN, FFI_TYPE_FLOAT }; -ffi_type ffi_type_double = { sizeof(double), DOUBLE_ALIGN, FFI_TYPE_DOUBLE }; - -#ifdef ffi_type_longdouble -#undef ffi_type_longdouble -#endif - /* This is already defined on OSX */ -ffi_type ffi_type_longdouble = { sizeof(long double), LONGDOUBLE_ALIGN, - FFI_TYPE_LONGDOUBLE }; - -ffi_type ffi_type_pointer = { sizeof(void *), VOID_P_ALIGN, FFI_TYPE_POINTER }; - /*---------------- EOF ----------------*/ diff --git a/Modules/expat/pyexpatns.h b/Modules/expat/pyexpatns.h index 515611c45203c1..8b149b9046e2ec 100644 --- a/Modules/expat/pyexpatns.h +++ b/Modules/expat/pyexpatns.h @@ -90,6 +90,8 @@ #define XmlPrologStateInit PyExpat_XmlPrologStateInit #define XmlPrologStateInitExternalEntity PyExpat_XmlPrologStateInitExternalEntity #define XML_ResumeParser PyExpat_XML_ResumeParser +#define XML_SetAllocTrackerActivationThreshold PyExpat_XML_SetAllocTrackerActivationThreshold +#define XML_SetAllocTrackerMaximumAmplification PyExpat_XML_SetAllocTrackerMaximumAmplification #define XML_SetAttlistDeclHandler PyExpat_XML_SetAttlistDeclHandler #define XML_SetBase PyExpat_XML_SetBase #define XML_SetBillionLaughsAttackProtectionActivationThreshold PyExpat_XML_SetBillionLaughsAttackProtectionActivationThreshold @@ -111,6 +113,7 @@ #define XML_SetExternalEntityRefHandler PyExpat_XML_SetExternalEntityRefHandler #define XML_SetExternalEntityRefHandlerArg PyExpat_XML_SetExternalEntityRefHandlerArg #define XML_SetHashSalt PyExpat_XML_SetHashSalt +#define XML_SetHashSalt16Bytes PyExpat_XML_SetHashSalt16Bytes #define XML_SetNamespaceDeclHandler PyExpat_XML_SetNamespaceDeclHandler #define XML_SetNotationDeclHandler PyExpat_XML_SetNotationDeclHandler #define XML_SetNotStandaloneHandler PyExpat_XML_SetNotStandaloneHandler diff --git a/PCbuild/_ctypes.vcxproj b/PCbuild/_ctypes.vcxproj index 69e4271a9bd8f8..6ac26f1916c9fd 100644 --- a/PCbuild/_ctypes.vcxproj +++ b/PCbuild/_ctypes.vcxproj @@ -94,7 +94,7 @@ - FFI_BUILDING;%(PreprocessorDefinitions) + USING_MALLOC_CLOSURE_DOT_C=1;%(PreprocessorDefinitions) /EXPORT:DllGetClassObject,PRIVATE /EXPORT:DllCanUnloadNow,PRIVATE %(AdditionalOptions)