From 08d141d814a7185999ad738738d41b92355157bb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2026 00:07:56 +0200 Subject: [PATCH 1/2] gh-157710: Detect overflow in PyUnicodeWriter_Finish() Check f the trailing null character has been modified to detect buffer overflow in C extensions. _PyUnicode_CheckConsistency() now always check if the trailing null character has been overridden to detect buffer overflow. Previously, it was only been checked if check_content parameter was non-zero. --- Lib/test/test_capi/test_unicode.py | 18 ++++++++++++++- ...-09-18-00-29-49.gh-issue-157710.Q_AD8D.rst | 3 +++ Modules/_testinternalcapi.c | 23 +++++++++++++++++++ Objects/unicode_writer.c | 15 ++++++++++++ Objects/unicodeobject.c | 9 ++++---- 5 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-09-18-00-29-49.gh-issue-157710.Q_AD8D.rst diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 0dcd8a25ad0128d..d25e298bfdb4a63 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1,7 +1,9 @@ -import unittest import sys +import textwrap +import unittest from test import support from test.support import threading_helper +from test.support.script_helper import assert_python_failure try: import _testcapi @@ -1981,6 +1983,20 @@ def test_substring_empty(self): writer.write_substring("abc", 1, 1) self.assertEqual(writer.finish(), '') + @unittest.skipUnless(support.Py_DEBUG, 'need debug build (Py_DEBUG)') + def test_detect_overflow(self): + # Test detection of buffer overflow + code = textwrap.dedent(''' + from test.support import SuppressCrashReport + import _testinternalcapi + + SuppressCrashReport().__enter__() + _testinternalcapi.unicodewriter_overflow() + ''') + proc = assert_python_failure('-c', code) + self.assertIn(b'Buffer overflow detected in PyUnicodeWriter', proc.err) + self.assertIn(f'at position 6'.encode(), proc.err) + @unittest.skipIf(ctypes is None, 'need ctypes') class PyUnicodeWriterFormatTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/C_API/2026-09-18-00-29-49.gh-issue-157710.Q_AD8D.rst b/Misc/NEWS.d/next/C_API/2026-09-18-00-29-49.gh-issue-157710.Q_AD8D.rst new file mode 100644 index 000000000000000..532978485af85a5 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-09-18-00-29-49.gh-issue-157710.Q_AD8D.rst @@ -0,0 +1,3 @@ +When Python is built in debug mode, :c:func:`PyUnicodeWriter_Finish` now +checks if the trailing null byte has been overridden to detect buffer +overflow. Patch by Victor Stinner. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 38e56ae70420985..d30affdbf621392 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -3206,6 +3206,28 @@ test_thread_state_ensure_from_view_interp_switch(PyObject *self, PyObject *unuse Py_RETURN_NONE; } +static PyObject * +unicodewriter_overflow(PyObject *self, PyObject *unused) +{ + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) { + return NULL; + } + if (PyUnicodeWriter_WriteASCII(writer, "hello", -1) < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } + + _PyUnicodeWriter *impl = (_PyUnicodeWriter*)writer; + PyObject *buffer = impl->buffer; + Py_ssize_t index = PyUnicode_GET_LENGTH(buffer); + PyUnicode_WRITE(impl->kind, impl->data, index, '#'); // overflow! + + // Spoiler: the function doesn't return if an overflow is detected + // in debug mode + return PyUnicodeWriter_Finish(writer); +} + /* Self interrupting context manager */ typedef struct { @@ -3393,6 +3415,7 @@ static PyMethodDef module_functions[] = { {"test_interp_guard_countdown", test_interp_guard_countdown, METH_NOARGS}, {"test_interp_view_countdown", test_interp_view_countdown, METH_NOARGS}, {"test_thread_state_ensure_from_view_interp_switch", test_thread_state_ensure_from_view_interp_switch, METH_NOARGS}, + {"unicodewriter_overflow", unicodewriter_overflow, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/unicode_writer.c b/Objects/unicode_writer.c index a753c9b971c702c..e4363b34d6e8d2e 100644 --- a/Objects/unicode_writer.c +++ b/Objects/unicode_writer.c @@ -602,6 +602,20 @@ _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) { PyObject *str; +#ifdef Py_DEBUG + // Check for buffer overflow + if (writer->buffer != NULL) { + Py_ssize_t pos = PyUnicode_GET_LENGTH(writer->buffer); + Py_UCS4 ch = PyUnicode_READ_CHAR(writer->buffer, pos); + if (ch != 0) { + _Py_FatalErrorFormat(__func__, + "Buffer overflow detected in " + "PyUnicodeWriter %p at position %zd", + writer, pos); + } + } +#endif + if (writer->pos == 0) { Py_CLEAR(writer->buffer); return _PyUnicode_GetEmpty(); @@ -612,6 +626,7 @@ _PyUnicodeWriter_Finish(_PyUnicodeWriter *writer) if (writer->readonly) { assert(PyUnicode_GET_LENGTH(str) == writer->pos); + assert(_PyUnicode_CheckConsistency(str, 1)); return str; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 86b9baadd0d8aa9..117ce876760e877 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -601,7 +601,6 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) # define CHECK_IF_FT(expr) (void)(expr) #endif - assert(op != NULL); CHECK(PyUnicode_Check(op)); @@ -647,13 +646,12 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) } /* check that the best kind is used: O(n) operation */ + const void *data = PyUnicode_DATA(ascii); if (check_content) { Py_ssize_t i; Py_UCS4 maxchar = 0; - const void *data; Py_UCS4 ch; - data = PyUnicode_DATA(ascii); for (i=0; i < ascii->length; i++) { ch = PyUnicode_READ(kind, data, i); @@ -676,9 +674,12 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) CHECK(maxchar >= 0x10000); CHECK(maxchar <= MAX_UNICODE); } - CHECK(PyUnicode_READ(kind, data, ascii->length) == 0); } + // Detect buffer overflow: check if the trailing null character + // has been overridden + CHECK(PyUnicode_READ(kind, data, ascii->length) == 0); + /* Check interning state */ #ifdef Py_DEBUG // Note that we do not check `_Py_IsImmortal(op)` in the GIL-enabled build From 85966a9039f825fc56bf5b66cd0c3f4ef04b087a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 18 Sep 2026 01:10:13 +0200 Subject: [PATCH 2/2] Fix test_capi on Windows --- Lib/test/test_capi/test_unicode.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index d25e298bfdb4a63..2cf508dbaf355e2 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -1995,7 +1995,9 @@ def test_detect_overflow(self): ''') proc = assert_python_failure('-c', code) self.assertIn(b'Buffer overflow detected in PyUnicodeWriter', proc.err) - self.assertIn(f'at position 6'.encode(), proc.err) + # Do not test the position value since it depends on the overallocation + # strategy which depends on the operating system + self.assertIn(f'at position '.encode(), proc.err) @unittest.skipIf(ctypes is None, 'need ctypes')