From 69d37ecfec0fb1903b7e2bf9983a9a3a28a55749 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 26 Aug 2026 15:05:58 +0530 Subject: [PATCH 1/3] gh-155725: Remove PyGILState_Ensure from tracemalloc tracemalloc no longer acquires the GIL nor creates a temporary thread state when tracing memory allocations. A thread with no attached thread state now records the trace with the "" traceback instead of attaching a thread state to capture the Python traceback. Threads without a thread state used to pay for a GIL acquisition plus a full thread state creation and destruction on every traced raw allocation, only to record an empty traceback anyway. --- Doc/c-api/memory.rst | 11 ++ Lib/test/test_tracemalloc.py | 13 ++- ...-08-26-10-00-00.gh-issue-155725.tRcMal.rst | 6 ++ Python/tracemalloc.c | 102 +++++------------- 4 files changed, 53 insertions(+), 79 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-26-10-00-00.gh-issue-155725.tRcMal.rst diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 73310670ac371c..746de7ce596b5b 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -780,6 +780,17 @@ tracemalloc C API If memory block is already tracked, update the existing trace. + The function can be called from any thread, with or without an + :term:`attached thread state`. If the calling thread has no attached + thread state, the traceback of the allocation is recorded as + ````. + + .. versionchanged:: next + The function no longer acquires the :term:`GIL`. When called from a + thread without an :term:`attached thread state`, the traceback is now + recorded as ```` instead of the Python traceback of the + calling thread. + .. c:function:: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) Untrack an allocated memory block in the :mod:`tracemalloc` module. diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 9d3ff8a620b6f2..5e7f83097dd803 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -1047,8 +1047,13 @@ def check_track(self, release_gil): size = tracemalloc.get_traced_memory()[0] frames = self.track(release_gil, nframe) - self.assertEqual(self.get_traceback(), - tracemalloc.Traceback(frames)) + if release_gil: + # PyTraceMalloc_Track() never acquires the GIL: without an + # attached thread state, the traceback is recorded as + expected = tracemalloc.Traceback([("", 0)]) + else: + expected = tracemalloc.Traceback(frames) + self.assertEqual(self.get_traceback(), expected) self.assertEqual(self.get_traced_memory(), self.size) @@ -1056,8 +1061,8 @@ def test_track(self): self.check_track(False) def test_track_without_gil(self): - # check that calling _PyTraceMalloc_Track() without holding the GIL - # works too + # check that calling PyTraceMalloc_Track() with the thread state + # detached (GIL released) records the trace with an unknown traceback self.check_track(True) def test_track_already_tracked(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-26-10-00-00.gh-issue-155725.tRcMal.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-26-10-00-00.gh-issue-155725.tRcMal.rst new file mode 100644 index 00000000000000..d93ef83bcc28b0 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-26-10-00-00.gh-issue-155725.tRcMal.rst @@ -0,0 +1,6 @@ +:mod:`tracemalloc` no longer acquires the :term:`GIL` nor creates a +temporary thread state when tracing memory allocations. Memory allocations +and :c:func:`PyTraceMalloc_Track` calls made by threads without an +:term:`attached thread state` now record the traceback as ```` +instead of temporarily attaching a thread state to capture the Python +traceback of the calling thread. diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 0afc84e021817c..ea3224f543d02d 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -32,9 +32,10 @@ static int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, #define allocators _PyRuntime.tracemalloc.allocators -/* This lock is needed because tracemalloc_free() is called without - the GIL held from PyMem_RawFree(). It cannot acquire the lock because it - would introduce a deadlock in _PyThreadState_DeleteCurrent(). */ +/* This lock protects the trace tables. It is acquired by threads which may + not have an attached thread state, such as tracemalloc_free() called from + PyMem_RawFree(): tracing never acquires the GIL nor attaches a thread + state. */ #define tables_lock _PyRuntime.tracemalloc.tables_lock #define TABLES_LOCK() PyMutex_LockFlags(&tables_lock, _Py_LOCK_DONT_DETACH) #define TABLES_UNLOCK() PyMutex_Unlock(&tables_lock) @@ -303,11 +304,8 @@ traceback_hash(traceback_t *traceback) static void -traceback_get_frames(traceback_t *traceback) +traceback_get_frames(traceback_t *traceback, PyThreadState *tstate) { - PyThreadState *tstate = _PyThreadState_GET(); - assert(tstate != NULL); - _PyInterpreterFrame *pyframe = _PyThreadState_GetFrame(tstate); while (pyframe) { if (traceback->nframe < tracemalloc_config.max_nframe) { @@ -329,13 +327,19 @@ traceback_new(void) traceback_t *traceback; _Py_hashtable_entry_t *entry; - _Py_AssertHoldsTstate(); + // A thread with no attached thread state cannot capture a Python + // traceback and must not use Python objects, such as the interned + // filenames: record the trace with the "" traceback instead. + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate == NULL) { + return tracemalloc_empty_traceback; + } /* get frames */ traceback = tracemalloc_traceback; traceback->nframe = 0; traceback->total_nframe = 0; - traceback_get_frames(traceback); + traceback_get_frames(traceback, tstate); if (traceback->nframe == 0) { return tracemalloc_empty_traceback; } @@ -497,8 +501,7 @@ tracemalloc_add_trace_unlocked(unsigned int domain, uintptr_t ptr, static void* -tracemalloc_alloc(int need_gil, int use_calloc, - void *ctx, size_t nelem, size_t elsize) +tracemalloc_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) { assert(elsize == 0 || nelem <= SIZE_MAX / elsize); @@ -506,12 +509,9 @@ tracemalloc_alloc(int need_gil, int use_calloc, // Ignore reentrant call. // - // For example, PyObjet_Malloc() calls + // For example, PyObject_Malloc() calls // PyMem_Malloc() for allocations larger than 512 bytes: don't trace the // same memory allocation twice. - // - // If reentrant calls are not ignored, PyGILState_Ensure() can call - // PyMem_RawMalloc() which would call PyGILState_Ensure() again in a loop. if (!reentrant) { set_reentrant(1); } @@ -532,10 +532,6 @@ tracemalloc_alloc(int need_gil, int use_calloc, goto done; } - PyGILState_STATE gil_state; - if (need_gil) { - gil_state = PyGILState_Ensure(); - } TABLES_LOCK(); if (tracemalloc_config.tracing) { @@ -548,9 +544,6 @@ tracemalloc_alloc(int need_gil, int use_calloc, // else: gh-128679: tracemalloc.stop() was called by another thread TABLES_UNLOCK(); - if (need_gil) { - PyGILState_Release(gil_state); - } done: if (!reentrant) { @@ -561,7 +554,7 @@ tracemalloc_alloc(int need_gil, int use_calloc, static void* -tracemalloc_realloc(int need_gil, void *ctx, void *ptr, size_t new_size) +tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) { int reentrant = get_reentrant(); @@ -582,10 +575,6 @@ tracemalloc_realloc(int need_gil, void *ctx, void *ptr, size_t new_size) goto done; } - PyGILState_STATE gil_state; - if (need_gil) { - gil_state = PyGILState_Ensure(); - } TABLES_LOCK(); if (!tracemalloc_config.tracing) { @@ -610,8 +599,8 @@ tracemalloc_realloc(int need_gil, void *ctx, void *ptr, size_t new_size) // This case is very unlikely: a hash entry has just been released, // so the hash table should have at least one free entry. // - // The GIL and the table lock ensures that only one thread is - // allocating memory. + // The table lock ensures that no other thread touched the trace + // tables in the meantime. Py_FatalError("tracemalloc_realloc() failed to allocate a trace"); } } @@ -627,9 +616,6 @@ tracemalloc_realloc(int need_gil, void *ctx, void *ptr, size_t new_size) unlock: TABLES_UNLOCK(); - if (need_gil) { - PyGILState_Release(gil_state); - } done: if (!reentrant) { @@ -665,44 +651,16 @@ tracemalloc_free(void *ctx, void *ptr) static void* -tracemalloc_malloc_gil(void *ctx, size_t size) -{ - return tracemalloc_alloc(0, 0, ctx, 1, size); -} - - -static void* -tracemalloc_calloc_gil(void *ctx, size_t nelem, size_t elsize) +tracemalloc_malloc(void *ctx, size_t size) { - return tracemalloc_alloc(0, 1, ctx, nelem, elsize); + return tracemalloc_alloc(0, ctx, 1, size); } static void* -tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) +tracemalloc_calloc(void *ctx, size_t nelem, size_t elsize) { - return tracemalloc_realloc(0, ctx, ptr, new_size); -} - - -static void* -tracemalloc_raw_malloc(void *ctx, size_t size) -{ - return tracemalloc_alloc(1, 0, ctx, 1, size); -} - - -static void* -tracemalloc_raw_calloc(void *ctx, size_t nelem, size_t elsize) -{ - return tracemalloc_alloc(1, 1, ctx, nelem, elsize); -} - - -static void* -tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) -{ - return tracemalloc_realloc(1, ctx, ptr, new_size); + return tracemalloc_alloc(1, ctx, nelem, elsize); } @@ -717,7 +675,8 @@ tracemalloc_clear_filename(void *value) static void tracemalloc_clear_traces_unlocked(void) { - // Clearing tracemalloc_filenames requires the GIL to call Py_DECREF() + // Clearing tracemalloc_filenames requires an attached thread state to + // call Py_DECREF() _Py_AssertHoldsTstate(); set_reentrant(1); @@ -829,20 +788,15 @@ _PyTraceMalloc_Start(int max_nframe) } PyMemAllocatorEx alloc; - alloc.malloc = tracemalloc_raw_malloc; - alloc.calloc = tracemalloc_raw_calloc; - alloc.realloc = tracemalloc_raw_realloc; + alloc.malloc = tracemalloc_malloc; + alloc.calloc = tracemalloc_calloc; + alloc.realloc = tracemalloc_realloc; alloc.free = tracemalloc_free; alloc.ctx = &allocators.raw; PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); - alloc.malloc = tracemalloc_malloc_gil; - alloc.calloc = tracemalloc_calloc_gil; - alloc.realloc = tracemalloc_realloc_gil; - alloc.free = tracemalloc_free; - alloc.ctx = &allocators.mem; PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); @@ -1221,7 +1175,6 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, /* tracemalloc is not tracing: do nothing */ return -2; } - PyGILState_STATE gil_state = PyGILState_Ensure(); TABLES_LOCK(); int result; @@ -1234,7 +1187,6 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, } TABLES_UNLOCK(); - PyGILState_Release(gil_state); return result; } From 5893aba17b1c50c13d59027526eac7601c955da9 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 26 Aug 2026 21:50:56 +0530 Subject: [PATCH 2/3] gh-155725: Capture tracemalloc tracebacks without a thread state attached Store traceback frame filenames as interned NUL terminated UTF-8 strings instead of Python str objects, so that capturing a traceback no longer uses or modifies Python objects. Threads without an attached thread state now capture their Python traceback by walking the frames of the thread state most recently bound to the thread; only threads which never had a thread state record the traceback as "". --- Doc/c-api/memory.rst | 13 +- Include/internal/pycore_tracemalloc.h | 9 +- Lib/test/test_tracemalloc.py | 13 +- ...-08-26-10-00-00.gh-issue-155725.tRcMal.rst | 10 +- Python/tracemalloc.c | 298 ++++++++++++++---- 5 files changed, 256 insertions(+), 87 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 746de7ce596b5b..cdaa7e0b4c7b6d 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -781,15 +781,14 @@ tracemalloc C API If memory block is already tracked, update the existing trace. The function can be called from any thread, with or without an - :term:`attached thread state`. If the calling thread has no attached - thread state, the traceback of the allocation is recorded as - ````. + :term:`attached thread state`. The traceback is captured using the + Python thread state associated with the calling thread, even if it is + not attached. If the thread has no Python thread state, the traceback + is recorded as ````. .. versionchanged:: next - The function no longer acquires the :term:`GIL`. When called from a - thread without an :term:`attached thread state`, the traceback is now - recorded as ```` instead of the Python traceback of the - calling thread. + The function no longer acquires the :term:`GIL` nor creates a + temporary thread state. .. c:function:: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index 9974ea3c4143fa..0522b24cd8fc67 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -44,9 +44,10 @@ struct __attribute__((packed)) #endif tracemalloc_frame { - /* filename cannot be NULL: "" is used if the Python frame - filename is NULL */ - PyObject *filename; + /* Interned NUL terminated UTF-8 (surrogatepass) string. + Cannot be NULL: "" is used if the Python frame filename + cannot be captured. */ + const char *filename; unsigned int lineno; }; @@ -85,7 +86,7 @@ struct _tracemalloc_runtime_state { Protected by TABLES_LOCK(). */ size_t peak_traced_memory; /* Hash table used as a set to intern filenames: - PyObject* => PyObject*. + char* (NUL terminated UTF-8 string) => NULL. Protected by the TABLES_LOCK(). */ _Py_hashtable_t *filenames; /* Buffer to store a new traceback in traceback_new(). diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 5e7f83097dd803..ee02fbd16af0c4 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -1047,13 +1047,8 @@ def check_track(self, release_gil): size = tracemalloc.get_traced_memory()[0] frames = self.track(release_gil, nframe) - if release_gil: - # PyTraceMalloc_Track() never acquires the GIL: without an - # attached thread state, the traceback is recorded as - expected = tracemalloc.Traceback([("", 0)]) - else: - expected = tracemalloc.Traceback(frames) - self.assertEqual(self.get_traceback(), expected) + self.assertEqual(self.get_traceback(), + tracemalloc.Traceback(frames)) self.assertEqual(self.get_traced_memory(), self.size) @@ -1061,8 +1056,8 @@ def test_track(self): self.check_track(False) def test_track_without_gil(self): - # check that calling PyTraceMalloc_Track() with the thread state - # detached (GIL released) records the trace with an unknown traceback + # check that calling PyTraceMalloc_Track() without the GIL + # (detached thread state) still captures the Python traceback self.check_track(True) def test_track_already_tracked(self): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-26-10-00-00.gh-issue-155725.tRcMal.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-26-10-00-00.gh-issue-155725.tRcMal.rst index d93ef83bcc28b0..3e157446954f61 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-26-10-00-00.gh-issue-155725.tRcMal.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-26-10-00-00.gh-issue-155725.tRcMal.rst @@ -1,6 +1,6 @@ :mod:`tracemalloc` no longer acquires the :term:`GIL` nor creates a -temporary thread state when tracing memory allocations. Memory allocations -and :c:func:`PyTraceMalloc_Track` calls made by threads without an -:term:`attached thread state` now record the traceback as ```` -instead of temporarily attaching a thread state to capture the Python -traceback of the calling thread. +temporary thread state to trace memory allocations: traceback frames now +store plain UTF-8 strings instead of Python str objects, and tracebacks are +captured using the Python thread state already associated with the calling +thread, even if it is not attached. Threads without a Python thread state +record the traceback as ````. diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index ea3224f543d02d..2cdc4ad3853333 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -8,7 +8,7 @@ #include "pycore_object.h" // _PyType_PreHeaderSize() #include "pycore_pymem.h" // _Py_tracemalloc_config #include "pycore_runtime.h" // _Py_ID() -#include "pycore_traceback.h" // _Py_DumpASCII() +#include "pycore_traceback.h" // _Py_DumpHexadecimal() #include // malloc() @@ -46,6 +46,10 @@ static int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, typedef struct tracemalloc_frame frame_t; typedef struct tracemalloc_traceback traceback_t; +/* Filename used when the Python frame filename cannot be captured */ +static const char tracemalloc_unknown_filename[] = ""; +#define UNKNOWN_FILENAME tracemalloc_unknown_filename + #define TRACEBACK_SIZE(NFRAME) \ (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME)) @@ -127,24 +131,19 @@ set_reentrant(int reentrant) static Py_uhash_t -hashtable_hash_pyobject(const void *key) +hashtable_hash_filename(const void *key) { - PyObject *obj = (PyObject *)key; - return PyObject_Hash(obj); + const char *filename = (const char *)key; + return (Py_uhash_t)Py_HashBuffer(filename, (Py_ssize_t)strlen(filename)); } static int -hashtable_compare_unicode(const void *key1, const void *key2) +hashtable_compare_filename(const void *key1, const void *key2) { - PyObject *obj1 = (PyObject *)key1; - PyObject *obj2 = (PyObject *)key2; - if (obj1 != NULL && obj2 != NULL) { - return (PyUnicode_Compare(obj1, obj2) == 0); - } - else { - return obj1 == obj2; - } + const char *filename1 = (const char *)key1; + const char *filename2 = (const char *)key2; + return (strcmp(filename1, filename2) == 0); } @@ -182,6 +181,100 @@ raw_free(void *ptr) } +/* Encode a str object to a NUL terminated UTF-8 (surrogatepass) string, + without using the Python C API. Return NULL on allocation failure. */ +static char * +tracemalloc_encode_filename(PyObject *obj) +{ + int kind = PyUnicode_KIND(obj); + const void *data = PyUnicode_DATA(obj); + Py_ssize_t length = PyUnicode_GET_LENGTH(obj); + + // worst case: 4 UTF-8 bytes per code point, plus the NUL terminator + if ((size_t)length > (SIZE_MAX - 1) / 4) { + return NULL; + } + char *buffer = raw_malloc((size_t)length * 4 + 1); + if (buffer == NULL) { + return NULL; + } + + char *p = buffer; + for (Py_ssize_t i = 0; i < length; i++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if (ch < 0x80) { + *p++ = (char)ch; + } + else if (ch < 0x800) { + *p++ = (char)(0xc0 | (ch >> 6)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } + else if (ch < 0x10000) { + *p++ = (char)(0xe0 | (ch >> 12)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } + else { + *p++ = (char)(0xf0 | (ch >> 18)); + *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); + *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); + *p++ = (char)(0x80 | (ch & 0x3f)); + } + } + *p = '\0'; + return buffer; +} + + +/* Intern a str object in the tracemalloc_filenames hash table as a NUL + terminated UTF-8 string. Return NULL on allocation failure. + The caller must hold the TABLES_LOCK(). */ +static const char * +tracemalloc_intern_filename(PyObject *obj) +{ + assert(PyUnicode_Check(obj)); + + const char *utf8; + char *encoded = NULL; + if (PyUnicode_IS_COMPACT_ASCII(obj)) { + // ASCII string data is valid UTF-8 and is NUL terminated + utf8 = (const char *)PyUnicode_DATA(obj); + } + else { + encoded = tracemalloc_encode_filename(obj); + if (encoded == NULL) { + return NULL; + } + utf8 = encoded; + } + + const char *result; + _Py_hashtable_entry_t *entry; + entry = _Py_hashtable_get_entry(tracemalloc_filenames, utf8); + if (entry != NULL) { + result = (const char *)entry->key; + } + else { + size_t size = strlen(utf8) + 1; + char *filename = raw_malloc(size); + if (filename == NULL) { + raw_free(encoded); + return NULL; + } + memcpy(filename, utf8, size); + + if (_Py_hashtable_set(tracemalloc_filenames, filename, NULL) < 0) { + raw_free(filename); + raw_free(encoded); + return NULL; + } + result = filename; + } + raw_free(encoded); + return result; +} + + static Py_uhash_t hashtable_hash_traceback(const void *key) { @@ -210,8 +303,8 @@ hashtable_compare_traceback(const void *key1, const void *key2) if (frame1->lineno != frame2->lineno) { return 0; } + // Filenames are interned: compare by pointer if (frame1->filename != frame2->filename) { - assert(PyUnicode_Compare(frame1->filename, frame2->filename) != 0); return 0; } } @@ -223,7 +316,7 @@ static void tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) { assert(PyStackRef_CodeCheck(pyframe->f_executable)); - frame->filename = &_Py_STR(anon_unknown); + frame->filename = UNKNOWN_FILENAME; int lineno = -1; PyCodeObject *code = _PyFrame_GetCode(pyframe); @@ -254,26 +347,15 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) } /* intern the filename */ - _Py_hashtable_entry_t *entry; - entry = _Py_hashtable_get_entry(tracemalloc_filenames, filename); - if (entry != NULL) { - filename = (PyObject *)entry->key; - } - else { - /* tracemalloc_filenames is responsible to keep a reference - to the filename */ - if (_Py_hashtable_set(tracemalloc_filenames, Py_NewRef(filename), - NULL) < 0) { - Py_DECREF(filename); + const char *filename_copy = tracemalloc_intern_filename(filename); + if (filename_copy == NULL) { #ifdef TRACE_DEBUG - tracemalloc_error("failed to intern the filename"); + tracemalloc_error("failed to intern the filename"); #endif - return; - } + return; } - /* the tracemalloc_filenames table keeps a reference to the filename */ - frame->filename = filename; + frame->filename = filename_copy; } @@ -289,7 +371,8 @@ traceback_hash(traceback_t *traceback) x = 0x345678UL; frame = traceback->frames; while (--len >= 0) { - y = (Py_uhash_t)PyObject_Hash(frame->filename); + // Filenames are interned: hash the pointer + y = (Py_uhash_t)Py_HashPointer(frame->filename); y ^= (Py_uhash_t)frame->lineno; frame++; @@ -327,12 +410,22 @@ traceback_new(void) traceback_t *traceback; _Py_hashtable_entry_t *entry; - // A thread with no attached thread state cannot capture a Python - // traceback and must not use Python objects, such as the interned - // filenames: record the trace with the "" traceback instead. + // Capturing a traceback needs a thread state to walk the frame stack, + // but the thread state doesn't need to be attached: only the thread + // itself pushes and pops its own frames, and no Python object is used + // or modified. If not attached (e.g. the GIL was released), fall back + // to the thread state most recently bound to the thread, if any. PyThreadState *tstate = _PyThreadState_GET(); if (tstate == NULL) { - return tracemalloc_empty_traceback; + if (_PyRuntimeState_GetFinalizing(&_PyRuntime) != NULL) { + // non-attached thread states can be cleared during finalization + return tracemalloc_empty_traceback; + } + tstate = PyGILState_GetThisThreadState(); + if (tstate == NULL) { + // the thread never had a thread state: no frames to capture + return tracemalloc_empty_traceback; + } } /* get frames */ @@ -664,21 +757,9 @@ tracemalloc_calloc(void *ctx, size_t nelem, size_t elsize) } -static void -tracemalloc_clear_filename(void *value) -{ - PyObject *filename = (PyObject *)value; - Py_DECREF(filename); -} - - static void tracemalloc_clear_traces_unlocked(void) { - // Clearing tracemalloc_filenames requires an attached thread state to - // call Py_DECREF() - _Py_AssertHoldsTstate(); - set_reentrant(1); _Py_hashtable_clear(tracemalloc_traces); @@ -704,9 +785,9 @@ _PyTraceMalloc_Init(void) return _PyStatus_NO_MEMORY(); } - tracemalloc_filenames = hashtable_new(hashtable_hash_pyobject, - hashtable_compare_unicode, - tracemalloc_clear_filename, NULL); + tracemalloc_filenames = hashtable_new(hashtable_hash_filename, + hashtable_compare_filename, + raw_free, NULL); tracemalloc_tracebacks = hashtable_new(hashtable_hash_traceback, hashtable_compare_traceback, @@ -729,8 +810,7 @@ _PyTraceMalloc_Init(void) tracemalloc_empty_traceback->nframe = 1; tracemalloc_empty_traceback->total_nframe = 1; - /* borrowed reference */ - tracemalloc_empty_traceback->frames[0].filename = &_Py_STR(anon_unknown); + tracemalloc_empty_traceback->frames[0].filename = UNKNOWN_FILENAME; tracemalloc_empty_traceback->frames[0].lineno = 0; tracemalloc_empty_traceback->hash = traceback_hash(tracemalloc_empty_traceback); @@ -852,17 +932,62 @@ _PyTraceMalloc_Stop(void) +/* Convert an interned filename to a str object. intern_filenames + (const char* => str object, can be NULL) shares the str objects. */ static PyObject* -frame_to_pyobject(frame_t *frame) +filename_to_pyobject(const char *filename, _Py_hashtable_t *intern_filenames) +{ + PyObject *filename_obj; + if (intern_filenames != NULL) { + filename_obj = _Py_hashtable_get(intern_filenames, filename); + if (filename_obj != NULL) { + return Py_NewRef(filename_obj); + } + } + + if (filename == UNKNOWN_FILENAME) { + filename_obj = Py_NewRef(&_Py_STR(anon_unknown)); + } + else { + filename_obj = PyUnicode_DecodeUTF8(filename, + (Py_ssize_t)strlen(filename), + "surrogatepass"); + if (filename_obj == NULL) { + return NULL; + } + } + + if (intern_filenames != NULL) { + if (_Py_hashtable_set(intern_filenames, filename, filename_obj) < 0) { + Py_DECREF(filename_obj); + PyErr_NoMemory(); + return NULL; + } + /* intern_filenames keeps a new reference to filename_obj */ + Py_INCREF(filename_obj); + } + return filename_obj; +} + + +static PyObject* +frame_to_pyobject(frame_t *frame, _Py_hashtable_t *intern_filenames) { assert(get_reentrant()); + PyObject *filename_obj = filename_to_pyobject(frame->filename, + intern_filenames); + if (filename_obj == NULL) { + return NULL; + } + PyObject *frame_obj = PyTuple_New(2); if (frame_obj == NULL) { + Py_DECREF(filename_obj); return NULL; } - PyTuple_SET_ITEM(frame_obj, 0, Py_NewRef(frame->filename)); + PyTuple_SET_ITEM(frame_obj, 0, filename_obj); PyObject *lineno_obj = PyLong_FromUnsignedLong(frame->lineno); if (lineno_obj == NULL) { @@ -876,7 +1001,8 @@ frame_to_pyobject(frame_t *frame) static PyObject* -traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) +traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table, + _Py_hashtable_t *intern_filenames) { PyObject *frames; if (intern_table != NULL) { @@ -892,7 +1018,8 @@ traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) } for (int i=0; i < traceback->nframe; i++) { - PyObject *frame = frame_to_pyobject(&traceback->frames[i]); + PyObject *frame = frame_to_pyobject(&traceback->frames[i], + intern_filenames); if (frame == NULL) { Py_DECREF(frames); return NULL; @@ -915,7 +1042,8 @@ traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) static PyObject* trace_to_pyobject(unsigned int domain, const trace_t *trace, - _Py_hashtable_t *intern_tracebacks) + _Py_hashtable_t *intern_tracebacks, + _Py_hashtable_t *intern_filenames) { assert(get_reentrant()); @@ -938,7 +1066,8 @@ trace_to_pyobject(unsigned int domain, const trace_t *trace, } PyTuple_SET_ITEM(trace_obj, 1, obj); - obj = traceback_to_pyobject(trace->traceback, intern_tracebacks); + obj = traceback_to_pyobject(trace->traceback, intern_tracebacks, + intern_filenames); if (obj == NULL) { Py_DECREF(trace_obj); return NULL; @@ -960,6 +1089,7 @@ typedef struct { _Py_hashtable_t *traces; _Py_hashtable_t *domains; _Py_hashtable_t *tracebacks; + _Py_hashtable_t *filenames; PyObject *list; unsigned int domain; } get_traces_t; @@ -1054,7 +1184,8 @@ tracemalloc_get_traces_fill(_Py_hashtable_t *traces, const trace_t *trace = (const trace_t *)value; PyObject *tuple = trace_to_pyobject(get_traces->domain, trace, - get_traces->tracebacks); + get_traces->tracebacks, + get_traces->filenames); if (tuple == NULL) { return 1; } @@ -1114,11 +1245,41 @@ tracemalloc_get_traceback_unlocked(unsigned int domain, uintptr_t ptr) #define PUTS(fd, str) (void)_Py_write_noraise(fd, str, (int)strlen(str)) +/* Dump an interned filename: write printable ASCII characters as-is, + escape the other bytes. The function is signal-safe. */ +static void +_PyMem_DumpFilename(int fd, const char *filename) +{ + const size_t max_length = 500; + size_t length = strlen(filename); + int truncated = 0; + if (length > max_length) { + length = max_length; + truncated = 1; + } + + for (size_t i = 0; i < length; i++) { + unsigned char ch = (unsigned char)filename[i]; + if (' ' <= ch && ch <= 126) { + /* printable ASCII character */ + char c = (char)ch; + (void)_Py_write_noraise(fd, &c, 1); + } + else { + PUTS(fd, "\\x"); + _Py_DumpHexadecimal(fd, ch, 2); + } + } + if (truncated) { + PUTS(fd, "..."); + } +} + static void _PyMem_DumpFrame(int fd, frame_t * frame) { PUTS(fd, " File \""); - _Py_DumpASCII(fd, frame->filename); + _PyMem_DumpFilename(fd, frame->filename); PUTS(fd, "\", line "); _Py_DumpDecimal(fd, frame->lineno); PUTS(fd, "\n"); @@ -1276,7 +1437,7 @@ _PyTraceMalloc_GetTraceback(unsigned int domain, uintptr_t ptr) PyObject *result; if (traceback) { set_reentrant(1); - result = traceback_to_pyobject(traceback, NULL); + result = traceback_to_pyobject(traceback, NULL, NULL); set_reentrant(0); } else { @@ -1317,6 +1478,7 @@ _PyTraceMalloc_GetTraces(void) get_traces.traces = NULL; get_traces.domains = NULL; get_traces.tracebacks = NULL; + get_traces.filenames = NULL; get_traces.list = PyList_New(0); if (get_traces.list == NULL) { goto finally; @@ -1335,6 +1497,15 @@ _PyTraceMalloc_GetTraces(void) goto no_memory; } + /* the filename hash table is used temporarily to share filename + str objects between tracebacks */ + get_traces.filenames = hashtable_new(_Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, tracemalloc_pyobject_decref); + if (get_traces.filenames == NULL) { + goto no_memory; + } + // Copy all traces so tracemalloc_get_traces_fill() doesn't have to disable // temporarily tracemalloc which would impact other threads and so would // miss allocations while get_traces() is called. @@ -1376,6 +1547,9 @@ _PyTraceMalloc_GetTraces(void) if (get_traces.tracebacks != NULL) { _Py_hashtable_destroy(get_traces.tracebacks); } + if (get_traces.filenames != NULL) { + _Py_hashtable_destroy(get_traces.filenames); + } if (get_traces.traces != NULL) { _Py_hashtable_destroy(get_traces.traces); } From b483beaf4f835324311381f37b5d39a3575bffd8 Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Wed, 26 Aug 2026 21:52:26 +0530 Subject: [PATCH 3/3] gh-155725: Revert Doc/c-api/memory.rst changes --- Doc/c-api/memory.rst | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index cdaa7e0b4c7b6d..73310670ac371c 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -780,16 +780,6 @@ tracemalloc C API If memory block is already tracked, update the existing trace. - The function can be called from any thread, with or without an - :term:`attached thread state`. The traceback is captured using the - Python thread state associated with the calling thread, even if it is - not attached. If the thread has no Python thread state, the traceback - is recorded as ````. - - .. versionchanged:: next - The function no longer acquires the :term:`GIL` nor creates a - temporary thread state. - .. c:function:: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) Untrack an allocated memory block in the :mod:`tracemalloc` module.