cpython: cef6a32d805f (original) (raw)

Mercurial > cpython

changeset 100551:cef6a32d805f

On memory error, dump the memory block traceback Issue #26564: _PyObject_DebugDumpAddress() now dumps the traceback where a memory block was allocated on memory block. Use the tracemalloc module to get the traceback. [#26564]

Victor Stinner victor.stinner@gmail.com
date	Tue, 15 Mar 2016 22:22:13 +0100
parents	5eb223e1638c
children	8215dae7ec3c
files	Doc/c-api/memory.rst Doc/whatsnew/3.6.rst Misc/NEWS Modules/_tracemalloc.c Modules/hashtable.c Objects/bytearrayobject.c Objects/obmalloc.c Parser/pgenmain.c
diffstat	8 files changed, 126 insertions(+), 19 deletions(-)[+] [-] Doc/c-api/memory.rst 7 Doc/whatsnew/3.6.rst 43 Misc/NEWS 4 Modules/_tracemalloc.c 67 Modules/hashtable.c 6 Objects/bytearrayobject.c 1 Objects/obmalloc.c 9 Parser/pgenmain.c 8

line wrap: on

line diff

--- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -349,12 +349,19 @@ Customize Memory Allocators allocator functions of the :c:data:PYMEM_DOMAIN_OBJ domain (ex: :c:func:PyObject_Malloc) are called

On error, the debug hooks use the :mod:tracemalloc module to get the
traceback where a memory block was allocated. The traceback is only
displayed if :mod:tracemalloc is tracing Python memory allocations and the
memory block was traced. + These hooks are installed by default if Python is compiled in debug mode. The :envvar:PYTHONMALLOC environment variable can be used to install debug hooks on a Python compiled in release mode. .. versionchanged:: 3.6 This function now also works on Python compiled in release mode.

 On error, the debug hooks now use :mod:`tracemalloc` to get the traceback[](#l1.18)

 where a memory block was allocated.[](#l1.19)

.. _pymalloc:

--- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -129,7 +129,48 @@ the C library for all Python memory allo It helps to use external memory debuggers like Valgrind on a Python compiled in release mode. -(Contributed by Victor Stinner in :issue:26516.) +On error, the debug hooks on Python memory allocators now use the +:mod:tracemalloc module to get the traceback where a memory block was +allocated. + +Example of fatal error on buffer overflow using +python3.6 -X tracemalloc=5 (store 5 frames in traces):: +

Debug memory block at address p=0x7fbcd41666f8: API 'o'

   4 bytes originally requested[](#l2.16)

   The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.[](#l2.17)

   The 8 pad bytes at tail=0x7fbcd41666fc are not all FORBIDDENBYTE (0xfb):[](#l2.18)

       at tail+0: 0x02 *** OUCH[](#l2.19)

```
       at tail+1: 0xfb[](#l2.20)
```
```
       at tail+2: 0xfb[](#l2.21)
```
```
       at tail+3: 0xfb[](#l2.22)
```
```
       at tail+4: 0xfb[](#l2.23)
```
```
       at tail+5: 0xfb[](#l2.24)
```
```
       at tail+6: 0xfb[](#l2.25)
```
```
       at tail+7: 0xfb[](#l2.26)
```

   The block was made by call #1233329 to debug malloc/realloc.[](#l2.27)

```
   Data at p: 1a 2b 30 00[](#l2.28)
```

Memory block allocated at (most recent call first):

 File "test/test_bytes.py", line 323[](#l2.31)

 File "unittest/case.py", line 600[](#l2.32)

 File "unittest/case.py", line 648[](#l2.33)

 File "unittest/suite.py", line 122[](#l2.34)

 File "unittest/suite.py", line 84[](#l2.35)

Fatal Python error: bad trailing pad byte

Current thread 0x00007fbcdbd32700 (most recent call first):

 File "test/test_bytes.py", line 323 in test_hex[](#l2.40)

 File "unittest/case.py", line 600 in run[](#l2.41)

 File "unittest/case.py", line 648 in __call__[](#l2.42)

 File "unittest/suite.py", line 122 in run[](#l2.43)

 File "unittest/suite.py", line 84 in __call__[](#l2.44)

 File "unittest/suite.py", line 122 in run[](#l2.45)

 File "unittest/suite.py", line 84 in __call__[](#l2.46)

```
 ...[](#l2.47)
```

+ +(Contributed by Victor Stinner in :issue:26516 and :issue:26564.) Other Language Changes

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,10 @@ Release date: tba Core and Builtins ----------------- +- Issue #26564: On error, the debug hooks on Python memory allocators now use

the :mod:tracemalloc module to get the traceback where a memory block was
allocated. +

Issue #26558: The debug hooks on Python memory allocator :c:func:PyObject_Malloc now detect when functions are called without holding the GIL.

--- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -1161,6 +1161,25 @@ finally: return get_traces.list; } +static traceback_t* +tracemalloc_get_traceback(const void *ptr) +{

trace_t trace;
int found;

if (!tracemalloc_config.tracing)
```
   return NULL;[](#l4.14)
```

TABLES_LOCK();
found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace);
TABLES_UNLOCK();

if (!found)
```
   return NULL;[](#l4.21)
```

return trace.traceback;

+} + PyDoc_STRVAR(tracemalloc_get_object_traceback_doc, "_get_object_traceback(obj)\n" "\n" @@ -1175,11 +1194,7 @@ py_tracemalloc_get_object_traceback(PyOb { PyTypeObject *type; void *ptr;

trace_t trace;
int found;

if (!tracemalloc_config.tracing)
```
   Py_RETURN_NONE;[](#l4.37)
```

traceback_t *traceback;

type = Py_TYPE(obj); if (PyType_IS_GC(type)) @@ -1187,16 +1202,46 @@ py_tracemalloc_get_object_traceback(PyOb else ptr = (void *)obj;

TABLES_LOCK();
found = _Py_HASHTABLE_GET(tracemalloc_traces, ptr, trace);
TABLES_UNLOCK();

if (!found)

traceback = tracemalloc_get_traceback(ptr);
if (traceback == NULL) Py_RETURN_NONE;

return traceback_to_pyobject(trace.traceback, NULL);

return traceback_to_pyobject(traceback, NULL);

+} + +#define PUTS(fd, str) _Py_write_noraise(fd, str, (int)strlen(str)) + +static void +_PyMem_DumpFrame(int fd, frame_t * frame) +{

PUTS(fd, " File "");
_Py_DumpASCII(fd, frame->filename);
PUTS(fd, "", line ");
_Py_DumpDecimal(fd, frame->lineno);
PUTS(fd, "\n");

} +/* Dump the traceback where a memory block was allocated into file descriptor

fd. The function may block on TABLES_LOCK() but it is unlikely. */ +void +_PyMem_DumpTraceback(int fd, const void *ptr) +{
traceback_t *traceback;
int i;

traceback = tracemalloc_get_traceback(ptr);
if (traceback == NULL)
```
   return;[](#l4.81)
```

PUTS(fd, "Memory block allocated at (most recent call first):\n");
for (i=0; i < traceback->nframe; i++) {

   _PyMem_DumpFrame(fd, &traceback->frames[i]);[](#l4.85)

}
PUTS(fd, "\n");

+} + +#undef PUTS + PyDoc_STRVAR(tracemalloc_start_doc, "start(nframe: int=1)\n" "\n"

--- a/Modules/hashtable.c +++ b/Modules/hashtable.c @@ -486,9 +486,9 @@ void void *data, *new_data; dst = _Py_hashtable_new_full(src->data_size, src->num_buckets,

                       src->hash_func, src->compare_func,[](#l5.7)

                       src->copy_data_func, src->free_data_func,[](#l5.8)

                       src->get_data_size_func, &src->alloc);[](#l5.9)

                            src->hash_func, src->compare_func,[](#l5.10)

                            src->copy_data_func, src->free_data_func,[](#l5.11)

                            src->get_data_size_func, &src->alloc);[](#l5.12)

if (dst == NULL) return NULL;

--- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -2820,6 +2820,7 @@ bytearray_hex(PyBytesObject self) { char argbuf = PyByteArray_AS_STRING(self); Py_ssize_t arglen = PyByteArray_GET_SIZE(self);

PyByteArray_AS_STRING(self)[arglen+1] = 2; return _Py_strhex(argbuf, arglen); }

--- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1,5 +1,10 @@ #include "Python.h" + +/* Defined in tracemalloc.c */ +extern void _PyMem_DumpTraceback(int fd, const void ptr); + + / Python's malloc wrappers (see pymem.h) / / @@ -2202,6 +2207,10 @@ static void } fputc('\n', stderr); }

fputc('\n', stderr);

fflush(stderr);
_PyMem_DumpTraceback(fileno(stderr), p);

}

--- a/Parser/pgenmain.c +++ b/Parser/pgenmain.c @@ -38,11 +38,11 @@ Py_Exit(int sts) } #ifdef WITH_THREAD -/* Needed by obmalloc.c / +/ Functions needed by obmalloc.c */ int PyGILState_Check(void) -{

return 1;

-} +{ return 1; } +void _PyMem_DumpTraceback(int fd, const void *ptr) +{} #endif int