cpython: 60655e543d8a (original) (raw)

Mercurial > cpython

changeset 100654:60655e543d8a

Add C functions _PyTraceMalloc_Track() Issue #26530: * Add C functions _PyTraceMalloc_Track() and _PyTraceMalloc_Untrack() to track memory blocks using the tracemalloc module. * Add _PyTraceMalloc_GetTraceback() to get the traceback of an object. [#26530]

Victor Stinner victor.stinner@gmail.com
date	Tue, 22 Mar 2016 13:39:05 +0100
parents	b86cdebe0e97
children	0da4532a8933
files	Include/pymem.h Lib/test/test_tracemalloc.py Misc/NEWS Modules/_testcapimodule.c Modules/_tracemalloc.c
diffstat	5 files changed, 300 insertions(+), 11 deletions(-)[+] [-] Include/pymem.h 34 Lib/test/test_tracemalloc.py 115 Misc/NEWS 5 Modules/_testcapimodule.c 75 Modules/_tracemalloc.c 82

line wrap: on

line diff

--- a/Include/pymem.h +++ b/Include/pymem.h @@ -25,6 +25,40 @@ PyAPI_FUNC(int) _PyMem_SetupAllocators(c PyAPI_FUNC(int) _PyMem_PymallocEnabled(void); #endif +/* Identifier of an address space (domain) in tracemalloc / +typedef unsigned int _PyTraceMalloc_domain_t; + +/ Track an allocated memory block in the tracemalloc module.

Return 0 on success, return -1 on error (failed to allocate memory to store
the trace). +
Return -2 if tracemalloc is disabled. +
If memory block was already tracked, begin by removing the old trace. */ +PyAPI_FUNC(int) _PyTraceMalloc_Track(
_PyTraceMalloc_domain_t domain,
Py_uintptr_t ptr,
size_t size);

+ +/* Untrack an allocated memory block in the tracemalloc module.

Do nothing if the block was not tracked. +
Return -2 if tracemalloc is disabled, otherwise return 0. */ +PyAPI_FUNC(int) _PyTraceMalloc_Untrack(
_PyTraceMalloc_domain_t domain,
Py_uintptr_t ptr);

+ +/* Get the traceback where a memory block was allocated. +

Return a tuple of (filename: str, lineno: int) tuples. +
Return None if the tracemalloc module is disabled or if the memory block
is not tracked by tracemalloc. +
Raise an exception and return NULL on error. / +PyAPI_FUNC(PyObject) _PyTraceMalloc_GetTraceback(
_PyTraceMalloc_domain_t domain,
Py_uintptr_t ptr);

#endif /* !Py_LIMITED_API */

--- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -11,9 +11,15 @@ try: import threading except ImportError: threading = None +try:

import _testcapi

+except ImportError:

_testcapi = None

+ EMPTY_STRING_SIZE = sys.getsizeof(b'') + def get_frames(nframe, lineno_delta): frames = [] frame = sys._getframe(1) @@ -866,12 +872,121 @@ class TestCommandLine(unittest.TestCase) assert_python_ok('-X', 'tracemalloc', '-c', code) +@unittest.skipIf(_testcapi is None, 'need _testcapi') +class TestCAPI(unittest.TestCase):

maxDiff = 80 * 20

def setUp(self):

   if tracemalloc.is_tracing():[](#l2.28)

       self.skipTest("tracemalloc must be stopped before the test")[](#l2.29)

```
   self.domain = 5[](#l2.31)
```
```
   self.size = 123[](#l2.32)
```

   self.obj = allocate_bytes(self.size)[0][](#l2.33)

   # for the type "object", id(obj) is the address of its memory block.[](#l2.35)

   # This type is not tracked by the garbage collector[](#l2.36)

```
   self.ptr = id(self.obj)[](#l2.37)
```

def tearDown(self):
```
   tracemalloc.stop()[](#l2.40)
```

def get_traceback(self):

   frames = _testcapi.tracemalloc_get_traceback(self.domain, self.ptr)[](#l2.43)

```
   if frames is not None:[](#l2.44)
```

       return tracemalloc.Traceback(frames)[](#l2.45)

```
   else:[](#l2.46)
```
```
       return None[](#l2.47)
```

def track(self, release_gil=False, nframe=1):

   frames = get_frames(nframe, 2)[](#l2.50)

   _testcapi.tracemalloc_track(self.domain, self.ptr, self.size,[](#l2.51)

                               release_gil)[](#l2.52)

```
   return frames[](#l2.53)
```

def untrack(self):

   _testcapi.tracemalloc_untrack(self.domain, self.ptr)[](#l2.56)

def get_traced_memory(self):

   # Get the traced size in the domain[](#l2.59)

   snapshot = tracemalloc.take_snapshot()[](#l2.60)

   domain_filter = tracemalloc.DomainFilter(True, self.domain)[](#l2.61)

   snapshot = snapshot.filter_traces([domain_filter])[](#l2.62)

   return sum(trace.size for trace in snapshot.traces)[](#l2.63)

def check_track(self, release_gil):
```
   nframe = 5[](#l2.66)
```
```
   tracemalloc.start(nframe)[](#l2.67)
```

   size = tracemalloc.get_traced_memory()[0][](#l2.69)

   frames = self.track(release_gil, nframe)[](#l2.71)

   self.assertEqual(self.get_traceback(),[](#l2.72)

                    tracemalloc.Traceback(frames))[](#l2.73)

   self.assertEqual(self.get_traced_memory(), self.size)[](#l2.75)

def test_track(self):
```
   self.check_track(False)[](#l2.78)
```

def test_track_without_gil(self):

   # check that calling _PyTraceMalloc_Track() without holding the GIL[](#l2.81)

```
   # works too[](#l2.82)
```
```
   self.check_track(True)[](#l2.83)
```

def test_track_already_tracked(self):
```
   nframe = 5[](#l2.86)
```
```
   tracemalloc.start(nframe)[](#l2.87)
```

```
   # track a first time[](#l2.89)
```
```
   self.track()[](#l2.90)
```

   # calling _PyTraceMalloc_Track() must remove the old trace and add[](#l2.92)

   # a new trace with the new traceback[](#l2.93)

   frames = self.track(nframe=nframe)[](#l2.94)

   self.assertEqual(self.get_traceback(),[](#l2.95)

                    tracemalloc.Traceback(frames))[](#l2.96)

def test_untrack(self):
```
   tracemalloc.start()[](#l2.99)
```

```
   self.track()[](#l2.101)
```

   self.assertIsNotNone(self.get_traceback())[](#l2.102)

   self.assertEqual(self.get_traced_memory(), self.size)[](#l2.103)

   # untrack must remove the trace[](#l2.105)

```
   self.untrack()[](#l2.106)
```

   self.assertIsNone(self.get_traceback())[](#l2.107)

   self.assertEqual(self.get_traced_memory(), 0)[](#l2.108)

   # calling _PyTraceMalloc_Untrack() multiple times must not crash[](#l2.110)

```
   self.untrack()[](#l2.111)
```
```
   self.untrack()[](#l2.112)
```

def test_stop_track(self):
```
   tracemalloc.start()[](#l2.115)
```
```
   tracemalloc.stop()[](#l2.116)
```

   with self.assertRaises(RuntimeError):[](#l2.118)

```
       self.track()[](#l2.119)
```

   self.assertIsNone(self.get_traceback())[](#l2.120)

def test_stop_untrack(self):
```
   tracemalloc.start()[](#l2.123)
```
```
   self.track()[](#l2.124)
```

```
   tracemalloc.stop()[](#l2.126)
```

   with self.assertRaises(RuntimeError):[](#l2.127)

```
       self.untrack()[](#l2.128)
```

+ + def test_main(): support.run_unittest( TestTracemallocEnabled, TestSnapshot, TestFilters, TestCommandLine,

```
   TestCAPI,[](#l2.137)
```
) if name == "main":

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -232,6 +232,11 @@ Core and Builtins Library ------- +- Issue #26530: Add C functions :c:func:_PyTraceMalloc_Track and

:c:func:_PyTraceMalloc_Untrack to track memory blocks using the
:mod:tracemalloc module. Add :c:func:_PyTraceMalloc_GetTraceback to get
the traceback of an object. +

Issue #26588: The _tracemalloc now supports tracing memory allocations of multiple address spaces (domains).

--- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3675,6 +3675,78 @@ pyobject_malloc_without_gil(PyObject *se Py_RETURN_NONE; } +static PyObject * +tracemalloc_track(PyObject *self, PyObject *args) +{

unsigned int domain;
PyObject *ptr_obj;
void *ptr;
Py_ssize_t size;
int release_gil = 0;
int res;

if (!PyArg_ParseTuple(args, "IOn|i", &domain, &ptr_obj, &size, &release_gil))
```
   return NULL;[](#l4.18)
```
ptr = PyLong_AsVoidPtr(ptr_obj);
if (PyErr_Occurred())
```
   return NULL;[](#l4.21)
```

if (release_gil) {
```
   Py_BEGIN_ALLOW_THREADS[](#l4.24)
```

   res = _PyTraceMalloc_Track(domain, (Py_uintptr_t)ptr, size);[](#l4.25)

```
   Py_END_ALLOW_THREADS[](#l4.26)
```
}
else {

   res = _PyTraceMalloc_Track(domain, (Py_uintptr_t)ptr, size);[](#l4.29)

}

if (res < 0) {

   PyErr_SetString(PyExc_RuntimeError, "_PyTraceMalloc_Track error");[](#l4.33)

```
   return NULL;[](#l4.34)
```
}

Py_RETURN_NONE;

+} + +static PyObject * +tracemalloc_untrack(PyObject *self, PyObject *args) +{

unsigned int domain;
PyObject *ptr_obj;
void *ptr;
int res;

if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj))
```
   return NULL;[](#l4.49)
```
ptr = PyLong_AsVoidPtr(ptr_obj);
if (PyErr_Occurred())
```
   return NULL;[](#l4.52)
```

res = _PyTraceMalloc_Untrack(domain, (Py_uintptr_t)ptr);
if (res < 0) {

   PyErr_SetString(PyExc_RuntimeError, "_PyTraceMalloc_Track error");[](#l4.56)

```
   return NULL;[](#l4.57)
```
}

Py_RETURN_NONE;

+} + +static PyObject * +tracemalloc_get_traceback(PyObject *self, PyObject *args) +{

unsigned int domain;
PyObject *ptr_obj;
void *ptr;

if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj))
```
   return NULL;[](#l4.71)
```
ptr = PyLong_AsVoidPtr(ptr_obj);
if (PyErr_Occurred())
```
   return NULL;[](#l4.74)
```

return _PyTraceMalloc_GetTraceback(domain, (Py_uintptr_t)ptr);

+} + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -3861,6 +3933,9 @@ static PyMethodDef TestMethods[] = { {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, {"pymem_malloc_without_gil", pymem_malloc_without_gil, METH_NOARGS}, {"pyobject_malloc_without_gil", pyobject_malloc_without_gil, METH_NOARGS},

{"tracemalloc_track", tracemalloc_track, METH_VARARGS},
{"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS},
{"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS}, {NULL, NULL} /* sentinel */ };

--- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -61,8 +61,6 @@ static PyThread_type_lock tables_lock; #define DEFAULT_DOMAIN 0 -typedef unsigned int domain_t; - /* Pack the frame_t structure to reduce the memory footprint. */ typedef struct #ifdef GNUC @@ -70,7 +68,7 @@ typedef struct #endif { Py_uintptr_t ptr;

domain_t domain;

_PyTraceMalloc_domain_t domain;

} pointer_t; /* Pack the frame_t structure to reduce the memory footprint on 64-bit @@ -519,11 +517,13 @@ traceback_new(void) static void -tracemalloc_remove_trace(domain_t domain, Py_uintptr_t ptr) +tracemalloc_remove_trace(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) { trace_t trace; int removed;

assert(tracemalloc_config.tracing);

+ if (tracemalloc_config.use_domain) { pointer_t key = {ptr, domain}; removed = _Py_HASHTABLE_POP(tracemalloc_traces, key, trace); @@ -544,12 +544,15 @@ tracemalloc_remove_trace(domain_t domain static int -tracemalloc_add_trace(domain_t domain, Py_uintptr_t ptr, size_t size) +tracemalloc_add_trace(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr,

```
                 size_t size)[](#l5.42)
```

{ traceback_t *traceback; trace_t trace; int res;

assert(tracemalloc_config.tracing);

+ /* first, remove the previous trace (if any) */ tracemalloc_remove_trace(domain, ptr); @@ -1183,7 +1186,7 @@ traceback_to_pyobject(traceback_t trace static PyObject -trace_to_pyobject(domain_t domain, trace_t *trace, +trace_to_pyobject(_PyTraceMalloc_domain_t domain, trace_t *trace, _Py_hashtable_t *intern_tracebacks) { PyObject *trace_obj = NULL; @@ -1229,7 +1232,7 @@ tracemalloc_get_traces_fill(_Py_hashtabl void *user_data) { get_traces_t *get_traces = user_data;

domain_t domain;

_PyTraceMalloc_domain_t domain; trace_t *trace; PyObject tracemalloc_obj; int res; @@ -1340,7 +1343,7 @@ finally: static traceback_t -tracemalloc_get_traceback(domain_t domain, const void *ptr) +tracemalloc_get_traceback(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) { trace_t trace; int found; @@ -1350,7 +1353,7 @@ tracemalloc_get_traceback(domain_t domai TABLES_LOCK(); if (tracemalloc_config.use_domain) {

   pointer_t key = {(Py_uintptr_t)ptr, domain};[](#l5.84)

   pointer_t key = {ptr, domain};[](#l5.85)
   found = _Py_HASHTABLE_GET(tracemalloc_traces, key, trace);[](#l5.86)

} else { @@ -1387,7 +1390,7 @@ py_tracemalloc_get_object_traceback(PyOb else ptr = (void *)obj;

traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr);

traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, (Py_uintptr_t)ptr); if (traceback == NULL) Py_RETURN_NONE;

@@ -1415,7 +1418,7 @@ void traceback_t *traceback; int i;

traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, ptr);

traceback = tracemalloc_get_traceback(DEFAULT_DOMAIN, (Py_uintptr_t)ptr); if (traceback == NULL) return;

@@ -1686,3 +1689,60 @@ void #endif tracemalloc_deinit(); } + +int +_PyTraceMalloc_Track(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr,

```
                size_t size)[](#l5.114)
```

int res;

+#ifdef WITH_THREAD

PyGILState_STATE gil_state;

+#endif +

if (!tracemalloc_config.tracing) {

   /* tracemalloc is not tracing: do nothing */[](#l5.122)

```
   return -2;[](#l5.123)
```
}

+ +#ifdef WITH_THREAD

gil_state = PyGILState_Ensure();

+#endif +

TABLES_LOCK();
res = tracemalloc_add_trace(domain, ptr, size);
TABLES_UNLOCK();

+ +#ifdef WITH_THREAD

PyGILState_Release(gil_state);

+#endif

return res;

+} + + +int +_PyTraceMalloc_Untrack(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) +{

if (!tracemalloc_config.tracing) {

   /* tracemalloc is not tracing: do nothing */[](#l5.145)

```
   return -2;[](#l5.146)
```
}

TABLES_LOCK();
tracemalloc_remove_trace(domain, ptr);
TABLES_UNLOCK();

return 0;

+} + + +PyObject* +_PyTraceMalloc_GetTraceback(_PyTraceMalloc_domain_t domain, Py_uintptr_t ptr) +{

traceback_t *traceback;

traceback = tracemalloc_get_traceback(domain, ptr);
if (traceback == NULL)
```
   Py_RETURN_NONE;[](#l5.164)
```

return traceback_to_pyobject(traceback, NULL);