cpython: 4e43e5b3f7fc (original) (raw)

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -9,6 +9,10 @@ What's New in Python 2.7.7? Core and Builtins ----------------- +- Issue #20494: Ensure that free()d memory arenas are really released on POSIX

systems supporting anonymous memory mappings. Patch by Charles-François
Natali. +

Issue #17825: Cursor "^" is correctly positioned for SyntaxError and IndentationError.

--- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2,6 +2,13 @@ #ifdef WITH_PYMALLOC +#ifdef HAVE_MMAP

#include <sys/mman.h>
#ifdef MAP_ANONYMOUS
#define ARENAS_USE_MMAP
#endif

+#endif + #ifdef WITH_VALGRIND #include <valgrind/valgrind.h> @@ -75,7 +82,8 @@ static int running_on_valgrind = -1;

Allocation strategy abstract: *
For small requests, the allocator sub-allocates blocks of memory.

- Requests greater than 256 bytes are routed to the system's allocator.

- Requests greater than SMALL_REQUEST_THRESHOLD bytes are routed to the
- system's allocator. *
- Small requests are grouped in size classes spaced 8 bytes apart, due
- to the required valid alignment of the returned address. Requests of @@ -107,10 +115,11 @@ static int running_on_valgrind = -1;
- ```
  57-64                   64                       7[](#l2.28)
```
- ```
  65-72                   72                       8[](#l2.29)
```
- ```
   ...                   ...                     ...[](#l2.30)
```

 241-248                 248                      30[](#l2.31)

 249-256                 256                      31[](#l2.32)

 497-504                 504                      62[](#l2.33)

 505-512                 512                      63 [](#l2.34)

 0, 257 and up: routed to the underlying allocator.[](#l2.36)

 0, SMALL_REQUEST_THRESHOLD + 1 and up: routed to the underlying[](#l2.37)

```
 allocator.[](#l2.38)
```

/ /==========================================================================*/ @@ -143,10 +152,13 @@ static int running_on_valgrind = -1;

 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 256[](#l2.43)

 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT[](#l2.44)

- Note: a size threshold of 512 guarantees that newly created dictionaries
- will be allocated from preallocated memory pools on 64-bit.
*
- Although not required, for better performance and space efficiency,
- it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2. */ -#define SMALL_REQUEST_THRESHOLD 256 +#define SMALL_REQUEST_THRESHOLD 512

#define NB_SMALL_SIZE_CLASSES (SMALL_REQUEST_THRESHOLD / ALIGNMENT) / @@ -174,15 +186,15 @@ static int running_on_valgrind = -1; /

The allocator sub-allocates blocks of memory (called arenas) aligned
on a page boundary. This is a reserved virtual address space for the

- current process (obtained through a malloc call). In no way this means
- that the memory arenas will be used entirely. A malloc() is usually
- an address range reservation for bytes, unless all pages within this
- space are referenced subsequently. So malloc'ing big blocks and not using
- them does not mean "wasting memory". It's an addressable range wastage...

- current process (obtained through a malloc()/mmap() call). In no way this
- means that the memory arenas will be used entirely. A malloc() is
- usually an address range reservation for bytes, unless all pages within
- this space are referenced subsequently. So malloc'ing big blocks and not
- using them does not mean "wasting memory". It's an addressable range
- wastage... *

- Therefore, allocating arenas with malloc is not optimal, because there is
- some address space wastage, but this is the most portable way to request
- memory from the system across various platforms.

- Arenas are allocated with mmap() on systems supporting anonymous memory
- mappings to reduce heap fragmentation. */

#define ARENA_SIZE (256 << 10) /* 256KB / @@ -440,6 +452,9 @@ static poolp usedpools[2 * ((NB_SMALL_SI , PT(48), PT(49), PT(50), PT(51), PT(52), PT(53), PT(54), PT(55) #if NB_SMALL_SIZE_CLASSES > 56 , PT(56), PT(57), PT(58), PT(59), PT(60), PT(61), PT(62), PT(63) +#if NB_SMALL_SIZE_CLASSES > 64 +#error "NB_SMALL_SIZE_CLASSES should be less than 64" +#endif / NB_SMALL_SIZE_CLASSES > 64 / #endif / NB_SMALL_SIZE_CLASSES > 56 / #endif / NB_SMALL_SIZE_CLASSES > 48 / #endif / NB_SMALL_SIZE_CLASSES > 40 */ @@ -577,7 +592,12 @@ new_arena(void) arenaobj = unused_arena_objects; unused_arena_objects = arenaobj->nextarena; assert(arenaobj->address == 0); +#ifdef ARENAS_USE_MMAP

arenaobj->address = (uptr)mmap(NULL, ARENA_SIZE, PROT_READ|PROT_WRITE,

                              MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);[](#l2.97)

+#else arenaobj->address = (uptr)malloc(ARENA_SIZE); +#endif if (arenaobj->address == 0) { /* The allocation failed: return NULL after putting the * arenaobj back. @@ -1054,7 +1074,11 @@ PyObject_Free(void p) unused_arena_objects = ao; / Free the entire arena. */ +#ifdef ARENAS_USE_MMAP

           munmap((void *)ao->address, ARENA_SIZE);[](#l2.109)

+#else free((void )ao->address); +#endif ao->address = 0; / mark unassociated */ --narenas_currently_allocated;

--- a/configure +++ b/configure @@ -10164,7 +10164,7 @@ for ac_func in alarm setitimer getitimer clock confstr ctermid execv fchmod fchown fork fpathconf ftime ftruncate [](#l3.4) gai_strerror getgroups getlogin getloadavg getpeername getpgid getpid [](#l3.5) getpriority getresuid getresgid getpwent getspnam getspent getsid getwd [](#l3.6)

initgroups kill killpg lchmod lchown lstat mkfifo mknod mktime [](#l3.7)

initgroups kill killpg lchmod lchown lstat mkfifo mknod mktime mmap [](#l3.8) mremap nice pathconf pause plock poll pthread_init [](#l3.9) putenv readlink realpath [](#l3.10) select sem_open sem_timedwait sem_getvalue sem_unlink setegid seteuid [](#l3.11)

--- a/configure.ac +++ b/configure.ac @@ -2905,7 +2905,7 @@ AC_CHECK_FUNCS(alarm setitimer getitimer clock confstr ctermid execv fchmod fchown fork fpathconf ftime ftruncate [](#l4.4) gai_strerror getgroups getlogin getloadavg getpeername getpgid getpid [](#l4.5) getpriority getresuid getresgid getpwent getspnam getspent getsid getwd [](#l4.6)

initgroups kill killpg lchmod lchown lstat mkfifo mknod mktime [](#l4.7)

initgroups kill killpg lchmod lchown lstat mkfifo mknod mktime mmap [](#l4.8) mremap nice pathconf pause plock poll pthread_init [](#l4.9) putenv readlink realpath [](#l4.10) select sem_open sem_timedwait sem_getvalue sem_unlink setegid seteuid [](#l4.11)

--- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -475,6 +475,9 @@ /* Define to 1 if you have the mktime' function. */[](#l5.4) #undef HAVE_MKTIME[](#l5.5) [](#l5.6) +/* Define to 1 if you have the mmap' function. / +#undef HAVE_MMAP + / Define to 1 if you have the `mremap' function. */ #undef HAVE_MREMAP