cpython: 10a79a33d09b (original) (raw)

--- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -1,4 +1,5 @@ -from test.support import verbose, run_unittest +from test.support import verbose, run_unittest, gc_collect +import io import re from re import Scanner import sys @@ -16,6 +17,17 @@ import unittest class ReTests(unittest.TestCase):

• def test_keep_buffer(self):

•    # See bug 14212[](#l1.15)

•    b = bytearray(b'x')[](#l1.16)

•    it = re.finditer(b'a', b)[](#l1.17)

•    with self.assertRaises(BufferError):[](#l1.18)

•        b.extend(b'x'*400)[](#l1.19)

•    list(it)[](#l1.20)

•    del it[](#l1.21)

•    gc_collect()[](#l1.22)

•    b.extend(b'x'*400)[](#l1.23)

+ def test_weakref(self): s = 'QabbbcR' x = re.compile('ab+c')

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -549,6 +549,9 @@ Tests Extension Modules ----------------- +- Issue #14212: The re module didn't retain a reference to buffers it was

• scanning, resulting in segfaults. +

• Issue #13840: The error message produced by ctypes.create_string_buffer when given a Unicode string has been fixed.

--- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -1664,7 +1664,7 @@ state_reset(SRE_STATE* state) } static void* -getstring(PyObject* string, Py_ssize_t* p_length, int* p_charsize) +getstring(PyObject* string, Py_ssize_t* p_length, int* p_charsize, Py_buffer view) { / given a python object, return a data pointer, a length (in characters), and a character size. return NULL if the object @@ -1674,7 +1674,6 @@ getstring(PyObject* string, Py_ssize_t* Py_ssize_t size, bytes; int charsize; void* ptr;

• Py_buffer view;

/* Unicode objects do not support the buffer API. So, get the data directly instead. / @@ -1686,26 +1685,21 @@ getstring(PyObject string, Py_ssize_t* } /* get pointer to string buffer */

• view.len = -1;

• view->len = -1; buffer = Py_TYPE(string)->tp_as_buffer; if (!buffer || !buffer->bf_getbuffer ||

•    (*buffer->bf_getbuffer)(string, &view, PyBUF_SIMPLE) < 0) {[](#l3.28)

•    (*buffer->bf_getbuffer)(string, view, PyBUF_SIMPLE) < 0) {[](#l3.29)
         PyErr_SetString(PyExc_TypeError, "expected string or buffer");[](#l3.30)
         return NULL;[](#l3.31)

  } /* determine buffer size */

• bytes = view.len;
• ptr = view.buf;

-

• /* Release the buffer immediately --- possibly dangerous

•   but doing something else would require some re-factoring[](#l3.39)

• */
• PyBuffer_Release(&view);

• bytes = view->len;
• ptr = view->buf;

if (bytes < 0) { PyErr_SetString(PyExc_TypeError, "buffer has negative size");

•    return NULL;[](#l3.47)

•    goto err;[](#l3.48)

  } /* determine character size / @@ -1719,7 +1713,7 @@ getstring(PyObject string, Py_ssize_t*

#endif else { PyErr_SetString(PyExc_TypeError, "buffer size mismatch");

•    return NULL;[](#l3.56)

•    goto err;[](#l3.57)

  } p_length = size; @@ -1728,8 +1722,13 @@ getstring(PyObject string, Py_ssize_t* if (ptr == NULL) { PyErr_SetString(PyExc_ValueError, "Buffer is NULL");

•        goto err;[](#l3.65)

  } return ptr;
• err:
• PyBuffer_Release(view);
• view->buf = NULL;
• return NULL;

} LOCAL(PyObject*) @@ -1747,20 +1746,21 @@ state_init(SRE_STATE* state, PatternObje state->lastmark = -1; state->lastindex = -1;

• ptr = getstring(string, &length, &charsize);

• state->buffer.buf = NULL;
• ptr = getstring(string, &length, &charsize, &state->buffer); if (!ptr)

•    return NULL;[](#l3.83)

- - if (charsize == 1 && pattern->charsize > 1) { - PyErr_SetString(PyExc_TypeError,

•    goto err;[](#l3.87)

+

• if (charsize == 1 && pattern->charsize > 1) {

•    PyErr_SetString(PyExc_TypeError,[](#l3.90)
         "can't use a string pattern on a bytes-like object");[](#l3.91)

•   return NULL;[](#l3.92)

• }
• if (charsize > 1 && pattern->charsize == 1) {

•   PyErr_SetString(PyExc_TypeError,[](#l3.95)

•    goto err;[](#l3.96)

• }
• if (charsize > 1 && pattern->charsize == 1) {

•    PyErr_SetString(PyExc_TypeError,[](#l3.99)
         "can't use a bytes pattern on a string-like object");[](#l3.100)

•   return NULL;[](#l3.101)

• }

•    goto err;[](#l3.103)

• }

/* adjust boundaries */ if (start < 0) @@ -1797,11 +1797,17 @@ state_init(SRE_STATE* state, PatternObje state->lower = sre_lower; return string;

• err:
• if (state->buffer.buf)

•    PyBuffer_Release(&state->buffer);[](#l3.114)

• return NULL;

} LOCAL(void) state_fini(SRE_STATE* state) {

• if (state->buffer.buf)

•    PyBuffer_Release(&state->buffer);[](#l3.122)

  Py_XDECREF(state->string); data_stack_dealloc(state); } @@ -1863,6 +1869,8 @@ pattern_dealloc(PatternObject* self) { if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self);
• if (self->view.buf)

•    PyBuffer_Release(&self->view);[](#l3.131)

  Py_XDECREF(self->pattern); Py_XDECREF(self->groupindex); Py_XDECREF(self->indexgroup); @@ -2297,6 +2305,7 @@ pattern_subx(PatternObject* self, PyObje Py_ssize_t i, b, e; int bint; int filter_is_callable;
• Py_buffer view;

if (PyCallable_Check(ptemplate)) { /* sub/subn takes either a function or a template / @@ -2306,7 +2315,8 @@ pattern_subx(PatternObject self, PyObje } else { /* if not callable, check if it's a literal string */ int literal;

•    ptr = getstring(ptemplate, &n, &bint);[](#l3.147)

•    view.buf = NULL;[](#l3.148)

•    ptr = getstring(ptemplate, &n, &bint, &view);[](#l3.149)
     b = bint;[](#l3.150)
     if (ptr) {[](#l3.151)
         if (b == 1) {[](#l3.152)

@@ -2320,6 +2330,8 @@ pattern_subx(PatternObject* self, PyObje PyErr_Clear(); literal = 0; }

•    if (view.buf)[](#l3.157)

•        PyBuffer_Release(&view);[](#l3.158)
     if (literal) {[](#l3.159)
         filter = ptemplate;[](#l3.160)
         Py_INCREF(filter);[](#l3.161)

@@ -2661,6 +2673,7 @@ static PyObject Py_ssize_t groups = 0; PyObject groupindex = NULL; PyObject* indexgroup = NULL; + if (!PyArg_ParseTuple(args, "OiO!|nOO", &pattern, &flags, &PyList_Type, &code, &groups, &groupindex, &indexgroup)) @@ -2675,6 +2688,7 @@ static PyObject * self->pattern = NULL; self->groupindex = NULL; self->indexgroup = NULL;

• self->view.buf = NULL;

self->codesize = n; @@ -2694,15 +2708,15 @@ static PyObject * return NULL; } - if (pattern == Py_None) - self->charsize = -1; - else { - Py_ssize_t p_length; - if (!getstring(pattern, &p_length, &self->charsize)) { - Py_DECREF(self); - return NULL; - } - }

• if (pattern == Py_None)

•    self->charsize = -1;[](#l3.192)

• else {

•    Py_ssize_t p_length;[](#l3.194)

•    if (!getstring(pattern, &p_length, &self->charsize, &self->view)) {[](#l3.195)

•        Py_DECREF(self);[](#l3.196)

•        return NULL;[](#l3.197)

•    }[](#l3.198)

• }

Py_INCREF(pattern); self->pattern = pattern;

--- a/Modules/sre.h +++ b/Modules/sre.h @@ -31,6 +31,7 @@ typedef struct { int flags; /* flags used when compiling pattern source */ PyObject weakreflist; / List of weak references / int charsize; / pattern charsize (or -1) */

• Py_buffer view; /* pattern code / Py_ssize_t codesize; SRE_CODE code[1]; @@ -80,6 +81,7 @@ typedef struct { char data_stack; size_t data_stack_size; size_t data_stack_base;
• Py_buffer buffer; /* current repeat context */ SRE_REPEAT repeat; / hooks */