cpython: ff0220c9d213 (original) (raw)

Mercurial > cpython

changeset 68383:ff0220c9d213

Issue #9935: Speed up pickling of instances of user-defined classes. [#9935]

line wrap: on

line diff

--- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3,6 +3,7 @@ import unittest import pickle import pickletools import copyreg +import weakref from http.cookies import SimpleCookie from test.support import TestFailed, TESTFN, run_with_locale, no_tracing @@ -842,6 +843,25 @@ class AbstractPickleTests(unittest.TestC self.assertEqual(B(x), B(y), detail) self.assertEqual(x.dict, y.dict, detail)

• def test_newobj_proxies(self):

•    # NEWOBJ should use the __class__ rather than the raw type[](#l1.16)

•    classes = myclasses[:][](#l1.17)

•    # Cannot create weakproxies to these classes[](#l1.18)

•    for c in (MyInt, MyTuple):[](#l1.19)

•        classes.remove(c)[](#l1.20)

•    for proto in protocols:[](#l1.21)

•        for C in classes:[](#l1.22)

•            B = C.__base__[](#l1.23)

•            x = C(C.sample)[](#l1.24)

•            x.foo = 42[](#l1.25)

•            p = weakref.proxy(x)[](#l1.26)

•            s = self.dumps(p, proto)[](#l1.27)

•            y = self.loads(s)[](#l1.28)

•            self.assertEqual(type(y), type(x))  # rather than type(p)[](#l1.29)

•            detail = (proto, C, B, x, y, type(y))[](#l1.30)

•            self.assertEqual(B(x), B(y), detail)[](#l1.31)

•            self.assertEqual(x.__dict__, y.__dict__, detail)[](#l1.32)

+ # Register a type with copyreg, with extension code extcode. Pickle # an object of that type. Check that the resulting pickle uses opcode # (EXT[124]) under proto 2, and not in proto 1. @@ -1009,7 +1029,6 @@ class AbstractPickleTests(unittest.TestC self.assertRaises(RuntimeError, self.dumps, x, proto) # protocol 2 don't raise a RuntimeError. d = self.dumps(x, 2)

•    self.assertRaises(RuntimeError, self.loads, d)[](#l1.41)

def test_reduce_bad_iterator(self): # Issue4176: crash when 4th and 5th items of reduce()

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -62,6 +62,8 @@ Core and Builtins Library ------- +- Issue #9935: Speed up pickling of instances of user-defined classes. +

• Issue #5622: Fix curses.wrapper to raise correct exception if curses initialization fails.

--- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -2842,6 +2842,28 @@ save_pers(PicklerObject *self, PyObject return status; } +static PyObject * +get_class(PyObject *obj) +{

• PyObject *cls;
• static PyObject *str_class;

+

• if (str_class == NULL) {

•    str_class = PyUnicode_InternFromString("__class__");[](#l3.14)

•    if (str_class == NULL)[](#l3.15)

•        return NULL;[](#l3.16)

• }
• cls = PyObject_GetAttr(obj, str_class);
• if (cls == NULL) {

•    if (PyErr_ExceptionMatches(PyExc_AttributeError)) {[](#l3.20)

•        PyErr_Clear();[](#l3.21)

•        cls = (PyObject *) Py_TYPE(obj);[](#l3.22)

•        Py_INCREF(cls);[](#l3.23)

•    }[](#l3.24)

• }
• return cls;

+} + /* We're saving obj, and args is the 2-thru-5 tuple returned by the

• appropriate reduce method for obj. */ @@ -2907,17 +2929,18 @@ save_reduce(PicklerObject self, PyObjec / Protocol 2 special case: if callable's name is newobj, use NEWOBJ. */ if (use_newobj) {

•    static PyObject *newobj_str = NULL;[](#l3.36)

•    PyObject *name_str;[](#l3.37)

•    static PyObject *newobj_str = NULL, *name_str = NULL;[](#l3.38)

•    PyObject *name;[](#l3.39)

if (newobj_str == NULL) { newobj_str = PyUnicode_InternFromString("newobj");

•        if (newobj_str == NULL)[](#l3.43)

•        name_str = PyUnicode_InternFromString("__name__");[](#l3.44)

•        if (newobj_str == NULL || name_str == NULL)[](#l3.45)
             return -1;[](#l3.46)
     }[](#l3.47)

•    name_str = PyObject_GetAttrString(callable, "__name__");[](#l3.49)

•    if (name_str == NULL) {[](#l3.50)

•    name = PyObject_GetAttr(callable, name_str);[](#l3.51)

•    if (name == NULL) {[](#l3.52)
         if (PyErr_ExceptionMatches(PyExc_AttributeError))[](#l3.53)
             PyErr_Clear();[](#l3.54)
         else[](#l3.55)

@@ -2925,9 +2948,9 @@ save_reduce(PicklerObject *self, PyObjec use_newobj = 0; } else {

•        use_newobj = PyUnicode_Check(name_str) && [](#l3.60)

•            PyUnicode_Compare(name_str, newobj_str) == 0;[](#l3.61)

•        Py_DECREF(name_str);[](#l3.62)

•        use_newobj = PyUnicode_Check(name) &&[](#l3.63)

•                     PyUnicode_Compare(name, newobj_str) == 0;[](#l3.64)

•        Py_DECREF(name);[](#l3.65)
     }[](#l3.66)

  } if (use_newobj) { @@ -2943,20 +2966,14 @@ save_reduce(PicklerObject *self, PyObjec }

cls = PyTuple_GET_ITEM(argtup, 0);

•    if (!PyObject_HasAttrString(cls, "__new__")) {[](#l3.73)

•    if (!PyType_Check(cls)) {[](#l3.74)
         PyErr_SetString(PicklingError, "args[0] from "[](#l3.75)

•                        "__newobj__ args has no __new__");[](#l3.76)

•                        "__newobj__ args is not a type");[](#l3.77)
         return -1;[](#l3.78)
     }[](#l3.79)

if (obj != NULL) {

•        obj_class = PyObject_GetAttrString(obj, "__class__");[](#l3.82)

•        if (obj_class == NULL) {[](#l3.83)

•            if (PyErr_ExceptionMatches(PyExc_AttributeError))[](#l3.84)

•                PyErr_Clear();[](#l3.85)

•            else[](#l3.86)

•                return -1;[](#l3.87)

•        }[](#l3.88)

•        obj_class = get_class(obj);[](#l3.89)
         p = obj_class != cls;    /* true iff a problem */[](#l3.90)
         Py_DECREF(obj_class);[](#l3.91)
         if (p) {[](#l3.92)

--- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3073,14 +3073,19 @@ static PyObject * import_copyreg(void) { static PyObject *copyreg_str;

• static PyObject *mod_copyreg = NULL;

if (!copyreg_str) { copyreg_str = PyUnicode_InternFromString("copyreg"); if (copyreg_str == NULL) return NULL; } -

• return PyImport_Import(copyreg_str);

• if (!mod_copyreg) {

•    mod_copyreg = PyImport_Import(copyreg_str);[](#l4.17)

• }

+

• Py_XINCREF(mod_copyreg);
• return mod_copyreg;

} static PyObject * @@ -3089,14 +3094,16 @@ slotnames(PyObject *cls) PyObject *clsdict; PyObject *copyreg; PyObject *slotnames; -

• if (!PyType_Check(cls)) {

•    Py_INCREF(Py_None);[](#l4.31)

•    return Py_None;[](#l4.32)

• static PyObject *str_slotnames;

+

• if (str_slotnames == NULL) {

•    str_slotnames = PyUnicode_InternFromString("__slotnames__");[](#l4.36)

•    if (str_slotnames == NULL)[](#l4.37)

•        return NULL;[](#l4.38)

  } clsdict = ((PyTypeObject *)cls)->tp_dict;

• slotnames = PyDict_GetItemString(clsdict, "slotnames");

• slotnames = PyDict_GetItem(clsdict, str_slotnames); if (slotnames != NULL && PyList_Check(slotnames)) { Py_INCREF(slotnames); return slotnames;

@@ -3130,12 +3137,20 @@ reduce_2(PyObject *obj) PyObject *slots = NULL, *listitems = NULL, *dictitems = NULL; PyObject *copyreg = NULL, *newobj = NULL, *res = NULL; Py_ssize_t i, n; -

• cls = PyObject_GetAttrString(obj, "class");
• if (cls == NULL)

•    return NULL;[](#l4.54)

-

• getnewargs = PyObject_GetAttrString(obj, "getnewargs");

• static PyObject *str_getnewargs = NULL, *str_getstate = NULL,

•                *str_newobj = NULL;[](#l4.58)

+

• if (str_getnewargs == NULL) {

•    str_getnewargs = PyUnicode_InternFromString("__getnewargs__");[](#l4.61)

•    str_getstate = PyUnicode_InternFromString("__getstate__");[](#l4.62)

•    str_newobj = PyUnicode_InternFromString("__newobj__");[](#l4.63)

•    if (!str_getnewargs || !str_getstate || !str_newobj)[](#l4.64)

•        return NULL;[](#l4.65)

• }

+

• cls = (PyObject *) Py_TYPE(obj);

+

• getnewargs = PyObject_GetAttr(obj, str_getnewargs); if (getnewargs != NULL) { args = PyObject_CallObject(getnewargs, NULL); Py_DECREF(getnewargs);

@@ -3153,7 +3168,7 @@ reduce_2(PyObject *obj) if (args == NULL) goto end;

• getstate = PyObject_GetAttrString(obj, "getstate");

• getstate = PyObject_GetAttr(obj, str_getstate); if (getstate != NULL) { state = PyObject_CallObject(getstate, NULL); Py_DECREF(getstate);

@@ -3161,17 +3176,18 @@ reduce_2(PyObject *obj) goto end; } else {

•    PyObject **dict;[](#l4.87)
     PyErr_Clear();[](#l4.88)

•    state = PyObject_GetAttrString(obj, "__dict__");[](#l4.89)

•    if (state == NULL) {[](#l4.90)

•        PyErr_Clear();[](#l4.91)

•    dict = _PyObject_GetDictPtr(obj);[](#l4.92)

•    if (dict && *dict)[](#l4.93)

•        state = *dict;[](#l4.94)

•    else[](#l4.95)
         state = Py_None;[](#l4.96)

•        Py_INCREF(state);[](#l4.97)

•    }[](#l4.98)

•    Py_INCREF(state);[](#l4.99)
     names = slotnames(cls);[](#l4.100)
     if (names == NULL)[](#l4.101)
         goto end;[](#l4.102)

•    if (names != Py_None) {[](#l4.103)

•    if (names != Py_None && PyList_GET_SIZE(names) > 0) {[](#l4.104)
         assert(PyList_Check(names));[](#l4.105)
         slots = PyDict_New();[](#l4.106)
         if (slots == NULL)[](#l4.107)

@@ -3230,7 +3246,7 @@ reduce_2(PyObject *obj) copyreg = import_copyreg(); if (copyreg == NULL) goto end;

• newobj = PyObject_GetAttrString(copyreg, "newobj");

• newobj = PyObject_GetAttr(copyreg, str_newobj); if (newobj == NULL) goto end;

@@ -3238,8 +3254,8 @@ reduce_2(PyObject *obj) args2 = PyTuple_New(n+1); if (args2 == NULL) goto end;

• Py_INCREF(cls); PyTuple_SET_ITEM(args2, 0, cls);

• cls = NULL; for (i = 0; i < n; i++) { PyObject *v = PyTuple_GET_ITEM(args, i); Py_INCREF(v);

@@ -3249,7 +3265,6 @@ reduce_2(PyObject *obj) res = PyTuple_Pack(5, newobj, args2, state, listitems, dictitems); end:

• Py_XDECREF(cls); Py_XDECREF(args); Py_XDECREF(args2); Py_XDECREF(slots); @@ -3309,31 +3324,34 @@ object_reduce(PyObject *self, PyObject * static PyObject * object_reduce_ex(PyObject *self, PyObject *args) {

• static PyObject *str_reduce = NULL, *objreduce; PyObject *reduce, *res; int proto = 0; if (!PyArg_ParseTuple(args, "|i:reduce_ex", &proto)) return NULL;

• reduce = PyObject_GetAttrString(self, "reduce");

• if (str_reduce == NULL) {

•    str_reduce = PyUnicode_InternFromString("__reduce__");[](#l4.148)

•    objreduce = PyDict_GetItemString(PyBaseObject_Type.tp_dict,[](#l4.149)

•                                     "__reduce__");[](#l4.150)

•    if (str_reduce == NULL || objreduce == NULL)[](#l4.151)

•        return NULL;[](#l4.152)

• }

+

• reduce = PyObject_GetAttr(self, str_reduce); if (reduce == NULL) PyErr_Clear(); else {

•    PyObject *cls, *clsreduce, *objreduce;[](#l4.159)

•    PyObject *cls, *clsreduce;[](#l4.160)
     int override;[](#l4.161)

•    cls = PyObject_GetAttrString(self, "__class__");[](#l4.162)

•    if (cls == NULL) {[](#l4.163)

•        Py_DECREF(reduce);[](#l4.164)

•        return NULL;[](#l4.165)

•    }[](#l4.166)

•    clsreduce = PyObject_GetAttrString(cls, "__reduce__");[](#l4.167)

•    Py_DECREF(cls);[](#l4.168)

+

•    cls = (PyObject *) Py_TYPE(self);[](#l4.170)

•    clsreduce = PyObject_GetAttr(cls, str_reduce);[](#l4.171)
     if (clsreduce == NULL) {[](#l4.172)
         Py_DECREF(reduce);[](#l4.173)
         return NULL;[](#l4.174)
     }[](#l4.175)

•    objreduce = PyDict_GetItemString(PyBaseObject_Type.tp_dict,[](#l4.176)

•                                     "__reduce__");[](#l4.177)
     override = (clsreduce != objreduce);[](#l4.178)
     Py_DECREF(clsreduce);[](#l4.179)
     if (override) {[](#l4.180)