cpython: 680959a3ae2e (original) (raw)

--- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -6,7 +6,8 @@ import cStringIO import pickletools import copy_reg -from test.test_support import TestFailed, have_unicode, TESTFN +from test.test_support import (TestFailed, have_unicode, TESTFN, _2G, _1M,

                          precisionbigmemtest)[](#l1.9)

Tests that try a number of pickle protocols should have a

for proto in protocols:

@@ -1280,3 +1281,31 @@ class AbstractPicklerUnpicklerObjectTest f.write(pickled2) f.seek(0) self.assertEqual(unpickler.load(), data2) + +class BigmemPickleTests(unittest.TestCase): +

Memory requirements: 1 byte per character for input strings, 1 byte
for pickled data, 1 byte for unpickled strings, 1 byte for internal
buffer and 1 byte of free space for resizing of internal buffer.

@precisionbigmemtest(size=_2G + 100*_1M, memuse=5)
def test_huge_strlist(self, size):
```
   chunksize = 2**20[](#l1.26)
```
```
   data = [][](#l1.27)
```
```
   while size > chunksize:[](#l1.28)
```

       data.append('x' * chunksize)[](#l1.29)

```
       size -= chunksize[](#l1.30)
```
```
       chunksize += 1[](#l1.31)
```
```
   data.append('y' * size)[](#l1.32)
```

```
   try:[](#l1.34)
```

       for proto in protocols:[](#l1.35)

```
           try:[](#l1.36)
```

               pickled = self.dumps(data, proto)[](#l1.37)

               res = self.loads(pickled)[](#l1.38)

               self.assertEqual(res, data)[](#l1.39)

```
           finally:[](#l1.40)
```
```
               res = None[](#l1.41)
```
```
               pickled = None[](#l1.42)
```
```
   finally:[](#l1.43)
```
```
       data = None[](#l1.44)
```

--- a/Lib/test/test_cpickle.py +++ b/Lib/test/test_cpickle.py @@ -1,7 +1,9 @@ import cPickle, unittest from cStringIO import StringIO -from test.pickletester import AbstractPickleTests, AbstractPickleModuleTests -from test.pickletester import AbstractPicklerUnpicklerObjectTests +from test.pickletester import (AbstractPickleTests,

                          AbstractPickleModuleTests,[](#l2.9)

                          AbstractPicklerUnpicklerObjectTests,[](#l2.10)

                          BigmemPickleTests)[](#l2.11)

from test import test_support class cPickleTests(AbstractPickleTests, AbstractPickleModuleTests): @@ -101,6 +103,16 @@ class cPicklePicklerUnpicklerObjectTests pickler_class = cPickle.Pickler unpickler_class = cPickle.Unpickler +class cPickleBigmemPickleTests(BigmemPickleTests): +

def dumps(self, arg, proto=0, fast=0):
```
   # Ignore fast[](#l2.22)
```

   return cPickle.dumps(arg, proto)[](#l2.23)

def loads(self, buf):
```
   # Ignore fast[](#l2.26)
```
```
   return cPickle.loads(buf)[](#l2.27)
```

+ class Node(object): pass @@ -133,6 +145,7 @@ def test_main(): cPickleFastPicklerTests, cPickleDeepRecursive, cPicklePicklerUnpicklerObjectTests,

```
   cPickleBigmemPickleTests,[](#l2.36)
```
) if name == "main":

--- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -3,10 +3,11 @@ from cStringIO import StringIO from test import test_support -from test.pickletester import AbstractPickleTests -from test.pickletester import AbstractPickleModuleTests -from test.pickletester import AbstractPersistentPicklerTests -from test.pickletester import AbstractPicklerUnpicklerObjectTests +from test.pickletester import (AbstractPickleTests,

                          AbstractPickleModuleTests,[](#l3.12)

                          AbstractPersistentPicklerTests,[](#l3.13)

                          AbstractPicklerUnpicklerObjectTests,[](#l3.14)

                          BigmemPickleTests)[](#l3.15)

class PickleTests(AbstractPickleTests, AbstractPickleModuleTests): @@ -66,6 +67,16 @@ class PicklerUnpicklerObjectTests(Abstra pickler_class = pickle.Pickler unpickler_class = pickle.Unpickler +class PickleBigmemPickleTests(BigmemPickleTests): +

def dumps(self, arg, proto=0, fast=0):
```
   # Ignore fast[](#l3.26)
```

   return pickle.dumps(arg, proto)[](#l3.27)

def loads(self, buf):
```
   # Ignore fast[](#l3.30)
```
```
   return pickle.loads(buf)[](#l3.31)
```

+ def test_main(): test_support.run_unittest( @@ -73,6 +84,7 @@ def test_main(): PicklerTests, PersPicklerTests, PicklerUnpicklerObjectTests,

```
   PickleBigmemPickleTests,[](#l3.40)
```
) test_support.run_doctest(pickle)

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -202,6 +202,8 @@ Core and Builtins Library ------- +- Issue #13555: cPickle now supports files larger than 2 GiB. +

Issue #17052: unittest discovery should use self.testLoader.
Issue #4591: Uid and gid values larger than 2**31 are supported now.

--- a/Modules/cPickle.c +++ b/Modules/cPickle.c @@ -139,15 +139,15 @@ static PyObject *__class___str, *__getin typedef struct { PyObject_HEAD

int length; /* number of initial slots in data currently used */
int size; /* number of slots in data allocated */

Py_ssize_t length; /* number of initial slots in data currently used */
Py_ssize_t size; /* number of slots in data allocated */ PyObject **data; } Pdata; static void Pdata_dealloc(Pdata *self) {

int i;

Py_ssize_t i; PyObject **p; for (i = self->length, p = self->data; --i >= 0; p++) { @@ -193,9 +193,9 @@ stackUnderflow(void)

number of items, this is a (non-erroneous) NOP. */ static int -Pdata_clear(Pdata *self, int clearto) +Pdata_clear(Pdata *self, Py_ssize_t clearto) {

int i;

Py_ssize_t i; PyObject **p; if (clearto < 0) return stackUnderflow(); @@ -214,18 +214,17 @@ Pdata_clear(Pdata *self, int clearto) static int Pdata_grow(Pdata *self) {

int bigger;
size_t nbytes;

Py_ssize_t bigger;
Py_ssize_t nbytes;

+ PyObject **tmp;

if (self->size > (PY_SSIZE_T_MAX >> 1))
```
   goto nomemory;[](#l5.46)
```
bigger = self->size << 1;

if (bigger <= 0) /* was 0, or new value overflows */
```
   goto nomemory;[](#l5.49)
```
if ((int)(size_t)bigger != bigger)

if (bigger > (PY_SSIZE_T_MAX / sizeof(PyObject *))) goto nomemory;

nbytes = (size_t)bigger * sizeof(PyObject *);
if (nbytes / sizeof(PyObject *) != (size_t)bigger)
```
   goto nomemory;[](#l5.55)
```

nbytes = bigger * sizeof(PyObject *); tmp = realloc(self->data, nbytes); if (tmp == NULL) goto nomemory;

@@ -280,10 +279,10 @@ Pdata_grow(Pdata *self) static PyObject * -Pdata_popTuple(Pdata *self, int start) +Pdata_popTuple(Pdata *self, Py_ssize_t start) { PyObject *r;

int i, j, l;

Py_ssize_t i, j, l;

l = self->length-start; r = PyTuple_New(l); @@ -297,10 +296,10 @@ Pdata_popTuple(Pdata *self, int start) } static PyObject * -Pdata_popList(Pdata *self, int start) +Pdata_popList(Pdata *self, Py_ssize_t start) { PyObject *r;

int i, j, l;

Py_ssize_t i, j, l;

l=self->length-start; if (!( r=PyList_New(l))) return NULL; @@ -347,9 +346,9 @@ typedef struct Picklerobject { int bin; int fast; /* Fast mode doesn't save in memo, don't use if circ ref */

int (*write_func)(struct Picklerobject *, const char *, Py_ssize_t);

Py_ssize_t (*write_func)(struct Picklerobject *, const char *, Py_ssize_t); char *write_buf;

int buf_size;

Py_ssize_t buf_size; PyObject dispatch_table; int fast_container; / count nested container dumps */ PyObject *fast_memo; @@ -373,12 +372,12 @@ typedef struct Unpicklerobject { PyObject *mark; PyObject *pers_func; PyObject *last_string;

int *marks;
int num_marks;
int marks_size;

Py_ssize_t *marks;
Py_ssize_t num_marks;
Py_ssize_t marks_size; Py_ssize_t (*read_func)(struct Unpicklerobject *, char **, Py_ssize_t); Py_ssize_t (*readline_func)(struct Unpicklerobject *, char **);

int buf_size;

Py_ssize_t buf_size; char *buf; PyObject *find_class; } Unpicklerobject; @@ -424,7 +423,7 @@ cPickle_ErrFormat(PyObject *ErrType, cha return NULL; }

-static int +static Py_ssize_t write_file(Picklerobject *self, const char *s, Py_ssize_t n) { size_t nbyteswritten; @@ -433,11 +432,6 @@ write_file(Picklerobject *self, const ch return 0; }

if (n > INT_MAX) {
```
   /* String too large */[](#l5.129)
```
```
   return -1;[](#l5.130)
```
}

- PyFile_IncUseCount((PyFileObject *)self->file); Py_BEGIN_ALLOW_THREADS nbyteswritten = fwrite(s, sizeof(char), n, self->fp); @@ -448,40 +442,44 @@ write_file(Picklerobject *self, const ch return -1; }

return (int)n;

return n;

} -static int +static Py_ssize_t write_cStringIO(Picklerobject *self, const char *s, Py_ssize_t n) {

Py_ssize_t len = n;

+ if (s == NULL) { return 0; }

while (n > INT_MAX) {

   if (PycStringIO->cwrite((PyObject *)self->file, s, INT_MAX) != INT_MAX) {[](#l5.155)

```
       return -1;[](#l5.156)
```
```
   }[](#l5.157)
```
```
   n -= INT_MAX;[](#l5.158)
```
}

+ if (PycStringIO->cwrite((PyObject *)self->file, s, n) != n) { return -1; }

return (int)n;

return len;

} -static int +static Py_ssize_t write_none(Picklerobject *self, const char *s, Py_ssize_t n) { if (s == NULL) return 0;

if (n > INT_MAX) return -1;
return (int)n;

return n;

} -static int -write_other(Picklerobject *self, const char *s, Py_ssize_t _n) +static Py_ssize_t +write_other(Picklerobject *self, const char *s, Py_ssize_t n) { PyObject *py_str = 0, *junk = 0;

int n;

if (_n > INT_MAX)
```
   return -1;[](#l5.188)
```
n = (int)_n;

+ if (s == NULL) { if (!( self->buf_size )) return 0; py_str = PyString_FromStringAndSize(self->write_buf, @@ -490,7 +488,7 @@ write_other(Picklerobject *self, const c return -1; } else {

   if (self->buf_size && (n + self->buf_size) > WRITE_BUF_SIZE) {[](#l5.198)

   if (self->buf_size && n > WRITE_BUF_SIZE - self->buf_size) {[](#l5.199)
       if (write_other(self, NULL, 0) < 0)[](#l5.200)
           return -1;[](#l5.201)
   }[](#l5.202)

@@ -531,7 +529,7 @@ read_file(Unpicklerobject *self, char ** size_t nbytesread; if (self->buf_size == 0) {

```
   int size;[](#l5.207)
```

```
   Py_ssize_t size;[](#l5.208)
```

size = ((n < 32) ? 32 : n); if (!( self->buf = (char *)malloc(size))) { @@ -575,7 +573,7 @@ read_file(Unpicklerobject *self, char ** static Py_ssize_t readline_file(Unpicklerobject *self, char **s) {

int i;

Py_ssize_t i;

if (self->buf_size == 0) { if (!( self->buf = (char *)malloc(40))) { @@ -587,7 +585,7 @@ readline_file(Unpicklerobject *self, cha i = 0; while (1) {

```
   int bigger;[](#l5.225)
```

   Py_ssize_t bigger;[](#l5.226)
   char *newbuf;[](#l5.227)
   for (; i < (self->buf_size - 1); i++) {[](#l5.228)
       if (feof(self->fp) ||[](#l5.229)

@@ -597,13 +595,13 @@ readline_file(Unpicklerobject *self, cha return i + 1; } }

   bigger = self->buf_size << 1;[](#l5.234)

   if (bigger <= 0) {              /* overflow */[](#l5.235)

   if (self->buf_size < (PY_SSIZE_T_MAX >> 1)) {[](#l5.236)
       PyErr_NoMemory();[](#l5.237)
       return -1;[](#l5.238)
   }[](#l5.239)

   bigger = self->buf_size << 1;[](#l5.240)
   newbuf = (char *)realloc(self->buf, bigger);[](#l5.241)

```
   if (!newbuf)  {[](#l5.242)
```

   if (newbuf == NULL)  {[](#l5.243)
       PyErr_NoMemory();[](#l5.244)
       return -1;[](#l5.245)
   }[](#l5.246)

@@ -616,30 +614,63 @@ readline_file(Unpicklerobject *self, cha static Py_ssize_t read_cStringIO(Unpicklerobject *self, char **s, Py_ssize_t n) {

char *ptr;

if (PycStringIO->cread((PyObject *)self->file, &ptr, n) != n) {

   PyErr_SetNone(PyExc_EOFError);[](#l5.254)

```
   return -1;[](#l5.255)
```
}

*s = ptr;

return n;

Py_ssize_t len = n;
char *start, *end = NULL;

while (1) {
```
   int k;[](#l5.265)
```
```
   char *ptr;[](#l5.266)
```
```
   if (n > INT_MAX)[](#l5.267)
```
```
       k = INT_MAX;[](#l5.268)
```
```
   else[](#l5.269)
```
```
       k = (int)n;[](#l5.270)
```

   if (PycStringIO->cread((PyObject *)self->file, &ptr, k) != k) {[](#l5.271)

       PyErr_SetNone(PyExc_EOFError);[](#l5.272)

```
       return -1;[](#l5.273)
```
```
   }[](#l5.274)
```
```
   if (end == NULL)[](#l5.275)
```
```
       start = ptr;[](#l5.276)
```
```
   else if (ptr != end) {[](#l5.277)
```

       /* non-continuous area */[](#l5.278)

```
       return -1;[](#l5.279)
```
```
   }[](#l5.280)
```
```
   if (n <= INT_MAX)[](#l5.281)
```
```
       break;[](#l5.282)
```
```
   end = ptr + INT_MAX;[](#l5.283)
```
```
   n -= INT_MAX;[](#l5.284)
```
}

*s = start;

return len;

} static Py_ssize_t readline_cStringIO(Unpicklerobject *self, char **s) {

Py_ssize_t n;
char *ptr;

if ((n = PycStringIO->creadline((PyObject *)self->file, &ptr)) < 0) {
```
   return -1;[](#l5.300)
```
}

*s = ptr;

Py_ssize_t n = 0;
char *start = NULL, *end = NULL;

while (1) {
```
   int k;[](#l5.308)
```
```
   char *ptr;[](#l5.309)
```

   if ((k = PycStringIO->creadline((PyObject *)self->file, &ptr)) < 0) {[](#l5.310)

```
       return -1;[](#l5.311)
```
```
   }[](#l5.312)
```
```
   n += k;[](#l5.313)
```
```
   if (end == NULL)[](#l5.314)
```
```
       start = ptr;[](#l5.315)
```
```
   else if (ptr != end) {[](#l5.316)
```

       /* non-continuous area */[](#l5.317)

```
       return -1;[](#l5.318)
```
```
   }[](#l5.319)
```

   if (k == 0 || ptr[k - 1] == '\n')[](#l5.320)

```
       break;[](#l5.321)
```
```
   end = ptr + k;[](#l5.322)
```
}

*s = start;

return n; } @@ -700,7 +731,7 @@ readline_other(Unpicklerobject *self, ch

The caller is responsible for free()'ing the return value. */ static char * -pystrndup(const char *s, int n) +pystrndup(const char *s, Py_ssize_t n) { char *r = (char *)malloc(n+1); if (r == NULL) @@ -715,7 +746,7 @@ static int get(Picklerobject *self, PyObject *id) { PyObject *value, *mv;

long c_value;

Py_ssize_t c_value; char s[30]; size_t len; @@ -735,7 +766,8 @@ get(Picklerobject *self, PyObject *id) if (!self->bin) { s[0] = GET;

   PyOS_snprintf(s + 1, sizeof(s) - 1, "%ld\n", c_value);[](#l5.351)

   PyOS_snprintf(s + 1, sizeof(s) - 1,[](#l5.352)

```
                 "%" PY_FORMAT_SIZE_T "d\n", c_value);[](#l5.353)
   len = strlen(s);[](#l5.354)
```
} else if (Pdata_Check(self->file)) { @@ -780,8 +812,7 @@ static int put2(Picklerobject *self, PyObject *ob) { char c_str[30];

int p;
size_t len;

Py_ssize_t len, p; int res = -1; PyObject *py_ob_id = 0, *memo_len = 0, *t = 0; @@ -818,7 +849,8 @@ put2(Picklerobject *self, PyObject *ob) if (!self->bin) { c_str[0] = PUT;

   PyOS_snprintf(c_str + 1, sizeof(c_str) - 1, "%d\n", p);[](#l5.371)

   PyOS_snprintf(c_str + 1, sizeof(c_str) - 1,[](#l5.372)

```
                 "%" PY_FORMAT_SIZE_T "d\n", p);[](#l5.373)
   len = strlen(c_str);[](#l5.374)
```
} else if (Pdata_Check(self->file)) { @@ -994,7 +1026,7 @@ save_int(Picklerobject *self, PyObject * { char c_str[32]; long l = PyInt_AS_LONG((PyIntObject *)args);

int len = 0;

Py_ssize_t len = 0;

if (!self->bin #if SIZEOF_LONG > 4 @@ -1201,7 +1233,7 @@ done: static int save_string(Picklerobject *self, PyObject *args, int doput) {

int size, len;

Py_ssize_t size, len; PyObject *repr=0; if ((size = PyString_Size(args)) < 0) @@ -1448,7 +1480,7 @@ save_unicode(Picklerobject *self, PyObje static int store_tuple_elements(Picklerobject *self, PyObject *t, int len) {

int i;

Py_ssize_t i; int res = -1; /* guilty until proved innocent */ assert(PyTuple_Size(t) == len); @@ -1477,7 +1509,7 @@ static int save_tuple(Picklerobject *self, PyObject *args) { PyObject *py_tuple_id = NULL;

int len, i;

Py_ssize_t len, i; int res = -1; static char tuple = TUPLE; @@ -1690,7 +1722,7 @@ save_list(Picklerobject *self, PyObject { int res = -1; char s[3];

int len;

Py_ssize_t len; PyObject *iter; if (self->fast && !fast_save_enter(self, args)) @@ -1943,7 +1975,7 @@ save_dict(Picklerobject *self, PyObject { int res = -1; char s[3];

int len;

Py_ssize_t len;

if (self->fast && !fast_save_enter(self, args)) goto finally; @@ -2027,7 +2059,7 @@ save_inst(Picklerobject *self, PyObject if ((getinitargs_func = PyObject_GetAttr(args, __getinitargs___str))) { PyObject *element = 0;

```
   int i, len;[](#l5.435)
```

```
   Py_ssize_t i, len;[](#l5.436)
```

if (!( class_args = PyObject_Call(getinitargs_func, empty_tuple, NULL))) @@ -2289,7 +2321,8 @@ static int save_pers(Picklerobject *self, PyObject *args, PyObject *f) { PyObject *pid = 0;

int size, res = -1;

Py_ssize_t size;
int res = -1;

static char persid = PERSID, binpersid = BINPERSID; @@ -2431,7 +2464,7 @@ save_reduce(Picklerobject *self, PyObjec if (use_newobj) { PyObject *cls; PyObject *newargtup;

```
   int n, i;[](#l5.454)
```

```
   Py_ssize_t n, i;[](#l5.455)
```

/* Sanity checks. */ n = PyTuple_Size(argtup); @@ -2815,7 +2848,7 @@ Pickle_clear_memo(Picklerobject *self, P static PyObject * Pickle_getvalue(Picklerobject *self, PyObject *args) {

int l, i, rsize, ssize, clear=1, lm;

Py_ssize_t l, i, rsize, ssize, clear=1, lm; long ik; PyObject *k, *r; char *s, *p, *have_get; @@ -3314,7 +3347,7 @@ find_class(PyObject *py_module_name, PyO return global; }

-static int +static Py_ssize_t marker(Unpicklerobject *self) { if (self->num_marks < 1) { @@ -3345,7 +3378,8 @@ load_int(Unpicklerobject *self) { PyObject *py_int = 0; char *endptr, *s;

int len, res = -1;

Py_ssize_t len;
int res = -1; long l; if ((len = self->readline_func(self, &s)) < 0) return -1; @@ -3477,7 +3511,8 @@ load_long(Unpicklerobject *self) { PyObject *l = 0; char *end, *s;

int len, res = -1;

Py_ssize_t len;
int res = -1;

if ((len = self->readline_func(self, &s)) < 0) return -1; if (len < 2) return bad_readline(); @@ -3541,7 +3576,8 @@ load_float(Unpicklerobject *self) { PyObject *py_float = 0; char *endptr, *s;

int len, res = -1;

Py_ssize_t len;
int res = -1; double d; if ((len = self->readline_func(self, &s)) < 0) return -1; @@ -3597,7 +3633,8 @@ static int load_string(Unpicklerobject *self) { PyObject *str = 0;

int len, res = -1;

Py_ssize_t len;
int res = -1; char *s, *p; if ((len = self->readline_func(self, &s)) < 0) return -1; @@ -3639,7 +3676,7 @@ static int load_binstring(Unpicklerobject *self) { PyObject *py_string = 0;

long l;

Py_ssize_t l; char *s; if (self->read_func(self, &s, 4) < 0) return -1; @@ -3691,20 +3728,17 @@ static int load_unicode(Unpicklerobject *self) { PyObject *str = 0;

int len, res = -1;

Py_ssize_t len; char *s; if ((len = self->readline_func(self, &s)) < 0) return -1; if (len < 1) return bad_readline(); if (!( str = PyUnicode_DecodeRawUnicodeEscape(s, len - 1, NULL)))

```
   goto finally;[](#l5.538)
```

```
   return -1;[](#l5.539)
```

PDATA_PUSH(self->stack, str, -1); return 0; -

finally:
return res;

} #endif @@ -3714,7 +3748,7 @@ static int load_binunicode(Unpicklerobject *self) { PyObject *unicode;

long l;

Py_ssize_t l; char *s; if (self->read_func(self, &s, 4) < 0) return -1; @@ -3745,7 +3779,7 @@ static int load_tuple(Unpicklerobject *self) { PyObject *tup;

int i;

Py_ssize_t i;

if ((i = marker(self)) < 0) return -1; if (!( tup=Pdata_popTuple(self->stack, i))) return -1; @@ -3798,7 +3832,7 @@ static int load_list(Unpicklerobject *self) { PyObject *list = 0;

int i;

Py_ssize_t i;

if ((i = marker(self)) < 0) return -1; if (!( list=Pdata_popList(self->stack, i))) return -1; @@ -3810,7 +3844,7 @@ static int load_dict(Unpicklerobject *self) { PyObject *dict, *key, *value;

int i, j, k;

Py_ssize_t i, j, k;

if ((i = marker(self)) < 0) return -1; j=self->stack->length; @@ -3886,7 +3920,7 @@ static int load_obj(Unpicklerobject *self) { PyObject *class, *tup, *obj=0;

int i;

Py_ssize_t i;

if ((i = marker(self)) < 0) return -1; if (!( tup=Pdata_popTuple(self->stack, i+1))) return -1; @@ -3907,7 +3941,7 @@ static int load_inst(Unpicklerobject *self) { PyObject *tup, *class=0, *obj=0, *module_name, *class_name;

int i, len;

Py_ssize_t i, len; char *s; if ((i = marker(self)) < 0) return -1; @@ -3993,7 +4027,7 @@ static int load_global(Unpicklerobject *self) { PyObject *class = 0, *module_name = 0, *class_name = 0;

int len;

Py_ssize_t len; char *s; if ((len = self->readline_func(self, &s)) < 0) return -1; @@ -4024,7 +4058,7 @@ static int load_persid(Unpicklerobject *self) { PyObject *pid = 0;

int len;

Py_ssize_t len; char *s; if (self->pers_func) { @@ -4102,7 +4136,7 @@ load_binpersid(Unpicklerobject *self) static int load_pop(Unpicklerobject *self) {

int len = self->stack->length;

Py_ssize_t len = self->stack->length;

/* Note that we split the (pickle.py) stack into two stacks, an object stack and a mark stack. We have to be clever and @@ -4127,7 +4161,7 @@ load_pop(Unpicklerobject *self) static int load_pop_mark(Unpicklerobject *self) {

int i;

Py_ssize_t i;

if ((i = marker(self)) < 0) return -1; @@ -4142,7 +4176,7 @@ static int load_dup(Unpicklerobject *self) { PyObject *last;

int len;

Py_ssize_t len;

if ((len = self->stack->length) <= 0) return stackUnderflow(); last=self->stack->data[len-1]; @@ -4156,7 +4190,7 @@ static int load_get(Unpicklerobject *self) { PyObject *py_str = 0, *value = 0;

int len;

Py_ssize_t len; char *s; int rc; @@ -4214,7 +4248,7 @@ load_long_binget(Unpicklerobject *self) PyObject *py_key = 0, *value = 0; unsigned char c; char *s;

long key;

Py_ssize_t key; int rc; if (self->read_func(self, &s, 4) < 0) return -1; @@ -4317,7 +4351,7 @@ static int load_put(Unpicklerobject *self) { PyObject *py_str = 0, *value = 0;

int len, l;

Py_ssize_t len, l; char *s; if ((l = self->readline_func(self, &s)) < 0) return -1; @@ -4337,7 +4371,7 @@ load_binput(Unpicklerobject *self) PyObject *py_key = 0, *value = 0; unsigned char key; char *s;

int len;

Py_ssize_t len;

if (self->read_func(self, &s, 1) < 0) return -1; if (!( (len=self->stack->length) > 0 )) return stackUnderflow(); @@ -4356,10 +4390,10 @@ static int load_long_binput(Unpicklerobject *self) { PyObject *py_key = 0, *value = 0;

long key;

Py_ssize_t key; unsigned char c; char *s;

int len;

Py_ssize_t len;

if (self->read_func(self, &s, 4) < 0) return -1; if (!( len=self->stack->length )) return stackUnderflow(); @@ -4382,10 +4416,10 @@ load_long_binput(Unpicklerobject *self) static int -do_append(Unpicklerobject *self, int x) +do_append(Unpicklerobject *self, Py_ssize_t x) { PyObject *value = 0, *list = 0, *append_method = 0;

int len, i;

Py_ssize_t len, i;

len=self->stack->length; if (!( len >= x && x > 0 )) return stackUnderflow(); @@ -4451,11 +4485,11 @@ load_appends(Unpicklerobject *self) } -static int -do_setitems(Unpicklerobject *self, int x) +static Py_ssize_t +do_setitems(Unpicklerobject *self, Py_ssize_t x) { PyObject *value = 0, *key = 0, *dict = 0;

int len, i, r=0;

Py_ssize_t len, i, r=0;

if (!( (len=self->stack->length) >= x && x > 0 )) return stackUnderflow(); @@ -4496,8 +4530,8 @@ load_build(Unpicklerobject *self) PyObject *state, *inst, *slotstate; PyObject *setstate; PyObject *d_key, *d_value;

int res = -1; Py_ssize_t i;

int res = -1;

/* Stack is ... instance, state. We want to leave instance at * the stack top, possibly mutated via instance.setstate(state). @@ -4596,7 +4630,7 @@ load_build(Unpicklerobject *self) static int load_mark(Unpicklerobject *self) {

int s;

Py_ssize_t s;

/* Note that we split the (pickle.py) stack into two stacks, an object stack and a mark stack. Here we push a mark onto the @@ -4604,14 +4638,14 @@ load_mark(Unpicklerobject *self) */ if ((self->num_marks + 1) >= self->marks_size) {

```
   int *marks;[](#l5.748)
```

   Py_ssize_t *marks;[](#l5.749)
   s=self->marks_size+20;[](#l5.750)
   if (s <= self->num_marks) s=self->num_marks + 1;[](#l5.751)
   if (self->marks == NULL)[](#l5.752)

       marks=(int *)malloc(s * sizeof(int));[](#l5.753)

       marks=(Py_ssize_t *)malloc(s * sizeof(Py_ssize_t));[](#l5.754)
   else[](#l5.755)

       marks=(int *)realloc(self->marks,[](#l5.756)

                                  s * sizeof(int));[](#l5.757)

       marks=(Py_ssize_t *)realloc(self->marks,[](#l5.758)

                                   s * sizeof(Py_ssize_t));[](#l5.759)
   if (!marks) {[](#l5.760)
       PyErr_NoMemory();[](#l5.761)
       return -1;[](#l5.762)

@@ -4981,7 +5015,7 @@ load(Unpicklerobject *self) static int noload_obj(Unpicklerobject *self) {

int i;

Py_ssize_t i;

if ((i = marker(self)) < 0) return -1; return Pdata_clear(self->stack, i+1); @@ -4991,7 +5025,7 @@ noload_obj(Unpicklerobject *self) static int noload_inst(Unpicklerobject *self) {

int i;

Py_ssize_t i; char *s; if ((i = marker(self)) < 0) return -1; @@ -5068,7 +5102,7 @@ noload_append(Unpicklerobject *self) static int noload_appends(Unpicklerobject *self) {

int i;

Py_ssize_t i; if ((i = marker(self)) < 0) return -1; return Pdata_clear(self->stack, i); }

@@ -5082,7 +5116,7 @@ noload_setitem(Unpicklerobject *self) static int noload_setitems(Unpicklerobject *self) {

int i;

Py_ssize_t i; if ((i = marker(self)) < 0) return -1; return Pdata_clear(self->stack, i); }

--- a/Modules/cStringIO.c +++ b/Modules/cStringIO.c @@ -210,11 +210,8 @@ IO_creadline(PyObject *self, char **outp if (n < end) n++; len = n - start;

if (len > INT_MAX) {

   PyErr_SetString(PyExc_OverflowError,[](#l6.8)

                   "length too large");[](#l6.9)

```
   return -1;[](#l6.10)
```
}

if (len > INT_MAX)
```
   len = INT_MAX;[](#l6.13)
```

*output=start;

cpython: 680959a3ae2e (original) (raw)

Tests that try a number of pickle protocols should have a

for proto in protocols:

Memory requirements: 1 byte per character for input strings, 1 byte

for pickled data, 1 byte for unpickled strings, 1 byte for internal

buffer and 1 byte of free space for resizing of internal buffer.