Add Unladden Swallow's optimizations to Python 3's pickle. - Code Review (original) (raw)
OLD
NEW
1 #include "Python.h"
1 #include "Python.h"
2 #include "structmember.h"
2 #include "structmember.h"
3
3
4 PyDoc_STRVAR(pickle_module_doc,
4 PyDoc_STRVAR(pickle_module_doc,
5 "Optimized C implementation for the Python pickle module.");
5 "Optimized C implementation for the Python pickle module.");
6
6
7 /* Bump this when new opcodes are added to the pickle protocol. */
7 /* Bump this when new opcodes are added to the pickle protocol. */
8 enum {
8 enum {
9 HIGHEST_PROTOCOL = 3,
9 HIGHEST_PROTOCOL = 3,
10 DEFAULT_PROTOCOL = 3
10 DEFAULT_PROTOCOL = 3
11 };
11 };
12
12
13
14 /* Pickle opcodes. These must be kept updated with pickle.py.
13 /* Pickle opcodes. These must be kept updated with pickle.py.
15 Extensive docs are in pickletools.py. */
14 Extensive docs are in pickletools.py. */
16 enum opcode {
15 enum opcode {
17 MARK = '(',
16 MARK = '(',
18 STOP = '.',
17 STOP = '.',
19 POP = '0',
18 POP = '0',
20 POP_MARK = '1',
19 POP_MARK = '1',
21 DUP = '2',
20 DUP = '2',
22 FLOAT = 'F',
21 FLOAT = 'F',
23 INT = 'I',
22 INT = 'I',
(...skipping 67 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
91 break if this gets out of synch with pickle.py, but it's unclear that woul d
90 break if this gets out of synch with pickle.py, but it's unclear that woul d
92 help anything either. */
91 help anything either. */
93 BATCHSIZE = 1000,
92 BATCHSIZE = 1000,
94
93
95 /* Nesting limit until Pickler, when running in "fast mode", starts
94 /* Nesting limit until Pickler, when running in "fast mode", starts
96 checking for self-referential data-structures. */
95 checking for self-referential data-structures. */
97 FAST_NESTING_LIMIT = 50,
96 FAST_NESTING_LIMIT = 50,
98
97
99 /* Size of the write buffer of Pickler. Higher values will reduce the
98 /* Size of the write buffer of Pickler. Higher values will reduce the
100 number of calls to the write() method of the output stream. */
99 number of calls to the write() method of the output stream. */
101 WRITE_BUF_SIZE = 256,
100 WRITE_BUF_SIZE = 4096,
102 };
101 };
103
102
104 /* Exception classes for pickle. These should override the ones defined in
103 /* Exception classes for pickle. These should override the ones defined in
105 pickle.py, when the C-optimized Pickler and Unpickler are used. */
104 pickle.py, when the C-optimized Pickler and Unpickler are used. */
106 static PyObject *PickleError = NULL;
105 static PyObject *PickleError = NULL;
107 static PyObject *PicklingError = NULL;
106 static PyObject *PicklingError = NULL;
108 static PyObject *UnpicklingError = NULL;
107 static PyObject *UnpicklingError = NULL;
109
108
110 /* copyreg.dispatch_table, {type_object: pickling_function} */
109 /* copyreg.dispatch_table, {type_object: pickling_function} */
111 static PyObject *dispatch_table = NULL;
110 static PyObject *dispatch_table = NULL;
(...skipping 21 matching lines...) Expand all Loading...
133
132
134 static int
133 static int
135 stack_underflow(void)
134 stack_underflow(void)
136 {
135 {
137 PyErr_SetString(UnpicklingError, "unpickling stack underflow");
136 PyErr_SetString(UnpicklingError, "unpickling stack underflow");
138 return -1;
137 return -1;
139 }
138 }
140
139
141 /* Internal data type used as the unpickling stack. */
140 /* Internal data type used as the unpickling stack. */
142 typedef struct {
141 typedef struct {
143 PyObject_HEAD
142 PyObject_VAR_HEAD
144 int length; /* number of initial slots in data currently used */
145 int size; /* number of slots in data allocated */
146 PyObject **data;
143 PyObject **data;
144 Py_ssize_t allocated; /* number of slots in data allocated */
147 } Pdata;
145 } Pdata;
148
146
149 static void
147 static void
150 Pdata_dealloc(Pdata *self)
148 Pdata_dealloc(Pdata *self)
151 {
149 {
152 int i;
150 int i = Py_SIZE(self);
153 PyObject **p;
151 while (--i >= 0) {
154
152 Py_DECREF(self->data[i]);
155 for (i = self->length, p = self->data; --i >= 0; p++) {
156 Py_DECREF(*p);
157 }
153 }
158 if (self->data)
154 PyMem_FREE(self->data);
159 PyMem_Free(self->data);
160 PyObject_Del(self);
155 PyObject_Del(self);
161 }
156 }
162
157
163 static PyTypeObject Pdata_Type = {
158 static PyTypeObject Pdata_Type = {
164 PyVarObject_HEAD_INIT(NULL, 0)
159 PyVarObject_HEAD_INIT(NULL, 0)
165 "_pickle.Pdata", /*tp_name*/
160 "_pickle.Pdata", /*tp_name*/
166 sizeof(Pdata), /*tp_basicsize*/
161 sizeof(Pdata), /*tp_basicsize*/
167 0, /*tp_itemsize*/
162 0, /*tp_itemsize*/
168 (destructor)Pdata_dealloc, /*tp_dealloc*/
163 (destructor)Pdata_dealloc, /*tp_dealloc*/
169 };
164 };
170
165
171 static PyObject *
166 static PyObject *
172 Pdata_New(void)
167 Pdata_New(void)
173 {
168 {
174 Pdata *self;
169 Pdata *self;
175
170
176 if (!(self = PyObject_New(Pdata, &Pdata_Type)))
171 if (!(self = PyObject_New(Pdata, &Pdata_Type)))
177 return NULL;
172 return NULL;
178 self->size = 8;
173 Py_SIZE(self) = 0;
179 self->length = 0;
174 self->allocated = 8;
180 self->data = PyMem_Malloc(self->size * sizeof(PyObject *));
175 self->data = PyMem_MALLOC(self->allocated * sizeof(PyObject *));
181 if (self->data)
176 if (self->data)
182 return (PyObject *)self;
177 return (PyObject *)self;
183 Py_DECREF(self);
178 Py_DECREF(self);
184 return PyErr_NoMemory();
179 return PyErr_NoMemory();
185 }
180 }
186
181
187
182
188 /* Retain only the initial clearto items. If clearto >= the current
183 /* Retain only the initial clearto items. If clearto >= the current
189 * number of items, this is a (non-erroneous) NOP.
184 * number of items, this is a (non-erroneous) NOP.
190 */
185 */
191 static int
186 static int
192 Pdata_clear(Pdata *self, int clearto)
187 Pdata_clear(Pdata *self, int clearto)
193 {
188 {
194 int i;
189 int i = Py_SIZE(self);
195 PyObject **p;
196
190
197 if (clearto < 0)
191 if (clearto < 0)
198 return stack_underflow();
192 return stack_underflow();
199 if (clearto >= self->length)
193 if (clearto >= i)
200 return 0;
194 return 0;
201
195
202 for (i = self->length, p = self->data + clearto; --i >= clearto; p++) {
196 while (--i >= clearto) {
203 Py_CLEAR(*p);
197 Py_CLEAR(self->data[i]);
204 }
198 }
205 self->length = clearto;
199 Py_SIZE(self) = clearto;
206
207 return 0;
200 return 0;
208 }
201 }
209
202
210 static int
203 static int
211 Pdata_grow(Pdata *self)
204 Pdata_grow(Pdata *self)
212 {
205 {
213 int bigger;
206 PyObject **data = self->data;
214 size_t nbytes;
207 Py_ssize_t allocated = self->allocated;
215 PyObject **tmp;
208 Py_ssize_t new_allocated;
216
209
217 bigger = (self->size << 1) + 1;
210 new_allocated = (allocated >> 3) + 6;
218 if (bigger <= 0) /* was 0, or new value overflows */
211 /* check for integer overflow */
212 if (new_allocated > PY_SSIZE_T_MAX - allocated)
219 goto nomemory;
213 goto nomemory;
220 if ((int)(size_t)bigger != bigger)
214 new_allocated += allocated;
215 if (new_allocated > (PY_SSIZE_T_MAX / sizeof(PyObject *)))
221 goto nomemory;
216 goto nomemory;
222 nbytes = (size_t)bigger * sizeof(PyObject *);
217 data = PyMem_REALLOC(data, new_allocated * sizeof(PyObject *));
223 if (nbytes / sizeof(PyObject *) != (size_t)bigger)
218 if (data == NULL)
224 goto nomemory;
219 goto nomemory;
225 tmp = PyMem_Realloc(self->data, nbytes);
220
226 if (tmp == NULL)
221 self->data = data;
227 goto nomemory;
222 self->allocated = new_allocated;
228 self->data = tmp;
229 self->size = bigger;
230 return 0;
223 return 0;
231
224
232 nomemory:
225 nomemory:
233 PyErr_NoMemory();
226 PyErr_NoMemory();
234 return -1;
227 return -1;
235 }
228 }
236
229
237 /* D is a Pdata*. Pop the topmost element and store it into V, which
230 /* D is a Pdata*. Pop the topmost element and store it into V, which
238 * must be an lvalue holding PyObject*. On stack underflow, UnpicklingError
231 * must be an lvalue holding PyObject*. On stack underflow, UnpicklingError
239 * is raised and V is set to NULL.
232 * is raised and V is set to NULL.
240 */
233 */
241 static PyObject *
234 static PyObject *
242 Pdata_pop(Pdata *self)
235 Pdata_pop(Pdata *self)
243 {
236 {
244 if (self->length == 0) {
237 if (Py_SIZE(self) == 0) {
245 PyErr_SetString(UnpicklingError, "bad pickle data");
238 PyErr_SetString(UnpicklingError, "bad pickle data");
246 return NULL;
239 return NULL;
247 }
240 }
248 return self->data[--(self->length)];
241 return self->data[--Py_SIZE(self)];
249 }
242 }
250 #define PDATA_POP(D, V) do { (V) = Pdata_pop((D)); } while (0)
243 #define PDATA_POP(D, V) do { (V) = Pdata_pop((D)); } while (0)
251
244
252 static int
245 static int
253 Pdata_push(Pdata *self, PyObject *obj)
246 Pdata_push(Pdata *self, PyObject *obj)
254 {
247 {
255 if (self->length == self->size && Pdata_grow(self) < 0) {
248 if (Py_SIZE(self) == self->allocated && Pdata_grow(self) < 0) {
256 return -1;
249 return -1;
257 }
250 }
258 self->data[self->length++] = obj;
251 self->data[Py_SIZE(self)++] = obj;
259 return 0;
252 return 0;
260 }
253 }
261
254
262 /* Push an object on stack, transferring its ownership to the stack. */
255 /* Push an object on stack, transferring its ownership to the stack. */
263 #define PDATA_PUSH(D, O, ER) do { \
256 #define PDATA_PUSH(D, O, ER) do { \
264 if (Pdata_push((D), (O)) < 0) return (ER); } while(0)
257 if (Pdata_push((D), (O)) < 0) return (ER); } while(0)
265
258
266 /* Push an object on stack, adding a new reference to the object. */
259 /* Push an object on stack, adding a new reference to the object. */
267 #define PDATA_APPEND(D, O, ER) do { \
260 #define PDATA_APPEND(D, O, ER) do { \
268 Py_INCREF((O)); \
261 Py_INCREF((O)); \
269 if (Pdata_push((D), (O)) < 0) return (ER); } while(0)
262 if (Pdata_push((D), (O)) < 0) return (ER); } while(0)
270
263
271 static PyObject *
264 static PyObject *
272 Pdata_poptuple(Pdata *self, Py_ssize_t start)
265 Pdata_poptuple(Pdata *self, Py_ssize_t start)
273 {
266 {
274 PyObject *tuple;
267 PyObject *tuple;
275 Py_ssize_t len, i, j;
268 Py_ssize_t len, i, j;
276
269
277 len = self->length - start;
270 len = Py_SIZE(self) - start;
278 tuple = PyTuple_New(len);
271 tuple = PyTuple_New(len);
279 if (tuple == NULL)
272 if (tuple == NULL)
280 return NULL;
273 return NULL;
281 for (i = start, j = 0; j < len; i++, j++)
274 for (i = start, j = 0; j < len; i++, j++)
282 PyTuple_SET_ITEM(tuple, j, self->data[i]);
275 PyTuple_SET_ITEM(tuple, j, self->data[i]);
283
276
284 self->length = start;
277 Py_SIZE(self) = start;
285 return tuple;
278 return tuple;
286 }
279 }
287
280
288 static PyObject *
281 static PyObject *
289 Pdata_poplist(Pdata *self, Py_ssize_t start)
282 Pdata_poplist(Pdata *self, Py_ssize_t start)
290 {
283 {
291 PyObject *list;
284 PyObject *list;
292 Py_ssize_t len, i, j;
285 Py_ssize_t len, i, j;
293
286
294 len = self->length - start;
287 len = Py_SIZE(self) - start;
295 list = PyList_New(len);
288 list = PyList_New(len);
296 if (list == NULL)
289 if (list == NULL)
297 return NULL;
290 return NULL;
298 for (i = start, j = 0; j < len; i++, j++)
291 for (i = start, j = 0; j < len; i++, j++)
299 PyList_SET_ITEM(list, j, self->data[i]);
292 PyList_SET_ITEM(list, j, self->data[i]);
300
293
301 self->length = start;
294 Py_SIZE(self) = start;
302 return list;
295 return list;
303 }
296 }
304
297
298 typedef struct {
299 PyObject *me_key;
300 long me_value;
301 } PyMemoEntry;
302
303 typedef struct {
304 Py_ssize_t mt_mask;
305 Py_ssize_t mt_used;
306 Py_ssize_t mt_allocated;
307 PyMemoEntry *mt_table;
308 } PyMemoTable;
309
305 typedef struct PicklerObject {
310 typedef struct PicklerObject {
306 PyObject_HEAD
311 PyObject_HEAD
307 PyObject *write; /* write() method of the output stream */
312 PyMemoTable *memo; /* Memo table, keep track of the seen
308 PyObject *memo; /* Memo dictionary, keep track of the seen
309 objects to support self-referential objects
313 objects to support self-referential objects
310 pickling. */
314 pickling. */
311 PyObject *pers_func; /* persistent_id() method, can be NULL */
315 PyObject *pers_func; /* persistent_id() method, can be NULL */
312 PyObject *arg;
316 PyObject *arg;
317
318 /* Write into a local buffer before flushing out to file. output_len
319 tracks the current size; when output_len >= max_output_len, we
320 PyMem_RESIZE. */
321 Py_ssize_t max_output_len;
322 Py_ssize_t output_len;
323 char *output_buffer;
324 PyObject *write; /* write() method of the output stream. */
325
313 int proto; /* Pickle protocol number, >= 0 */
326 int proto; /* Pickle protocol number, >= 0 */
314 int bin; /* Boolean, true if proto > 0 */
327 int bin; /* Boolean, true if proto > 0 */
315 int buf_size; /* Size of the current buffered pickle data */
328 int buf_size; /* Size of the current buffered pickle data */
316 char *write_buf; /* Write buffer, this is to avoid calling the
317 write() method of the output stream too
318 often. */
319 int fast; /* Enable fast mode if set to a true value.
329 int fast; /* Enable fast mode if set to a true value.
320 The fast mode disable the usage of memo,
330 The fast mode disable the usage of memo,
321 therefore speeding the pickling process by
331 therefore speeding the pickling process by
322 not generating superfluous PUT opcodes. It
332 not generating superfluous PUT opcodes. It
323 should not be used if with self-referential
333 should not be used if with self-referential
324 objects. */
334 objects. */
325 int fast_nesting;
335 int fast_nesting;
326 int fix_imports; /* Indicate whether Pickler should fix
336 int fix_imports; /* Indicate whether Pickler should fix
327 the name of globals for Python 2.x. */
337 the name of globals for Python 2.x. */
328 PyObject *fast_memo;
338 PyObject *fast_memo;
329 } PicklerObject;
339 } PicklerObject;
330
340
331 typedef struct UnpicklerObject {
341 typedef struct UnpicklerObject {
332 PyObject_HEAD
342 PyObject_HEAD
333 Pdata *stack; /* Pickle data stack, store unpickled objects. * /
343 Pdata *stack; /* Pickle data stack, store unpickled objects. * /
334 PyObject *readline; /* readline() method of the output stream */
344
335 PyObject *read; /* read() method of the output stream */
345 /* The unpickler memo is just an array of PyObject *s. Using a dict
336 PyObject *memo; /* Memo dictionary, provide the objects stored
346 is unnecessary, since the keys are contiguous ints. */
337 using the PUT opcodes. */
347 PyObject **memo;
348 Py_ssize_t memo_size;
349
338 PyObject *arg;
350 PyObject *arg;
339 PyObject *pers_func; /* persistent_load() method, can be NULL. */
351 PyObject *pers_func; /* persistent_load() method, can be NULL. */
340 PyObject *last_string; /* Reference to the last string read by the
352
341 readline() method. */
353 /* We have to hold on to the Python object, since freeing it will free
342 char *buffer; /* Reading buffer. */
354 input_buffer. */
355 PyObject *py_input;
356 char *input_buffer;
357 Py_ssize_t input_len;
358 Py_ssize_t next_read_idx;
359 PyObject *read; /* read() method of the input stream. */
360 PyObject *readline; /* readline() method of the input stream. */
361
343 char *encoding; /* Name of the encoding to be used for
362 char *encoding; /* Name of the encoding to be used for
344 decoding strings pickled using Python
363 decoding strings pickled using Python
345 2.x. The default value is "ASCII" */
364 2.x. The default value is "ASCII" */
346 char *errors; /* Name of errors handling scheme to used when
365 char *errors; /* Name of errors handling scheme to used when
347 decoding strings. The default value is
366 decoding strings. The default value is
348 "strict". */
367 "strict". */
349 int *marks; /* Mark stack, used for unpickling container
368 int *marks; /* Mark stack, used for unpickling container
350 objects. */
369 objects. */
351 Py_ssize_t num_marks; /* Number of marks in the mark stack. */
370 Py_ssize_t num_marks; /* Number of marks in the mark stack. */
352 Py_ssize_t marks_size; /* Current allocated size of the mark stack. */
371 Py_ssize_t marks_size; /* Current allocated size of the mark stack. */
353 int proto; /* Protocol of the pickle loaded. */
372 int proto; /* Protocol of the pickle loaded. */
354 int fix_imports; /* Indicate whether Unpickler should fix
373 int fix_imports; /* Indicate whether Unpickler should fix
355 the name of globals pickled by Python 2.x. */
374 the name of globals pickled by Python 2.x. */
356 } UnpicklerObject;
375 } UnpicklerObject;
357
376
358 /* Forward declarations */
377 /* Forward declarations */
359 static int save(PicklerObject *, PyObject *, int);
378 static int save(PicklerObject *, PyObject *, int);
360 static int save_reduce(PicklerObject *, PyObject *, PyObject *);
379 static int save_reduce(PicklerObject *, PyObject *, PyObject *);
361 static PyTypeObject Pickler_Type;
380 static PyTypeObject Pickler_Type;
362 static PyTypeObject Unpickler_Type;
381 static PyTypeObject Unpickler_Type;
363
382
364
383
384 /*************************************************************************
385 A custom hashtable mapping void* to longs. This is used by the pickler for
386 memoization. Using a custom hashtable rather than PyDict allows us to skip
387 a bunch of unnecessary object creation. This makes a huge performance
388 difference. */
389
390 #define MT_MINSIZE 8
391 #define PERTURB_SHIFT 5
392
393
394 static PyMemoTable *
395 PyMemoTable_New(void)
396 {
397 PyMemoTable *memo = PyMem_MALLOC(sizeof(PyMemoTable));
398 if (memo == NULL) {
399 PyErr_NoMemory();
400 return NULL;
401 }
402
403 memo->mt_used = 0;
404 memo->mt_allocated = MT_MINSIZE;
405 memo->mt_mask = MT_MINSIZE - 1;
406 memo->mt_table = PyMem_MALLOC(MT_MINSIZE * sizeof(PyMemoEntry));
407 if (memo->mt_table == NULL) {
408 PyMem_FREE(memo);
409 PyErr_NoMemory();
410 return NULL;
411 }
412 memset(memo->mt_table, 0, MT_MINSIZE * sizeof(PyMemoEntry));
413
414 return memo;
415 }
416
417 static PyMemoTable *
418 PyMemoTable_Copy(PyMemoTable *self)
419 {
420 Py_ssize_t i;
421 PyMemoTable *new = PyMemoTable_New();
422 if (new == NULL)
423 return NULL;
424
425 new->mt_used = self->mt_used;
426 new->mt_allocated = self->mt_allocated;
427 new->mt_mask = self->mt_mask;
428 /* The table we get from _New() is probably smaller than we wanted.
429 Free it and allocate one that's the right size. */
430 PyMem_FREE(new->mt_table);
431 new->mt_table = PyMem_MALLOC(self->mt_allocated * sizeof(PyMemoEntry));
432 if (new->mt_table == NULL) {
433 PyMem_FREE(new);
434 return NULL;
435 }
436 for (i = 0; i < self->mt_allocated; i++) {
437 Py_XINCREF(self->mt_table[i].me_key);
438 }
439 memcpy(new->mt_table, self->mt_table,
440 sizeof(PyMemoEntry) * self->mt_allocated);
441
442 return new;
443 }
444
445 static Py_ssize_t
446 PyMemoTable_Size(PyMemoTable *self)
447 {
448 return self->mt_used;
449 }
450
451 static int
452 PyMemoTable_Clear(PyMemoTable *self)
453 {
454 Py_ssize_t i = self->mt_allocated;
455
456 while (--i >= 0) {
457 Py_XDECREF(self->mt_table[i].me_key);
458 }
459 self->mt_used = 0;
460 memset(self->mt_table, 0, self->mt_allocated * sizeof(PyMemoEntry));
461 return 0;
462 }
463
464 static void
465 PyMemoTable_Del(PyMemoTable *self)
466 {
467 if (self == NULL)
468 return;
469 PyMemoTable_Clear(self);
470
471 PyMem_FREE(self->mt_table);
472 PyMem_FREE(self);
473 }
474
475 /* Since entries cannot be deleted from this hashtable, _PyMemoTable_Lookup()
476 can be considerably simpler than dictobject.c's lookdict(). */
477 static PyMemoEntry *
478 _PyMemoTable_Lookup(PyMemoTable *self, PyObject *key)
479 {
480 size_t i;
481 size_t perturb;
482 size_t mask = (size_t)self->mt_mask;
483 PyMemoEntry *table = self->mt_table;
484 PyMemoEntry *entry;
485 long hash = (long)key >> 3;
486
487 i = hash & mask;
488 entry = &table[i];
489 if (entry->me_key == NULL || entry->me_key == key)
490 return entry;
491
492 for (perturb = hash; ; perturb >>= PERTURB_SHIFT) {
493 i = (i << 2) + i + perturb + 1;
494 entry = &table[i & mask];
495 if (entry->me_key == NULL || entry->me_key == key)
496 return entry;
497 }
498 assert(0); /* Never reached */
499 return NULL;
500 }
501
502 /* Returns -1 on failure, 0 on success. */
503 static int
504 _PyMemoTable_ResizeTable(PyMemoTable *self, Py_ssize_t min_size)
505 {
506 PyMemoEntry *oldtable = NULL;
507 PyMemoEntry *oldentry, *newentry;
508 Py_ssize_t new_size = MT_MINSIZE;
509 Py_ssize_t to_process;
510
511 assert(min_size > 0);
512
513 /* Find the smallest valid table size >= min_size. */
514 while (new_size < min_size && new_size > 0)
515 new_size <<= 1;
516 if (new_size <= 0) {
517 PyErr_NoMemory();
518 return -1;
519 }
520 /* new_size needs to be a power of two. */
521 assert((new_size & (new_size - 1)) == 0);
522
523 /* Allocate new table. */
524 oldtable = self->mt_table;
525 self->mt_table = PyMem_MALLOC(new_size * sizeof(PyMemoEntry));
526 if (self->mt_table == NULL) {
527 PyMem_FREE(oldtable);
528 PyErr_NoMemory();
529 return -1;
530 }
531 self->mt_allocated = new_size;
532 self->mt_mask = new_size - 1;
533 memset(self->mt_table, 0, sizeof(PyMemoEntry) * new_size);
534
535 /* Copy entries from the old table. */
536 to_process = self->mt_used;
537 for (oldentry = oldtable; to_process > 0; oldentry++) {
538 if (oldentry->me_key != NULL) {
539 to_process--;
540 /* newentry is a pointer to a chunk of the new
541 mt_table, so we're setting the key:value pair
542 in-place. */
543 newentry = _PyMemoTable_Lookup(self, oldentry->me_key);
544 newentry->me_key = oldentry->me_key;
545 newentry->me_value = oldentry->me_value;
546 }
547 }
548
549 /* Deallocate the old table. */
550 PyMem_FREE(oldtable);
551 return 0;
552 }
553
554 /* Returns NULL on failure, a pointer to the value otherwise. */
555 static long *
556 PyMemoTable_Get(PyMemoTable *self, PyObject *key)
557 {
558 PyMemoEntry *entry = _PyMemoTable_Lookup(self, key);
559 if (entry->me_key == NULL)
560 return NULL;
561 return &entry->me_value;
562 }
563
564 /* Returns -1 on failure, 0 on success. */
565 static int
566 PyMemoTable_Set(PyMemoTable *self, PyObject *key, long value)
567 {
568 PyMemoEntry *entry;
569
570 assert(key != NULL);
571
572 entry = _PyMemoTable_Lookup(self, key);
573 if (entry->me_key != NULL) {
574 entry->me_value = value;
575 return 0;
576 }
577 Py_INCREF(key);
578 entry->me_key = key;
579 entry->me_value = value;
580 self->mt_used++;
581
582 /* If we added a key, we can safely resize. Otherwise just return!
583 * If used >= 2/3 size, adjust size. Normally, this quaduples the size.
584 *
585 * Quadrupling the size improves average table sparseness
586 * (reducing collisions) at the cost of some memory. It also halves
587 * the number of expensive resize operations in a growing memo table.
588 *
589 * Very large memo tables (over 50K items) use doubling instead.
590 * This may help applications with severe memory constraints.
591 */
592 if (!(self->mt_used * 3 >= (self->mt_mask + 1) * 2))
593 return 0;
594 return _PyMemoTable_ResizeTable(self,
595 (self->mt_used > 50000 ? 2 : 4) * self->mt_used);
596 }
597
598 #undef MT_MINSIZE
599 #undef PERTURB_SHIFT
600
601 /*************************************************************************/
602
365 /* Helpers for creating the argument tuple passed to functions. This has the
603 /* Helpers for creating the argument tuple passed to functions. This has the
366 performance advantage of calling PyTuple_New() only once. */
604 performance advantage of calling PyTuple_New() only once.
367
605
606 XXX(avassalotti): Inline directly in _Pickler_FastCall() and
607 _Unpickler_FastCall(). */
368 #define ARG_TUP(self, obj) do { \
608 #define ARG_TUP(self, obj) do { \
369 if ((self)->arg || ((self)->arg=PyTuple_New(1))) { \
609 if ((self)->arg || ((self)->arg=PyTuple_New(1))) { \
370 Py_XDECREF(PyTuple_GET_ITEM((self)->arg, 0)); \
610 Py_XDECREF(PyTuple_GET_ITEM((self)->arg, 0)); \
371 PyTuple_SET_ITEM((self)->arg, 0, (obj)); \
611 PyTuple_SET_ITEM((self)->arg, 0, (obj)); \
372 } \
612 } \
373 else { \
613 else { \
374 Py_DECREF((obj)); \
614 Py_DECREF((obj)); \
375 } \
615 } \
376 } while (0)
616 } while (0)
377
617
(...skipping 16 matching lines...) Expand all Loading...
394 overwhelmed by other factors, like I/O throughput, the GC traversal and
634 overwhelmed by other factors, like I/O throughput, the GC traversal and
395 object allocation overhead. So, I really doubt these functions provide any
635 object allocation overhead. So, I really doubt these functions provide any
396 real benefits.
636 real benefits.
397
637
398 On the other hand, oprofile reports that pickle spends a lot of time in
638 On the other hand, oprofile reports that pickle spends a lot of time in
399 these functions. But, that is probably more related to the function call
639 these functions. But, that is probably more related to the function call
400 overhead, than the argument tuple allocation.
640 overhead, than the argument tuple allocation.
401
641
402 XXX: And, what is the reference behavior of these? Steal, borrow? At first
642 XXX: And, what is the reference behavior of these? Steal, borrow? At first
403 glance, it seems to steal the reference of 'arg' and borrow the reference
643 glance, it seems to steal the reference of 'arg' and borrow the reference
404 of 'func'.
644 of 'func'. */
405 */
406 static PyObject *
645 static PyObject *
407 pickler_call(PicklerObject *self, PyObject *func, PyObject *arg)
646 _Pickler_FastCall(PicklerObject *self, PyObject *func, PyObject *arg)
408 {
647 {
409 PyObject *result = NULL;
648 PyObject *result = NULL;
410
649
411 ARG_TUP(self, arg);
650 ARG_TUP(self, arg);
412 if (self->arg) {
651 if (self->arg) {
413 result = PyObject_Call(func, self->arg, NULL);
652 result = PyObject_Call(func, self->arg, NULL);
414 FREE_ARG_TUP(self);
653 FREE_ARG_TUP(self);
415 }
654 }
416 return result;
655 return result;
417 }
656 }
418
657
658 static int
659 _Pickler_ClearBuffer(PicklerObject *self)
660 {
661 memset(self->output_buffer, 0, sizeof(char) * self->output_len);
662 self->output_len = 0;
663 return 0;
664 }
665
419 static PyObject *
666 static PyObject *
420 unpickler_call(UnpicklerObject *self, PyObject *func, PyObject *arg)
667 _Pickler_GetString(PicklerObject *self)
668 {
669 return PyBytes_FromStringAndSize(self->output_buffer,
670 self->output_len);
671 }
672
673 static int
674 _Pickler_FlushToFile(PicklerObject *self)
675 {
676 PyObject *output, *result;
677
678 assert(self->write != NULL);
679
680 output = _Pickler_GetString(self);
681 if (output == NULL)
682 return -1;
683
684 result = _Pickler_FastCall(self, self->write, output);
685 Py_XDECREF(result);
686 return (result == NULL) ? -1 : 0;
687 }
688
689 static int
690 _Pickler_Write(PicklerObject *self, const char *s, Py_ssize_t n)
691 {
692 Py_ssize_t i;
693 char *buffer;
694
695 assert(s != NULL);
696
697 if (self->output_len + n > self->max_output_len) {
698 self->max_output_len = (self->output_len + n) * 2;
699 buffer = PyMem_REALLOC(self->output_buffer, self->max_output_len);
700 if (buffer == NULL) {
701 PyErr_NoMemory();
702 return -1;
703 }
704 self->output_buffer = buffer;
705 }
706 if (n < 8) {
707 /* This is faster than memcpy when the string is short. */
708 for (i = 0; i < n; i++) {
709 self->output_buffer[self->output_len + i] = s[i];
710 }
711 }
712 else {
713 memcpy(self->output_buffer + self->output_len, s, n);
714 }
715 self->output_len += n;
716 return n;
717 }
718
719 static PicklerObject *
720 _Pickler_New(void)
721 {
722 PicklerObject *self;
723
724 self = PyObject_GC_New(PicklerObject, &Pickler_Type);
725 if (self == NULL)
726 return NULL;
727
728 self->memo = PyMemoTable_New();
729 if (self->memo == NULL)
730 return NULL;
731 self->pers_func = NULL;
732 self->arg = NULL;
733 self->max_output_len = WRITE_BUF_SIZE;
734 self->output_len = 0;
735 self->output_buffer = PyMem_MALLOC(WRITE_BUF_SIZE);
736 if (self->output_buffer == NULL) {
737 PyErr_NoMemory();
738 return NULL;
739 }
740 self->write = NULL;
741 self->proto = 0;
742 self->bin = 0;
743 self->fast = 0;
744 self->fast_nesting = 0;
745 self->fix_imports = 0;
746 self->fast_memo = NULL;
747
748 return self;
749 }
750
751 static int
752 _Pickler_SetProtocol(PicklerObject *self, PyObject *proto_obj,
753 PyObject *fix_imports_obj)
754 {
755 long proto = 0;
756 int fix_imports;
757
758 if (proto_obj == NULL || proto_obj == Py_None)
759 proto = DEFAULT_PROTOCOL;
760 else {
761 proto = PyLong_AsLong(proto_obj);
762 if (proto == -1 && PyErr_Occurred())
763 return -1;
764 }
765 if (proto < 0)
766 proto = HIGHEST_PROTOCOL;
767 if (proto > HIGHEST_PROTOCOL) {
768 PyErr_Format(PyExc_ValueError, "pickle protocol must be <= %d",
769 HIGHEST_PROTOCOL);
770 return -1;
771 }
772 fix_imports = PyObject_IsTrue(fix_imports_obj);
773 if (fix_imports == -1)
774 return -1;
775 ····
776 self->proto = proto;
777 self->bin = proto > 0;
778 self->fix_imports = fix_imports && proto < 3;
779
780 return 0;
781 }
782
783 /* Returns -1 (with an exception set) on failure, 0 on success. This may
784 be called once on a freshly created Pickler. */
785 static int
786 _Pickler_SetOutputStream(PicklerObject *self, PyObject *file)
787 {
788 assert(file != NULL);
789 self->write = PyObject_GetAttrString(file, "write");
790 if (self->write == NULL) {
791 if (PyErr_ExceptionMatches(PyExc_AttributeError))
792 PyErr_SetString(PyExc_TypeError,
793 "file must have a 'write' attribute");
794 return -1;
795 }
796
797 return 0;
798 }
799
800 /* See documentation for _Pickler_FastCall(). */
801 static PyObject *
802 _Unpickler_FastCall(UnpicklerObject *self, PyObject *func, PyObject *arg)
421 {
803 {
422 PyObject *result = NULL;
804 PyObject *result = NULL;
423
805
424 ARG_TUP(self, arg);
806 ARG_TUP(self, arg);
425 if (self->arg) {
807 if (self->arg) {
426 result = PyObject_Call(func, self->arg, NULL);
808 result = PyObject_Call(func, self->arg, NULL);
427 FREE_ARG_TUP(self);
809 FREE_ARG_TUP(self);
428 }
810 }
429 return result;
811 return result;
430 }
812 }
431
813
814 /* Returns the size of the input on success, -1 on failure. This takes its
815 own reference to `input`. */
432 static Py_ssize_t
816 static Py_ssize_t
433 pickler_write(PicklerObject *self, const char *s, Py_ssize_t n)
817 _Unpickler_SetStringInput(UnpicklerObject *self, PyObject *input)
434 {
818 {
435 PyObject *data, *result;
819 /* XXX(avassalotti): This should support all objects implementing the
436
820 buffer protocol. */
437 if (self->write_buf == NULL) {
821 if (!PyBytes_Check(input)) {
438 PyErr_SetString(PyExc_SystemError, "invalid write buffer");
822 PyErr_Format(PyExc_TypeError,
439 return -1;
823 "input must be bytes, not %.200s",
440 }
824 Py_TYPE(input)->tp_name);
441
825 return -1;
442 if (s == NULL) {
826 }
443 if (!(self->buf_size))
827
444 return 0;
828 /* Decrefing py_input will free input_buffer. */
445 data = PyBytes_FromStringAndSize(self->write_buf, self->buf_size);
829 Py_XDECREF(self->py_input);
446 if (data == NULL)
830 Py_INCREF(input);
831 self->py_input = input;
832 self->input_buffer = PyBytes_AS_STRING(input);
833 self->input_len = Py_SIZE(input);
834 self->next_read_idx = 0;
835 return self->input_len;
836 }
837
838 static const Py_ssize_t READ_WHOLE_LINE = -1;
839
840 /* If reading from a file, we need to only pull the bytes we need, since there
841 may be multiple pickle objects arranged contiguously in the same input
842 buffer.
843
844 If `n` is READ_WHOLE_LINE, read a whole line. Otherwise, read up to `n`
845 bytes from the input stream/buffer.
846
847 Update the unpickler's input buffer with the newly-read data. Returns -1 on
848 failure; on success, returns the number of bytes read from the file.
849
850 On success, self->input_len will be 0; this is intentional so that when
851 unpickling from a file, the "we've run out of data" code paths will trigger,
852 causing the Unpickler to go back to the file for more data. Use the returned
853 size to tell you how much data you can process. */
854 static Py_ssize_t
855 _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n)
856 {
857 PyObject *data;
858 Py_ssize_t read_size;
859
860 assert(self->read != NULL);
861 assert(self->next_read_idx == 0);
862
863 if (n == READ_WHOLE_LINE)
864 data = PyObject_Call(self->readline, empty_tuple, NULL);
865 else {
866 PyObject *len = PyLong_FromSsize_t(n);
867 if (len == NULL)
447 return -1;
868 return -1;
448 }
869 data = _Unpickler_FastCall(self, self->read, len);
449 else {
870 }
450 if (self->buf_size && (n + self->buf_size) > WRITE_BUF_SIZE) {
871 if (data == NULL)
451 if (pickler_write(self, NULL, 0) < 0)
872 return -1;
873
874 read_size = _Unpickler_SetStringInput(self, data);
875 self->input_len = 0;
876 Py_DECREF(data);
877 return read_size;
878 }
879
880 /* Read `n` bytes from the unpickler's data source, storing the result in `*s`.
881
882 This should be used for all data reads, rather than accessing the unpickler's
883 input buffer directly. This method deals correctly with reading from input
884 streams, which the input buffer doesn't deal with.
885
886 Note that when reading from a file-like object, self->next_read_idx won't
887 be updated (it should remain at 0 for the entire unpickling process). You
888 should use this function's return value to know how many bytes you can
889 consume.
890
891 Returns -1 (with an exception set) on failure. On success, return the
892 number of chars read. */
893 static Py_ssize_t
894 _Unpickler_Read(UnpicklerObject *self, char **s, Py_ssize_t n)
895 {
896 if (n == 0) {
897 *s = NULL;
898 return 0;
899 }
900
901 /* This condition will always be true if self->read. */
902 if (self->next_read_idx + n > self->input_len) {
903 if (self->read) {
904 Py_ssize_t num_read;
905 assert(self->next_read_idx == self->input_len);
906 num_read = _Unpickler_ReadFromFile(self, n);
907 if (n < 0)
452 return -1;
908 return -1;
909 if (num_read == n) {
910 *s = self->input_buffer;
911 return num_read;
912 }
453 }
913 }
454
914 PyErr_Format(PyExc_EOFError, "Ran out of input");
455 if (n > WRITE_BUF_SIZE) {
915 return -1;
456 if (!(data = PyBytes_FromStringAndSize(s, n)))
916 }
457 return -1;
917 assert(self->read == NULL);
918 *s = &self->input_buffer[self->next_read_idx];
919 self->next_read_idx += n;
920 return n;
921 }
922
923 /* Read a line from the input stream/buffer. If we run off the end of the input
924 before hitting \n, return the data we found.
925
926 Returns the number of chars read, or -1 on failure. */
927 static Py_ssize_t
928 _Unpickler_Readline(UnpicklerObject *self, char **s)
929 {
930 Py_ssize_t i, num_read;
931
932 /* This loop will never be entered if self->read is not NULL. */
933 for (i = self->next_read_idx; i < self->input_len; i++) {
934 assert(self->read == NULL);
935 if (self->input_buffer[i] == '\n') {
936 *s = &self->input_buffer[self->next_read_idx];
937 num_read = i - self->next_read_idx + 1;
938 self->next_read_idx = i + 1;
939 return num_read;
458 }
940 }
459 else {
941 }
460 memcpy(self->write_buf + self->buf_size, s, n);
942 if (self->read) {
461 self->buf_size += n;
943 assert(self->next_read_idx == self->input_len);
462 return n;
944 num_read = _Unpickler_ReadFromFile(self, READ_WHOLE_LINE);
463 }
945 if (num_read < 0)
464 }
946 return -1;
465
947 *s = self->input_buffer;
466 /* object with write method */
948 return num_read;
467 result = pickler_call(self, self->write, data);
949 }
468 if (result == NULL)
950
469 return -1;
951 /* If we get here, we've run off the end of the input string. Return the
470
952 remaining string and let the caller figure it out. */
471 Py_DECREF(result);
953 *s = &self->input_buffer[self->next_read_idx];
472 self->buf_size = 0;
954 num_read = i - self->next_read_idx;
473 return n;
955 self->next_read_idx = i;
474 }
956 return num_read;
475
957 }
476 /* XXX: These read/readline functions ought to be optimized. Buffered I/O
958
477 might help a lot, especially with the new (but much slower) io library.
959 /* Returns -1 (with an exception set) on failure, 0 on success. The memo array
478 On the other hand, the added complexity might not worth it.
960 will be modified in place. */
479 */
961 static int
480
962 _Unpickler_ResizeMemoList(UnpicklerObject *self, Py_ssize_t new_size)
481 /* Read at least n characters from the input stream and set s to the current
963 {
482 reading position. */
964 Py_ssize_t i;
483 static Py_ssize_t
965 PyObject **memo;
484 unpickler_read(UnpicklerObject *self, char **s, Py_ssize_t n)
966
485 {
967 assert(new_size > self->memo_size);
486 PyObject *len;
968
487 PyObject *data;
969 memo = PyMem_REALLOC(self->memo, new_size * sizeof(PyObject *));
488
970 if (memo == NULL) {
489 len = PyLong_FromSsize_t(n);
971 PyErr_NoMemory();
490 if (len == NULL)
972 return -1;
491 return -1;
973 }
492
974 self->memo = memo;
493 data = unpickler_call(self, self->read, len);
975 for (i = self->memo_size; i < new_size; i++)
494 if (data == NULL)
976 self->memo[i] = NULL;
495 return -1;
977 self->memo_size = new_size;
496
978 return 0;
497 /* XXX: Should bytearray be supported too? */
979 }
498 if (!PyBytes_Check(data)) {
980
499 PyErr_SetString(PyExc_ValueError,
981 /* Returns NULL if idx is out of bounds. */
500 "read() from the underlying stream did not"
982 static PyObject *
501 "return bytes");
983 _Unpickler_MemoGet(UnpicklerObject *self, Py_ssize_t idx)
502 Py_DECREF(data);
984 {
503 return -1;
985 if (idx < 0 || idx >= self->memo_size)
504 }
986 return NULL;
505
987
506 if (PyBytes_GET_SIZE(data) != n) {
988 return self->memo[idx];
507 PyErr_SetNone(PyExc_EOFError);
989 }
508 Py_DECREF(data);
990
509 return -1;
991 /* Returns -1 (with an exception set) on failure, 0 on success.
510 }
992 This takes its own reference to `value`. */
511
993 static int
512 Py_XDECREF(self->last_string);
994 _Unpickler_MemoPut(UnpicklerObject *self, Py_ssize_t idx, PyObject *value)
513 self->last_string = data;
995 {
514
996 PyObject *old_item;
515 if (!(*s = PyBytes_AS_STRING(data)))
997
516 return -1;
998 if (idx >= self->memo_size) {
517
999 if (_Unpickler_ResizeMemoList(self, idx * 2) < 0)
518 return n;
1000 return -1;
519 }
1001 assert(idx < self->memo_size);
520
1002 }
521 static Py_ssize_t
1003 Py_INCREF(value);
522 unpickler_readline(UnpicklerObject *self, char **s)
1004 old_item = self->memo[idx];
523 {
1005 self->memo[idx] = value;
524 PyObject *data;
1006 Py_XDECREF(old_item);
525
1007 return 0;
526 data = PyObject_CallObject(self->readline, empty_tuple);
1008 }
527 if (data == NULL)
1009
528 return -1;
1010 static PyObject **
529
1011 _Unpickler_NewMemo(Py_ssize_t new_size)
530 /* XXX: Should bytearray be supported too? */
1012 {
531 if (!PyBytes_Check(data)) {
1013 PyObject **memo = PyMem_MALLOC(new_size * sizeof(PyObject *));
532 PyErr_SetString(PyExc_ValueError,
1014 if (memo == NULL)
533 "readline() from the underlying stream did not"
1015 return NULL;
534 "return bytes");
1016 memset(memo, 0, new_size * sizeof(PyObject *));
535 return -1;
1017 return memo;
536 }
1018 }
537
1019
538 Py_XDECREF(self->last_string);
1020 /* Free the unpickler's memo, taking care to decref any items left in it. */
539 self->last_string = data;
1021 static void
540
1022 _Unpickler_MemoCleanup(UnpicklerObject *self)
541 if (!(*s = PyBytes_AS_STRING(data)))
1023 {
542 return -1;
1024 Py_ssize_t i;
543
1025
544 return PyBytes_GET_SIZE(data);
1026 if (self->memo == NULL)
545 }
1027 return;
546
1028
547 /* Generate a GET opcode for an object stored in the memo. The 'key' argument
1029 i = self->memo_size;
548 should be the address of the object as returned by PyLong_FromVoidPtr(). */
1030 while (--i >= 0) {
1031 Py_XDECREF(self->memo[i]);
1032 }
1033 PyMem_FREE(self->memo);
1034 self->memo = NULL;
1035 }
1036
1037 static UnpicklerObject *
1038 _Unpickler_New(void)
1039 {
1040 UnpicklerObject *self;
1041
1042 self = PyObject_GC_New(UnpicklerObject, &Unpickler_Type);
1043 if (self == NULL)
1044 return NULL;
1045
1046 self->stack = (Pdata *)Pdata_New();
1047 if (self->stack == NULL)
1048 return NULL;
1049
1050 self->memo_size = 32;
1051 self->memo = _Unpickler_NewMemo(self->memo_size);
1052 if (self->memo == NULL)
1053 return NULL;
1054
1055 self->arg = NULL;
1056 self->pers_func = NULL;
1057 self->py_input = NULL;
1058 self->input_buffer = NULL;
1059 self->input_len = 0;
1060 self->next_read_idx = 0;
1061 self->read = NULL;
1062 self->readline = NULL;
1063 self->encoding = NULL;
1064 self->errors = NULL;
1065 self->marks = NULL;
1066 self->num_marks = 0;
1067 self->marks_size = 0;
1068 self->proto = 0;
1069 self->fix_imports = 0;
1070
1071 return self;
1072 }
1073
1074 /* Returns -1 (with an exception set) on failure, 0 on success. This may
1075 be called once on a freshly created Pickler. */
1076 static int
1077 _Unpickler_SetInputStream(UnpicklerObject *self, PyObject *file)
1078 {
1079 self->read = PyObject_GetAttrString(file, "read");
1080 self->readline = PyObject_GetAttrString(file, "readline");
1081 if (self->readline == NULL || self->read == NULL) {
1082 if (PyErr_ExceptionMatches(PyExc_AttributeError))
1083 PyErr_SetString(PyExc_TypeError,
1084 "file must have 'read' and 'readline' attributes");
1085 Py_CLEAR(self->read);
1086 Py_CLEAR(self->readline);
1087 return -1;
1088 }
1089 return 0;
1090 }
1091
1092 /* Returns -1 (with an exception set) on failure, 0 on success. This may
1093 be called once on a freshly created Pickler. */
1094 static int
1095 _Unpickler_SetInputEncoding(UnpicklerObject *self,
1096 const char *encoding,
1097 const char *errors)
1098 {
1099 if (encoding == NULL)
1100 encoding = "ASCII";
1101 if (errors == NULL)
1102 errors = "strict";
1103
1104 self->encoding = strdup(encoding);
1105 self->errors = strdup(errors);
1106 if (self->encoding == NULL || self->errors == NULL) {
1107 PyErr_NoMemory();
1108 return -1;
1109 }
1110 return 0;
1111 }
1112
1113 /* Generate a GET opcode for an object stored in the memo. */
549 static int
1114 static int
550 memo_get(PicklerObject *self, PyObject *key)
1115 memo_get(PicklerObject *self, PyObject *key)
551 {
1116 {
552 PyObject *value;
1117 long *value;
553 PyObject *memo_id;
554 long x;
555 char pdata[30];
1118 char pdata[30];
556 int len;
1119 int len;
557
1120
558 value = PyDict_GetItemWithError(self->memo, key);
1121 value = PyMemoTable_Get(self->memo, key);
559 if (value == NULL) {
1122 if (value == NULL) {
560 if (!PyErr_Occurred())
1123 PyErr_SetObject(PyExc_KeyError, key);
561 PyErr_SetObject(PyExc_KeyError, key);
1124 return -1;
562 return -1;
1125 }
563 }
564
565 memo_id = PyTuple_GetItem(value, 0);
566 if (memo_id == NULL)
567 return -1;
568
569 if (!PyLong_Check(memo_id)) {
570 PyErr_SetString(PicklingError, "memo id must be an integer");
571 return -1;
572 }
573 x = PyLong_AsLong(memo_id);
574 if (x == -1 && PyErr_Occurred())
575 return -1;
576
1126
577 if (!self->bin) {
1127 if (!self->bin) {
578 pdata[0] = GET;
1128 pdata[0] = GET;
579 PyOS_snprintf(pdata + 1, sizeof(pdata) - 1, "%ld\n", x);
1129 PyOS_snprintf(pdata + 1, sizeof(pdata) - 1, "%ld\n", *value);
580 len = (int)strlen(pdata);
1130 len = (int)strlen(pdata);
581 }
1131 }
582 else {
1132 else {
583 if (x < 256) {
1133 if (*value < 256) {
584 pdata[0] = BINGET;
1134 pdata[0] = BINGET;
585 pdata[1] = (unsigned char)(x & 0xff);
1135 pdata[1] = (unsigned char)(*value & 0xff);
586 len = 2;
1136 len = 2;
587 }
1137 }
588 else if (x <= 0xffffffffL) {
1138 else if (*value <= 0xffffffffL) {
589 pdata[0] = LONG_BINGET;
1139 pdata[0] = LONG_BINGET;
590 pdata[1] = (unsigned char)(x & 0xff);
1140 pdata[1] = (unsigned char)(*value & 0xff);
591 pdata[2] = (unsigned char)((x >> 8) & 0xff);
1141 pdata[2] = (unsigned char)((*value >> 8) & 0xff);
592 pdata[3] = (unsigned char)((x >> 16) & 0xff);
1142 pdata[3] = (unsigned char)((*value >> 16) & 0xff);
593 pdata[4] = (unsigned char)((x >> 24) & 0xff);
1143 pdata[4] = (unsigned char)((*value >> 24) & 0xff);
594 len = 5;
1144 len = 5;
595 }
1145 }
596 else { /* unlikely */
1146 else { /* unlikely */
597 PyErr_SetString(PicklingError,
1147 PyErr_SetString(PicklingError,
598 "memo id too large for LONG_BINGET");
1148 "memo id too large for LONG_BINGET");
599 return -1;
1149 return -1;
600 }
1150 }
601 }
1151 }
602
1152
603 if (pickler_write(self, pdata, len) < 0)
1153 if (_Pickler_Write(self, pdata, len) < 0)
604 return -1;
1154 return -1;
605
1155
606 return 0;
1156 return 0;
607 }
1157 }
608
1158
609 /* Store an object in the memo, assign it a new unique ID based on the number
1159 /* Store an object in the memo, assign it a new unique ID based on the number
610 of objects currently stored in the memo and generate a PUT opcode. */
1160 of objects currently stored in the memo and generate a PUT opcode. */
611 static int
1161 static int
612 memo_put(PicklerObject *self, PyObject *obj)
1162 memo_put(PicklerObject *self, PyObject *obj)
613 {
1163 {
614 PyObject *key = NULL;
615 PyObject *memo_id = NULL;
616 PyObject *tuple = NULL;
617 long x;
1164 long x;
618 char pdata[30];
1165 char pdata[30];
619 int len;
1166 int len;
620 int status = 0;
1167 int status = 0;
621
1168
622 if (self->fast)
1169 if (self->fast)
623 return 0;
1170 return 0;
624
1171
625 key = PyLong_FromVoidPtr(obj);
1172 x = PyMemoTable_Size(self->memo);
626 if (key == NULL)
1173 if (PyMemoTable_Set(self->memo, obj, x) < 0)
627 goto error;
628 if ((x = PyDict_Size(self->memo)) < 0)
629 goto error;
630 memo_id = PyLong_FromLong(x);
631 if (memo_id == NULL)
632 goto error;
633 tuple = PyTuple_New(2);
634 if (tuple == NULL)
635 goto error;
636
637 Py_INCREF(memo_id);
638 PyTuple_SET_ITEM(tuple, 0, memo_id);
639 Py_INCREF(obj);
640 PyTuple_SET_ITEM(tuple, 1, obj);
641 if (PyDict_SetItem(self->memo, key, tuple) < 0)
642 goto error;
1174 goto error;
643
1175
644 if (!self->bin) {
1176 if (!self->bin) {
645 pdata[0] = PUT;
1177 pdata[0] = PUT;
646 PyOS_snprintf(pdata + 1, sizeof(pdata) - 1, "%ld\n", x);
1178 PyOS_snprintf(pdata + 1, sizeof(pdata) - 1, "%ld\n", x);
647 len = strlen(pdata);
1179 len = strlen(pdata);
648 }
1180 }
649 else {
1181 else {
650 if (x < 256) {
1182 if (x < 256) {
651 pdata[0] = BINPUT;
1183 pdata[0] = BINPUT;
652 pdata[1] = (unsigned char)x;
1184 pdata[1] = (unsigned char)x;
653 len = 2;
1185 len = 2;
654 }
1186 }
655 else if (x <= 0xffffffffL) {
1187 else if (x <= 0xffffffffL) {
656 pdata[0] = LONG_BINPUT;
1188 pdata[0] = LONG_BINPUT;
657 pdata[1] = (unsigned char)(x & 0xff);
1189 pdata[1] = (unsigned char)(x & 0xff);
658 pdata[2] = (unsigned char)((x >> 8) & 0xff);
1190 pdata[2] = (unsigned char)((x >> 8) & 0xff);
659 pdata[3] = (unsigned char)((x >> 16) & 0xff);
1191 pdata[3] = (unsigned char)((x >> 16) & 0xff);
660 pdata[4] = (unsigned char)((x >> 24) & 0xff);
1192 pdata[4] = (unsigned char)((x >> 24) & 0xff);
661 len = 5;
1193 len = 5;
662 }
1194 }
663 else { /* unlikely */
1195 else { /* unlikely */
664 PyErr_SetString(PicklingError,
1196 PyErr_SetString(PicklingError,
665 "memo id too large for LONG_BINPUT");
1197 "memo id too large for LONG_BINPUT");
666 return -1;
1198 return -1;
667 }
1199 }
668 }
1200 }
669
1201
670 if (pickler_write(self, pdata, len) < 0)
1202 if (_Pickler_Write(self, pdata, len) < 0)
671 goto error;
1203 goto error;
672
1204
673 if (0) {
1205 if (0) {
674 error:
1206 error:
675 status = -1;
1207 status = -1;
676 }
1208 }
677
1209
678 Py_XDECREF(key);
679 Py_XDECREF(memo_id);
680 Py_XDECREF(tuple);
681
682 return status;
1210 return status;
683 }
1211 }
684
1212
685 static PyObject *
1213 static PyObject *
686 whichmodule(PyObject *global, PyObject *global_name)
1214 whichmodule(PyObject *global, PyObject *global_name)
687 {
1215 {
688 Py_ssize_t i, j;
1216 Py_ssize_t i, j;
689 static PyObject *module_str = NULL;
1217 static PyObject *module_str = NULL;
690 static PyObject *main_str = NULL;
1218 static PyObject *main_str = NULL;
691 PyObject *module_name;
1219 PyObject *module_name;
(...skipping 122 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
814 }
1342 }
815 Py_DECREF(key);
1343 Py_DECREF(key);
816 }
1344 }
817 return 1;
1345 return 1;
818 }
1346 }
819
1347
820 static int
1348 static int
821 save_none(PicklerObject *self, PyObject *obj)
1349 save_none(PicklerObject *self, PyObject *obj)
822 {
1350 {
823 const char none_op = NONE;
1351 const char none_op = NONE;
824 if (pickler_write(self, &none_op, 1) < 0)
1352 if (_Pickler_Write(self, &none_op, 1) < 0)
825 return -1;
1353 return -1;
826
1354
827 return 0;
1355 return 0;
828 }
1356 }
829
1357
830 static int
1358 static int
831 save_bool(PicklerObject *self, PyObject *obj)
1359 save_bool(PicklerObject *self, PyObject *obj)
832 {
1360 {
833 static const char *buf[2] = { FALSE, TRUE };
1361 static const char *buf[2] = { FALSE, TRUE };
834 const char len[2] = {sizeof(FALSE) - 1, sizeof(TRUE) - 1};
1362 const char len[2] = {sizeof(FALSE) - 1, sizeof(TRUE) - 1};
835 int p = (obj == Py_True);
1363 int p = (obj == Py_True);
836
1364
837 if (self->proto >= 2) {
1365 if (self->proto >= 2) {
838 const char bool_op = p ? NEWTRUE : NEWFALSE;
1366 const char bool_op = p ? NEWTRUE : NEWFALSE;
839 if (pickler_write(self, &bool_op, 1) < 0)
1367 if (_Pickler_Write(self, &bool_op, 1) < 0)
840 return -1;
1368 return -1;
841 }
1369 }
842 else if (pickler_write(self, buf[p], len[p]) < 0)
1370 else if (_Pickler_Write(self, buf[p], len[p]) < 0)
843 return -1;
1371 return -1;
844
1372
845 return 0;
1373 return 0;
846 }
1374 }
847
1375
848 static int
1376 static int
849 save_int(PicklerObject *self, long x)
1377 save_int(PicklerObject *self, long x)
850 {
1378 {
851 char pdata[32];
1379 char pdata[32];
852 int len = 0;
1380 int len = 0;
853
1381
854 if (!self->bin
1382 if (!self->bin
855 #if SIZEOF_LONG > 4
1383 #if SIZEOF_LONG > 4
856 || x > 0x7fffffffL || x < -0x80000000L
1384 || x > 0x7fffffffL || x < -0x80000000L
857 #endif
1385 #endif
858 ) {
1386 ) {
859 /* Text-mode pickle, or long too big to fit in the 4-byte
1387 /* Text-mode pickle, or long too big to fit in the 4-byte
860 * signed BININT format: store as a string.
1388 * signed BININT format: store as a string.
861 */
1389 */
862 pdata[0] = LONG; /* use LONG for consistency with pickle.py */
1390 pdata[0] = LONG; /* use LONG for consistency with pickle.py */
863 PyOS_snprintf(pdata + 1, sizeof(pdata) - 1, "%ldL\n", x);
1391 PyOS_snprintf(pdata + 1, sizeof(pdata) - 1, "%ldL\n", x);
864 if (pickler_write(self, pdata, strlen(pdata)) < 0)
1392 if (_Pickler_Write(self, pdata, strlen(pdata)) < 0)
865 return -1;
1393 return -1;
866 }
1394 }
867 else {
1395 else {
868 /* Binary pickle and x fits in a signed 4-byte int. */
1396 /* Binary pickle and x fits in a signed 4-byte int. */
869 pdata[1] = (unsigned char)(x & 0xff);
1397 pdata[1] = (unsigned char)(x & 0xff);
870 pdata[2] = (unsigned char)((x >> 8) & 0xff);
1398 pdata[2] = (unsigned char)((x >> 8) & 0xff);
871 pdata[3] = (unsigned char)((x >> 16) & 0xff);
1399 pdata[3] = (unsigned char)((x >> 16) & 0xff);
872 pdata[4] = (unsigned char)((x >> 24) & 0xff);
1400 pdata[4] = (unsigned char)((x >> 24) & 0xff);
873
1401
874 if ((pdata[4] == 0) && (pdata[3] == 0)) {
1402 if ((pdata[4] == 0) && (pdata[3] == 0)) {
875 if (pdata[2] == 0) {
1403 if (pdata[2] == 0) {
876 pdata[0] = BININT1;
1404 pdata[0] = BININT1;
877 len = 2;
1405 len = 2;
878 }
1406 }
879 else {
1407 else {
880 pdata[0] = BININT2;
1408 pdata[0] = BININT2;
881 len = 3;
1409 len = 3;
882 }
1410 }
883 }
1411 }
884 else {
1412 else {
885 pdata[0] = BININT;
1413 pdata[0] = BININT;
886 len = 5;
1414 len = 5;
887 }
1415 }
888
1416
889 if (pickler_write(self, pdata, len) < 0)
1417 if (_Pickler_Write(self, pdata, len) < 0)
890 return -1;
1418 return -1;
891 }
1419 }
892
1420
893 return 0;
1421 return 0;
894 }
1422 }
895
1423
896 static int
1424 static int
897 save_long(PicklerObject *self, PyObject *obj)
1425 save_long(PicklerObject *self, PyObject *obj)
898 {
1426 {
899 PyObject *repr = NULL;
1427 PyObject *repr = NULL;
(...skipping 15 matching lines...) Expand all Loading...
915 size_t nbits;
1443 size_t nbits;
916 size_t nbytes;
1444 size_t nbytes;
917 unsigned char *pdata;
1445 unsigned char *pdata;
918 char header[5];
1446 char header[5];
919 int i;
1447 int i;
920 int sign = _PyLong_Sign(obj);
1448 int sign = _PyLong_Sign(obj);
921
1449
922 if (sign == 0) {
1450 if (sign == 0) {
923 header[0] = LONG1;
1451 header[0] = LONG1;
924 header[1] = 0; /* It's 0 -- an empty bytestring. */
1452 header[1] = 0; /* It's 0 -- an empty bytestring. */
925 if (pickler_write(self, header, 2) < 0)
1453 if (_Pickler_Write(self, header, 2) < 0)
926 goto error;
1454 goto error;
927 return 0;
1455 return 0;
928 }
1456 }
929 nbits = _PyLong_NumBits(obj);
1457 nbits = _PyLong_NumBits(obj);
930 if (nbits == (size_t)-1 && PyErr_Occurred())
1458 if (nbits == (size_t)-1 && PyErr_Occurred())
931 goto error;
1459 goto error;
932 /* How many bytes do we need? There are nbits >> 3 full
1460 /* How many bytes do we need? There are nbits >> 3 full
933 * bytes of data, and nbits & 7 leftover bits. If there
1461 * bytes of data, and nbits & 7 leftover bits. If there
934 * are any leftover bits, then we clearly need another
1462 * are any leftover bits, then we clearly need another
935 * byte. Wnat's not so obvious is that we *probably*
1463 * byte. Wnat's not so obvious is that we *probably*
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
976 }
1504 }
977 else {
1505 else {
978 header[0] = LONG4;
1506 header[0] = LONG4;
979 size = (int)nbytes;
1507 size = (int)nbytes;
980 for (i = 1; i < 5; i++) {
1508 for (i = 1; i < 5; i++) {
981 header[i] = (unsigned char)(size & 0xff);
1509 header[i] = (unsigned char)(size & 0xff);
982 size >>= 8;
1510 size >>= 8;
983 }
1511 }
984 size = 5;
1512 size = 5;
985 }
1513 }
986 if (pickler_write(self, header, size) < 0 ||
1514 if (_Pickler_Write(self, header, size) < 0 ||
987 pickler_write(self, (char *)pdata, (int)nbytes) < 0)
1515 _Pickler_Write(self, (char *)pdata, (int)nbytes) < 0)
988 goto error;
1516 goto error;
989 }
1517 }
990 else {
1518 else {
991 char *string;
1519 char *string;
992
1520
993 /* proto < 2: write the repr and newline. This is quadratic-time (in
1521 /* proto < 2: write the repr and newline. This is quadratic-time (in
994 the number of digits), in both directions. We add a trailing 'L'
1522 the number of digits), in both directions. We add a trailing 'L'
995 to the repr, for compatibility with Python 2.x. */
1523 to the repr, for compatibility with Python 2.x. */
996
1524
997 repr = PyObject_Repr(obj);
1525 repr = PyObject_Repr(obj);
998 if (repr == NULL)
1526 if (repr == NULL)
999 goto error;
1527 goto error;
1000
1528
1001 string = _PyUnicode_AsStringAndSize(repr, &size);
1529 string = _PyUnicode_AsStringAndSize(repr, &size);
1002 if (string == NULL)
1530 if (string == NULL)
1003 goto error;
1531 goto error;
1004
1532
1005 if (pickler_write(self, &long_op, 1) < 0 ||
1533 if (_Pickler_Write(self, &long_op, 1) < 0 ||
1006 pickler_write(self, string, size) < 0 ||
1534 _Pickler_Write(self, string, size) < 0 ||
1007 pickler_write(self, "L\n", 2) < 0)
1535 _Pickler_Write(self, "L\n", 2) < 0)
1008 goto error;
1536 goto error;
1009 }
1537 }
1010
1538
1011 if (0) {
1539 if (0) {
1012 error:
1540 error:
1013 status = -1;
1541 status = -1;
1014 }
1542 }
1015 Py_XDECREF(repr);
1543 Py_XDECREF(repr);
1016
1544
1017 return status;
1545 return status;
1018 }
1546 }
1019
1547
1020 static int
1548 static int
1021 save_float(PicklerObject *self, PyObject *obj)
1549 save_float(PicklerObject *self, PyObject *obj)
1022 {
1550 {
1023 double x = PyFloat_AS_DOUBLE((PyFloatObject *)obj);
1551 double x = PyFloat_AS_DOUBLE((PyFloatObject *)obj);
1024
1552
1025 if (self->bin) {
1553 if (self->bin) {
1026 char pdata[9];
1554 char pdata[9];
1027 pdata[0] = BINFLOAT;
1555 pdata[0] = BINFLOAT;
1028 if (_PyFloat_Pack8(x, (unsigned char *)&pdata[1], 0) < 0)
1556 if (_PyFloat_Pack8(x, (unsigned char *)&pdata[1], 0) < 0)
1029 return -1;
1557 return -1;
1030 if (pickler_write(self, pdata, 9) < 0)
1558 if (_Pickler_Write(self, pdata, 9) < 0)
1031 return -1;
1559 return -1;
1032 }·
1560 }·
1033 else {
1561 else {
1034 int result = -1;
1562 int result = -1;
1035 char *buf = NULL;
1563 char *buf = NULL;
1036 char op = FLOAT;
1564 char op = FLOAT;
1037
1565
1038 if (pickler_write(self, &op, 1) < 0)
1566 if (_Pickler_Write(self, &op, 1) < 0)
1039 goto done;
1567 goto done;
1040
1568
1041 buf = PyOS_double_to_string(x, 'g', 17, 0, NULL);
1569 buf = PyOS_double_to_string(x, 'g', 17, 0, NULL);
1042 if (!buf) {
1570 if (!buf) {
1043 PyErr_NoMemory();
1571 PyErr_NoMemory();
1044 goto done;
1572 goto done;
1045 }
1573 }
1046
1574
1047 if (pickler_write(self, buf, strlen(buf)) < 0)
1575 if (_Pickler_Write(self, buf, strlen(buf)) < 0)
1048 goto done;
1576 goto done;
1049
1577
1050 if (pickler_write(self, "\n", 1) < 0)
1578 if (_Pickler_Write(self, "\n", 1) < 0)
1051 goto done;
1579 goto done;
1052
1580
1053 result = 0;
1581 result = 0;
1054 done:
1582 done:
1055 PyMem_Free(buf);
1583 PyMem_Free(buf);
1056 return result;
1584 return result;
1057 }
1585 }
1058
1586
1059 return 0;
1587 return 0;
1060 }
1588 }
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
1106 header[1] = (unsigned char)(size & 0xff);
1634 header[1] = (unsigned char)(size & 0xff);
1107 header[2] = (unsigned char)((size >> 8) & 0xff);
1635 header[2] = (unsigned char)((size >> 8) & 0xff);
1108 header[3] = (unsigned char)((size >> 16) & 0xff);
1636 header[3] = (unsigned char)((size >> 16) & 0xff);
1109 header[4] = (unsigned char)((size >> 24) & 0xff);
1637 header[4] = (unsigned char)((size >> 24) & 0xff);
1110 len = 5;
1638 len = 5;
1111 }
1639 }
1112 else {
1640 else {
1113 return -1; /* string too large */
1641 return -1; /* string too large */
1114 }
1642 }
1115
1643
1116 if (pickler_write(self, header, len) < 0)
1644 if (_Pickler_Write(self, header, len) < 0)
1117 return -1;
1645 return -1;
1118
1646
1119 if (pickler_write(self, PyBytes_AS_STRING(obj), size) < 0)
1647 if (_Pickler_Write(self, PyBytes_AS_STRING(obj), size) < 0)
1120 return -1;
1648 return -1;
1121
1649
1122 if (memo_put(self, obj) < 0)
1650 if (memo_put(self, obj) < 0)
1123 return -1;
1651 return -1;
1124
1652
1125 return 0;
1653 return 0;
1126 }
1654 }
1127 }
1655 }
1128
1656
1129 /* A copy of PyUnicode_EncodeRawUnicodeEscape() that also translates
1657 /* A copy of PyUnicode_EncodeRawUnicodeEscape() that also translates
(...skipping 106 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
1236 size = PyBytes_GET_SIZE(encoded);
1764 size = PyBytes_GET_SIZE(encoded);
1237 if (size < 0 || size > 0xffffffffL)
1765 if (size < 0 || size > 0xffffffffL)
1238 goto error; /* string too large */
1766 goto error; /* string too large */
1239
1767
1240 pdata[0] = BINUNICODE;
1768 pdata[0] = BINUNICODE;
1241 pdata[1] = (unsigned char)(size & 0xff);
1769 pdata[1] = (unsigned char)(size & 0xff);
1242 pdata[2] = (unsigned char)((size >> 8) & 0xff);
1770 pdata[2] = (unsigned char)((size >> 8) & 0xff);
1243 pdata[3] = (unsigned char)((size >> 16) & 0xff);
1771 pdata[3] = (unsigned char)((size >> 16) & 0xff);
1244 pdata[4] = (unsigned char)((size >> 24) & 0xff);
1772 pdata[4] = (unsigned char)((size >> 24) & 0xff);
1245
1773
1246 if (pickler_write(self, pdata, 5) < 0)
1774 if (_Pickler_Write(self, pdata, 5) < 0)
1247 goto error;
1775 goto error;
1248
1776
1249 if (pickler_write(self, PyBytes_AS_STRING(encoded), size) < 0)
1777 if (_Pickler_Write(self, PyBytes_AS_STRING(encoded), size) < 0)
1250 goto error;
1778 goto error;
1251 }
1779 }
1252 else {
1780 else {
1253 const char unicode_op = UNICODE;
1781 const char unicode_op = UNICODE;
1254
1782
1255 encoded = raw_unicode_escape(PyUnicode_AS_UNICODE(obj),
1783 encoded = raw_unicode_escape(PyUnicode_AS_UNICODE(obj),
1256 PyUnicode_GET_SIZE(obj));
1784 PyUnicode_GET_SIZE(obj));
1257 if (encoded == NULL)
1785 if (encoded == NULL)
1258 goto error;
1786 goto error;
1259
1787
1260 if (pickler_write(self, &unicode_op, 1) < 0)
1788 if (_Pickler_Write(self, &unicode_op, 1) < 0)
1261 goto error;
1789 goto error;
1262
1790
1263 size = PyBytes_GET_SIZE(encoded);
1791 size = PyBytes_GET_SIZE(encoded);
1264 if (pickler_write(self, PyBytes_AS_STRING(encoded), size) < 0)
1792 if (_Pickler_Write(self, PyBytes_AS_STRING(encoded), size) < 0)
1265 goto error;
1793 goto error;
1266
1794
1267 if (pickler_write(self, "\n", 1) < 0)
1795 if (_Pickler_Write(self, "\n", 1) < 0)
1268 goto error;
1796 goto error;
1269 }
1797 }
1270 if (memo_put(self, obj) < 0)
1798 if (memo_put(self, obj) < 0)
1271 goto error;
1799 goto error;
1272
1800
1273 Py_DECREF(encoded);
1801 Py_DECREF(encoded);
1274 return 0;
1802 return 0;
1275
1803
1276 error:
1804 error:
1277 Py_XDECREF(encoded);
1805 Py_XDECREF(encoded);
(...skipping 22 matching lines...) Expand all Loading...
1300
1828
1301 /* Tuples are ubiquitous in the pickle protocols, so many techniques are
1829 /* Tuples are ubiquitous in the pickle protocols, so many techniques are
1302 * used across protocols to minimize the space needed to pickle them.
1830 * used across protocols to minimize the space needed to pickle them.
1303 * Tuples are also the only builtin immutable type that can be recursive
1831 * Tuples are also the only builtin immutable type that can be recursive
1304 * (a tuple can be reached from itself), and that requires some subtle
1832 * (a tuple can be reached from itself), and that requires some subtle
1305 * magic so that it works in all cases. IOW, this is a long routine.
1833 * magic so that it works in all cases. IOW, this is a long routine.
1306 */
1834 */
1307 static int
1835 static int
1308 save_tuple(PicklerObject *self, PyObject *obj)
1836 save_tuple(PicklerObject *self, PyObject *obj)
1309 {
1837 {
1310 PyObject *memo_key = NULL;
1311 int len, i;
1838 int len, i;
1312 int status = 0;
1313
1839
1314 const char mark_op = MARK;
1840 const char mark_op = MARK;
1315 const char tuple_op = TUPLE;
1841 const char tuple_op = TUPLE;
1316 const char pop_op = POP;
1842 const char pop_op = POP;
1317 const char pop_mark_op = POP_MARK;
1843 const char pop_mark_op = POP_MARK;
1318 const char len2opcode[] = {EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3};
1844 const char len2opcode[] = {EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3};
1319
1845
1320 if ((len = PyTuple_Size(obj)) < 0)
1846 if ((len = PyTuple_Size(obj)) < 0)
1321 return -1;
1847 return -1;
1322
1848
1323 if (len == 0) {
1849 if (len == 0) {
1324 char pdata[2];
1850 char pdata[2];
1325
1851
1326 if (self->proto) {
1852 if (self->proto) {
1327 pdata[0] = EMPTY_TUPLE;
1853 pdata[0] = EMPTY_TUPLE;
1328 len = 1;
1854 len = 1;
1329 }
1855 }
1330 else {
1856 else {
1331 pdata[0] = MARK;
1857 pdata[0] = MARK;
1332 pdata[1] = TUPLE;
1858 pdata[1] = TUPLE;
1333 len = 2;
1859 len = 2;
1334 }
1860 }
1335 if (pickler_write(self, pdata, len) < 0)
1861 if (_Pickler_Write(self, pdata, len) < 0)
1336 return -1;
1862 return -1;
1337 return 0;
1863 return 0;
1338 }
1864 }
1339
1865
1340 /* id(tuple) isn't in the memo now. If it shows up there after
1866 /* The tuple isn't in the memo now. If it shows up there after
1341 * saving the tuple elements, the tuple must be recursive, in
1867 * saving the tuple elements, the tuple must be recursive, in
1342 * which case we'll pop everything we put on the stack, and fetch
1868 * which case we'll pop everything we put on the stack, and fetch
1343 * its value from the memo.
1869 * its value from the memo.
1344 */
1870 */
1345 memo_key = PyLong_FromVoidPtr(obj);
1346 if (memo_key == NULL)
1347 return -1;
1348
1349 if (len <= 3 && self->proto >= 2) {
1871 if (len <= 3 && self->proto >= 2) {
1350 /* Use TUPLE{1,2,3} opcodes. */
1872 /* Use TUPLE{1,2,3} opcodes. */
1351 if (store_tuple_elements(self, obj, len) < 0)
1873 if (store_tuple_elements(self, obj, len) < 0)
1352 goto error;
1874 return -1;
1353
1875
1354 if (PyDict_GetItem(self->memo, memo_key)) {
1876 if (PyMemoTable_Get(self->memo, obj)) {
1355 /* pop the len elements */
1877 /* pop the len elements */
1356 for (i = 0; i < len; i++)
1878 for (i = 0; i < len; i++)
1357 if (pickler_write(self, &pop_op, 1) < 0)
1879 if (_Pickler_Write(self, &pop_op, 1) < 0)
1358 goto error;
1880 return -1;
1359 /* fetch from memo */
1881 /* fetch from memo */
1360 if (memo_get(self, memo_key) < 0)
1882 if (memo_get(self, obj) < 0)
1361 goto error;
1883 return -1;
1362
1884
1363 Py_DECREF(memo_key);
1364 return 0;
1885 return 0;
1365 }
1886 }
1366 else { /* Not recursive. */
1887 else { /* Not recursive. */
1367 if (pickler_write(self, len2opcode + len, 1) < 0)
1888 if (_Pickler_Write(self, len2opcode + len, 1) < 0)
1368 goto error;
1889 return -1;
1369 }
1890 }
1370 goto memoize;
1891 goto memoize;
1371 }
1892 }
1372
1893
1373 /* proto < 2 and len > 0, or proto >= 2 and len > 3.
1894 /* proto < 2 and len > 0, or proto >= 2 and len > 3.
1374 * Generate MARK e1 e2 ... TUPLE
1895 * Generate MARK e1 e2 ... TUPLE
1375 */
1896 */
1376 if (pickler_write(self, &mark_op, 1) < 0)
1897 if (_Pickler_Write(self, &mark_op, 1) < 0)
1377 goto error;
1898 return -1;
1378
1899
1379 if (store_tuple_elements(self, obj, len) < 0)
1900 if (store_tuple_elements(self, obj, len) < 0)
1380 goto error;
1901 return -1;
1381
1902
1382 if (PyDict_GetItem(self->memo, memo_key)) {
1903 if (PyMemoTable_Get(self->memo, obj)) {
1383 /* pop the stack stuff we pushed */
1904 /* pop the stack stuff we pushed */
1384 if (self->bin) {
1905 if (self->bin) {
1385 if (pickler_write(self, &pop_mark_op, 1) < 0)
1906 if (_Pickler_Write(self, &pop_mark_op, 1) < 0)
1386 goto error;
1907 return -1;
1387 }
1908 }
1388 else {
1909 else {
1389 /* Note that we pop one more than len, to remove
1910 /* Note that we pop one more than len, to remove
1390 * the MARK too.
1911 * the MARK too.
1391 */
1912 */
1392 for (i = 0; i <= len; i++)
1913 for (i = 0; i <= len; i++)
1393 if (pickler_write(self, &pop_op, 1) < 0)
1914 if (_Pickler_Write(self, &pop_op, 1) < 0)
1394 goto error;
1915 return -1;
1395 }
1916 }
1396 /* fetch from memo */
1917 /* fetch from memo */
1397 if (memo_get(self, memo_key) < 0)
1918 if (memo_get(self, obj) < 0)
1398 goto error;
1919 return -1;
1399
1920
1400 Py_DECREF(memo_key);
1401 return 0;
1921 return 0;
1402 }
1922 }
1403 else { /* Not recursive. */
1923 else { /* Not recursive. */
1404 if (pickler_write(self, &tuple_op, 1) < 0)
1924 if (_Pickler_Write(self, &tuple_op, 1) < 0)
1405 goto error;
1925 return -1;
1406 }
1926 }
1407
1927
1408 memoize:
1928 memoize:
1409 if (memo_put(self, obj) < 0)
1929 if (memo_put(self, obj) < 0)
1410 goto error;
1930 return -1;
1411
1931
1412 if (0) {
1932 return 0;
1413 error:
1414 status = -1;
1415 }
1416
1417 Py_DECREF(memo_key);
1418 return status;
1419 }
1933 }
1420
1934
1421 /* iter is an iterator giving items, and we batch up chunks of
1935 /* iter is an iterator giving items, and we batch up chunks of
1422 * MARK item item ... item APPENDS
1936 * MARK item item ... item APPENDS
1423 * opcode sequences. Calling code should have arranged to first create an
1937 * opcode sequences. Calling code should have arranged to first create an
1424 * empty list, or list-like object, for the APPENDS to operate on.
1938 * empty list, or list-like object, for the APPENDS to operate on.
1425 * Returns 0 on success, <0 on error.
1939 * Returns 0 on success, <0 on error.
1426 */
1940 */
1427 static int
1941 static int
1428 batch_list(PicklerObject *self, PyObject *iter)
1942 batch_list(PicklerObject *self, PyObject *iter)
(...skipping 19 matching lines...) Expand all Loading...
1448 obj = PyIter_Next(iter);
1962 obj = PyIter_Next(iter);
1449 if (obj == NULL) {
1963 if (obj == NULL) {
1450 if (PyErr_Occurred())
1964 if (PyErr_Occurred())
1451 return -1;
1965 return -1;
1452 break;
1966 break;
1453 }
1967 }
1454 i = save(self, obj, 0);
1968 i = save(self, obj, 0);
1455 Py_DECREF(obj);
1969 Py_DECREF(obj);
1456 if (i < 0)
1970 if (i < 0)
1457 return -1;
1971 return -1;
1458 if (pickler_write(self, &append_op, 1) < 0)
1972 if (_Pickler_Write(self, &append_op, 1) < 0)
1459 return -1;
1973 return -1;
1460 }
1974 }
1461 return 0;
1975 return 0;
1462 }
1976 }
1463
1977
1464 /* proto > 0: write in batches of BATCHSIZE. */
1978 /* proto > 0: write in batches of BATCHSIZE. */
1465 do {
1979 do {
1466 /* Get first item */
1980 /* Get first item */
1467 firstitem = PyIter_Next(iter);
1981 firstitem = PyIter_Next(iter);
1468 if (firstitem == NULL) {
1982 if (firstitem == NULL) {
1469 if (PyErr_Occurred())
1983 if (PyErr_Occurred())
1470 goto error;
1984 goto error;
1471
1985
1472 /* nothing more to add */
1986 /* nothing more to add */
1473 break;
1987 break;
1474 }
1988 }
1475
1989
1476 /* Try to get a second item */
1990 /* Try to get a second item */
1477 obj = PyIter_Next(iter);
1991 obj = PyIter_Next(iter);
1478 if (obj == NULL) {
1992 if (obj == NULL) {
1479 if (PyErr_Occurred())
1993 if (PyErr_Occurred())
1480 goto error;
1994 goto error;
1481
1995
1482 /* Only one item to write */
1996 /* Only one item to write */
1483 if (save(self, firstitem, 0) < 0)
1997 if (save(self, firstitem, 0) < 0)
1484 goto error;
1998 goto error;
1485 if (pickler_write(self, &append_op, 1) < 0)
1999 if (_Pickler_Write(self, &append_op, 1) < 0)
1486 goto error;
2000 goto error;
1487 Py_CLEAR(firstitem);
2001 Py_CLEAR(firstitem);
1488 break;
2002 break;
1489 }
2003 }
1490
2004
1491 /* More than one item to write */
2005 /* More than one item to write */
1492
2006
1493 /* Pump out MARK, items, APPENDS. */
2007 /* Pump out MARK, items, APPENDS. */
1494 if (pickler_write(self, &mark_op, 1) < 0)
2008 if (_Pickler_Write(self, &mark_op, 1) < 0)
1495 goto error;
2009 goto error;
1496
2010
1497 if (save(self, firstitem, 0) < 0)
2011 if (save(self, firstitem, 0) < 0)
1498 goto error;
2012 goto error;
1499 Py_CLEAR(firstitem);
2013 Py_CLEAR(firstitem);
1500 n = 1;
2014 n = 1;
1501
2015
1502 /* Fetch and save up to BATCHSIZE items */
2016 /* Fetch and save up to BATCHSIZE items */
1503 while (obj) {
2017 while (obj) {
1504 if (save(self, obj, 0) < 0)
2018 if (save(self, obj, 0) < 0)
1505 goto error;
2019 goto error;
1506 Py_CLEAR(obj);
2020 Py_CLEAR(obj);
1507 n += 1;
2021 n += 1;
1508
2022
1509 if (n == BATCHSIZE)
2023 if (n == BATCHSIZE)
1510 break;
2024 break;
1511
2025
1512 obj = PyIter_Next(iter);
2026 obj = PyIter_Next(iter);
1513 if (obj == NULL) {
2027 if (obj == NULL) {
1514 if (PyErr_Occurred())
2028 if (PyErr_Occurred())
1515 goto error;
2029 goto error;
1516 break;
2030 break;
1517 }
2031 }
1518 }
2032 }
1519
2033
1520 if (pickler_write(self, &appends_op, 1) < 0)
2034 if (_Pickler_Write(self, &appends_op, 1) < 0)
1521 goto error;
2035 goto error;
1522
2036
1523 } while (n == BATCHSIZE);
2037 } while (n == BATCHSIZE);
1524 return 0;
2038 return 0;
1525
2039
1526 error:
2040 error:
1527 Py_XDECREF(firstitem);
2041 Py_XDECREF(firstitem);
1528 Py_XDECREF(obj);
2042 Py_XDECREF(obj);
1529 return -1;
2043 return -1;
1530 }
2044 }
1531
2045
2046 /* This is a variant of batch_list() above, specialized for lists (with no
2047 * support for list subclasses). Like batch_list(), we batch up chunks of
2048 * MARK item item ... item APPENDS
2049 * opcode sequences. Calling code should have arranged to first create an
2050 * empty list, or list-like object, for the APPENDS to operate on.
2051 * Returns 0 on success, -1 on error.
2052 *
2053 * This version is considerably faster than batch_list(), if less general.
2054 *
2055 * Note that this only works for protocols > 0.
2056 */
2057 static int
2058 batch_list_exact(PicklerObject *self, PyObject *obj)
2059 {
2060 PyObject *item = NULL;
2061 int this_batch, total;
2062
2063 const char append_op = APPEND;
2064 const char appends_op = APPENDS;
2065 const char mark_op = MARK;
2066
2067 assert(obj != NULL);
2068 assert(self->proto > 0);
2069 assert(PyList_CheckExact(obj));
2070
2071 if (PyList_GET_SIZE(obj) == 1) {
2072 item = PyList_GET_ITEM(obj, 0);
2073 if (save(self, item, 0) < 0)
2074 return -1;
2075 if (_Pickler_Write(self, &append_op, 1) < 0)
2076 return -1;
2077 return 0;
2078 }
2079
2080 /* Write in batches of BATCHSIZE. */
2081 total = 0;
2082 do {
2083 this_batch = 0;
2084 if (_Pickler_Write(self, &mark_op, 1) < 0)
2085 return -1;
2086 while (total < PyList_GET_SIZE(obj)) {
2087 item = PyList_GET_ITEM(obj, total);
2088 if (save(self, item, 0) < 0)
2089 return -1;
2090 total++;
2091 if (++this_batch == BATCHSIZE)
2092 break;
2093 }
2094 if (_Pickler_Write(self, &appends_op, 1) < 0)
2095 return -1;
2096
2097 } while (total < PyList_GET_SIZE(obj));
2098
2099 return 0;
2100 }
2101
1532 static int
2102 static int
1533 save_list(PicklerObject *self, PyObject *obj)
2103 save_list(PicklerObject *self, PyObject *obj)
1534 {
2104 {
1535 PyObject *iter;
1536 char header[3];
2105 char header[3];
1537 int len;
2106 int len;
1538 int status = 0;
2107 int status = 0;
1539
2108
1540 if (self->fast && !fast_save_enter(self, obj))
2109 if (self->fast && !fast_save_enter(self, obj))
1541 goto error;
2110 goto error;
1542
2111
1543 /* Create an empty list. */
2112 /* Create an empty list. */
1544 if (self->bin) {
2113 if (self->bin) {
1545 header[0] = EMPTY_LIST;
2114 header[0] = EMPTY_LIST;
1546 len = 1;
2115 len = 1;
1547 }
2116 }
1548 else {
2117 else {
1549 header[0] = MARK;
2118 header[0] = MARK;
1550 header[1] = LIST;
2119 header[1] = LIST;
1551 len = 2;
2120 len = 2;
1552 }
2121 }
1553
2122
1554 if (pickler_write(self, header, len) < 0)
2123 if (_Pickler_Write(self, header, len) < 0)
1555 goto error;
2124 goto error;
1556
2125
1557 /* Get list length, and bow out early if empty. */
2126 /* Get list length, and bow out early if empty. */
1558 if ((len = PyList_Size(obj)) < 0)
2127 if ((len = PyList_Size(obj)) < 0)
1559 goto error;
2128 goto error;
1560
2129
1561 if (memo_put(self, obj) < 0)
2130 if (memo_put(self, obj) < 0)
1562 goto error;
2131 goto error;
1563
2132
1564 if (len != 0) {
2133 if (len != 0) {
1565 /* Save the list elements. */
2134 /* Materialize the list elements. */
1566 iter = PyObject_GetIter(obj);
2135 if (PyList_CheckExact(obj) && self->proto > 0) {
1567 if (iter == NULL)
2136 if (Py_EnterRecursiveCall(" while pickling an object") == 0) {
1568 goto error;
2137 status = batch_list_exact(self, obj);
1569 status = batch_list(self, iter);
2138 Py_LeaveRecursiveCall();
1570 Py_DECREF(iter);
2139 }
2140 } else {
2141 PyObject *iter = PyObject_GetIter(obj);
2142 if (iter == NULL)
2143 goto error;
2144
2145 if (Py_EnterRecursiveCall(" while pickling an object") == 0) {
2146 status = batch_list(self, iter);
2147 Py_LeaveRecursiveCall();
2148 }
2149 Py_DECREF(iter);
2150 }
1571 }
2151 }
1572
1573 if (0) {
2152 if (0) {
1574 error:
2153 error:
1575 status = -1;
2154 status = -1;
1576 }
2155 }
1577
2156
1578 if (self->fast && !fast_save_leave(self, obj))
2157 if (self->fast && !fast_save_leave(self, obj))
1579 status = -1;
2158 status = -1;
1580
2159
1581 return status;
2160 return status;
1582 }
2161 }
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
1618 PyErr_SetString(PyExc_TypeError, "dict items "
2197 PyErr_SetString(PyExc_TypeError, "dict items "
1619 "iterator must return 2-tuples");
2198 "iterator must return 2-tuples");
1620 return -1;
2199 return -1;
1621 }
2200 }
1622 i = save(self, PyTuple_GET_ITEM(obj, 0), 0);
2201 i = save(self, PyTuple_GET_ITEM(obj, 0), 0);
1623 if (i >= 0)
2202 if (i >= 0)
1624 i = save(self, PyTuple_GET_ITEM(obj, 1), 0);
2203 i = save(self, PyTuple_GET_ITEM(obj, 1), 0);
1625 Py_DECREF(obj);
2204 Py_DECREF(obj);
1626 if (i < 0)
2205 if (i < 0)
1627 return -1;
2206 return -1;
1628 if (pickler_write(self, &setitem_op, 1) < 0)
2207 if (_Pickler_Write(self, &setitem_op, 1) < 0)
1629 return -1;
2208 return -1;
1630 }
2209 }
1631 return 0;
2210 return 0;
1632 }
2211 }
1633
2212
1634 /* proto > 0: write in batches of BATCHSIZE. */
2213 /* proto > 0: write in batches of BATCHSIZE. */
1635 do {
2214 do {
1636 /* Get first item */
2215 /* Get first item */
1637 firstitem = PyIter_Next(iter);
2216 firstitem = PyIter_Next(iter);
1638 if (firstitem == NULL) {
2217 if (firstitem == NULL) {
(...skipping 13 matching lines...) Expand all Loading...
1652 obj = PyIter_Next(iter);
2231 obj = PyIter_Next(iter);
1653 if (obj == NULL) {
2232 if (obj == NULL) {
1654 if (PyErr_Occurred())
2233 if (PyErr_Occurred())
1655 goto error;
2234 goto error;
1656
2235
1657 /* Only one item to write */
2236 /* Only one item to write */
1658 if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
2237 if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
1659 goto error;
2238 goto error;
1660 if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
2239 if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
1661 goto error;
2240 goto error;
1662 if (pickler_write(self, &setitem_op, 1) < 0)
2241 if (_Pickler_Write(self, &setitem_op, 1) < 0)
1663 goto error;
2242 goto error;
1664 Py_CLEAR(firstitem);
2243 Py_CLEAR(firstitem);
1665 break;
2244 break;
1666 }
2245 }
1667
2246
1668 /* More than one item to write */
2247 /* More than one item to write */
1669
2248
1670 /* Pump out MARK, items, SETITEMS. */
2249 /* Pump out MARK, items, SETITEMS. */
1671 if (pickler_write(self, &mark_op, 1) < 0)
2250 if (_Pickler_Write(self, &mark_op, 1) < 0)
1672 goto error;
2251 goto error;
1673
2252
1674 if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
2253 if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0)
1675 goto error;
2254 goto error;
1676 if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
2255 if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0)
1677 goto error;
2256 goto error;
1678 Py_CLEAR(firstitem);
2257 Py_CLEAR(firstitem);
1679 n = 1;
2258 n = 1;
1680
2259
1681 /* Fetch and save up to BATCHSIZE items */
2260 /* Fetch and save up to BATCHSIZE items */
1682 while (obj) {
2261 while (obj) {
1683 if (!PyTuple_Check(obj) || PyTuple_Size(obj) != 2) {
2262 if (!PyTuple_Check(obj) || PyTuple_Size(obj) != 2) {
1684 PyErr_SetString(PyExc_TypeError, "dict items "
2263 PyErr_SetString(PyExc_TypeError, "dict items "
1685 "iterator must return 2-tuples");
2264 "iterator must return 2-tuples");
1686 goto error;
2265 goto error;
1687 » » » }
2266 }
1688 if (save(self, PyTuple_GET_ITEM(obj, 0), 0) < 0 ||
2267 if (save(self, PyTuple_GET_ITEM(obj, 0), 0) < 0 ||
1689 save(self, PyTuple_GET_ITEM(obj, 1), 0) < 0)
2268 save(self, PyTuple_GET_ITEM(obj, 1), 0) < 0)
1690 goto error;
2269 goto error;
1691 Py_CLEAR(obj);
2270 Py_CLEAR(obj);
1692 n += 1;
2271 n += 1;
1693
2272
1694 if (n == BATCHSIZE)
2273 if (n == BATCHSIZE)
1695 break;
2274 break;
1696
2275
1697 obj = PyIter_Next(iter);
2276 obj = PyIter_Next(iter);
1698 if (obj == NULL) {
2277 if (obj == NULL) {
1699 if (PyErr_Occurred())
2278 if (PyErr_Occurred())
1700 goto error;
2279 goto error;
1701 break;
2280 break;
1702 }
2281 }
1703 }
2282 }
1704
2283
1705 if (pickler_write(self, &setitems_op, 1) < 0)
2284 if (_Pickler_Write(self, &setitems_op, 1) < 0)
1706 goto error;
2285 goto error;
1707
2286
1708 } while (n == BATCHSIZE);
2287 } while (n == BATCHSIZE);
1709 return 0;
2288 return 0;
1710
2289
1711 error:
2290 error:
1712 Py_XDECREF(firstitem);
2291 Py_XDECREF(firstitem);
1713 Py_XDECREF(obj);
2292 Py_XDECREF(obj);
1714 return -1;
2293 return -1;
1715 }
2294 }
(...skipping 23 matching lines...) Expand all Loading...
1739
2318
1740 dict_size = PyDict_Size(obj);
2319 dict_size = PyDict_Size(obj);
1741
2320
1742 /* Special-case len(d) == 1 to save space. */
2321 /* Special-case len(d) == 1 to save space. */
1743 if (dict_size == 1) {
2322 if (dict_size == 1) {
1744 PyDict_Next(obj, &ppos, &key, &value);
2323 PyDict_Next(obj, &ppos, &key, &value);
1745 if (save(self, key, 0) < 0)
2324 if (save(self, key, 0) < 0)
1746 return -1;
2325 return -1;
1747 if (save(self, value, 0) < 0)
2326 if (save(self, value, 0) < 0)
1748 return -1;
2327 return -1;
1749 if (pickler_write(self, &setitem_op, 1) < 0)
2328 if (_Pickler_Write(self, &setitem_op, 1) < 0)
1750 return -1;
2329 return -1;
1751 return 0;
2330 return 0;
1752 }
2331 }
1753
2332
1754 /* Write in batches of BATCHSIZE. */
2333 /* Write in batches of BATCHSIZE. */
1755 do {
2334 do {
1756 i = 0;
2335 i = 0;
1757 if (pickler_write(self, &mark_op, 1) < 0)
2336 if (_Pickler_Write(self, &mark_op, 1) < 0)
1758 return -1;
2337 return -1;
1759 while (PyDict_Next(obj, &ppos, &key, &value)) {
2338 while (PyDict_Next(obj, &ppos, &key, &value)) {
1760 if (save(self, key, 0) < 0)
2339 if (save(self, key, 0) < 0)
1761 return -1;
2340 return -1;
1762 if (save(self, value, 0) < 0)
2341 if (save(self, value, 0) < 0)
1763 return -1;
2342 return -1;
1764 if (++i == BATCHSIZE)
2343 if (++i == BATCHSIZE)
1765 break;
2344 break;
1766 }
2345 }
1767 if (pickler_write(self, &setitems_op, 1) < 0)
2346 if (_Pickler_Write(self, &setitems_op, 1) < 0)
1768 return -1;
2347 return -1;
1769 if (PyDict_Size(obj) != dict_size) {
2348 if (PyDict_Size(obj) != dict_size) {
1770 PyErr_Format(
2349 PyErr_Format(
1771 PyExc_RuntimeError,
2350 PyExc_RuntimeError,
1772 "dictionary changed size during iteration");
2351 "dictionary changed size during iteration");
1773 return -1;
2352 return -1;
1774 }
2353 }
1775
2354
1776 } while (i == BATCHSIZE);
2355 } while (i == BATCHSIZE);
1777 return 0;
2356 return 0;
(...skipping 14 matching lines...) Expand all Loading...
1792 if (self->bin) {
2371 if (self->bin) {
1793 header[0] = EMPTY_DICT;
2372 header[0] = EMPTY_DICT;
1794 len = 1;
2373 len = 1;
1795 }
2374 }
1796 else {
2375 else {
1797 header[0] = MARK;
2376 header[0] = MARK;
1798 header[1] = DICT;
2377 header[1] = DICT;
1799 len = 2;
2378 len = 2;
1800 }
2379 }
1801
2380
1802 if (pickler_write(self, header, len) < 0)
2381 if (_Pickler_Write(self, header, len) < 0)
1803 goto error;
2382 goto error;
1804
2383
1805 /* Get dict size, and bow out early if empty. */
2384 /* Get dict size, and bow out early if empty. */
1806 if ((len = PyDict_Size(obj)) < 0)
2385 if ((len = PyDict_Size(obj)) < 0)
1807 goto error;
2386 goto error;
1808
2387
1809 if (memo_put(self, obj) < 0)
2388 if (memo_put(self, obj) < 0)
1810 goto error;
2389 goto error;
1811
2390
1812 if (len != 0) {
2391 if (len != 0) {
(...skipping 142 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
1955 }
2534 }
1956 else {
2535 else {
1957 pdata[0] = EXT4;
2536 pdata[0] = EXT4;
1958 pdata[1] = (unsigned char)(code & 0xff);
2537 pdata[1] = (unsigned char)(code & 0xff);
1959 pdata[2] = (unsigned char)((code >> 8) & 0xff);
2538 pdata[2] = (unsigned char)((code >> 8) & 0xff);
1960 pdata[3] = (unsigned char)((code >> 16) & 0xff);
2539 pdata[3] = (unsigned char)((code >> 16) & 0xff);
1961 pdata[4] = (unsigned char)((code >> 24) & 0xff);
2540 pdata[4] = (unsigned char)((code >> 24) & 0xff);
1962 n = 5;
2541 n = 5;
1963 }
2542 }
1964
2543
1965 if (pickler_write(self, pdata, n) < 0)
2544 if (_Pickler_Write(self, pdata, n) < 0)
1966 goto error;
2545 goto error;
1967 }
2546 }
1968 else {
2547 else {
1969 /* Generate a normal global opcode if we are using a pickle
2548 /* Generate a normal global opcode if we are using a pickle
1970 protocol <= 2, or if the object is not registered in the
2549 protocol <= 2, or if the object is not registered in the
1971 extension registry. */
2550 extension registry. */
1972 PyObject *encoded;
2551 PyObject *encoded;
1973 PyObject *(*unicode_encoder)(PyObject *);
2552 PyObject *(*unicode_encoder)(PyObject *);
1974
2553
1975 gen_global:
2554 gen_global:
1976 if (pickler_write(self, &global_op, 1) < 0)
2555 if (_Pickler_Write(self, &global_op, 1) < 0)
1977 goto error;
2556 goto error;
1978
2557
1979 /* Since Python 3.0 now supports non-ASCII identifiers, we encode both
2558 /* Since Python 3.0 now supports non-ASCII identifiers, we encode both
1980 the module name and the global name using UTF-8. We do so only when
2559 the module name and the global name using UTF-8. We do so only when
1981 we are using the pickle protocol newer than version 3. This is to
2560 we are using the pickle protocol newer than version 3. This is to
1982 ensure compatibility with older Unpickler running on Python 2.x. */
2561 ensure compatibility with older Unpickler running on Python 2.x. */
1983 if (self->proto >= 3) {
2562 if (self->proto >= 3) {
1984 unicode_encoder = PyUnicode_AsUTF8String;
2563 unicode_encoder = PyUnicode_AsUTF8String;
1985 }
2564 }
1986 else {
2565 else {
(...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
2046
2625
2047 /* Save the name of the module. */
2626 /* Save the name of the module. */
2048 encoded = unicode_encoder(module_name);
2627 encoded = unicode_encoder(module_name);
2049 if (encoded == NULL) {
2628 if (encoded == NULL) {
2050 if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError))
2629 if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError))
2051 PyErr_Format(PicklingError,
2630 PyErr_Format(PicklingError,
2052 "can't pickle module identifier '%S' using "
2631 "can't pickle module identifier '%S' using "
2053 "pickle protocol %i", module_name, self->proto);
2632 "pickle protocol %i", module_name, self->proto);
2054 goto error;
2633 goto error;
2055 }
2634 }
2056 if (pickler_write(self, PyBytes_AS_STRING(encoded),
2635 if (_Pickler_Write(self, PyBytes_AS_STRING(encoded),
2057 PyBytes_GET_SIZE(encoded)) < 0) {
2636 PyBytes_GET_SIZE(encoded)) < 0) {
2058 Py_DECREF(encoded);
2637 Py_DECREF(encoded);
2059 goto error;
2638 goto error;
2060 }
2639 }
2061 Py_DECREF(encoded);
2640 Py_DECREF(encoded);
2062 if(pickler_write(self, "\n", 1) < 0)
2641 if(_Pickler_Write(self, "\n", 1) < 0)
2063 goto error;
2642 goto error;
2064
2643
2065 /* Save the name of the module. */
2644 /* Save the name of the module. */
2066 encoded = unicode_encoder(global_name);
2645 encoded = unicode_encoder(global_name);
2067 if (encoded == NULL) {
2646 if (encoded == NULL) {
2068 if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError))
2647 if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError))
2069 PyErr_Format(PicklingError,
2648 PyErr_Format(PicklingError,
2070 "can't pickle global identifier '%S' using "
2649 "can't pickle global identifier '%S' using "
2071 "pickle protocol %i", global_name, self->proto);
2650 "pickle protocol %i", global_name, self->proto);
2072 goto error;
2651 goto error;
2073 }
2652 }
2074 if (pickler_write(self, PyBytes_AS_STRING(encoded),
2653 if (_Pickler_Write(self, PyBytes_AS_STRING(encoded),
2075 PyBytes_GET_SIZE(encoded)) < 0) {
2654 PyBytes_GET_SIZE(encoded)) < 0) {
2076 Py_DECREF(encoded);
2655 Py_DECREF(encoded);
2077 goto error;
2656 goto error;
2078 }
2657 }
2079 Py_DECREF(encoded);
2658 Py_DECREF(encoded);
2080 if(pickler_write(self, "\n", 1) < 0)
2659 if(_Pickler_Write(self, "\n", 1) < 0)
2081 goto error;
2660 goto error;
2082
2661
2083 /* Memoize the object. */
2662 /* Memoize the object. */
2084 if (memo_put(self, obj) < 0)
2663 if (memo_put(self, obj) < 0)
2085 goto error;
2664 goto error;
2086 }
2665 }
2087
2666
2088 if (0) {
2667 if (0) {
2089 error:
2668 error:
2090 status = -1;
2669 status = -1;
2091 }
2670 }
2092 Py_XDECREF(module_name);
2671 Py_XDECREF(module_name);
2093 Py_XDECREF(global_name);
2672 Py_XDECREF(global_name);
2094 Py_XDECREF(module);
2673 Py_XDECREF(module);
2095
2674
2096 return status;
2675 return status;
2097 }
2676 }
2098
2677
2099 static int
2678 static int
2100 save_pers(PicklerObject *self, PyObject *obj, PyObject *func)
2679 save_pers(PicklerObject *self, PyObject *obj, PyObject *func)
2101 {
2680 {
2102 PyObject *pid = NULL;
2681 PyObject *pid = NULL;
2103 int status = 0;
2682 int status = 0;
2104
2683
2105 const char persid_op = PERSID;
2684 const char persid_op = PERSID;
2106 const char binpersid_op = BINPERSID;
2685 const char binpersid_op = BINPERSID;
2107
2686
2108 Py_INCREF(obj);
2687 Py_INCREF(obj);
2109 pid = pickler_call(self, func, obj);
2688 pid = _Pickler_FastCall(self, func, obj);
2110 if (pid == NULL)
2689 if (pid == NULL)
2111 return -1;
2690 return -1;
2112
2691
2113 if (pid != Py_None) {
2692 if (pid != Py_None) {
2114 if (self->bin) {
2693 if (self->bin) {
2115 if (save(self, pid, 1) < 0 ||
2694 if (save(self, pid, 1) < 0 ||
2116 pickler_write(self, &binpersid_op, 1) < 0)
2695 _Pickler_Write(self, &binpersid_op, 1) < 0)
2117 goto error;
2696 goto error;
2118 }
2697 }
2119 else {
2698 else {
2120 PyObject *pid_str = NULL;
2699 PyObject *pid_str = NULL;
2121 char *pid_ascii_bytes;
2700 char *pid_ascii_bytes;
2122 Py_ssize_t size;
2701 Py_ssize_t size;
2123
2702
2124 pid_str = PyObject_Str(pid);
2703 pid_str = PyObject_Str(pid);
2125 if (pid_str == NULL)
2704 if (pid_str == NULL)
2126 goto error;
2705 goto error;
2127
2706
2128 /* XXX: Should it check whether the persistent id only contains
2707 /* XXX: Should it check whether the persistent id only contains
2129 ASCII characters? And what if the pid contains embedded
2708 ASCII characters? And what if the pid contains embedded
2130 newlines? */
2709 newlines? */
2131 pid_ascii_bytes = _PyUnicode_AsStringAndSize(pid_str, &size);
2710 pid_ascii_bytes = _PyUnicode_AsStringAndSize(pid_str, &size);
2132 Py_DECREF(pid_str);
2711 Py_DECREF(pid_str);
2133 if (pid_ascii_bytes == NULL)
2712 if (pid_ascii_bytes == NULL)
2134 goto error;
2713 goto error;
2135
2714
2136 if (pickler_write(self, &persid_op, 1) < 0 ||
2715 if (_Pickler_Write(self, &persid_op, 1) < 0 ||
2137 pickler_write(self, pid_ascii_bytes, size) < 0 ||
2716 _Pickler_Write(self, pid_ascii_bytes, size) < 0 ||
2138 pickler_write(self, "\n", 1) < 0)
2717 _Pickler_Write(self, "\n", 1) < 0)
2139 goto error;
2718 goto error;
2140 }
2719 }
2141 status = 1;
2720 status = 1;
2142 }
2721 }
2143
2722
2144 if (0) {
2723 if (0) {
2145 error:
2724 error:
2146 status = -1;
2725 status = -1;
2147 }
2726 }
2148 Py_XDECREF(pid);
2727 Py_XDECREF(pid);
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
2213 }
2792 }
2214
2793
2215 /* Protocol 2 special case: if callable's name is __newobj__, use
2794 /* Protocol 2 special case: if callable's name is __newobj__, use
2216 NEWOBJ. */
2795 NEWOBJ. */
2217 if (use_newobj) {
2796 if (use_newobj) {
2218 static PyObject *newobj_str = NULL;
2797 static PyObject *newobj_str = NULL;
2219 PyObject *name_str;
2798 PyObject *name_str;
2220
2799
2221 if (newobj_str == NULL) {
2800 if (newobj_str == NULL) {
2222 newobj_str = PyUnicode_InternFromString("__newobj__");
2801 newobj_str = PyUnicode_InternFromString("__newobj__");
2802 if (newobj_str == NULL)
2803 return -1;
2223 }
2804 }
2224
2805
2225 name_str = PyObject_GetAttrString(callable, "__name__");
2806 name_str = PyObject_GetAttrString(callable, "__name__");
2226 if (name_str == NULL) {
2807 if (name_str == NULL) {
2227 if (PyErr_ExceptionMatches(PyExc_AttributeError))
2808 if (PyErr_ExceptionMatches(PyExc_AttributeError))
2228 PyErr_Clear();
2809 PyErr_Clear();
2229 else
2810 else
2230 return -1;
2811 return -1;
2231 use_newobj = 0;
2812 use_newobj = 0;
2232 }
2813 }
(...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
2305 newargtup = PyTuple_GetSlice(argtup, 1, Py_SIZE(argtup));
2886 newargtup = PyTuple_GetSlice(argtup, 1, Py_SIZE(argtup));
2306 if (newargtup == NULL)
2887 if (newargtup == NULL)
2307 return -1;
2888 return -1;
2308
2889
2309 p = save(self, newargtup, 0);
2890 p = save(self, newargtup, 0);
2310 Py_DECREF(newargtup);
2891 Py_DECREF(newargtup);
2311 if (p < 0)
2892 if (p < 0)
2312 return -1;
2893 return -1;
2313
2894
2314 /* Add NEWOBJ opcode. */
2895 /* Add NEWOBJ opcode. */
2315 if (pickler_write(self, &newobj_op, 1) < 0)
2896 if (_Pickler_Write(self, &newobj_op, 1) < 0)
2316 return -1;
2897 return -1;
2317 }
2898 }
2318 else { /* Not using NEWOBJ. */
2899 else { /* Not using NEWOBJ. */
2319 if (save(self, callable, 0) < 0 ||
2900 if (save(self, callable, 0) < 0 ||
2320 save(self, argtup, 0) < 0 ||
2901 save(self, argtup, 0) < 0 ||
2321 pickler_write(self, &reduce_op, 1) < 0)
2902 _Pickler_Write(self, &reduce_op, 1) < 0)
2322 return -1;
2903 return -1;
2323 }
2904 }
2324
2905
2325 /* obj can be NULL when save_reduce() is used directly. A NULL obj means
2906 /* obj can be NULL when save_reduce() is used directly. A NULL obj means
2326 the caller do not want to memoize the object. Not particularly useful,
2907 the caller do not want to memoize the object. Not particularly useful,
2327 but that is to mimic the behavior save_reduce() in pickle.py when
2908 but that is to mimic the behavior save_reduce() in pickle.py when
2328 obj is None. */
2909 obj is None. */
2329 if (obj && memo_put(self, obj) < 0)
2910 if (obj && memo_put(self, obj) < 0)
2330 return -1;
2911 return -1;
2331
2912
2332 if (listitems && batch_list(self, listitems) < 0)
2913 if (listitems && batch_list(self, listitems) < 0)
2333 return -1;
2914 return -1;
2334
2915
2335 if (dictitems && batch_dict(self, dictitems) < 0)
2916 if (dictitems && batch_dict(self, dictitems) < 0)
2336 return -1;
2917 return -1;
2337
2918
2338 if (state) {
2919 if (state) {
2339 if (save(self, state, 0) < 0 ||·
2920 if (save(self, state, 0) < 0 ||·
2340 pickler_write(self, &build_op, 1) < 0)
2921 _Pickler_Write(self, &build_op, 1) < 0)
2341 return -1;
2922 return -1;
2342 }
2923 }
2343
2924
2344 return 0;
2925 return 0;
2345 }
2926 }
2346
2927
2347 static int
2928 static int
2348 save(PicklerObject *self, PyObject *obj, int pers_save)
2929 save(PicklerObject *self, PyObject *obj, int pers_save)
2349 {
2930 {
2350 PyTypeObject *type;
2931 PyTypeObject *type;
2351 PyObject *reduce_func = NULL;
2932 PyObject *reduce_func = NULL;
2352 PyObject *reduce_value = NULL;
2933 PyObject *reduce_value = NULL;
2353 PyObject *memo_key = NULL;
2354 int status = 0;
2934 int status = 0;
2355
2935
2356 if (Py_EnterRecursiveCall(" while pickling an object") < 0)
2936 if (Py_EnterRecursiveCall(" while pickling an object") < 0)
2357 return -1;
2937 return -1;
2358
2938
2359 /* The extra pers_save argument is necessary to avoid calling save_pers()
2939 /* The extra pers_save argument is necessary to avoid calling save_pers()
2360 on its returned object. */
2940 on its returned object. */
2361 if (!pers_save && self->pers_func) {
2941 if (!pers_save && self->pers_func) {
2362 /* save_pers() returns:
2942 /* save_pers() returns:
2363 -1 to signal an error;
2943 -1 to signal an error;
2364 0 if it did nothing successfully;
2944 0 if it did nothing successfully;
2365 1 if a persistent id was saved.
2945 1 if a persistent id was saved.
2366 */
2946 */
2367 if ((status = save_pers(self, obj, self->pers_func)) != 0)
2947 if ((status = save_pers(self, obj, self->pers_func)) != 0)
2368 goto done;
2948 goto done;
2369 }
2949 }
2370
2950
2371 type = Py_TYPE(obj);
2951 type = Py_TYPE(obj);
2372
2952
2373 /* XXX: The old cPickle had an optimization that used switch-case
2953 /* The old cPickle had an optimization that used switch-case statement
2374 statement dispatching on the first letter of the type name. It was
2954 dispatching on the first letter of the type name. This has was removed
2375 probably not a bad idea after all. If benchmarks shows that particular
2955 since benchmarks shown that this optimization was actually slowing
2376 optimization had some real benefits, it would be nice to add it
2956 things down. */
2377 back. */
2378
2957
2379 /* Atom types; these aren't memoized, so don't check the memo. */
2958 /* Atom types; these aren't memoized, so don't check the memo. */
2380
2959
2381 if (obj == Py_None) {
2960 if (obj == Py_None) {
2382 status = save_none(self, obj);
2961 status = save_none(self, obj);
2383 goto done;
2962 goto done;
2384 }
2963 }
2385 else if (obj == Py_False || obj == Py_True) {
2964 else if (obj == Py_False || obj == Py_True) {
2386 status = save_bool(self, obj);
2965 status = save_bool(self, obj);
2387 goto done;
2966 goto done;
2388 }
2967 }
2389 else if (type == &PyLong_Type) {
2968 else if (type == &PyLong_Type) {
2390 status = save_long(self, obj);
2969 status = save_long(self, obj);
2391 goto done;
2970 goto done;
2392 }
2971 }
2393 else if (type == &PyFloat_Type) {
2972 else if (type == &PyFloat_Type) {
2394 status = save_float(self, obj);
2973 status = save_float(self, obj);
2395 goto done;
2974 goto done;
2396 }
2975 }
2397
2976
2398 /* Check the memo to see if it has the object. If so, generate
2977 /* Check the memo to see if it has the object. If so, generate
2399 a GET (or BINGET) opcode, instead of pickling the object
2978 a GET (or BINGET) opcode, instead of pickling the object
2400 once again. */
2979 once again. */
2401 memo_key = PyLong_FromVoidPtr(obj);
2980 if (PyMemoTable_Get(self->memo, obj)) {
2402 if (memo_key == NULL)
2981 if (memo_get(self, obj) < 0)
2403 goto error;
2404 if (PyDict_GetItem(self->memo, memo_key)) {
2405 if (memo_get(self, memo_key) < 0)
2406 goto error;
2982 goto error;
2407 goto done;
2983 goto done;
2408 }
2984 }
2409
2985
2410 if (type == &PyBytes_Type) {
2986 if (type == &PyBytes_Type) {
2411 status = save_bytes(self, obj);
2987 status = save_bytes(self, obj);
2412 goto done;
2988 goto done;
2413 }
2989 }
2414 else if (type == &PyUnicode_Type) {
2990 else if (type == &PyUnicode_Type) {
2415 status = save_unicode(self, obj);
2991 status = save_unicode(self, obj);
(...skipping 42 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
2458 */
3034 */
2459 reduce_func = PyDict_GetItem(dispatch_table, (PyObject *)type);
3035 reduce_func = PyDict_GetItem(dispatch_table, (PyObject *)type);
2460 if (reduce_func != NULL) {
3036 if (reduce_func != NULL) {
2461 /* Here, the reference count of the reduce_func object returned by
3037 /* Here, the reference count of the reduce_func object returned by
2462 PyDict_GetItem needs to be increased to be consistent with the one
3038 PyDict_GetItem needs to be increased to be consistent with the one
2463 returned by PyObject_GetAttr. This is allow us to blindly DECREF
3039 returned by PyObject_GetAttr. This is allow us to blindly DECREF
2464 reduce_func at the end of the save() routine.
3040 reduce_func at the end of the save() routine.
2465 */
3041 */
2466 Py_INCREF(reduce_func);
3042 Py_INCREF(reduce_func);
2467 Py_INCREF(obj);
3043 Py_INCREF(obj);
2468 reduce_value = pickler_call(self, reduce_func, obj);
3044 reduce_value = _Pickler_FastCall(self, reduce_func, obj);
2469 }
3045 }
2470 else {
3046 else {
2471 static PyObject *reduce_str = NULL;
3047 static PyObject *reduce_str = NULL;
2472 static PyObject *reduce_ex_str = NULL;
3048 static PyObject *reduce_ex_str = NULL;
2473
3049
2474 /* Cache the name of the reduce methods. */
3050 /* Cache the name of the reduce methods. */
2475 if (reduce_str == NULL) {
3051 if (reduce_str == NULL) {
2476 reduce_str = PyUnicode_InternFromString("__reduce__");
3052 reduce_str = PyUnicode_InternFromString("__reduce__");
2477 if (reduce_str == NULL)
3053 if (reduce_str == NULL)
2478 goto error;
3054 goto error;
2479 reduce_ex_str = PyUnicode_InternFromString("__reduce_ex__");
3055 reduce_ex_str = PyUnicode_InternFromString("__reduce_ex__");
2480 if (reduce_ex_str == NULL)
3056 if (reduce_ex_str == NULL)
2481 goto error;
3057 goto error;
2482 }
3058 }
2483
3059
2484 /* XXX: If the __reduce__ method is defined, __reduce_ex__ is
3060 /* XXX: If the __reduce__ method is defined, __reduce_ex__ is
2485 automatically defined as __reduce__. While this is convenient, this
3061 automatically defined as __reduce__. While this is convenient, this
2486 make it impossible to know which method was actually called. Of
3062 make it impossible to know which method was actually called. Of
2487 course, this is not a big deal. But still, it would be nice to let
3063 course, this is not a big deal. But still, it would be nice to let
2488 the user know which method was called when something go
3064 the user know which method was called when something go
2489 wrong. Incidentally, this means if __reduce_ex__ is not defined, we
3065 wrong. Incidentally, this means if __reduce_ex__ is not defined, we
2490 don't actually have to check for a __reduce__ method. */
3066 don't actually have to check for a __reduce__ method. */
2491
3067
2492 /* Check for a __reduce_ex__ method. */
3068 /* Check for a __reduce_ex__ method. */
2493 reduce_func = PyObject_GetAttr(obj, reduce_ex_str);
3069 reduce_func = PyObject_GetAttr(obj, reduce_ex_str);
2494 if (reduce_func != NULL) {
3070 if (reduce_func != NULL) {
2495 PyObject *proto;
3071 PyObject *proto;
2496 proto = PyLong_FromLong(self->proto);
3072 proto = PyLong_FromLong(self->proto);
2497 if (proto != NULL) {
3073 if (proto != NULL) {
2498 reduce_value = pickler_call(self, reduce_func, proto);
3074 reduce_value = _Pickler_FastCall(self, reduce_func, proto);
2499 }
3075 }
2500 }
3076 }
2501 else {
3077 else {
2502 if (PyErr_ExceptionMatches(PyExc_AttributeError))
3078 if (PyErr_ExceptionMatches(PyExc_AttributeError))
2503 PyErr_Clear();
3079 PyErr_Clear();
2504 else
3080 else
2505 goto error;
3081 goto error;
2506 /* Check for a __reduce__ method. */
3082 /* Check for a __reduce__ method. */
2507 reduce_func = PyObject_GetAttr(obj, reduce_str);
3083 reduce_func = PyObject_GetAttr(obj, reduce_str);
2508 if (reduce_func != NULL) {
3084 if (reduce_func != NULL) {
(...skipping 22 matching lines...) Expand all Loading...
2531 }
3107 }
2532
3108
2533 status = save_reduce(self, reduce_value, obj);
3109 status = save_reduce(self, reduce_value, obj);
2534
3110
2535 if (0) {
3111 if (0) {
2536 error:
3112 error:
2537 status = -1;
3113 status = -1;
2538 }
3114 }
2539 done:
3115 done:
2540 Py_LeaveRecursiveCall();
3116 Py_LeaveRecursiveCall();
2541 Py_XDECREF(memo_key);
2542 Py_XDECREF(reduce_func);
3117 Py_XDECREF(reduce_func);
2543 Py_XDECREF(reduce_value);
3118 Py_XDECREF(reduce_value);
2544
3119
2545 return status;
3120 return status;
2546 }
3121 }
2547
3122
2548 static int
3123 static int
2549 dump(PicklerObject *self, PyObject *obj)
3124 dump(PicklerObject *self, PyObject *obj)
2550 {
3125 {
2551 const char stop_op = STOP;
3126 const char stop_op = STOP;
2552
3127
2553 if (self->proto >= 2) {
3128 if (self->proto >= 2) {
2554 char header[2];
3129 char header[2];
2555
3130
2556 header[0] = PROTO;
3131 header[0] = PROTO;
2557 assert(self->proto >= 0 && self->proto < 256);
3132 assert(self->proto >= 0 && self->proto < 256);
2558 header[1] = (unsigned char)self->proto;
3133 header[1] = (unsigned char)self->proto;
2559 if (pickler_write(self, header, 2) < 0)
3134 if (_Pickler_Write(self, header, 2) < 0)
2560 return -1;
3135 return -1;
2561 }
3136 }
2562
3137
2563 if (save(self, obj, 0) < 0 ||
3138 if (save(self, obj, 0) < 0 ||
2564 pickler_write(self, &stop_op, 1) < 0 ||
3139 _Pickler_Write(self, &stop_op, 1) < 0)
2565 pickler_write(self, NULL, 0) < 0)
2566 return -1;
3140 return -1;
2567
3141
2568 return 0;
3142 return 0;
2569 }
3143 }
2570
3144
2571 PyDoc_STRVAR(Pickler_clear_memo_doc,
3145 PyDoc_STRVAR(Pickler_clear_memo_doc,
2572 "clear_memo() -> None. Clears the pickler's \"memo\"."
3146 "clear_memo() -> None. Clears the pickler's \"memo\"."
2573 "\n"
3147 "\n"
2574 "The memo is the data structure that remembers which objects the\n"
3148 "The memo is the data structure that remembers which objects the\n"
2575 "pickler has already seen, so that shared or recursive objects are\n"
3149 "pickler has already seen, so that shared or recursive objects are\n"
2576 "pickled by reference and not by value. This method is useful when\n"
3150 "pickled by reference and not by value. This method is useful when\n"
2577 "re-using picklers.");
3151 "re-using picklers.");
2578
3152
2579 static PyObject *
3153 static PyObject *
2580 Pickler_clear_memo(PicklerObject *self)
3154 Pickler_clear_memo(PicklerObject *self)
2581 {
3155 {
2582 if (self->memo)
3156 if (self->memo)
2583 PyDict_Clear(self->memo);
3157 PyMemoTable_Clear(self->memo);
2584
3158
2585 Py_RETURN_NONE;
3159 Py_RETURN_NONE;
2586 }
3160 }
2587
3161
2588 PyDoc_STRVAR(Pickler_dump_doc,
3162 PyDoc_STRVAR(Pickler_dump_doc,
2589 "dump(obj) -> None. Write a pickled representation of obj to the open file.");
3163 "dump(obj) -> None. Write a pickled representation of obj to the open file.");
2590
3164
2591 static PyObject *
3165 static PyObject *
2592 Pickler_dump(PicklerObject *self, PyObject *args)
3166 Pickler_dump(PicklerObject *self, PyObject *args)
2593 {
3167 {
2594 PyObject *obj;
3168 PyObject *obj;
2595
3169
2596 /* Check whether the Pickler was initialized correctly (issue3664).
3170 /* Check whether the Pickler was initialized correctly (issue3664).
2597 Developers often forget to call __init__() in their subclasses, which
3171 Developers often forget to call __init__() in their subclasses, which
2598 would trigger a segfault without this check. */
3172 would trigger a segfault without this check. */
2599 if (self->write == NULL) {
3173 if (self->write == NULL) {
2600 PyErr_Format(PicklingError,·
3174 PyErr_Format(PicklingError,·
2601 "Pickler.__init__() was not called by %s.__init__()",
3175 "Pickler.__init__() was not called by %s.__init__()",
2602 Py_TYPE(self)->tp_name);
3176 Py_TYPE(self)->tp_name);
2603 return NULL;
3177 return NULL;
2604 }
3178 }
2605
3179
2606 if (!PyArg_ParseTuple(args, "O:dump", &obj))
3180 if (!PyArg_ParseTuple(args, "O:dump", &obj))
2607 return NULL;
3181 return NULL;
2608
3182
3183 if (_Pickler_ClearBuffer(self) < 0)
3184 return NULL;
3185
2609 if (dump(self, obj) < 0)
3186 if (dump(self, obj) < 0)
2610 return NULL;
3187 return NULL;
2611
3188
3189 if (_Pickler_FlushToFile(self) < 0)
3190 return NULL;
3191
2612 Py_RETURN_NONE;
3192 Py_RETURN_NONE;
2613 }
3193 }
2614
3194
2615 static struct PyMethodDef Pickler_methods[] = {
3195 static struct PyMethodDef Pickler_methods[] = {
2616 {"dump", (PyCFunction)Pickler_dump, METH_VARARGS,
3196 {"dump", (PyCFunction)Pickler_dump, METH_VARARGS,
2617 Pickler_dump_doc},
3197 Pickler_dump_doc},
2618 {"clear_memo", (PyCFunction)Pickler_clear_memo, METH_NOARGS,
3198 {"clear_memo", (PyCFunction)Pickler_clear_memo, METH_NOARGS,
2619 Pickler_clear_memo_doc},
3199 Pickler_clear_memo_doc},
2620 {NULL, NULL} /* sentinel */
3200 {NULL, NULL} /* sentinel */
2621 };
3201 };
2622
3202
2623 static void
3203 static void
2624 Pickler_dealloc(PicklerObject *self)
3204 Pickler_dealloc(PicklerObject *self)
2625 {
3205 {
2626 PyObject_GC_UnTrack(self);
3206 PyObject_GC_UnTrack(self);
2627
3207
2628 Py_XDECREF(self->write);
3208 Py_XDECREF(self->write);
2629 Py_XDECREF(self->memo);
2630 Py_XDECREF(self->pers_func);
3209 Py_XDECREF(self->pers_func);
2631 Py_XDECREF(self->arg);
3210 Py_XDECREF(self->arg);
2632 Py_XDECREF(self->fast_memo);
3211 Py_XDECREF(self->fast_memo);
2633
3212
2634 PyMem_Free(self->write_buf);
3213 PyMemoTable_Del(self->memo);
3214 PyMem_FREE(self->output_buffer);
2635
3215
2636 Py_TYPE(self)->tp_free((PyObject *)self);
3216 Py_TYPE(self)->tp_free((PyObject *)self);
2637 }
3217 }
2638
3218
2639 static int
3219 static int
2640 Pickler_traverse(PicklerObject *self, visitproc visit, void *arg)
3220 Pickler_traverse(PicklerObject *self, visitproc visit, void *arg)
2641 {
3221 {
2642 Py_VISIT(self->write);
3222 Py_VISIT(self->write);
2643 Py_VISIT(self->memo);
2644 Py_VISIT(self->pers_func);
3223 Py_VISIT(self->pers_func);
2645 Py_VISIT(self->arg);
3224 Py_VISIT(self->arg);
2646 Py_VISIT(self->fast_memo);
3225 Py_VISIT(self->fast_memo);
2647 return 0;
3226 return 0;
2648 }
3227 }
2649
3228
2650 static int
3229 static int
2651 Pickler_clear(PicklerObject *self)
3230 Pickler_clear(PicklerObject *self)
2652 {
3231 {
2653 Py_CLEAR(self->write);
3232 Py_CLEAR(self->write);
2654 Py_CLEAR(self->memo);
2655 Py_CLEAR(self->pers_func);
3233 Py_CLEAR(self->pers_func);
2656 Py_CLEAR(self->arg);
3234 Py_CLEAR(self->arg);
2657 Py_CLEAR(self->fast_memo);
3235 Py_CLEAR(self->fast_memo);
2658
3236
2659 PyMem_Free(self->write_buf);
3237 PyMemoTable_Del(self->memo);
2660 self->write_buf = NULL;
3238 self->memo = NULL;
2661
3239 PyMem_FREE(self->output_buffer);
3240 self->output_buffer = NULL;
2662 return 0;
3241 return 0;
2663 }
3242 }
2664
3243
3244
2665 PyDoc_STRVAR(Pickler_doc,
3245 PyDoc_STRVAR(Pickler_doc,
2666 "Pickler(file, protocol=None)"
3246 "Pickler(file, protocol=None)"
2667 "\n"
3247 "\n"
2668 "This takes a binary file for writing a pickle data stream.\n"
3248 "This takes a binary file for writing a pickle data stream.\n"
2669 "\n"
3249 "\n"
2670 "The optional protocol argument tells the pickler to use the\n"
3250 "The optional protocol argument tells the pickler to use the\n"
2671 "given protocol; supported protocols are 0, 1, 2, 3. The default\n"
3251 "given protocol; supported protocols are 0, 1, 2, 3. The default\n"
2672 "protocol is 3; a backward-incompatible protocol designed for\n"
3252 "protocol is 3; a backward-incompatible protocol designed for\n"
2673 "Python 3.0.\n"
3253 "Python 3.0.\n"
2674 "\n"
3254 "\n"
(...skipping 10 matching lines...) Expand all Loading...
2685 "If fix_imports is True and protocol is less than 3, pickle will try to\n"
3265 "If fix_imports is True and protocol is less than 3, pickle will try to\n"
2686 "map the new Python 3.x names to the old module names used in Python\n"
3266 "map the new Python 3.x names to the old module names used in Python\n"
2687 "2.x, so that the pickle data stream is readable with Python 2.x.\n");
3267 "2.x, so that the pickle data stream is readable with Python 2.x.\n");
2688
3268
2689 static int
3269 static int
2690 Pickler_init(PicklerObject *self, PyObject *args, PyObject *kwds)
3270 Pickler_init(PicklerObject *self, PyObject *args, PyObject *kwds)
2691 {
3271 {
2692 static char *kwlist[] = {"file", "protocol", "fix_imports", 0};
3272 static char *kwlist[] = {"file", "protocol", "fix_imports", 0};
2693 PyObject *file;
3273 PyObject *file;
2694 PyObject *proto_obj = NULL;
3274 PyObject *proto_obj = NULL;
2695 long proto = 0;
3275 PyObject *fix_imports = Py_True;
2696 int fix_imports = 1;
2697
3276
2698 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|Oi:Pickler",
3277 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO:Pickler",
2699 kwlist, &file, &proto_obj, &fix_imports))
3278 kwlist, &file, &proto_obj, &fix_imports))
2700 return -1;
3279 return -1;
2701
3280
2702 /* In case of multiple __init__() calls, clear previous content. */
3281 /* In case of multiple __init__() calls, clear previous content. */
2703 if (self->write != NULL)
3282 if (self->write != NULL)
2704 (void)Pickler_clear(self);
3283 (void)Pickler_clear(self);
2705
3284
2706 if (proto_obj == NULL || proto_obj == Py_None)
3285 if (_Pickler_SetProtocol(self, proto_obj, fix_imports) < 0)
2707 proto = DEFAULT_PROTOCOL;
3286 return -1;
2708 else {
2709 proto = PyLong_AsLong(proto_obj);
2710 if (proto == -1 && PyErr_Occurred())
2711 return -1;
2712 }
2713
3287
2714 if (proto < 0)
3288 if (_Pickler_SetOutputStream(self, file) < 0)
2715 proto = HIGHEST_PROTOCOL;
2716 if (proto > HIGHEST_PROTOCOL) {
2717 PyErr_Format(PyExc_ValueError, "pickle protocol must be <= %d",
2718 HIGHEST_PROTOCOL);
2719 return -1;
3289 return -1;
2720 }
2721
3290
2722 self->proto = proto;
2723 self->bin = proto > 0;
2724 self->arg = NULL;
3291 self->arg = NULL;
2725 self->fast = 0;
3292 self->fast = 0;
2726 self->fast_nesting = 0;
3293 self->fast_nesting = 0;
2727 self->fast_memo = NULL;
3294 self->fast_memo = NULL;
2728 self->fix_imports = fix_imports && proto < 3;
2729
3295
2730 if (!PyObject_HasAttrString(file, "write")) {
3296 self->max_output_len = WRITE_BUF_SIZE;
2731 PyErr_SetString(PyExc_TypeError,
3297 self->output_len = 0;
2732 "file must have a 'write' attribute");
3298 self->output_buffer = PyMem_MALLOC(WRITE_BUF_SIZE);
2733 return -1;
3299 if (self->output_buffer == NULL) {
2734 }
2735 self->write = PyObject_GetAttrString(file, "write");
2736 if (self->write == NULL)
2737 return -1;
2738 » self->buf_size = 0;
2739 self->write_buf = (char *)PyMem_Malloc(WRITE_BUF_SIZE);
2740 if (self->write_buf == NULL) {
2741 PyErr_NoMemory();
3300 PyErr_NoMemory();
2742 return -1;
3301 return -1;
2743 }
3302 }
2744 self->pers_func = NULL;
3303 self->pers_func = NULL;
2745 if (PyObject_HasAttrString((PyObject *)self, "persistent_id")) {
3304 if (PyObject_HasAttrString((PyObject *)self, "persistent_id")) {
2746 self->pers_func = PyObject_GetAttrString((PyObject *)self,
3305 self->pers_func = PyObject_GetAttrString((PyObject *)self,
2747 "persistent_id");
3306 "persistent_id");
2748 if (self->pers_func == NULL)
3307 if (self->pers_func == NULL)
2749 return -1;
3308 return -1;
2750 }
3309 }
2751 self->memo = PyDict_New();
3310 self->memo = PyMemoTable_New();
2752 if (self->memo == NULL)
3311 if (self->memo == NULL)
2753 return -1;
3312 return -1;
2754
3313
2755 return 0;
3314 return 0;
2756 }
3315 }
2757
3316
3317 /* Define a proxy object for the Pickler's internal memo object. This is to
3318 * avoid breaking code like:
3319 * pickler.memo.clear()
3320 * and
3321 * pickler.memo = saved_memo
3322 * Is this a good idea? Not really, but we don't want to break code that uses
3323 * it. Note that we don't implement the entire mapping API here. This is
3324 * intentional, as these should be treated as black-box implementation details.
3325 */
3326
3327 typedef struct {
3328 PyObject_HEAD
3329 PicklerObject *pickler; /* Pickler whose memo table we're proxying. */
3330 } PicklerMemoProxyObject;
3331
3332 PyDoc_STRVAR(pmp_clear_doc,
3333 "memo.clear() -> None. Remove all items from memo.");
3334
3335 static PyObject *
3336 pmp_clear(PicklerMemoProxyObject *self)
3337 {
3338 if (self->pickler->memo)
3339 PyMemoTable_Clear(self->pickler->memo);
3340 Py_RETURN_NONE;
3341 }
3342
3343 PyDoc_STRVAR(pmp_copy_doc,
3344 "memo.copy() -> new_memo. Copy the memo to a new object.");
3345
3346 static PyObject *
3347 pmp_copy(PicklerMemoProxyObject *self)
3348 {
3349 Py_ssize_t i;
3350 PyMemoTable *memo;
3351 PyObject *new_memo = PyDict_New();
3352 if (new_memo == NULL)
3353 return NULL;
3354
3355 memo = self->pickler->memo;
3356 for (i = 0; i < memo->mt_allocated; ++i) {
3357 PyMemoEntry entry = memo->mt_table[i];
3358 if (entry.me_key != NULL) {
3359 int status;
3360 PyObject *key, *value;
3361
3362 key = PyLong_FromVoidPtr(entry.me_key);
3363 value = Py_BuildValue("lO", entry.me_value, entry.me_key);
3364
3365 if (key == NULL || value == NULL) {
3366 Py_XDECREF(key);
3367 Py_XDECREF(value);
3368 goto error;
3369 }
3370 status = PyDict_SetItem(new_memo, key, value);
3371 Py_DECREF(key);
3372 Py_DECREF(value);
3373 if (status < 0)
3374 goto error;
3375 }
3376 }
3377 return new_memo;
3378
3379 error:
3380 Py_XDECREF(new_memo);
3381 return NULL;
3382 }
3383
3384 PyDoc_STRVAR(pmp_reduce_doc,
3385 "memo.__reduce__(). Pickling support.");
3386
3387 static PyObject *
3388 pmp_reduce(PicklerMemoProxyObject *self, PyObject *args)
3389 {
3390 PyObject *reduce_value, *dict_args;
3391 PyObject *contents = pmp_copy(self);
3392 if (contents == NULL)
3393 return NULL;
3394
3395 reduce_value = PyTuple_New(2);
3396 if (reduce_value == NULL) {
3397 Py_DECREF(contents);
3398 return NULL;
3399 }
3400 dict_args = PyTuple_New(1);
3401 if (dict_args == NULL) {
3402 Py_DECREF(contents);
3403 Py_DECREF(reduce_value);
3404 return NULL;
3405 }
3406 PyTuple_SET_ITEM(dict_args, 0, contents);
3407 Py_INCREF((PyObject *)&PyDict_Type);
3408 PyTuple_SET_ITEM(reduce_value, 0, (PyObject *)&PyDict_Type);
3409 PyTuple_SET_ITEM(reduce_value, 1, dict_args);
3410 return reduce_value;
3411 }
3412
3413 static PyMethodDef picklerproxy_methods[] = {
3414 {"clear", (PyCFunction)pmp_clear, METH_NOARGS, pmp_clear_doc},
3415 {"copy", (PyCFunction)pmp_copy, METH_NOARGS, pmp_copy_doc},
3416 {"__reduce__", (PyCFunction)pmp_reduce, METH_VARARGS, pmp_reduce_doc},
3417 {NULL, NULL} /* sentinel */
3418 };
3419
3420 static void
3421 PicklerMemoProxy_dealloc(PicklerMemoProxyObject *self)
3422 {
3423 PyObject_GC_UnTrack(self);
3424 Py_XDECREF(self->pickler);
3425 PyObject_GC_Del((PyObject *)self);
3426 }
3427
3428 static int
3429 PicklerMemoProxy_traverse(PicklerMemoProxyObject *self,
3430 visitproc visit, void *arg)
3431 {
3432 Py_VISIT(self->pickler);
3433 return 0;
3434 }
3435
3436 static int
3437 PicklerMemoProxy_clear(PicklerMemoProxyObject *self)
3438 {
3439 Py_CLEAR(self->pickler);
3440 return 0;
3441 }
3442
3443 static PyTypeObject PicklerMemoProxyType = {
3444 PyVarObject_HEAD_INIT(NULL, 0)
3445 "_pickle.PicklerMemoProxy", /*tp_name*/
3446 sizeof(PicklerMemoProxyObject), /*tp_basicsize*/
3447 0,
3448 (destructor)PicklerMemoProxy_dealloc, /* tp_dealloc */
3449 0, /* tp_print */
3450 0, /* tp_getattr */
3451 0, /* tp_setattr */
3452 0, /* tp_compare */
3453 0, /* tp_repr */
3454 0, /* tp_as_number */
3455 0, /* tp_as_sequence */
3456 0, /* tp_as_mapping */
3457 (hashfunc)PyObject_HashNotImplemented, /* tp_hash */
3458 0, /* tp_call */
3459 0, /* tp_str */
3460 PyObject_GenericGetAttr, /* tp_getattro */
3461 PyObject_GenericSetAttr, /* tp_setattro */
3462 0, /* tp_as_buffer */
3463 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
3464 0, /* tp_doc */
3465 (traverseproc)PicklerMemoProxy_traverse, /* tp_traverse */
3466 (inquiry)PicklerMemoProxy_clear, /* tp_clear */
3467 0, /* tp_richcompare */
3468 0, /* tp_weaklistoffset */
3469 0, /* tp_iter */
3470 0, /* tp_iternext */
3471 picklerproxy_methods, /* tp_methods */
3472 };
3473
3474 static PyObject *
3475 PicklerMemoProxy_New(PicklerObject *pickler)
3476 {
3477 PicklerMemoProxyObject *self;
3478
3479 self = PyObject_GC_New(PicklerMemoProxyObject, &PicklerMemoProxyType);
3480 if (self == NULL)
3481 return NULL;
3482 Py_INCREF(pickler);
3483 self->pickler = pickler;
3484 PyObject_GC_Track(self);
3485 return (PyObject *)self;
3486 }
3487
3488 /*****************************************************************************/
3489
2758 static PyObject *
3490 static PyObject *
2759 Pickler_get_memo(PicklerObject *self)
3491 Pickler_get_memo(PicklerObject *self)
2760 {
3492 {
2761 if (self->memo == NULL)
3493 return PicklerMemoProxy_New(self);
2762 PyErr_SetString(PyExc_AttributeError, "memo");
2763 else
2764 Py_INCREF(self->memo);
2765 return self->memo;
2766 }
3494 }
2767
3495
2768 static int
3496 static int
2769 Pickler_set_memo(PicklerObject *self, PyObject *value)
3497 Pickler_set_memo(PicklerObject *self, PyObject *obj)
2770 {
3498 {
2771 PyObject *tmp;
3499 PyMemoTable *new_memo = NULL;
2772
3500
2773 if (value == NULL) {
3501 if (obj == NULL) {
2774 PyErr_SetString(PyExc_TypeError,
3502 PyErr_SetString(PyExc_TypeError,
2775 "attribute deletion is not supported");
3503 "attribute deletion is not supported");
2776 return -1;
3504 return -1;
2777 }
3505 }
2778 if (!PyDict_Check(value)) {
3506
2779 PyErr_SetString(PyExc_TypeError, "memo must be a dictionary");
3507 if (Py_TYPE(obj) == &PicklerMemoProxyType) {
3508 PicklerObject *pickler =
3509 ((PicklerMemoProxyObject *)obj)->pickler;
3510
3511 new_memo = PyMemoTable_Copy(pickler->memo);
3512 if (new_memo == NULL)
3513 return -1;
3514 }
3515 else if (PyDict_Check(obj)) {
3516 Py_ssize_t i = 0;
3517 PyObject *key, *value;
3518
3519 new_memo = PyMemoTable_New();
3520 if (new_memo == NULL)
3521 return -1;
3522
3523 while (PyDict_Next(obj, &i, &key, &value)) {
3524 long memo_id;
3525 PyObject *memo_obj;
3526
3527 if (!PyTuple_Check(value) || Py_SIZE(value) != 2) {
3528 PyErr_SetString(PyExc_TypeError,
3529 "'memo' values must be 2-item tuples");
3530 goto error;
3531 }
3532 memo_id = PyLong_AsLong(PyTuple_GET_ITEM(value, 0));
3533 if (memo_id == -1 && PyErr_Occurred())
3534 goto error;
3535 memo_obj = PyTuple_GET_ITEM(value, 1);
3536 if (PyMemoTable_Set(new_memo, memo_obj, memo_id) < 0)
3537 goto error;
3538 }
3539 }
3540 else {
3541 PyErr_Format(PyExc_TypeError,
3542 "'memo' attribute must be an PicklerMemoProxy object"
3543 "or dict, not %.200s", Py_TYPE(obj)->tp_name);
2780 return -1;
3544 return -1;
2781 }
3545 }
2782
3546
2783 tmp = self->memo;
3547 PyMemoTable_Del(self->memo);
2784 Py_INCREF(value);
3548 self->memo = new_memo;
2785 self->memo = value;
3549
2786 Py_XDECREF(tmp);
3550 return 0;
2787
3551
2788 return 0;
3552 error:
2789 }
3553 if (new_memo)
2790
3554 PyMemoTable_Del(new_memo);
2791 static PyObject *
3555 return -1;
3556 }
3557
3558 static PyObject *
2792 Pickler_get_persid(PicklerObject *self)
3559 Pickler_get_persid(PicklerObject *self)
2793 {
3560 {
2794 if (self->pers_func == NULL)
3561 if (self->pers_func == NULL)
2795 PyErr_SetString(PyExc_AttributeError, "persistent_id");
3562 PyErr_SetString(PyExc_AttributeError, "persistent_id");
2796 else
3563 else
2797 Py_INCREF(self->pers_func);
3564 Py_INCREF(self->pers_func);
2798 return self->pers_func;
3565 return self->pers_func;
2799 }
3566 }
2800
3567
2801 static int
3568 static int
(...skipping 117 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
2919 }
3686 }
2920
3687
2921 static int
3688 static int
2922 load_int(UnpicklerObject *self)
3689 load_int(UnpicklerObject *self)
2923 {
3690 {
2924 PyObject *value;
3691 PyObject *value;
2925 char *endptr, *s;
3692 char *endptr, *s;
2926 Py_ssize_t len;
3693 Py_ssize_t len;
2927 long x;
3694 long x;
2928
3695
2929 if ((len = unpickler_readline(self, &s)) < 0)
3696 if ((len = _Unpickler_Readline(self, &s)) < 0)
2930 return -1;
3697 return -1;
2931 if (len < 2)
3698 if (len < 2)
2932 return bad_readline();
3699 return bad_readline();
2933
3700
2934 errno = 0;
3701 errno = 0;
2935 /* XXX: Should the base argument of strtol() be explicitly set to 10? */
3702 /* XXX: Should the base argument of strtol() be explicitly set to 10?
3703 XXX(avassalotti): Should this uses PyOS_strtol()? */
2936 x = strtol(s, &endptr, 0);
3704 x = strtol(s, &endptr, 0);
2937
3705
2938 if (errno || (*endptr != '\n') || (endptr[1] != '\0')) {
3706 if (errno || (*endptr != '\n') || (endptr[1] != '\0')) {
2939 /* Hm, maybe we've got something long. Let's try reading
3707 /* Hm, maybe we've got something long. Let's try reading
2940 * it as a Python long object. */
3708 * it as a Python long object. */
2941 errno = 0;
3709 errno = 0;
2942 /* XXX: Same thing about the base here. */
3710 /* XXX: Same thing about the base here. */
2943 value = PyLong_FromString(s, NULL, 0);·
3711 value = PyLong_FromString(s, NULL, 0);·
2944 if (value == NULL) {
3712 if (value == NULL) {
2945 PyErr_SetString(PyExc_ValueError,
3713 PyErr_SetString(PyExc_ValueError,
(...skipping 64 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
3010
3778
3011 PDATA_PUSH(self->stack, value, -1);
3779 PDATA_PUSH(self->stack, value, -1);
3012 return 0;
3780 return 0;
3013 }
3781 }
3014
3782
3015 static int
3783 static int
3016 load_binint(UnpicklerObject *self)
3784 load_binint(UnpicklerObject *self)
3017 {
3785 {
3018 char *s;
3786 char *s;
3019
3787
3020 if (unpickler_read(self, &s, 4) < 0)
3788 if (_Unpickler_Read(self, &s, 4) < 0)
3021 return -1;
3789 return -1;
3022
3790
3023 return load_binintx(self, s, 4);
3791 return load_binintx(self, s, 4);
3024 }
3792 }
3025
3793
3026 static int
3794 static int
3027 load_binint1(UnpicklerObject *self)
3795 load_binint1(UnpicklerObject *self)
3028 {
3796 {
3029 char *s;
3797 char *s;
3030
3798
3031 if (unpickler_read(self, &s, 1) < 0)
3799 if (_Unpickler_Read(self, &s, 1) < 0)
3032 return -1;
3800 return -1;
3033
3801
3034 return load_binintx(self, s, 1);
3802 return load_binintx(self, s, 1);
3035 }
3803 }
3036
3804
3037 static int
3805 static int
3038 load_binint2(UnpicklerObject *self)
3806 load_binint2(UnpicklerObject *self)
3039 {
3807 {
3040 char *s;
3808 char *s;
3041
3809
3042 if (unpickler_read(self, &s, 2) < 0)
3810 if (_Unpickler_Read(self, &s, 2) < 0)
3043 return -1;
3811 return -1;
3044
3812
3045 return load_binintx(self, s, 2);
3813 return load_binintx(self, s, 2);
3046 }
3814 }
3047
3815
3048 static int
3816 static int
3049 load_long(UnpicklerObject *self)
3817 load_long(UnpicklerObject *self)
3050 {
3818 {
3051 PyObject *value;
3819 PyObject *value;
3052 char *s;
3820 char *s;
3053 Py_ssize_t len;
3821 Py_ssize_t len;
3054
3822
3055 if ((len = unpickler_readline(self, &s)) < 0)
3823 if ((len = _Unpickler_Readline(self, &s)) < 0)
3056 return -1;
3824 return -1;
3057 if (len < 2)
3825 if (len < 2)
3058 return bad_readline();
3826 return bad_readline();
3059
3827
3060 /* s[len-2] will usually be 'L' (and s[len-1] is '\n'); we need to remove
3828 /* s[len-2] will usually be 'L' (and s[len-1] is '\n'); we need to remove
3061 the 'L' before calling PyLong_FromString. In order to maintain
3829 the 'L' before calling PyLong_FromString. In order to maintain
3062 compatibility with Python 3.0.0, we don't actually *require*
3830 compatibility with Python 3.0.0, we don't actually *require*
3063 the 'L' to be present. */
3831 the 'L' to be present. */
3064 if (s[len-2] == 'L') {
3832 if (s[len-2] == 'L') {
3065 s[len-2] = '\0';
3833 s[len-2] = '\0';
(...skipping 11 matching lines...) Expand all Loading...
3077 * data following.
3845 * data following.
3078 */
3846 */
3079 static int
3847 static int
3080 load_counted_long(UnpicklerObject *self, int size)
3848 load_counted_long(UnpicklerObject *self, int size)
3081 {
3849 {
3082 PyObject *value;
3850 PyObject *value;
3083 char *nbytes;
3851 char *nbytes;
3084 char *pdata;
3852 char *pdata;
3085
3853
3086 assert(size == 1 || size == 4);
3854 assert(size == 1 || size == 4);
3087 if (unpickler_read(self, &nbytes, size) < 0)
3855 if (_Unpickler_Read(self, &nbytes, size) < 0)
3088 return -1;
3856 return -1;
3089
3857
3090 size = calc_binint(nbytes, size);
3858 size = calc_binint(nbytes, size);
3091 if (size < 0) {
3859 if (size < 0) {
3092 /* Corrupt or hostile pickle -- we never write one like this */
3860 /* Corrupt or hostile pickle -- we never write one like this */
3093 PyErr_SetString(UnpicklingError,
3861 PyErr_SetString(UnpicklingError,
3094 "LONG pickle has negative byte count");
3862 "LONG pickle has negative byte count");
3095 return -1;
3863 return -1;
3096 }
3864 }
3097
3865
3098 if (size == 0)
3866 if (size == 0)
3099 value = PyLong_FromLong(0L);
3867 value = PyLong_FromLong(0L);
3100 else {
3868 else {
3101 /* Read the raw little-endian bytes and convert. */
3869 /* Read the raw little-endian bytes and convert. */
3102 if (unpickler_read(self, &pdata, size) < 0)
3870 if (_Unpickler_Read(self, &pdata, size) < 0)
3103 return -1;
3871 return -1;
3104 value = _PyLong_FromByteArray((unsigned char *)pdata, (size_t)size,
3872 value = _PyLong_FromByteArray((unsigned char *)pdata, (size_t)size,
3105 1 /* little endian */ , 1 /* signed */ );
3873 1 /* little endian */ , 1 /* signed */ );
3106 }
3874 }
3107 if (value == NULL)
3875 if (value == NULL)
3108 return -1;
3876 return -1;
3109 PDATA_PUSH(self->stack, value, -1);
3877 PDATA_PUSH(self->stack, value, -1);
3110 return 0;
3878 return 0;
3111 }
3879 }
3112
3880
3113 static int
3881 static int
3114 load_float(UnpicklerObject *self)
3882 load_float(UnpicklerObject *self)
3115 {
3883 {
3116 PyObject *value;
3884 PyObject *value;
3117 char *endptr, *s;
3885 char *endptr, *s;
3118 Py_ssize_t len;
3886 Py_ssize_t len;
3119 double d;
3887 double d;
3120
3888
3121 if ((len = unpickler_readline(self, &s)) < 0)
3889 if ((len = _Unpickler_Readline(self, &s)) < 0)
3122 return -1;
3890 return -1;
3123 if (len < 2)
3891 if (len < 2)
3124 return bad_readline();
3892 return bad_readline();
3125
3893
3126 errno = 0;
3894 errno = 0;
3127 d = PyOS_string_to_double(s, &endptr, PyExc_OverflowError);
3895 d = PyOS_string_to_double(s, &endptr, PyExc_OverflowError);
3128 if (d == -1.0 && PyErr_Occurred())
3896 if (d == -1.0 && PyErr_Occurred())
3129 return -1;
3897 return -1;
3130 if ((endptr[0] != '\n') || (endptr[1] != '\0')) {
3898 if ((endptr[0] != '\n') || (endptr[1] != '\0')) {
3131 PyErr_SetString(PyExc_ValueError, "could not convert string to float");
3899 PyErr_SetString(PyExc_ValueError, "could not convert string to float");
3132 return -1;
3900 return -1;
3133 }
3901 }
3134 value = PyFloat_FromDouble(d);
3902 value = PyFloat_FromDouble(d);
3135 if (value == NULL)
3903 if (value == NULL)
3136 return -1;
3904 return -1;
3137
3905
3138 PDATA_PUSH(self->stack, value, -1);
3906 PDATA_PUSH(self->stack, value, -1);
3139 return 0;
3907 return 0;
3140 }
3908 }
3141
3909
3142 static int
3910 static int
3143 load_binfloat(UnpicklerObject *self)
3911 load_binfloat(UnpicklerObject *self)
3144 {
3912 {
3145 PyObject *value;
3913 PyObject *value;
3146 double x;
3914 double x;
3147 char *s;
3915 char *s;
3148
3916
3149 if (unpickler_read(self, &s, 8) < 0)
3917 if (_Unpickler_Read(self, &s, 8) < 0)
3150 return -1;
3918 return -1;
3151
3919
3152 x = _PyFloat_Unpack8((unsigned char *)s, 0);
3920 x = _PyFloat_Unpack8((unsigned char *)s, 0);
3153 if (x == -1.0 && PyErr_Occurred())
3921 if (x == -1.0 && PyErr_Occurred())
3154 return -1;
3922 return -1;
3155
3923
3156 if ((value = PyFloat_FromDouble(x)) == NULL)
3924 if ((value = PyFloat_FromDouble(x)) == NULL)
3157 return -1;
3925 return -1;
3158
3926
3159 PDATA_PUSH(self->stack, value, -1);
3927 PDATA_PUSH(self->stack, value, -1);
3160 return 0;
3928 return 0;
3161 }
3929 }
3162
3930
3163 static int
3931 static int
3164 load_string(UnpicklerObject *self)
3932 load_string(UnpicklerObject *self)
3165 {
3933 {
3166 PyObject *bytes;
3934 PyObject *bytes;
3167 PyObject *str = NULL;
3935 PyObject *str = NULL;
3168 Py_ssize_t len;
3936 Py_ssize_t len;
3169 char *s, *p;
3937 char *s, *p;
3170
3938
3171 if ((len = unpickler_readline(self, &s)) < 0)
3939 if ((len = _Unpickler_Readline(self, &s)) < 0)
3172 return -1;
3940 return -1;
3173 if (len < 3)
3941 if (len < 3)
3174 return bad_readline();
3942 return bad_readline();
3175 if ((s = strdup(s)) == NULL) {
3943 if ((s = strdup(s)) == NULL) {
3176 PyErr_NoMemory();
3944 PyErr_NoMemory();
3177 return -1;
3945 return -1;
3178 }
3946 }
3179
3947
3180 /* Strip outermost quotes */
3948 /* Strip outermost quotes */
3181 while (s[len - 1] <= ' ')
3949 while (s[len - 1] <= ' ')
(...skipping 29 matching lines...) Expand all Loading...
3211 return 0;
3979 return 0;
3212 }
3980 }
3213
3981
3214 static int
3982 static int
3215 load_binbytes(UnpicklerObject *self)
3983 load_binbytes(UnpicklerObject *self)
3216 {
3984 {
3217 PyObject *bytes;
3985 PyObject *bytes;
3218 long x;
3986 long x;
3219 char *s;
3987 char *s;
3220
3988
3221 if (unpickler_read(self, &s, 4) < 0)
3989 if (_Unpickler_Read(self, &s, 4) < 0)
3222 return -1;
3990 return -1;
3223
3991
3224 x = calc_binint(s, 4);
3992 x = calc_binint(s, 4);
3225 if (x < 0) {
3993 if (x < 0) {
3226 PyErr_SetString(UnpicklingError,·
3994 PyErr_SetString(UnpicklingError,·
3227 "BINBYTES pickle has negative byte count");
3995 "BINBYTES pickle has negative byte count");
3228 return -1;
3996 return -1;
3229 }
3997 }
3230
3998
3231 if (unpickler_read(self, &s, x) < 0)
3999 if (_Unpickler_Read(self, &s, x) < 0)
3232 return -1;
4000 return -1;
3233 bytes = PyBytes_FromStringAndSize(s, x);
4001 bytes = PyBytes_FromStringAndSize(s, x);
3234 if (bytes == NULL)
4002 if (bytes == NULL)
3235 return -1;
4003 return -1;
3236
4004
3237 PDATA_PUSH(self->stack, bytes, -1);
4005 PDATA_PUSH(self->stack, bytes, -1);
3238 return 0;
4006 return 0;
3239 }
4007 }
3240
4008
3241 static int
4009 static int
3242 load_short_binbytes(UnpicklerObject *self)
4010 load_short_binbytes(UnpicklerObject *self)
3243 {
4011 {
3244 PyObject *bytes;
4012 PyObject *bytes;
3245 unsigned char x;
4013 unsigned char x;
3246 char *s;
4014 char *s;
3247
4015
3248 if (unpickler_read(self, &s, 1) < 0)
4016 if (_Unpickler_Read(self, &s, 1) < 0)
3249 return -1;
4017 return -1;
3250
4018
3251 x = (unsigned char)s[0];
4019 x = (unsigned char)s[0];
3252
4020
3253 if (unpickler_read(self, &s, x) < 0)
4021 if (_Unpickler_Read(self, &s, x) < 0)
3254 return -1;
4022 return -1;
3255
4023
3256 bytes = PyBytes_FromStringAndSize(s, x);
4024 bytes = PyBytes_FromStringAndSize(s, x);
3257 if (bytes == NULL)
4025 if (bytes == NULL)
3258 return -1;
4026 return -1;
3259
4027
3260 PDATA_PUSH(self->stack, bytes, -1);
4028 PDATA_PUSH(self->stack, bytes, -1);
3261 return 0;
4029 return 0;
3262 }
4030 }
3263
4031
3264 static int
4032 static int
3265 load_binstring(UnpicklerObject *self)
4033 load_binstring(UnpicklerObject *self)
3266 {
4034 {
3267 PyObject *str;
4035 PyObject *str;
3268 long x;
4036 long x;
3269 char *s;
4037 char *s;
3270
4038
3271 if (unpickler_read(self, &s, 4) < 0)
4039 if (_Unpickler_Read(self, &s, 4) < 0)
3272 return -1;
4040 return -1;
3273
4041
3274 x = calc_binint(s, 4);
4042 x = calc_binint(s, 4);
3275 if (x < 0) {
4043 if (x < 0) {
3276 PyErr_SetString(UnpicklingError,·
4044 PyErr_SetString(UnpicklingError,·
3277 "BINSTRING pickle has negative byte count");
4045 "BINSTRING pickle has negative byte count");
3278 return -1;
4046 return -1;
3279 }
4047 }
3280
4048
3281 if (unpickler_read(self, &s, x) < 0)
4049 if (_Unpickler_Read(self, &s, x) < 0)
3282 return -1;
4050 return -1;
3283
4051
3284 /* Convert Python 2.x strings to unicode. */
4052 /* Convert Python 2.x strings to unicode. */
3285 str = PyUnicode_Decode(s, x, self->encoding, self->errors);
4053 str = PyUnicode_Decode(s, x, self->encoding, self->errors);
3286 if (str == NULL)
4054 if (str == NULL)
3287 return -1;
4055 return -1;
3288
4056
3289 PDATA_PUSH(self->stack, str, -1);
4057 PDATA_PUSH(self->stack, str, -1);
3290 return 0;
4058 return 0;
3291 }
4059 }
3292
4060
3293 static int
4061 static int
3294 load_short_binstring(UnpicklerObject *self)
4062 load_short_binstring(UnpicklerObject *self)
3295 {
4063 {
3296 PyObject *str;
4064 PyObject *str;
3297 unsigned char x;
4065 unsigned char x;
3298 char *s;
4066 char *s;
3299
4067
3300 if (unpickler_read(self, &s, 1) < 0)
4068 if (_Unpickler_Read(self, &s, 1) < 0)
3301 return -1;
4069 return -1;
3302
4070
3303 x = (unsigned char)s[0];
4071 x = (unsigned char)s[0];
3304
4072
3305 if (unpickler_read(self, &s, x) < 0)
4073 if (_Unpickler_Read(self, &s, x) < 0)
3306 return -1;
4074 return -1;
3307
4075
3308 /* Convert Python 2.x strings to unicode. */
4076 /* Convert Python 2.x strings to unicode. */
3309 str = PyUnicode_Decode(s, x, self->encoding, self->errors);
4077 str = PyUnicode_Decode(s, x, self->encoding, self->errors);
3310 if (str == NULL)
4078 if (str == NULL)
3311 return -1;
4079 return -1;
3312
4080
3313 PDATA_PUSH(self->stack, str, -1);
4081 PDATA_PUSH(self->stack, str, -1);
3314 return 0;
4082 return 0;
3315 }
4083 }
3316
4084
3317 static int
4085 static int
3318 load_unicode(UnpicklerObject *self)
4086 load_unicode(UnpicklerObject *self)
3319 {
4087 {
3320 PyObject *str;
4088 PyObject *str;
3321 Py_ssize_t len;
4089 Py_ssize_t len;
3322 char *s;
4090 char *s;
3323
4091
3324 if ((len = unpickler_readline(self, &s)) < 0)
4092 if ((len = _Unpickler_Readline(self, &s)) < 0)
3325 return -1;
4093 return -1;
3326 if (len < 1)
4094 if (len < 1)
3327 return bad_readline();
4095 return bad_readline();
3328
4096
3329 str = PyUnicode_DecodeRawUnicodeEscape(s, len - 1, NULL);
4097 str = PyUnicode_DecodeRawUnicodeEscape(s, len - 1, NULL);
3330 if (str == NULL)
4098 if (str == NULL)
3331 return -1;
4099 return -1;
3332
4100
3333 PDATA_PUSH(self->stack, str, -1);
4101 PDATA_PUSH(self->stack, str, -1);
3334 return 0;
4102 return 0;
3335 }
4103 }
3336
4104
3337 static int
4105 static int
3338 load_binunicode(UnpicklerObject *self)
4106 load_binunicode(UnpicklerObject *self)
3339 {
4107 {
3340 PyObject *str;
4108 PyObject *str;
3341 long size;
4109 long size;
3342 char *s;
4110 char *s;
3343
4111
3344 if (unpickler_read(self, &s, 4) < 0)
4112 if (_Unpickler_Read(self, &s, 4) < 0)
3345 return -1;
4113 return -1;
3346
4114
3347 size = calc_binint(s, 4);
4115 size = calc_binint(s, 4);
3348 if (size < 0) {
4116 if (size < 0) {
3349 PyErr_SetString(UnpicklingError,·
4117 PyErr_SetString(UnpicklingError,·
3350 "BINUNICODE pickle has negative byte count");
4118 "BINUNICODE pickle has negative byte count");
3351 return -1;
4119 return -1;
3352 }
4120 }
3353
4121
3354 if (unpickler_read(self, &s, size) < 0)
4122 if (_Unpickler_Read(self, &s, size) < 0)
3355 return -1;
4123 return -1;
3356
4124
3357 str = PyUnicode_DecodeUTF8(s, size, "surrogatepass");
4125 str = PyUnicode_DecodeUTF8(s, size, "surrogatepass");
3358 if (str == NULL)
4126 if (str == NULL)
3359 return -1;
4127 return -1;
3360
4128
3361 PDATA_PUSH(self->stack, str, -1);
4129 PDATA_PUSH(self->stack, str, -1);
3362 return 0;
4130 return 0;
3363 }
4131 }
3364
4132
(...skipping 73 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
3438 }
4206 }
3439
4207
3440 static int
4208 static int
3441 load_dict(UnpicklerObject *self)
4209 load_dict(UnpicklerObject *self)
3442 {
4210 {
3443 PyObject *dict, *key, *value;
4211 PyObject *dict, *key, *value;
3444 int i, j, k;
4212 int i, j, k;
3445
4213
3446 if ((i = marker(self)) < 0)
4214 if ((i = marker(self)) < 0)
3447 return -1;
4215 return -1;
3448 j = self->stack->length;
4216 j = Py_SIZE(self->stack);
3449
4217
3450 if ((dict = PyDict_New()) == NULL)
4218 if ((dict = PyDict_New()) == NULL)
3451 return -1;
4219 return -1;
3452
4220
3453 for (k = i + 1; k < j; k += 2) {
4221 for (k = i + 1; k < j; k += 2) {
3454 key = self->stack->data[k - 1];
4222 key = self->stack->data[k - 1];
3455 value = self->stack->data[k];
4223 value = self->stack->data[k];
3456 if (PyDict_SetItem(dict, key, value) < 0) {
4224 if (PyDict_SetItem(dict, key, value) < 0) {
3457 Py_DECREF(dict);
4225 Py_DECREF(dict);
3458 return -1;
4226 return -1;
(...skipping 55 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
3514 PyObject *args = NULL;
4282 PyObject *args = NULL;
3515 PyObject *obj = NULL;
4283 PyObject *obj = NULL;
3516 PyObject *module_name;
4284 PyObject *module_name;
3517 PyObject *class_name;
4285 PyObject *class_name;
3518 Py_ssize_t len;
4286 Py_ssize_t len;
3519 int i;
4287 int i;
3520 char *s;
4288 char *s;
3521
4289
3522 if ((i = marker(self)) < 0)
4290 if ((i = marker(self)) < 0)
3523 return -1;
4291 return -1;
3524 if ((len = unpickler_readline(self, &s)) < 0)
4292 if ((len = _Unpickler_Readline(self, &s)) < 0)
3525 return -1;
4293 return -1;
3526 if (len < 2)
4294 if (len < 2)
3527 return bad_readline();
4295 return bad_readline();
3528
4296
3529 /* Here it is safe to use PyUnicode_DecodeASCII(), even though non-ASCII
4297 /* Here it is safe to use PyUnicode_DecodeASCII(), even though non-ASCII
3530 identifiers are permitted in Python 3.0, since the INST opcode is only
4298 identifiers are permitted in Python 3.0, since the INST opcode is only
3531 supported by older protocols on Python 2.x. */
4299 supported by older protocols on Python 2.x. */
3532 module_name = PyUnicode_DecodeASCII(s, len - 1, "strict");
4300 module_name = PyUnicode_DecodeASCII(s, len - 1, "strict");
3533 if (module_name == NULL)
4301 if (module_name == NULL)
3534 return -1;
4302 return -1;
3535
4303
3536 if ((len = unpickler_readline(self, &s)) >= 0) {
4304 if ((len = _Unpickler_Readline(self, &s)) >= 0) {
3537 if (len < 2)
4305 if (len < 2)
3538 return bad_readline();
4306 return bad_readline();
3539 class_name = PyUnicode_DecodeASCII(s, len - 1, "strict");
4307 class_name = PyUnicode_DecodeASCII(s, len - 1, "strict");
3540 if (class_name != NULL) {
4308 if (class_name != NULL) {
3541 cls = find_class(self, module_name, class_name);
4309 cls = find_class(self, module_name, class_name);
3542 Py_DECREF(class_name);
4310 Py_DECREF(class_name);
3543 }
4311 }
3544 }
4312 }
3545 Py_DECREF(module_name);
4313 Py_DECREF(module_name);
3546
4314
(...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
3612
4380
3613 static int
4381 static int
3614 load_global(UnpicklerObject *self)
4382 load_global(UnpicklerObject *self)
3615 {
4383 {
3616 PyObject *global = NULL;
4384 PyObject *global = NULL;
3617 PyObject *module_name;
4385 PyObject *module_name;
3618 PyObject *global_name;
4386 PyObject *global_name;
3619 Py_ssize_t len;
4387 Py_ssize_t len;
3620 char *s;
4388 char *s;
3621
4389
3622 if ((len = unpickler_readline(self, &s)) < 0)
4390 if ((len = _Unpickler_Readline(self, &s)) < 0)
3623 return -1;
4391 return -1;
3624 if (len < 2)
4392 if (len < 2)
3625 return bad_readline();
4393 return bad_readline();
3626 module_name = PyUnicode_DecodeUTF8(s, len - 1, "strict");
4394 module_name = PyUnicode_DecodeUTF8(s, len - 1, "strict");
3627 if (!module_name)
4395 if (!module_name)
3628 return -1;
4396 return -1;
3629
4397
3630 if ((len = unpickler_readline(self, &s)) >= 0) {
4398 if ((len = _Unpickler_Readline(self, &s)) >= 0) {
3631 if (len < 2) {
4399 if (len < 2) {
3632 Py_DECREF(module_name);
4400 Py_DECREF(module_name);
3633 return bad_readline();
4401 return bad_readline();
3634 }
4402 }
3635 global_name = PyUnicode_DecodeUTF8(s, len - 1, "strict");
4403 global_name = PyUnicode_DecodeUTF8(s, len - 1, "strict");
3636 if (global_name) {
4404 if (global_name) {
3637 global = find_class(self, module_name, global_name);
4405 global = find_class(self, module_name, global_name);
3638 Py_DECREF(global_name);
4406 Py_DECREF(global_name);
3639 }
4407 }
3640 }
4408 }
3641 Py_DECREF(module_name);
4409 Py_DECREF(module_name);
3642
4410
3643 if (global == NULL)
4411 if (global == NULL)
3644 return -1;
4412 return -1;
3645 PDATA_PUSH(self->stack, global, -1);
4413 PDATA_PUSH(self->stack, global, -1);
3646 return 0;
4414 return 0;
3647 }
4415 }
3648
4416
3649 static int
4417 static int
3650 load_persid(UnpicklerObject *self)
4418 load_persid(UnpicklerObject *self)
3651 {
4419 {
3652 PyObject *pid;
4420 PyObject *pid;
3653 Py_ssize_t len;
4421 Py_ssize_t len;
3654 char *s;
4422 char *s;
3655
4423
3656 if (self->pers_func) {
4424 if (self->pers_func) {
3657 if ((len = unpickler_readline(self, &s)) < 0)
4425 if ((len = _Unpickler_Readline(self, &s)) < 0)
3658 return -1;
4426 return -1;
3659 if (len < 2)
4427 if (len < 2)
3660 return bad_readline();
4428 return bad_readline();
3661
4429
3662 pid = PyBytes_FromStringAndSize(s, len - 1);
4430 pid = PyBytes_FromStringAndSize(s, len - 1);
3663 if (pid == NULL)
4431 if (pid == NULL)
3664 return -1;
4432 return -1;
3665
4433
3666 /* Ugh... this does not leak since unpickler_call() steals the
4434 /* Ugh... this does not leak since _Unpickler_FastCall() steals the
3667 reference to pid first. */
4435 reference to pid first. */
3668 pid = unpickler_call(self, self->pers_func, pid);
4436 pid = _Unpickler_FastCall(self, self->pers_func, pid);
3669 if (pid == NULL)
4437 if (pid == NULL)
3670 return -1;
4438 return -1;
3671
4439
3672 PDATA_PUSH(self->stack, pid, -1);
4440 PDATA_PUSH(self->stack, pid, -1);
3673 return 0;
4441 return 0;
3674 }
4442 }
3675 else {
4443 else {
3676 PyErr_SetString(UnpicklingError,
4444 PyErr_SetString(UnpicklingError,
3677 "A load persistent id instruction was encountered,\n"
4445 "A load persistent id instruction was encountered,\n"
3678 "but no persistent_load function was specified.");
4446 "but no persistent_load function was specified.");
3679 return -1;
4447 return -1;
3680 }
4448 }
3681 }
4449 }
3682
4450
3683 static int
4451 static int
3684 load_binpersid(UnpicklerObject *self)
4452 load_binpersid(UnpicklerObject *self)
3685 {
4453 {
3686 PyObject *pid;
4454 PyObject *pid;
3687
4455
3688 if (self->pers_func) {
4456 if (self->pers_func) {
3689 PDATA_POP(self->stack, pid);
4457 PDATA_POP(self->stack, pid);
3690 if (pid == NULL)
4458 if (pid == NULL)
3691 return -1;
4459 return -1;
3692
4460
3693 /* Ugh... this does not leak since unpickler_call() steals the
4461 /* Ugh... this does not leak since _Unpickler_FastCall() steals the
3694 reference to pid first. */
4462 reference to pid first. */
3695 pid = unpickler_call(self, self->pers_func, pid);
4463 pid = _Unpickler_FastCall(self, self->pers_func, pid);
3696 if (pid == NULL)
4464 if (pid == NULL)
3697 return -1;
4465 return -1;
3698
4466
3699 PDATA_PUSH(self->stack, pid, -1);
4467 PDATA_PUSH(self->stack, pid, -1);
3700 return 0;
4468 return 0;
3701 }
4469 }
3702 else {
4470 else {
3703 PyErr_SetString(UnpicklingError,
4471 PyErr_SetString(UnpicklingError,
3704 "A load persistent id instruction was encountered,\n"
4472 "A load persistent id instruction was encountered,\n"
3705 "but no persistent_load function was specified.");
4473 "but no persistent_load function was specified.");
3706 return -1;
4474 return -1;
3707 }
4475 }
3708 }
4476 }
3709
4477
3710 static int
4478 static int
3711 load_pop(UnpicklerObject *self)
4479 load_pop(UnpicklerObject *self)
3712 {
4480 {
3713 int len = self->stack->length;
4481 int len = Py_SIZE(self->stack);
3714
4482
3715 /* Note that we split the (pickle.py) stack into two stacks,
4483 /* Note that we split the (pickle.py) stack into two stacks,
3716 * an object stack and a mark stack. We have to be clever and
4484 * an object stack and a mark stack. We have to be clever and
3717 * pop the right one. We do this by looking at the top of the
4485 * pop the right one. We do this by looking at the top of the
3718 * mark stack first, and only signalling a stack underflow if
4486 * mark stack first, and only signalling a stack underflow if
3719 * the object stack is empty and the mark stack doesn't match
4487 * the object stack is empty and the mark stack doesn't match
3720 * our expectations.
4488 * our expectations.
3721 */
4489 */
3722 if (self->num_marks > 0 && self->marks[self->num_marks - 1] == len) {
4490 if (self->num_marks > 0 && self->marks[self->num_marks - 1] == len) {
3723 self->num_marks--;
4491 self->num_marks--;
3724 } else if (len > 0) {
4492 } else if (len > 0) {
3725 len--;
4493 len--;
3726 Py_DECREF(self->stack->data[len]);
4494 Py_DECREF(self->stack->data[len]);
3727 self->stack->length = len;
4495 Py_SIZE(self->stack) = len;
3728 } else {
4496 } else {
3729 return stack_underflow();
4497 return stack_underflow();
3730 }
4498 }
3731 return 0;
4499 return 0;
3732 }
4500 }
3733
4501
3734 static int
4502 static int
3735 load_pop_mark(UnpicklerObject *self)
4503 load_pop_mark(UnpicklerObject *self)
3736 {
4504 {
3737 int i;
4505 int i;
3738
4506
3739 if ((i = marker(self)) < 0)
4507 if ((i = marker(self)) < 0)
3740 return -1;
4508 return -1;
3741
4509
3742 Pdata_clear(self->stack, i);
4510 Pdata_clear(self->stack, i);
3743
4511
3744 return 0;
4512 return 0;
3745 }
4513 }
3746
4514
3747 static int
4515 static int
3748 load_dup(UnpicklerObject *self)
4516 load_dup(UnpicklerObject *self)
3749 {
4517 {
3750 PyObject *last;
4518 PyObject *last;
3751 int len;
4519 int len;
3752
4520
3753 if ((len = self->stack->length) <= 0)
4521 if ((len = Py_SIZE(self->stack)) <= 0)
3754 return stack_underflow();
4522 return stack_underflow();
3755 last = self->stack->data[len - 1];
4523 last = self->stack->data[len - 1];
3756 PDATA_APPEND(self->stack, last, -1);
4524 PDATA_APPEND(self->stack, last, -1);
3757 return 0;
4525 return 0;
3758 }
4526 }
3759
4527
3760 static int
4528 static int
3761 load_get(UnpicklerObject *self)
4529 load_get(UnpicklerObject *self)
3762 {
4530 {
3763 PyObject *key, *value;
4531 PyObject *key, *value;
4532 Py_ssize_t idx;
3764 Py_ssize_t len;
4533 Py_ssize_t len;
3765 char *s;
4534 char *s;
3766
4535
3767 if ((len = unpickler_readline(self, &s)) < 0)
4536 if ((len = _Unpickler_Readline(self, &s)) < 0)
3768 return -1;
4537 return -1;
3769 if (len < 2)
4538 if (len < 2)
3770 return bad_readline();
4539 return bad_readline();
3771
4540
3772 key = PyLong_FromString(s, NULL, 10);
4541 key = PyLong_FromString(s, NULL, 10);
3773 if (key == NULL)
4542 if (key == NULL)
3774 return -1;
4543 return -1;
4544 idx = PyLong_AsSsize_t(key);
4545 if (idx == -1 && PyErr_Occurred()) {
4546 Py_DECREF(key);
4547 return -1;
4548 }
3775
4549
3776 value = PyDict_GetItemWithError(self->memo, key);
4550 value = _Unpickler_MemoGet(self, idx);
3777 if (value == NULL) {
4551 if (value == NULL) {
3778 if (!PyErr_Occurred())
4552 if (!PyErr_Occurred())
3779 PyErr_SetObject(PyExc_KeyError, key);
4553 PyErr_SetObject(PyExc_KeyError, key);
3780 Py_DECREF(key);
4554 Py_DECREF(key);
3781 return -1;
4555 return -1;
3782 }
4556 }
3783 Py_DECREF(key);
4557 Py_DECREF(key);
3784
4558
3785 PDATA_APPEND(self->stack, value, -1);
4559 PDATA_APPEND(self->stack, value, -1);
3786 return 0;
4560 return 0;
3787 }
4561 }
3788
4562
3789 static int
4563 static int
3790 load_binget(UnpicklerObject *self)
4564 load_binget(UnpicklerObject *self)
3791 {
4565 {
3792 PyObject *key, *value;
4566 PyObject *value;
4567 Py_ssize_t idx;
3793 char *s;
4568 char *s;
3794
4569
3795 if (unpickler_read(self, &s, 1) < 0)
4570 if (_Unpickler_Read(self, &s, 1) < 0)
3796 return -1;
4571 return -1;
3797
4572
3798 /* Here, the unsigned cast is necessary to avoid negative values. */
4573 idx = Py_CHARMASK(s[0]);
3799 key = PyLong_FromLong((long)(unsigned char)s[0]);
3800 if (key == NULL)
3801 return -1;
3802
4574
3803 value = PyDict_GetItemWithError(self->memo, key);
4575 value = _Unpickler_MemoGet(self, idx);
3804 if (value == NULL) {
4576 if (value == NULL) {
4577 PyObject *key = PyLong_FromSsize_t(idx);
3805 if (!PyErr_Occurred())
4578 if (!PyErr_Occurred())
3806 PyErr_SetObject(PyExc_KeyError, key);
4579 PyErr_SetObject(PyExc_KeyError, key);
3807 Py_DECREF(key);
4580 Py_DECREF(key);
3808 return -1;
4581 return -1;
3809 }
4582 }
3810 Py_DECREF(key);
3811
4583
3812 PDATA_APPEND(self->stack, value, -1);
4584 PDATA_APPEND(self->stack, value, -1);
3813 return 0;
4585 return 0;
3814 }
4586 }
3815
4587
3816 static int
4588 static int
3817 load_long_binget(UnpicklerObject *self)
4589 load_long_binget(UnpicklerObject *self)
3818 {
4590 {
3819 PyObject *key, *value;
4591 PyObject *value;
4592 Py_ssize_t idx;
3820 char *s;
4593 char *s;
3821 long k;
3822
4594
3823 if (unpickler_read(self, &s, 4) < 0)
4595 if (_Unpickler_Read(self, &s, 4) < 0)
3824 return -1;
4596 return -1;
3825
4597
3826 k = (long)(unsigned char)s[0];
4598 idx = (long)Py_CHARMASK(s[0]);
3827 k |= (long)(unsigned char)s[1] << 8;
4599 idx |= (long)Py_CHARMASK(s[1]) << 8;
3828 k |= (long)(unsigned char)s[2] << 16;
4600 idx |= (long)Py_CHARMASK(s[2]) << 16;
3829 k |= (long)(unsigned char)s[3] << 24;
4601 idx |= (long)Py_CHARMASK(s[3]) << 24;
3830
4602
3831 key = PyLong_FromLong(k);
4603 value = _Unpickler_MemoGet(self, idx);
3832 if (key == NULL)
3833 return -1;
3834
3835 value = PyDict_GetItemWithError(self->memo, key);
3836 if (value == NULL) {
4604 if (value == NULL) {
4605 PyObject *key = PyLong_FromSsize_t(idx);
3837 if (!PyErr_Occurred())
4606 if (!PyErr_Occurred())
3838 PyErr_SetObject(PyExc_KeyError, key);
4607 PyErr_SetObject(PyExc_KeyError, key);
3839 Py_DECREF(key);
4608 Py_DECREF(key);
3840 return -1;
4609 return -1;
3841 }
4610 }
3842 Py_DECREF(key);
3843
4611
3844 PDATA_APPEND(self->stack, value, -1);
4612 PDATA_APPEND(self->stack, value, -1);
3845 return 0;
4613 return 0;
3846 }
4614 }
3847
4615
3848 /* Push an object from the extension registry (EXT[124]). nbytes is
4616 /* Push an object from the extension registry (EXT[124]). nbytes is
3849 * the number of bytes following the opcode, holding the index (code) value.
4617 * the number of bytes following the opcode, holding the index (code) value.
3850 */
4618 */
3851 static int
4619 static int
3852 load_extension(UnpicklerObject *self, int nbytes)
4620 load_extension(UnpicklerObject *self, int nbytes)
3853 {
4621 {
3854 char *codebytes; /* the nbytes bytes after the opcode */
4622 char *codebytes; /* the nbytes bytes after the opcode */
3855 long code; /* calc_binint returns long */
4623 long code; /* calc_binint returns long */
3856 PyObject *py_code; /* code as a Python int */
4624 PyObject *py_code; /* code as a Python int */
3857 PyObject *obj; /* the object to push */
4625 PyObject *obj; /* the object to push */
3858 PyObject *pair; /* (module_name, class_name) */
4626 PyObject *pair; /* (module_name, class_name) */
3859 PyObject *module_name, *class_name;
4627 PyObject *module_name, *class_name;
3860
4628
3861 assert(nbytes == 1 || nbytes == 2 || nbytes == 4);
4629 assert(nbytes == 1 || nbytes == 2 || nbytes == 4);
3862 if (unpickler_read(self, &codebytes, nbytes) < 0)
4630 if (_Unpickler_Read(self, &codebytes, nbytes) < 0)
3863 return -1;
4631 return -1;
3864 code = calc_binint(codebytes, nbytes);
4632 code = calc_binint(codebytes, nbytes);
3865 if (code <= 0) { /* note that 0 is forbidden */
4633 if (code <= 0) { /* note that 0 is forbidden */
3866 /* Corrupt or hostile pickle. */
4634 /* Corrupt or hostile pickle. */
3867 PyErr_SetString(UnpicklingError, "EXT specifies code <= 0");
4635 PyErr_SetString(UnpicklingError, "EXT specifies code <= 0");
3868 return -1;
4636 return -1;
3869 }
4637 }
3870
4638
3871 /* Look for the code in the cache. */
4639 /* Look for the code in the cache. */
3872 py_code = PyLong_FromLong(code);
4640 py_code = PyLong_FromLong(code);
(...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
3913 return -1;
4681 return -1;
3914 }
4682 }
3915 PDATA_PUSH(self->stack, obj, -1);
4683 PDATA_PUSH(self->stack, obj, -1);
3916 return 0;
4684 return 0;
3917 }
4685 }
3918
4686
3919 static int
4687 static int
3920 load_put(UnpicklerObject *self)
4688 load_put(UnpicklerObject *self)
3921 {
4689 {
3922 PyObject *key, *value;
4690 PyObject *key, *value;
4691 Py_ssize_t idx;
3923 Py_ssize_t len;
4692 Py_ssize_t len;
3924 char *s;
4693 char *s;
3925 int x;
3926
4694
3927 if ((len = unpickler_readline(self, &s)) < 0)
4695 if ((len = _Unpickler_Readline(self, &s)) < 0)
3928 return -1;
4696 return -1;
3929 if (len < 2)
4697 if (len < 2)
3930 return bad_readline();
4698 return bad_readline();
3931 if ((x = self->stack->length) <= 0)
4699 if (Py_SIZE(self->stack) <= 0)
3932 return stack_underflow();
4700 return stack_underflow();
4701 value = self->stack->data[Py_SIZE(self->stack) - 1];
3933
4702
3934 key = PyLong_FromString(s, NULL, 10);
4703 key = PyLong_FromString(s, NULL, 10);
3935 if (key == NULL)
4704 if (key == NULL)
3936 return -1;
4705 return -1;
3937 value = self->stack->data[x - 1];
4706 idx = PyLong_AsSsize_t(key);
4707 Py_DECREF(key);
4708 if (idx == -1 && PyErr_Occurred())
4709 return -1;
3938
4710
3939 x = PyDict_SetItem(self->memo, key, value);
4711 return _Unpickler_MemoPut(self, idx, value);
3940 Py_DECREF(key);
3941 return x;
3942 }
4712 }
3943
4713
3944 static int
4714 static int
3945 load_binput(UnpicklerObject *self)
4715 load_binput(UnpicklerObject *self)
3946 {
4716 {
3947 PyObject *key, *value;
4717 PyObject *value;
4718 Py_ssize_t idx;
3948 char *s;
4719 char *s;
3949 int x;
3950
4720
3951 if (unpickler_read(self, &s, 1) < 0)
4721 if (_Unpickler_Read(self, &s, 1) < 0)
3952 return -1;
4722 return -1;
3953 if ((x = self->stack->length) <= 0)
4723
4724 if (Py_SIZE(self->stack) <= 0)
3954 return stack_underflow();
4725 return stack_underflow();
4726 value = self->stack->data[Py_SIZE(self->stack) - 1];
3955
4727
3956 key = PyLong_FromLong((long)(unsigned char)s[0]);
4728 idx = Py_CHARMASK(s[0]);
3957 if (key == NULL)
3958 return -1;
3959 value = self->stack->data[x - 1];
3960
4729
3961 x = PyDict_SetItem(self->memo, key, value);
4730 return _Unpickler_MemoPut(self, idx, value);
3962 Py_DECREF(key);
3963 return x;
3964 }
4731 }
3965
4732
3966 static int
4733 static int
3967 load_long_binput(UnpicklerObject *self)
4734 load_long_binput(UnpicklerObject *self)
3968 {
4735 {
3969 PyObject *key, *value;
4736 PyObject *value;
3970 long k;
4737 Py_ssize_t idx;
3971 char *s;
4738 char *s;
3972 int x;
3973
4739
3974 if (unpickler_read(self, &s, 4) < 0)
4740 if (_Unpickler_Read(self, &s, 4) < 0)
3975 return -1;
4741 return -1;
3976 if ((x = self->stack->length) <= 0)
4742
4743 if (Py_SIZE(self->stack) <= 0)
3977 return stack_underflow();
4744 return stack_underflow();
4745 value = self->stack->data[Py_SIZE(self->stack) - 1];
3978
4746
3979 k = (long)(unsigned char)s[0];
4747 idx = (long)Py_CHARMASK(s[0]);
3980 k |= (long)(unsigned char)s[1] << 8;
4748 idx |= (long)Py_CHARMASK(s[1]) << 8;
3981 k |= (long)(unsigned char)s[2] << 16;
4749 idx |= (long)Py_CHARMASK(s[2]) << 16;
3982 k |= (long)(unsigned char)s[3] << 24;
4750 idx |= (long)Py_CHARMASK(s[3]) << 24;
3983
4751
3984 key = PyLong_FromLong(k);
4752 return _Unpickler_MemoPut(self, idx, value);
3985 if (key == NULL)
3986 return -1;
3987 value = self->stack->data[x - 1];
3988
3989 x = PyDict_SetItem(self->memo, key, value);
3990 Py_DECREF(key);
3991 return x;
3992 }
4753 }
3993
4754
3994 static int
4755 static int
3995 do_append(UnpicklerObject *self, int x)
4756 do_append(UnpicklerObject *self, int x)
3996 {
4757 {
3997 PyObject *value;
4758 PyObject *value;
3998 PyObject *list;
4759 PyObject *list;
3999 int len, i;
4760 int len, i;
4000
4761
4001 len = self->stack->length;
4762 len = Py_SIZE(self->stack);
4002 if (x > len || x <= 0)
4763 if (x > len || x <= 0)
4003 return stack_underflow();
4764 return stack_underflow();
4004 if (len == x) /* nothing to do */
4765 if (len == x) /* nothing to do */
4005 return 0;
4766 return 0;
4006
4767
4007 list = self->stack->data[x - 1];
4768 list = self->stack->data[x - 1];
4008
4769
4009 if (PyList_Check(list)) {
4770 if (PyList_Check(list)) {
4010 PyObject *slice;
4771 PyObject *slice;
4011 Py_ssize_t list_len;
4772 Py_ssize_t list_len;
4012
4773
4013 slice = Pdata_poplist(self->stack, x);
4774 slice = Pdata_poplist(self->stack, x);
4014 if (!slice)
4775 if (!slice)
4015 return -1;
4776 return -1;
4016 list_len = PyList_GET_SIZE(list);
4777 list_len = PyList_GET_SIZE(list);
4017 i = PyList_SetSlice(list, list_len, list_len, slice);
4778 i = PyList_SetSlice(list, list_len, list_len, slice);
4018 Py_DECREF(slice);
4779 Py_DECREF(slice);
4019 return i;
4780 return i;
4020 }
4781 }
4021 else {
4782 else {
4022 PyObject *append_func;
4783 PyObject *append_func;
4023
4784
4024 append_func = PyObject_GetAttrString(list, "append");
4785 append_func = PyObject_GetAttrString(list, "append");
4025 if (append_func == NULL)
4786 if (append_func == NULL)
4026 return -1;
4787 return -1;
4027 for (i = x; i < len; i++) {
4788 for (i = x; i < len; i++) {
4028 PyObject *result;
4789 PyObject *result;
4029
4790
4030 value = self->stack->data[i];
4791 value = self->stack->data[i];
4031 result = unpickler_call(self, append_func, value);
4792 result = _Unpickler_FastCall(self, append_func, value);
4032 if (result == NULL) {
4793 if (result == NULL) {
4033 Pdata_clear(self->stack, i + 1);
4794 Pdata_clear(self->stack, i + 1);
4034 self->stack->length = x;
4795 Py_SIZE(self->stack) = x;
4035 return -1;
4796 return -1;
4036 }
4797 }
4037 Py_DECREF(result);
4798 Py_DECREF(result);
4038 }
4799 }
4039 self->stack->length = x;
4800 Py_SIZE(self->stack) = x;
4040 }
4801 }
4041
4802
4042 return 0;
4803 return 0;
4043 }
4804 }
4044
4805
4045 static int
4806 static int
4046 load_append(UnpicklerObject *self)
4807 load_append(UnpicklerObject *self)
4047 {
4808 {
4048 return do_append(self, self->stack->length - 1);
4809 return do_append(self, Py_SIZE(self->stack) - 1);
4049 }
4810 }
4050
4811
4051 static int
4812 static int
4052 load_appends(UnpicklerObject *self)
4813 load_appends(UnpicklerObject *self)
4053 {
4814 {
4054 return do_append(self, marker(self));
4815 return do_append(self, marker(self));
4055 }
4816 }
4056
4817
4057 static int
4818 static int
4058 do_setitems(UnpicklerObject *self, int x)
4819 do_setitems(UnpicklerObject *self, int x)
4059 {
4820 {
4060 PyObject *value, *key;
4821 PyObject *value, *key;
4061 PyObject *dict;
4822 PyObject *dict;
4062 int len, i;
4823 int len, i;
4063 int status = 0;
4824 int status = 0;
4064
4825
4065 len = self->stack->length;
4826 len = Py_SIZE(self->stack);
4066 if (x > len || x <= 0)
4827 if (x > len || x <= 0)
4067 return stack_underflow();
4828 return stack_underflow();
4068 if (len == x) /* nothing to do */
4829 if (len == x) /* nothing to do */
4069 return 0;
4830 return 0;
4070 if ((len - x) % 2 != 0) {·
4831 if ((len - x) % 2 != 0) {·
4071 /* Currupt or hostile pickle -- we never write one like this. */
4832 /* Currupt or hostile pickle -- we never write one like this. */
4072 PyErr_SetString(UnpicklingError, "odd number of items for SETITEMS");
4833 PyErr_SetString(UnpicklingError, "odd number of items for SETITEMS");
4073 return -1;
4834 return -1;
4074 }
4835 }
4075
4836
(...skipping 10 matching lines...) Expand all Loading...
4086 }
4847 }
4087 }
4848 }
4088
4849
4089 Pdata_clear(self->stack, x);
4850 Pdata_clear(self->stack, x);
4090 return status;
4851 return status;
4091 }
4852 }
4092
4853
4093 static int
4854 static int
4094 load_setitem(UnpicklerObject *self)
4855 load_setitem(UnpicklerObject *self)
4095 {
4856 {
4096 return do_setitems(self, self->stack->length - 2);
4857 return do_setitems(self, Py_SIZE(self->stack) - 2);
4097 }
4858 }
4098
4859
4099 static int
4860 static int
4100 load_setitems(UnpicklerObject *self)
4861 load_setitems(UnpicklerObject *self)
4101 {
4862 {
4102 return do_setitems(self, marker(self));
4863 return do_setitems(self, marker(self));
4103 }
4864 }
4104
4865
4105 static int
4866 static int
4106 load_build(UnpicklerObject *self)
4867 load_build(UnpicklerObject *self)
4107 {
4868 {
4108 PyObject *state, *inst, *slotstate;
4869 PyObject *state, *inst, *slotstate;
4109 PyObject *setstate;
4870 PyObject *setstate;
4110 int status = 0;
4871 int status = 0;
4111
4872
4112 /* Stack is ... instance, state. We want to leave instance at
4873 /* Stack is ... instance, state. We want to leave instance at
4113 * the stack top, possibly mutated via instance.__setstate__(state).
4874 * the stack top, possibly mutated via instance.__setstate__(state).
4114 */
4875 */
4115 if (self->stack->length < 2)
4876 if (Py_SIZE(self->stack) < 2)
4116 return stack_underflow();
4877 return stack_underflow();
4117
4878
4118 PDATA_POP(self->stack, state);
4879 PDATA_POP(self->stack, state);
4119 if (state == NULL)
4880 if (state == NULL)
4120 return -1;
4881 return -1;
4121
4882
4122 inst = self->stack->data[self->stack->length - 1];
4883 inst = self->stack->data[Py_SIZE(self->stack) - 1];
4123
4884
4124 setstate = PyObject_GetAttrString(inst, "__setstate__");
4885 setstate = PyObject_GetAttrString(inst, "__setstate__");
4125 if (setstate == NULL) {
4886 if (setstate == NULL) {
4126 if (PyErr_ExceptionMatches(PyExc_AttributeError))
4887 if (PyErr_ExceptionMatches(PyExc_AttributeError))
4127 PyErr_Clear();
4888 PyErr_Clear();
4128 else {
4889 else {
4129 Py_DECREF(state);
4890 Py_DECREF(state);
4130 return -1;
4891 return -1;
4131 }
4892 }
4132 }
4893 }
4133 else {
4894 else {
4134 PyObject *result;
4895 PyObject *result;
4135
4896
4136 /* The explicit __setstate__ is responsible for everything. */
4897 /* The explicit __setstate__ is responsible for everything. */
4137 /* Ugh... this does not leak since unpickler_call() steals the
4898 /* Ugh... this does not leak since _Unpickler_FastCall() steals the
4138 reference to state first. */
4899 reference to state first. */
4139 result = unpickler_call(self, setstate, state);
4900 result = _Unpickler_FastCall(self, setstate, state);
4140 Py_DECREF(setstate);
4901 Py_DECREF(setstate);
4141 if (result == NULL)
4902 if (result == NULL)
4142 return -1;
4903 return -1;
4143 Py_DECREF(result);
4904 Py_DECREF(result);
4144 return 0;
4905 return 0;
4145 }
4906 }
4146
4907
4147 /* A default __setstate__. First see whether state embeds a
4908 /* A default __setstate__. First see whether state embeds a
4148 * slot state dict too (a proto 2 addition).
4909 * slot state dict too (a proto 2 addition).
4149 */
4910 */
(...skipping 92 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
4242 else
5003 else
4243 marks = (int *)PyMem_Realloc(self->marks, alloc * sizeof(int));
5004 marks = (int *)PyMem_Realloc(self->marks, alloc * sizeof(int));
4244 if (marks == NULL) {
5005 if (marks == NULL) {
4245 PyErr_NoMemory();
5006 PyErr_NoMemory();
4246 return -1;
5007 return -1;
4247 }
5008 }
4248 self->marks = marks;
5009 self->marks = marks;
4249 self->marks_size = (Py_ssize_t)alloc;
5010 self->marks_size = (Py_ssize_t)alloc;
4250 }
5011 }
4251
5012
4252 self->marks[self->num_marks++] = self->stack->length;
5013 self->marks[self->num_marks++] = Py_SIZE(self->stack);
4253
5014
4254 return 0;
5015 return 0;
4255 }
5016 }
4256
5017
4257 static int
5018 static int
4258 load_reduce(UnpicklerObject *self)
5019 load_reduce(UnpicklerObject *self)
4259 {
5020 {
4260 PyObject *callable = NULL;
5021 PyObject *callable = NULL;
4261 PyObject *argtup = NULL;
5022 PyObject *argtup = NULL;
4262 PyObject *obj = NULL;
5023 PyObject *obj = NULL;
(...skipping 17 matching lines...) Expand all Loading...
4280
5041
4281 /* Just raises an error if we don't know the protocol specified. PROTO
5042 /* Just raises an error if we don't know the protocol specified. PROTO
4282 * is the first opcode for protocols >= 2.
5043 * is the first opcode for protocols >= 2.
4283 */
5044 */
4284 static int
5045 static int
4285 load_proto(UnpicklerObject *self)
5046 load_proto(UnpicklerObject *self)
4286 {
5047 {
4287 char *s;
5048 char *s;
4288 int i;
5049 int i;
4289
5050
4290 if (unpickler_read(self, &s, 1) < 0)
5051 if (_Unpickler_Read(self, &s, 1) < 0)
4291 return -1;
5052 return -1;
4292
5053
4293 i = (unsigned char)s[0];
5054 i = (unsigned char)s[0];
4294 if (i <= HIGHEST_PROTOCOL) {
5055 if (i <= HIGHEST_PROTOCOL) {
4295 self->proto = i;
5056 self->proto = i;
4296 return 0;
5057 return 0;
4297 }
5058 }
4298
5059
4299 PyErr_Format(PyExc_ValueError, "unsupported pickle protocol: %d", i);
5060 PyErr_Format(PyExc_ValueError, "unsupported pickle protocol: %d", i);
4300 return -1;
5061 return -1;
4301 }
5062 }
4302
5063
4303 static PyObject *
5064 static PyObject *
4304 load(UnpicklerObject *self)
5065 load(UnpicklerObject *self)
4305 {
5066 {
4306 PyObject *err;
5067 PyObject *err;
4307 PyObject *value = NULL;
5068 PyObject *value = NULL;
4308 char *s;
5069 char *s;
4309
5070
4310 self->num_marks = 0;
5071 self->num_marks = 0;
4311 if (self->stack->length)
5072 if (Py_SIZE(self->stack))
4312 Pdata_clear(self->stack, 0);
5073 Pdata_clear(self->stack, 0);
4313
5074
4314 /* Convenient macros for the dispatch while-switch loop just below. */
5075 /* Convenient macros for the dispatch while-switch loop just below. */
4315 #define OP(opcode, load_func) \
5076 #define OP(opcode, load_func) \
4316 case opcode: if (load_func(self) < 0) break; continue;
5077 case opcode: if (load_func(self) < 0) break; continue;
4317
5078
4318 #define OP_ARG(opcode, load_func, arg) \
5079 #define OP_ARG(opcode, load_func, arg) \
4319 case opcode: if (load_func(self, (arg)) < 0) break; continue;
5080 case opcode: if (load_func(self, (arg)) < 0) break; continue;
4320
5081
4321 while (1) {
5082 while (1) {
4322 if (unpickler_read(self, &s, 1) < 0)
5083 if (_Unpickler_Read(self, &s, 1) < 0)
4323 break;
5084 break;
4324
5085
4325 switch ((enum opcode)s[0]) {
5086 switch ((enum opcode)s[0]) {
4326 OP(NONE, load_none)
5087 OP(NONE, load_none)
4327 OP(BININT, load_binint)
5088 OP(BININT, load_binint)
4328 OP(BININT1, load_binint1)
5089 OP(BININT1, load_binint1)
4329 OP(BININT2, load_binint2)
5090 OP(BININT2, load_binint2)
4330 OP(INT, load_int)
5091 OP(INT, load_int)
4331 OP(LONG, load_long)
5092 OP(LONG, load_long)
4332 OP_ARG(LONG1, load_counted_long, 1)
5093 OP_ARG(LONG1, load_counted_long, 1)
(...skipping 203 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
4536 Unpickler_find_class_doc},
5297 Unpickler_find_class_doc},
4537 {NULL, NULL} /* sentinel */
5298 {NULL, NULL} /* sentinel */
4538 };
5299 };
4539
5300
4540 static void
5301 static void
4541 Unpickler_dealloc(UnpicklerObject *self)
5302 Unpickler_dealloc(UnpicklerObject *self)
4542 {
5303 {
4543 PyObject_GC_UnTrack((PyObject *)self);
5304 PyObject_GC_UnTrack((PyObject *)self);
4544 Py_XDECREF(self->readline);
5305 Py_XDECREF(self->readline);
4545 Py_XDECREF(self->read);
5306 Py_XDECREF(self->read);
4546 Py_XDECREF(self->memo);
4547 Py_XDECREF(self->stack);
5307 Py_XDECREF(self->stack);
4548 Py_XDECREF(self->pers_func);
5308 Py_XDECREF(self->pers_func);
4549 Py_XDECREF(self->arg);
5309 Py_XDECREF(self->arg);
4550 Py_XDECREF(self->last_string);
5310 Py_XDECREF(self->py_input);
4551
5311
5312 _Unpickler_MemoCleanup(self);
4552 PyMem_Free(self->marks);
5313 PyMem_Free(self->marks);
4553 free(self->encoding);
5314 free(self->encoding);
4554 free(self->errors);
5315 free(self->errors);
4555
5316
4556 Py_TYPE(self)->tp_free((PyObject *)self);
5317 Py_TYPE(self)->tp_free((PyObject *)self);
4557 }
5318 }
4558
5319
4559 static int
5320 static int
4560 Unpickler_traverse(UnpicklerObject *self, visitproc visit, void *arg)
5321 Unpickler_traverse(UnpicklerObject *self, visitproc visit, void *arg)
4561 {
5322 {
4562 Py_VISIT(self->readline);
5323 Py_VISIT(self->readline);
4563 Py_VISIT(self->read);
5324 Py_VISIT(self->read);
4564 Py_VISIT(self->memo);
4565 Py_VISIT(self->stack);
5325 Py_VISIT(self->stack);
4566 Py_VISIT(self->pers_func);
5326 Py_VISIT(self->pers_func);
4567 Py_VISIT(self->arg);
5327 Py_VISIT(self->arg);
4568 Py_VISIT(self->last_string);
5328 Py_VISIT(self->py_input);
4569 return 0;
5329 return 0;
4570 }
5330 }
4571
5331
4572 static int
5332 static int
4573 Unpickler_clear(UnpicklerObject *self)
5333 Unpickler_clear(UnpicklerObject *self)
4574 {
5334 {
4575 Py_CLEAR(self->readline);
5335 Py_CLEAR(self->readline);
4576 Py_CLEAR(self->read);
5336 Py_CLEAR(self->read);
4577 Py_CLEAR(self->memo);
4578 Py_CLEAR(self->stack);
5337 Py_CLEAR(self->stack);
4579 Py_CLEAR(self->pers_func);
5338 Py_CLEAR(self->pers_func);
4580 Py_CLEAR(self->arg);
5339 Py_CLEAR(self->arg);
4581 Py_CLEAR(self->last_string);
5340 Py_CLEAR(self->py_input);
4582
5341
5342 _Unpickler_MemoCleanup(self);
4583 PyMem_Free(self->marks);
5343 PyMem_Free(self->marks);
4584 self->marks = NULL;
5344 self->marks = NULL;
4585 free(self->encoding);
5345 free(self->encoding);
4586 self->encoding = NULL;
5346 self->encoding = NULL;
4587 free(self->errors);
5347 free(self->errors);
4588 self->errors = NULL;
5348 self->errors = NULL;
4589
5349
4590 return 0;
5350 return 0;
4591 }
5351 }
4592
5352
(...skipping 18 matching lines...) Expand all Loading...
4611 "map the old Python 2.x names to the new names used in Python 3.x. The\n"
5371 "map the old Python 2.x names to the new names used in Python 3.x. The\n"
4612 "*encoding* and *errors* tell pickle how to decode 8-bit string\n"
5372 "*encoding* and *errors* tell pickle how to decode 8-bit string\n"
4613 "instances pickled by Python 2.x; these default to 'ASCII' and\n"
5373 "instances pickled by Python 2.x; these default to 'ASCII' and\n"
4614 "'strict', respectively.\n");
5374 "'strict', respectively.\n");
4615
5375
4616 static int
5376 static int
4617 Unpickler_init(UnpicklerObject *self, PyObject *args, PyObject *kwds)
5377 Unpickler_init(UnpicklerObject *self, PyObject *args, PyObject *kwds)
4618 {
5378 {
4619 static char *kwlist[] = {"file", "fix_imports", "encoding", "errors", 0};
5379 static char *kwlist[] = {"file", "fix_imports", "encoding", "errors", 0};
4620 PyObject *file;
5380 PyObject *file;
4621 int fix_imports = 1;
5381 PyObject *fix_imports = Py_True;
4622 char *encoding = NULL;
5382 char *encoding = NULL;
4623 char *errors = NULL;
5383 char *errors = NULL;
4624
5384
4625 /* XXX: That is an horrible error message. But, I don't know how to do
5385 /* XXX: That is an horrible error message. But, I don't know how to do
4626 better... */
5386 better... */
4627 if (Py_SIZE(args) != 1) {
5387 if (Py_SIZE(args) != 1) {
4628 PyErr_Format(PyExc_TypeError,
5388 PyErr_Format(PyExc_TypeError,
4629 "%s takes exactly one positional argument (%zd given)",
5389 "%s takes exactly one positional argument (%zd given)",
4630 Py_TYPE(self)->tp_name, Py_SIZE(args));
5390 Py_TYPE(self)->tp_name, Py_SIZE(args));
4631 return -1;
5391 return -1;
4632 }
5392 }
4633
5393
4634 /* Arguments parsing needs to be done in the __init__() method to allow
5394 /* Arguments parsing needs to be done in the __init__() method to allow
4635 subclasses to define their own __init__() method, which may (or may
5395 subclasses to define their own __init__() method, which may (or may
4636 not) support Unpickler arguments. However, this means we need to be
5396 not) support Unpickler arguments. However, this means we need to be
4637 extra careful in the other Unpickler methods, since a subclass could
5397 extra careful in the other Unpickler methods, since a subclass could
4638 forget to call Unpickler.__init__() thus breaking our internal
5398 forget to call Unpickler.__init__() thus breaking our internal
4639 invariants. */
5399 invariants. */
4640 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|iss:Unpickler", kwlist,
5400 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|Oss:Unpickler", kwlist,
4641 &file, &fix_imports, &encoding, &errors))
5401 &file, &fix_imports, &encoding, &errors))
4642 return -1;
5402 return -1;
4643
5403
4644 /* In case of multiple __init__() calls, clear previous content. */
5404 /* In case of multiple __init__() calls, clear previous content. */
4645 if (self->read != NULL)
5405 if (self->read != NULL)
4646 (void)Unpickler_clear(self);
5406 (void)Unpickler_clear(self);
4647
5407
4648 self->read = PyObject_GetAttrString(file, "read");
5408 if (_Unpickler_SetInputStream(self, file) < 0)
4649 self->readline = PyObject_GetAttrString(file, "readline");
4650 if (self->readline == NULL || self->read == NULL)
4651 return -1;
5409 return -1;
4652
5410
4653 if (encoding == NULL)
5411 if (_Unpickler_SetInputEncoding(self, encoding, errors) < 0)
4654 encoding = "ASCII";
5412 return -1;
4655 if (errors == NULL)
4656 errors = "strict";
4657
5413
4658 self->encoding = strdup(encoding);
5414 self->fix_imports = PyObject_IsTrue(fix_imports);
4659 self->errors = strdup(errors);
5415 if (self->fix_imports == -1)
4660 if (self->encoding == NULL || self->errors == NULL) {
4661 PyErr_NoMemory();
4662 return -1;
5416 return -1;
4663 }
4664
5417
4665 if (PyObject_HasAttrString((PyObject *)self, "persistent_load")) {
5418 if (PyObject_HasAttrString((PyObject *)self, "persistent_load")) {
4666 self->pers_func = PyObject_GetAttrString((PyObject *)self,
5419 self->pers_func = PyObject_GetAttrString((PyObject *)self,
4667 "persistent_load");
5420 "persistent_load");
4668 if (self->pers_func == NULL)
5421 if (self->pers_func == NULL)
4669 return -1;
5422 return -1;
4670 }
5423 }
4671 else {
5424 else {
4672 self->pers_func = NULL;
5425 self->pers_func = NULL;
4673 }
5426 }
4674
5427
4675 self->stack = (Pdata *)Pdata_New();
5428 self->stack = (Pdata *)Pdata_New();
4676 if (self->stack == NULL)
5429 if (self->stack == NULL)
4677 return -1;
5430 return -1;
4678
5431
4679 self->memo = PyDict_New();
5432 self->memo_size = 32;
5433 self->memo = _Unpickler_NewMemo(self->memo_size);
4680 if (self->memo == NULL)
5434 if (self->memo == NULL)
4681 return -1;
5435 return -1;
4682
5436
4683 self->last_string = NULL;
5437 self->py_input = NULL;
4684 self->arg = NULL;
5438 self->arg = NULL;
4685 self->proto = 0;
5439 self->proto = 0;
4686 self->fix_imports = fix_imports;
5440
4687
5441 return 0;
4688 return 0;
5442 }
4689 }
5443
5444 /* Define a proxy object for the Unpickler's internal memo object. This is to
5445 * avoid breaking code like:
5446 * unpickler.memo.clear()
5447 * and
5448 * unpickler.memo = saved_memo
5449 * Is this a good idea? Not really, but we don't want to break code that uses
5450 * it. Note that we don't implement the entire mapping API here. This is
5451 * intentional, as these should be treated as black-box implementation details.
5452 *
5453 * We do, however, have to implement pickling/unpickling support because of
5454 * real-world code like cvs2svn.·
5455 */
5456
5457 typedef struct {
5458 PyObject_HEAD
5459 UnpicklerObject *unpickler;
5460 } UnpicklerMemoProxyObject;
5461
5462 PyDoc_STRVAR(ump_clear_doc,
5463 "memo.clear() -> None. Remove all items from memo.");
5464
5465 static PyObject *
5466 ump_clear(UnpicklerMemoProxyObject *self)
5467 {
5468 _Unpickler_MemoCleanup(self->unpickler);
5469 self->unpickler->memo = _Unpickler_NewMemo(self->unpickler->memo_size);
5470 if (self->unpickler->memo == NULL)
5471 return NULL;
5472 Py_RETURN_NONE;
5473 }
5474
5475 PyDoc_STRVAR(ump_copy_doc,
5476 "memo.copy() -> new_memo. Copy the memo to a new object.");
5477
5478 static PyObject *
5479 ump_copy(UnpicklerMemoProxyObject *self)
5480 {
5481 Py_ssize_t i;
5482 PyObject *new_memo = PyDict_New();
5483 if (new_memo == NULL)
5484 return NULL;
5485
5486 for (i = 0; i < self->unpickler->memo_size; i++) {
5487 int status;
5488 PyObject *key, *value;
5489
5490 value = self->unpickler->memo[i];
5491 if (value == NULL)
5492 continue;
5493
5494 key = PyLong_FromSsize_t(i);
5495 if (key == NULL)
5496 goto error;
5497 status = PyDict_SetItem(new_memo, key, value);
5498 Py_DECREF(key);
5499 if (status < 0)
5500 goto error;
5501 }
5502 return new_memo;
5503
5504 error:
5505 Py_DECREF(new_memo);
5506 return NULL;
5507 }
5508
5509 PyDoc_STRVAR(ump_reduce_doc,
5510 "memo.__reduce__(). Pickling support.");
5511
5512 static PyObject *
5513 ump_reduce(UnpicklerMemoProxyObject *self, PyObject *args)
5514 {
5515 PyObject *reduce_value;
5516 PyObject *constructor_args;
5517 PyObject *contents = ump_copy(self);
5518 if (contents == NULL)
5519 return NULL;
5520
5521 reduce_value = PyTuple_New(2);
5522 if (reduce_value == NULL) {
5523 Py_DECREF(contents);
5524 return NULL;
5525 }
5526 constructor_args = PyTuple_New(1);
5527 if (constructor_args == NULL) {
5528 Py_DECREF(contents);
5529 Py_DECREF(reduce_value);
5530 return NULL;
5531 }
5532 PyTuple_SET_ITEM(constructor_args, 0, contents);
5533 Py_INCREF((PyObject *)&PyDict_Type);
5534 PyTuple_SET_ITEM(reduce_value, 0, (PyObject *)&PyDict_Type);
5535 PyTuple_SET_ITEM(reduce_value, 1, constructor_args);
5536 return reduce_value;
5537 }
5538
5539 static PyMethodDef unpicklerproxy_methods[] = {
5540 {"clear", (PyCFunction)ump_clear, METH_NOARGS, ump_clear_doc},
5541 {"copy", (PyCFunction)ump_copy, METH_NOARGS, ump_copy_doc},
5542 {"__reduce__", (PyCFunction)ump_reduce, METH_VARARGS, ump_reduce_doc},
5543 {NULL, NULL} /* sentinel */
5544 };
5545
5546 static void
5547 UnpicklerMemoProxy_dealloc(UnpicklerMemoProxyObject *self)
5548 {
5549 PyObject_GC_UnTrack(self);
5550 Py_XDECREF(self->unpickler);
5551 PyObject_GC_Del((PyObject *)self);
5552 }
5553
5554 static int
5555 UnpicklerMemoProxy_traverse(UnpicklerMemoProxyObject *self,
5556 visitproc visit, void *arg)
5557 {
5558 Py_VISIT(self->unpickler);
5559 return 0;
5560 }
5561
5562 static int
5563 UnpicklerMemoProxy_clear(UnpicklerMemoProxyObject *self)
5564 {
5565 Py_CLEAR(self->unpickler);
5566 return 0;
5567 }
5568
5569 static PyTypeObject UnpicklerMemoProxyType = {
5570 PyVarObject_HEAD_INIT(NULL, 0)
5571 "_pickle.UnpicklerMemoProxy", /*tp_name*/
5572 sizeof(UnpicklerMemoProxyObject), /*tp_basicsize*/
5573 0,
5574 (destructor)UnpicklerMemoProxy_dealloc, /* tp_dealloc */
5575 0, /* tp_print */
5576 0, /* tp_getattr */
5577 0, /* tp_setattr */
5578 0, /* tp_compare */
5579 0, /* tp_repr */
5580 0, /* tp_as_number */
5581 0, /* tp_as_sequence */
5582 0, /* tp_as_mapping */
5583 (hashfunc)PyObject_HashNotImplemented, /* tp_hash */
5584 0, /* tp_call */
5585 0, /* tp_str */
5586 PyObject_GenericGetAttr, /* tp_getattro */
5587 PyObject_GenericSetAttr, /* tp_setattro */
5588 0, /* tp_as_buffer */
5589 Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC,
5590 0, /* tp_doc */
5591 (traverseproc)UnpicklerMemoProxy_traverse, /* tp_traverse */
5592 (inquiry)UnpicklerMemoProxy_clear, /* tp_clear */
5593 0, /* tp_richcompare */
5594 0, /* tp_weaklistoffset */
5595 0, /* tp_iter */
5596 0, /* tp_iternext */
5597 unpicklerproxy_methods, /* tp_methods */
5598 };
5599
5600 static PyObject *
5601 UnpicklerMemoProxy_New(UnpicklerObject *unpickler)
5602 {
5603 UnpicklerMemoProxyObject *self;
5604
5605 self = PyObject_GC_New(UnpicklerMemoProxyObject,
5606 &UnpicklerMemoProxyType);
5607 if (self == NULL)
5608 return NULL;
5609 Py_INCREF(unpickler);
5610 self->unpickler = unpickler;
5611 PyObject_GC_Track(self);
5612 return (PyObject *)self;
5613 }
5614
5615 /*****************************************************************************/
5616
4690
5617
4691 static PyObject *
5618 static PyObject *
4692 Unpickler_get_memo(UnpicklerObject *self)
5619 Unpickler_get_memo(UnpicklerObject *self)
4693 {
5620 {
4694 if (self->memo == NULL)
5621 return UnpicklerMemoProxy_New(self);
4695 PyErr_SetString(PyExc_AttributeError, "memo");
4696 else
4697 Py_INCREF(self->memo);
4698 return self->memo;
4699 }
5622 }
4700
5623
4701 static int
5624 static int
4702 Unpickler_set_memo(UnpicklerObject *self, PyObject *value)
5625 Unpickler_set_memo(UnpicklerObject *self, PyObject *obj)
4703 {
5626 {
4704 PyObject *tmp;
5627 PyObject **new_memo;
4705
5628 Py_ssize_t new_memo_size = 0;
4706 if (value == NULL) {
5629 Py_ssize_t i;
5630
5631 if (obj == NULL) {
4707 PyErr_SetString(PyExc_TypeError,
5632 PyErr_SetString(PyExc_TypeError,
4708 "attribute deletion is not supported");
5633 "attribute deletion is not supported");
4709 return -1;
5634 return -1;
4710 }
5635 }
4711 if (!PyDict_Check(value)) {
5636
4712 PyErr_SetString(PyExc_TypeError, "memo must be a dictionary");
5637 if (Py_TYPE(obj) == &UnpicklerMemoProxyType) {
4713 return -1;
5638 UnpicklerObject *unpickler =
4714 }
5639 ((UnpicklerMemoProxyObject *)obj)->unpickler;
4715
5640
4716 tmp = self->memo;
5641 new_memo_size = unpickler->memo_size;
4717 Py_INCREF(value);
5642 new_memo = _Unpickler_NewMemo(new_memo_size);
4718 self->memo = value;
5643 if (new_memo == NULL)
4719 Py_XDECREF(tmp);
5644 return -1;
4720
5645
4721 return 0;
5646 for (i = 0; i < new_memo_size; i++) {
4722 }
5647 Py_XINCREF(unpickler->memo[i]);
4723
5648 new_memo[i] = unpickler->memo[i];
4724 static PyObject *
5649 }
5650 }
5651 else if (PyDict_Check(obj)) {
5652 Py_ssize_t i = 0;
5653 PyObject *key, *value;
5654
5655 new_memo_size = PyDict_Size(obj);
5656 new_memo = _Unpickler_NewMemo(new_memo_size);
5657 if (new_memo == NULL)
5658 return -1;
5659
5660 while (PyDict_Next(obj, &i, &key, &value)) {
5661 Py_ssize_t idx;
5662 if (!PyLong_Check(key)) {
5663 PyErr_SetString(PyExc_TypeError,
5664 "memo key must be integers");
5665 goto error;
5666 }
5667 idx = PyLong_AsSsize_t(key);
5668 if (idx == -1 && PyErr_Occurred())
5669 goto error;
5670 if (_Unpickler_MemoPut(self, idx, value) < 0)
5671 goto error;
5672 }
5673 }
5674 else {
5675 PyErr_Format(PyExc_TypeError,
5676 "'memo' attribute must be an UnpicklerMemoProxy object"
5677 "or dict, not %.200s", Py_TYPE(obj)->tp_name);
5678 return -1;
5679 }
5680
5681 _Unpickler_MemoCleanup(self);
5682 self->memo_size = new_memo_size;
5683 self->memo = new_memo;
5684
5685 return 0;
5686
5687 error:
5688 if (new_memo_size) {
5689 i = new_memo_size;
5690 while (--i >= 0) {
5691 Py_XDECREF(new_memo[i]);
5692 }
5693 PyMem_FREE(new_memo);
5694 }
5695 return -1;
5696 }
5697
5698 static PyObject *
4725 Unpickler_get_persload(UnpicklerObject *self)
5699 Unpickler_get_persload(UnpicklerObject *self)
4726 {
5700 {
4727 if (self->pers_func == NULL)
5701 if (self->pers_func == NULL)
4728 PyErr_SetString(PyExc_AttributeError, "persistent_load");
5702 PyErr_SetString(PyExc_AttributeError, "persistent_load");
4729 else
5703 else
4730 Py_INCREF(self->pers_func);
5704 Py_INCREF(self->pers_func);
4731 return self->pers_func;
5705 return self->pers_func;
4732 }
5706 }
4733
5707
4734 static int
5708 static int
(...skipping 29 matching lines...) Expand all Loading...
4764 };
5738 };
4765
5739
4766 static PyTypeObject Unpickler_Type = {
5740 static PyTypeObject Unpickler_Type = {
4767 PyVarObject_HEAD_INIT(NULL, 0)
5741 PyVarObject_HEAD_INIT(NULL, 0)
4768 "_pickle.Unpickler", /*tp_name*/
5742 "_pickle.Unpickler", /*tp_name*/
4769 sizeof(UnpicklerObject), /*tp_basicsize*/
5743 sizeof(UnpicklerObject), /*tp_basicsize*/
4770 0, /*tp_itemsize*/
5744 0, /*tp_itemsize*/
4771 (destructor)Unpickler_dealloc, /*tp_dealloc*/
5745 (destructor)Unpickler_dealloc, /*tp_dealloc*/
4772 0, /*tp_print*/
5746 0, /*tp_print*/
4773 0, /*tp_getattr*/
5747 0, /*tp_getattr*/
4774 0,» /*tp_setattr*/
5748 0, /*tp_setattr*/
4775 0, /*tp_reserved*/
5749 0, /*tp_reserved*/
4776 0, /*tp_repr*/
5750 0, /*tp_repr*/
4777 0, /*tp_as_number*/
5751 0, /*tp_as_number*/
4778 0, /*tp_as_sequence*/
5752 0, /*tp_as_sequence*/
4779 0, /*tp_as_mapping*/
5753 0, /*tp_as_mapping*/
4780 0, /*tp_hash*/
5754 0, /*tp_hash*/
4781 0, /*tp_call*/
5755 0, /*tp_call*/
4782 0, /*tp_str*/
5756 0, /*tp_str*/
4783 0, /*tp_getattro*/
5757 0, /*tp_getattro*/
4784 0, /*tp_setattro*/
5758 0, /*tp_setattro*/
(...skipping 14 matching lines...) Expand all Loading...
4799 0, /*tp_descr_get*/
5773 0, /*tp_descr_get*/
4800 0, /*tp_descr_set*/
5774 0, /*tp_descr_set*/
4801 0, /*tp_dictoffset*/
5775 0, /*tp_dictoffset*/
4802 (initproc)Unpickler_init, /*tp_init*/
5776 (initproc)Unpickler_init, /*tp_init*/
4803 PyType_GenericAlloc, /*tp_alloc*/
5777 PyType_GenericAlloc, /*tp_alloc*/
4804 PyType_GenericNew, /*tp_new*/
5778 PyType_GenericNew, /*tp_new*/
4805 PyObject_GC_Del, /*tp_free*/
5779 PyObject_GC_Del, /*tp_free*/
4806 0, /*tp_is_gc*/
5780 0, /*tp_is_gc*/
4807 };
5781 };
4808
5782
5783 PyDoc_STRVAR(pickle_dump_doc,
5784 "dump(obj, file, protocol=None, *, fix_imports=True) -> None\n"
5785 "\n"
5786 "Write a pickled representation of obj to the open file object file. This\n"
5787 "is equivalent to ``Pickler(file, protocol).dump(obj)``, but may be more\n"
5788 "efficient.\n"
5789 "\n"
5790 "The optional protocol argument tells the pickler to use the given protocol;\n"
5791 "supported protocols are 0, 1, 2, 3. The default protocol is 3; a\n"
5792 "backward-incompatible protocol designed for Python 3.0.\n"
5793 "\n"
5794 "Specifying a negative protocol version selects the highest protocol version\n"
5795 "supported. The higher the protocol used, the more recent the version of\n"
5796 "Python needed to read the pickle produced.\n"
5797 "\n"
5798 "The file argument must have a write() method that accepts a single bytes\n"
5799 "argument. It can thus be a file object opened for binary writing, a\n"
5800 "io.BytesIO instance, or any other custom object that meets this interface.\n"
5801 "\n"
5802 "If fix_imports is True and protocol is less than 3, pickle will try to\n"
5803 "map the new Python 3.x names to the old module names used in Python 2.x,\n"
5804 "so that the pickle data stream is readable with Python 2.x.\n");
5805
5806 static PyObject *
5807 pickle_dump(PyObject *self, PyObject *args, PyObject *kwds)
5808 {
5809 static char *kwlist[] = {"obj", "file", "protocol", "fix_imports", 0};
5810 PyObject *obj;
5811 PyObject *file;
5812 PyObject *proto = NULL;
5813 PyObject *fix_imports = Py_True;
5814 PicklerObject *pickler;
5815
5816 /* fix_imports is a keyword-only argument. */
5817 if (Py_SIZE(args) > 3) {
5818 PyErr_Format(PyExc_TypeError,
5819 "pickle.dump() takes at most 3 positional "
5820 "argument (%zd given)", Py_SIZE(args));
5821 return NULL;
5822 }
5823
5824 if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO|OO:dump", kwlist,
5825 &obj, &file, &proto, &fix_imports))
5826 return NULL;
5827
5828 pickler = _Pickler_New();
5829 if (pickler == NULL)
5830 return NULL;
5831
5832 if (_Pickler_SetProtocol(pickler, proto, fix_imports) < 0)
5833 goto error;
5834
5835 if (_Pickler_SetOutputStream(pickler, file) < 0)
5836 goto error;
5837
5838 if (dump(pickler, obj) < 0)
5839 goto error;
5840
5841 if (_Pickler_FlushToFile(pickler) < 0)
5842 goto error;
5843
5844 Py_DECREF(pickler);
5845 Py_RETURN_NONE;
5846
5847 error:
5848 Py_XDECREF(pickler);
5849 return NULL;
5850 }
5851
5852 PyDoc_STRVAR(pickle_dumps_doc,
5853 "dumps(obj, protocol=None, *, fix_imports=True) -> bytes\n"
5854 "\n"
5855 "Return the pickled representation of the object as a bytes\n"
5856 "object, instead of writing it to a file.\n"
5857 "\n"
5858 "The optional protocol argument tells the pickler to use the given protocol;\n"
5859 "supported protocols are 0, 1, 2, 3. The default protocol is 3; a\n"
5860 "backward-incompatible protocol designed for Python 3.0.\n"
5861 "\n"
5862 "Specifying a negative protocol version selects the highest protocol version\n"
5863 "supported. The higher the protocol used, the more recent the version of\n"
5864 "Python needed to read the pickle produced.\n"
5865 "\n"
5866 "If fix_imports is True and *protocol* is less than 3, pickle will try to\n"
5867 "map the new Python 3.x names to the old module names used in Python 2.x,\n"
5868 "so that the pickle data stream is readable with Python 2.x.\n");
5869
5870 static PyObject *
5871 pickle_dumps(PyObject *self, PyObject *args, PyObject *kwds)
5872 {
5873 static char *kwlist[] = {"obj", "protocol", "fix_imports", 0};
5874 PyObject *obj;
5875 PyObject *proto = NULL;
5876 PyObject *result;
5877 PyObject *fix_imports = Py_True;
5878 PicklerObject *pickler;
5879
5880 /* fix_imports is a keyword-only argument. */
5881 if (Py_SIZE(args) > 2) {
5882 PyErr_Format(PyExc_TypeError,
5883 "pickle.dumps() takes at most 2 positional "
5884 "argument (%zd given)", Py_SIZE(args));
5885 return NULL;
5886 }
5887
5888 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO:dumps", kwlist,
5889 &obj, &proto, &fix_imports))
5890 return NULL;
5891
5892 pickler = _Pickler_New();
5893 if (pickler == NULL)
5894 return NULL;
5895
5896 if (_Pickler_SetProtocol(pickler, proto, fix_imports) < 0)
5897 goto error;
5898
5899 if (dump(pickler, obj) < 0)
5900 goto error;
5901
5902 result = _Pickler_GetString(pickler);
5903 Py_DECREF(pickler);
5904 return result;
5905
5906 error:
5907 Py_XDECREF(pickler);
5908 return NULL;
5909 }
5910
5911 PyDoc_STRVAR(pickle_load_doc,
5912 "load(file, *, fix_imports=True, encoding='ASCII', errors='strict') -> object\n"
5913 "\n"
5914 "Read a pickled object representation from the open file object file and\n"
5915 "return the reconstituted object hierarchy specified therein. This is\n"
5916 "equivalent to ``Unpickler(file).load()``, but may be more efficient.\n"
5917 "\n"
5918 "The protocol version of the pickle is detected automatically, so no protocol\n"
5919 "argument is needed. Bytes past the pickled object's representation are\n"
5920 "ignored.\n"
5921 "\n"
5922 "The argument file must have two methods, a read() method that takes an\n"
5923 "integer argument, and a readline() method that requires no arguments. Both\n"
5924 "methods should return bytes. Thus *file* can be a binary file object opened\n"
5925 "for reading, a BytesIO object, or any other custom object that meets this\n"
5926 "interface.\n"
5927 "\n"
5928 "Optional keyword arguments are fix_imports, encoding and errors,\n"
5929 "which are used to control compatiblity support for pickle stream generated\n"
5930 "by Python 2.x. If fix_imports is True, pickle will try to map the old\n"
5931 "Python 2.x names to the new names used in Python 3.x. The encoding and\n"
5932 "errors tell pickle how to decode 8-bit string instances pickled by Python\n"
5933 "2.x; these default to 'ASCII' and 'strict', respectively.\n");
5934
5935 static PyObject *
5936 pickle_load(PyObject *self, PyObject *args, PyObject *kwds)
5937 {
5938 static char *kwlist[] = {"file", "fix_imports", "encoding", "errors", 0};
5939 PyObject *file;
5940 PyObject *fix_imports = Py_True;
5941 PyObject *result;
5942 char *encoding = NULL;
5943 char *errors = NULL;
5944 UnpicklerObject *unpickler;
5945
5946 /* fix_imports, encoding and errors are a keyword-only argument. */
5947 if (Py_SIZE(args) != 1) {
5948 PyErr_Format(PyExc_TypeError,
5949 "pickle.load() takes exactly one positional "
5950 "argument (%zd given)", Py_SIZE(args));
5951 return NULL;
5952 }
5953
5954 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|Oss:load", kwlist,
5955 &file, &fix_imports, &encoding, &errors))
5956 return NULL;
5957
5958 unpickler = _Unpickler_New();
5959 if (unpickler == NULL)
5960 return NULL;
5961
5962 if (_Unpickler_SetInputStream(unpickler, file) < 0)
5963 goto error;
5964
5965 if (_Unpickler_SetInputEncoding(unpickler, encoding, errors) < 0)
5966 goto error;
5967
5968 unpickler->fix_imports = PyObject_IsTrue(fix_imports);
5969 if (unpickler->fix_imports == -1)
5970 goto error;
5971
5972 result = load(unpickler);
5973 Py_DECREF(unpickler);
5974 return result;
5975
5976 error:
5977 Py_XDECREF(unpickler);
5978 return NULL;
5979 }
5980
5981 PyDoc_STRVAR(pickle_loads_doc,
5982 "loads(input, *, fix_imports=True, encoding='ASCII', errors='strict') -> object\ n"
5983 "\n"
5984 "Read a pickled object hierarchy from a bytes object and return the\n"
5985 "reconstituted object hierarchy specified therein\n"
5986 "\n"
5987 "The protocol version of the pickle is detected automatically, so no protocol\n"
5988 "argument is needed. Bytes past the pickled object's representation are\n"
5989 "ignored.\n"
5990 "\n"
5991 "Optional keyword arguments are fix_imports, encoding and errors, which\n"
5992 "are used to control compatiblity support for pickle stream generated\n"
5993 "by Python 2.x. If fix_imports is True, pickle will try to map the old\n"
5994 "Python 2.x names to the new names used in Python 3.x. The encoding and\n"
5995 "errors tell pickle how to decode 8-bit string instances pickled by Python\n"
5996 "2.x; these default to 'ASCII' and 'strict', respectively.\n");
5997
5998 static PyObject *
5999 pickle_loads(PyObject *self, PyObject *args, PyObject *kwds)
6000 {
6001 static char *kwlist[] = {"input", "fix_imports", "encoding", "errors", 0};
6002 PyObject *input;
6003 PyObject *fix_imports = Py_True;
6004 PyObject *result;
6005 char *encoding = NULL;
6006 char *errors = NULL;
6007 UnpicklerObject *unpickler;
6008
6009 /* fix_imports, encoding and errors are a keyword-only argument. */
6010 if (Py_SIZE(args) != 1) {
6011 PyErr_Format(PyExc_TypeError,
6012 "pickle.load() takes exactly one positional "
6013 "argument (%zd given)", Py_SIZE(args));
6014 return NULL;
6015 }
6016
6017 if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|Oss:loads", kwlist,
6018 &input, &fix_imports, &encoding, &errors))
6019 return NULL;
6020
6021 unpickler = _Unpickler_New();
6022 if (unpickler == NULL)
6023 return NULL;
6024
6025 if (_Unpickler_SetStringInput(unpickler, input) < 0)
6026 goto error;
6027
6028 if (_Unpickler_SetInputEncoding(unpickler, encoding, errors) < 0)
6029 goto error;
6030
6031 unpickler->fix_imports = PyObject_IsTrue(fix_imports);
6032 if (unpickler->fix_imports == -1)
6033 goto error;
6034
6035 result = load(unpickler);
6036 Py_DECREF(unpickler);
6037 return result;
6038
6039 error:
6040 Py_XDECREF(unpickler);
6041 return NULL;
6042 }
6043
6044
6045 static struct PyMethodDef pickle_methods[] = {
6046 {"dump", (PyCFunction)pickle_dump, METH_VARARGS|METH_KEYWORDS,
6047 pickle_dump_doc},
6048 {"dumps", (PyCFunction)pickle_dumps, METH_VARARGS|METH_KEYWORDS,
6049 pickle_dumps_doc},
6050 {"load", (PyCFunction)pickle_load, METH_VARARGS|METH_KEYWORDS,
6051 pickle_load_doc},
6052 {"loads", (PyCFunction)pickle_loads, METH_VARARGS|METH_KEYWORDS,
6053 pickle_loads_doc},
6054 {NULL, NULL} /* sentinel */
6055 };
6056
4809 static int
6057 static int
4810 initmodule(void)
6058 initmodule(void)
4811 {
6059 {
4812 PyObject *copyreg = NULL;
6060 PyObject *copyreg = NULL;
4813 PyObject *compat_pickle = NULL;
6061 PyObject *compat_pickle = NULL;
4814
6062
4815 /* XXX: We should ensure that the types of the dictionaries imported are
6063 /* XXX: We should ensure that the types of the dictionaries imported are
4816 exactly PyDict objects. Otherwise, it is possible to crash the pickle
6064 exactly PyDict objects. Otherwise, it is possible to crash the pickle
4817 since we use the PyDict API directly to access these dictionaries. */
6065 since we use the PyDict API directly to access these dictionaries. */
4818
6066
(...skipping 89 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading...
4908 Py_CLEAR(empty_tuple);
6156 Py_CLEAR(empty_tuple);
4909 Py_CLEAR(two_tuple);
6157 Py_CLEAR(two_tuple);
4910 return -1;
6158 return -1;
4911 }
6159 }
4912
6160
4913 static struct PyModuleDef _picklemodule = {
6161 static struct PyModuleDef _picklemodule = {
4914 PyModuleDef_HEAD_INIT,
6162 PyModuleDef_HEAD_INIT,
4915 "_pickle",
6163 "_pickle",
4916 pickle_module_doc,
6164 pickle_module_doc,
4917 -1,
6165 -1,
4918 NULL,
6166 pickle_methods,
4919 NULL,
6167 NULL,
4920 NULL,
6168 NULL,
4921 NULL,
6169 NULL,
4922 NULL
6170 NULL
4923 };
6171 };
4924
6172
4925 PyMODINIT_FUNC
6173 PyMODINIT_FUNC
4926 PyInit__pickle(void)
6174 PyInit__pickle(void)
4927 {
6175 {
4928 PyObject *m;
6176 PyObject *m;
4929
6177
4930 if (PyType_Ready(&Unpickler_Type) < 0)
6178 if (PyType_Ready(&Unpickler_Type) < 0)
4931 return NULL;
6179 return NULL;
4932 if (PyType_Ready(&Pickler_Type) < 0)
6180 if (PyType_Ready(&Pickler_Type) < 0)
4933 return NULL;
6181 return NULL;
4934 if (PyType_Ready(&Pdata_Type) < 0)
6182 if (PyType_Ready(&Pdata_Type) < 0)
4935 return NULL;
6183 return NULL;
6184 if (PyType_Ready(&PicklerMemoProxyType) < 0)
6185 return NULL;
6186 if (PyType_Ready(&UnpicklerMemoProxyType) < 0)
6187 return NULL;
4936
6188
4937 /* Create the module and add the functions. */
6189 /* Create the module and add the functions. */
4938 m = PyModule_Create(&_picklemodule);
6190 m = PyModule_Create(&_picklemodule);
4939 if (m == NULL)
6191 if (m == NULL)
4940 return NULL;
6192 return NULL;
4941
6193
4942 if (PyModule_AddObject(m, "Pickler", (PyObject *)&Pickler_Type) < 0)
6194 if (PyModule_AddObject(m, "Pickler", (PyObject *)&Pickler_Type) < 0)
4943 return NULL;
6195 return NULL;
4944 if (PyModule_AddObject(m, "Unpickler", (PyObject *)&Unpickler_Type) < 0)
6196 if (PyModule_AddObject(m, "Unpickler", (PyObject *)&Unpickler_Type) < 0)
4945 return NULL;
6197 return NULL;
(...skipping 16 matching lines...) Expand all Loading...
4962 if (PyModule_AddObject(m, "PicklingError", PicklingError) < 0)
6214 if (PyModule_AddObject(m, "PicklingError", PicklingError) < 0)
4963 return NULL;
6215 return NULL;
4964 if (PyModule_AddObject(m, "UnpicklingError", UnpicklingError) < 0)
6216 if (PyModule_AddObject(m, "UnpicklingError", UnpicklingError) < 0)
4965 return NULL;
6217 return NULL;
4966
6218
4967 if (initmodule() < 0)
6219 if (initmodule() < 0)
4968 return NULL;
6220 return NULL;
4969
6221
4970 return m;
6222 return m;
4971 }
6223 }
OLD
NEW