cpython: 42dd11028e94 (original) (raw)

--- a/Lib/importlib/test/source/test_file_loader.py +++ b/Lib/importlib/test/source/test_file_loader.py @@ -214,7 +214,7 @@ class BadBytecodeTest(unittest.TestCase) lambda bc: bc[:8] + b'', del_source=del_source) file_path = mapping['_temp'] if not del_source else bytecode_path

       with self.assertRaises(ValueError):[](#l1.7)

       with self.assertRaises(EOFError):[](#l1.8)
           self.import_(file_path, '_temp')[](#l1.9)

def _test_bad_magic(self, test, *, del_source=False):

--- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -228,6 +228,30 @@ class BugsTestCase(unittest.TestCase): invalid_string = b'l\x02\x00\x00\x00\x00\x00\x00\x00' self.assertRaises(ValueError, marshal.loads, invalid_string)

def test_multiple_dumps_and_loads(self):

   # Issue 12291: marshal.load() should be callable multiple times[](#l2.8)

   # with interleaved data written by non-marshal code[](#l2.9)

   # Adapted from a patch by Engelbert Gruber.[](#l2.10)

   data = (1, 'abc', b'def', 1.0, (2, 'a', ['b', b'c']))[](#l2.11)

   for interleaved in (b'', b'0123'):[](#l2.12)

       ilen = len(interleaved)[](#l2.13)

```
       positions = [][](#l2.14)
```
```
       try:[](#l2.15)
```

           with open(support.TESTFN, 'wb') as f:[](#l2.16)

```
               for d in data:[](#l2.17)
```

                   marshal.dump(d, f)[](#l2.18)

```
                   if ilen:[](#l2.19)
```

                       f.write(interleaved)[](#l2.20)

                   positions.append(f.tell())[](#l2.21)

           with open(support.TESTFN, 'rb') as f:[](#l2.22)

               for i, d in enumerate(data):[](#l2.23)

                   self.assertEqual(d, marshal.load(f))[](#l2.24)

```
                   if ilen:[](#l2.25)
```

                       f.read(ilen)[](#l2.26)

                   self.assertEqual(positions[i], f.tell())[](#l2.27)

```
       finally:[](#l2.28)
```

           support.unlink(support.TESTFN)[](#l2.29)

+ def test_main(): support.run_unittest(IntTestCase,

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,9 @@ What's New in Python 3.3 Alpha 1? Core and Builtins ----------------- +- Issue #12291: You can now load multiple marshalled objects from a stream,

with other data interleaved between marshalled objects. +

Issue #12356: When required positional or keyword-only arguments are not given, produce a informative error message which includes the name(s) of the missing arguments.

--- a/Python/marshal.c +++ b/Python/marshal.c @@ -57,6 +57,7 @@ typedef struct { int error; /* see WFERR_* values / int depth; / If fp == NULL, the following are valid: */

PyObject * readable; /* Stream-like object being read from */ PyObject *str; PyObject *current_filename; char ptr; @@ -463,27 +464,75 @@ typedef WFILE RFILE; / Same struct with

#define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF) -#define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p)) - static int r_string(char *s, int n, RFILE *p) {

if (p->fp != NULL)

   /* The result fits into int because it must be <=n. */[](#l4.21)

   return (int)fread(s, 1, n, p->fp);[](#l4.22)

if (p->end - p->ptr < n)

   n = (int)(p->end - p->ptr);[](#l4.24)

memcpy(s, p->ptr, n);
p->ptr += n;
return n;

char * ptr;
int read, left;

if (!p->readable) {
```
   if (p->fp != NULL)[](#l4.32)
```

       /* The result fits into int because it must be <=n. */[](#l4.33)

       read = (int) fread(s, 1, n, p->fp);[](#l4.34)

```
   else {[](#l4.35)
```

       left = (int)(p->end - p->ptr);[](#l4.36)

       read = (left < n) ? left : n;[](#l4.37)

       memcpy(s, p->ptr, read);[](#l4.38)

```
       p->ptr += read;[](#l4.39)
```
```
   }[](#l4.40)
```
}
else {

   PyObject *data = PyObject_CallMethod(p->readable, "read", "i", n);[](#l4.43)

```
   read = 0;[](#l4.44)
```
```
   if (data != NULL) {[](#l4.45)
```

       if (!PyBytes_Check(data)) {[](#l4.46)

           PyErr_Format(PyExc_TypeError,[](#l4.47)

                        "f.read() returned not bytes but %.100s",[](#l4.48)

                        data->ob_type->tp_name);[](#l4.49)

```
       }[](#l4.50)
```
```
       else {[](#l4.51)
```

           read = PyBytes_GET_SIZE(data);[](#l4.52)

```
           if (read > 0) {[](#l4.53)
```

               ptr = PyBytes_AS_STRING(data);[](#l4.54)

               memcpy(s, ptr, read);[](#l4.55)

```
           }[](#l4.56)
```
```
       }[](#l4.57)
```
```
       Py_DECREF(data);[](#l4.58)
```
```
   }[](#l4.59)
```
}
if (!PyErr_Occurred() && (read < n)) {

   PyErr_SetString(PyExc_EOFError, "EOF read where not expected");[](#l4.62)

}
return read;

+} + + +static int +r_byte(RFILE *p) +{

int c = EOF;
unsigned char ch;
int n;

if (!p->readable)

   c = p->fp ? getc(p->fp) : rs_byte(p);[](#l4.76)

else {

   n = r_string((char *) &ch, 1, p);[](#l4.78)

```
   if (n > 0)[](#l4.79)
```
```
       c = ch;[](#l4.80)
```
}
return c;

} static int r_short(RFILE *p) { register short x;

x = r_byte(p);
x |= r_byte(p) << 8;

unsigned char buffer[2];

r_string((char *) buffer, 2, p);
x = buffer[0];
x |= buffer[1] << 8; /* Sign-extension, in case short greater than 16 bits */ x |= -(x & 0x8000); return x; @@ -493,19 +542,13 @@ static long r_long(RFILE *p) { register long x;

register FILE *fp = p->fp;
if (fp) {
```
   x = getc(fp);[](#l4.105)
```
```
   x |= (long)getc(fp) << 8;[](#l4.106)
```

   x |= (long)getc(fp) << 16;[](#l4.107)

   x |= (long)getc(fp) << 24;[](#l4.108)

}
else {
```
   x = rs_byte(p);[](#l4.111)
```

   x |= (long)rs_byte(p) << 8;[](#l4.112)

   x |= (long)rs_byte(p) << 16;[](#l4.113)

   x |= (long)rs_byte(p) << 24;[](#l4.114)

}

unsigned char buffer[4];

r_string((char *) buffer, 4, p);
x = buffer[0];
x |= (long)buffer[1] << 8;
x |= (long)buffer[2] << 16;
x |= (long)buffer[3] << 24;

#if SIZEOF_LONG > 4 /* Sign extension for 64-bit machines */ x |= -(x & 0x80000000L); @@ -523,25 +566,30 @@ r_long(RFILE *p) static PyObject * r_long64(RFILE *p) {

PyObject * result = NULL; long lo4 = r_long(p); long hi4 = r_long(p); +
if (!PyErr_Occurred()) {

#if SIZEOF_LONG > 4

long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
return PyLong_FromLong(x);

   long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);[](#l4.138)

   result = PyLong_FromLong(x);[](#l4.139)

#else

unsigned char buf[8];
int one = 1;
int is_little_endian = (int)(char)&one;
if (is_little_endian) {
```
   memcpy(buf, &lo4, 4);[](#l4.145)
```
```
   memcpy(buf+4, &hi4, 4);[](#l4.146)
```

```
   unsigned char buf[8];[](#l4.147)
```
```
   int one = 1;[](#l4.148)
```

   int is_little_endian = (int)*(char*)&one;[](#l4.149)

```
   if (is_little_endian) {[](#l4.150)
```
```
       memcpy(buf, &lo4, 4);[](#l4.151)
```

       memcpy(buf+4, &hi4, 4);[](#l4.152)

```
   }[](#l4.153)
```
```
   else {[](#l4.154)
```
```
       memcpy(buf, &hi4, 4);[](#l4.155)
```

       memcpy(buf+4, &lo4, 4);[](#l4.156)

```
   }[](#l4.157)
```

   result = _PyLong_FromByteArray(buf, 8, is_little_endian, 1);[](#l4.158)

+#endif }

else {
```
   memcpy(buf, &hi4, 4);[](#l4.162)
```
```
   memcpy(buf+4, &lo4, 4);[](#l4.163)
```
}
return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);

-#endif

return result;

} static PyObject * @@ -553,6 +601,8 @@ r_PyLong(RFILE *p) digit d; n = r_long(p);

if (PyErr_Occurred())
```
   return NULL;[](#l4.176)
```
if (n == 0) return (PyObject *)_PyLong_New(0); if (n < -INT_MAX || n > INT_MAX) { @@ -572,6 +622,8 @@ r_PyLong(RFILE *p) d = 0; for (j=0; j < PyLong_MARSHAL_RATIO; j++) { md = r_short(p);
```
       if (PyErr_Occurred())[](#l4.184)
```

           break;[](#l4.185)
       if (md < 0 || md > PyLong_MARSHAL_BASE)[](#l4.186)
           goto bad_digit;[](#l4.187)
       d += (digit)md << j*PyLong_MARSHAL_SHIFT;[](#l4.188)

@@ -581,6 +633,8 @@ r_PyLong(RFILE *p) d = 0; for (j=0; j < shorts_in_top_digit; j++) { md = r_short(p);

```
   if (PyErr_Occurred())[](#l4.193)
```

       break;[](#l4.194)
   if (md < 0 || md > PyLong_MARSHAL_BASE)[](#l4.195)
       goto bad_digit;[](#l4.196)
   /* topmost marshal digit should be nonzero */[](#l4.197)

@@ -592,6 +646,10 @@ r_PyLong(RFILE p) } d += (digit)md << jPyLong_MARSHAL_SHIFT; }

if (PyErr_Occurred()) {
```
   Py_DECREF(ob);[](#l4.203)
```
```
   return NULL;[](#l4.204)
```
} /* top digit should be nonzero, else the resulting PyLong won't be normalized */ ob->ob_digit[size-1] = d; @@ -660,7 +718,8 @@ r_object(RFILE *p) break;

case TYPE_INT:

   retval = PyLong_FromLong(r_long(p));[](#l4.213)

```
   n = r_long(p);[](#l4.214)
```

   retval = PyErr_Occurred() ? NULL : PyLong_FromLong(n);[](#l4.215)
   break;[](#l4.216)

case TYPE_INT64: @@ -770,6 +829,10 @@ r_object(RFILE *p) case TYPE_STRING: n = r_long(p);

```
   if (PyErr_Occurred()) {[](#l4.223)
```
```
       retval = NULL;[](#l4.224)
```
```
       break;[](#l4.225)
```

   }[](#l4.226)
   if (n < 0 || n > INT_MAX) {[](#l4.227)
       PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)");[](#l4.228)
       retval = NULL;[](#l4.229)

@@ -795,6 +858,10 @@ r_object(RFILE *p) char *buffer; n = r_long(p);

```
   if (PyErr_Occurred()) {[](#l4.234)
```
```
       retval = NULL;[](#l4.235)
```
```
       break;[](#l4.236)
```

   }[](#l4.237)
   if (n < 0 || n > INT_MAX) {[](#l4.238)
       PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)");[](#l4.239)
       retval = NULL;[](#l4.240)

@@ -820,6 +887,10 @@ r_object(RFILE *p) case TYPE_TUPLE: n = r_long(p);

```
   if (PyErr_Occurred()) {[](#l4.245)
```
```
       retval = NULL;[](#l4.246)
```
```
       break;[](#l4.247)
```

   }[](#l4.248)
   if (n < 0 || n > INT_MAX) {[](#l4.249)
       PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)");[](#l4.250)
       retval = NULL;[](#l4.251)

@@ -847,6 +918,10 @@ r_object(RFILE *p) case TYPE_LIST: n = r_long(p);

```
   if (PyErr_Occurred()) {[](#l4.256)
```
```
       retval = NULL;[](#l4.257)
```
```
       break;[](#l4.258)
```

   }[](#l4.259)
   if (n < 0 || n > INT_MAX) {[](#l4.260)
       PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)");[](#l4.261)
       retval = NULL;[](#l4.262)

@@ -899,6 +974,10 @@ r_object(RFILE *p) case TYPE_SET: case TYPE_FROZENSET: n = r_long(p);

```
   if (PyErr_Occurred()) {[](#l4.267)
```
```
       retval = NULL;[](#l4.268)
```
```
       break;[](#l4.269)
```

   }[](#l4.270)
   if (n < 0 || n > INT_MAX) {[](#l4.271)
       PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)");[](#l4.272)
       retval = NULL;[](#l4.273)

@@ -952,10 +1031,20 @@ r_object(RFILE p) / XXX ignore long->int overflows for now */ argcount = (int)r_long(p);

```
       if (PyErr_Occurred())[](#l4.278)
```

           goto code_error;[](#l4.279)
       kwonlyargcount = (int)r_long(p);[](#l4.280)

```
       if (PyErr_Occurred())[](#l4.281)
```

           goto code_error;[](#l4.282)
       nlocals = (int)r_long(p);[](#l4.283)

```
       if (PyErr_Occurred())[](#l4.284)
```

           goto code_error;[](#l4.285)
       stacksize = (int)r_long(p);[](#l4.286)

```
       if (PyErr_Occurred())[](#l4.287)
```

           goto code_error;[](#l4.288)
       flags = (int)r_long(p);[](#l4.289)

```
       if (PyErr_Occurred())[](#l4.290)
```

           goto code_error;[](#l4.291)
       code = r_object(p);[](#l4.292)
       if (code == NULL)[](#l4.293)
           goto code_error;[](#l4.294)

@@ -1049,6 +1138,7 @@ PyMarshal_ReadShortFromFile(FILE *fp) { RFILE rf; assert(fp);

rf.readable = NULL; rf.fp = fp; rf.current_filename = NULL; rf.end = rf.ptr = NULL; @@ -1060,6 +1150,7 @@ PyMarshal_ReadLongFromFile(FILE *fp) { RFILE rf; rf.fp = fp;
rf.readable = NULL; rf.current_filename = NULL; rf.ptr = rf.end = NULL; return r_long(&rf); @@ -1121,6 +1212,7 @@ PyMarshal_ReadObjectFromFile(FILE *fp) RFILE rf; PyObject *result; rf.fp = fp;
rf.readable = NULL; rf.current_filename = NULL; rf.depth = 0; rf.ptr = rf.end = NULL; @@ -1134,6 +1226,7 @@ PyMarshal_ReadObjectFromString(char *str RFILE rf; PyObject *result; rf.fp = NULL;
rf.readable = NULL; rf.current_filename = NULL; rf.ptr = str; rf.end = str + len; @@ -1149,6 +1242,7 @@ PyMarshal_WriteObjectToString(PyObject * PyObject *res = NULL; wf.fp = NULL;
wf.readable = NULL; wf.str = PyBytes_FromStringAndSize((char *)NULL, 50); if (wf.str == NULL) return NULL;

@@ -1224,32 +1318,33 @@ The version argument indicates the data static PyObject * marshal_load(PyObject *self, PyObject *f) {

/* XXX Quick hack -- need to do this differently */ PyObject *data, *result; RFILE rf;
data = PyObject_CallMethod(f, "read", "");

char *p;
int n;

* Make a call to the read method, but read zero bytes.[](#l4.347)

* This is to ensure that the object passed in at least[](#l4.348)

* has a read method which returns bytes.[](#l4.349)

```
*/[](#l4.350)
```
data = PyObject_CallMethod(f, "read", "i", 0); if (data == NULL) return NULL;

rf.fp = NULL;
rf.current_filename = NULL;
if (PyBytes_Check(data)) {

   rf.ptr = PyBytes_AS_STRING(data);[](#l4.357)

   rf.end = rf.ptr + PyBytes_GET_SIZE(data);[](#l4.358)

}
else if (PyBytes_Check(data)) {

   rf.ptr = PyBytes_AS_STRING(data);[](#l4.361)

   rf.end = rf.ptr + PyBytes_GET_SIZE(data);[](#l4.362)

if (!PyBytes_Check(data)) {

   PyErr_Format(PyExc_TypeError,[](#l4.364)

                "f.read() returned not bytes but %.100s",[](#l4.365)

                data->ob_type->tp_name);[](#l4.366)

```
   result = NULL;[](#l4.367)
```
} else {

   PyErr_Format(PyExc_TypeError,[](#l4.370)

                "f.read() returned neither string "[](#l4.371)

                "nor bytes but %.100s",[](#l4.372)

                data->ob_type->tp_name);[](#l4.373)

```
   Py_DECREF(data);[](#l4.374)
```
```
   return NULL;[](#l4.375)
```

   rf.strings = PyList_New(0);[](#l4.376)

```
   rf.depth = 0;[](#l4.377)
```
```
   rf.fp = NULL;[](#l4.378)
```
```
   rf.readable = f;[](#l4.379)
```

   result = read_object(&rf);[](#l4.380)

```
   Py_DECREF(rf.strings);[](#l4.381)
```
}

rf.depth = 0;
result = read_object(&rf); Py_DECREF(data); return result; }

@@ -1300,6 +1395,7 @@ marshal_loads(PyObject *self, PyObject * s = p.buf; n = p.len; rf.fp = NULL;

rf.readable = NULL; rf.current_filename = NULL; rf.ptr = s; rf.end = s + n;