cpython: 95b9c07b27f7 (original) (raw)

Mercurial > cpython

changeset 98993:95b9c07b27f7 3.5

Issue #24802: Merge null termination fixes from 3.4 into 3.5 [#24802]

Martin Panter vadmium+py@gmail.com
date	Sat, 07 Nov 2015 02:56:11 +0000
parents	de79e483565c(current diff)a75336ac40e0(diff)
children	4df1eaecb506 f33ce913220b
files	Lib/test/test_compile.py Lib/test/test_float.py Lib/test/test_int.py Misc/NEWS Objects/abstract.c Objects/complexobject.c Objects/floatobject.c Python/bltinmodule.c Python/clinic/bltinmodule.c.h
diffstat	8 files changed, 161 insertions(+), 36 deletions(-)[+] [-] Lib/test/test_compile.py 21 Lib/test/test_float.py 38 Lib/test/test_int.py 42 Misc/NEWS 4 Objects/abstract.c 22 Objects/complexobject.c 7 Objects/floatobject.c 15 Python/bltinmodule.c 48

line wrap: on

line diff

--- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -542,6 +542,27 @@ if 1: check_limit("a", "[0]") check_limit("a", "*a")

def test_null_terminated(self):

   # The source code is null-terminated internally, but bytes-like[](#l1.8)

   # objects are accepted, which could be not terminated.[](#l1.9)

   # Exception changed from TypeError to ValueError in 3.5[](#l1.10)

   with self.assertRaisesRegex(Exception, "cannot contain null"):[](#l1.11)

       compile("123\x00", "<dummy>", "eval")[](#l1.12)

   with self.assertRaisesRegex(Exception, "cannot contain null"):[](#l1.13)

       compile(memoryview(b"123\x00"), "<dummy>", "eval")[](#l1.14)

   code = compile(memoryview(b"123\x00")[1:-1], "<dummy>", "eval")[](#l1.15)

   self.assertEqual(eval(code), 23)[](#l1.16)

   code = compile(memoryview(b"1234")[1:-1], "<dummy>", "eval")[](#l1.17)

   self.assertEqual(eval(code), 23)[](#l1.18)

   code = compile(memoryview(b"$23$")[1:-1], "<dummy>", "eval")[](#l1.19)

   self.assertEqual(eval(code), 23)[](#l1.20)

   # Also test when eval() and exec() do the compilation step[](#l1.22)

   self.assertEqual(eval(memoryview(b"1234")[1:-1]), 23)[](#l1.23)

```
   namespace = dict()[](#l1.24)
```

   exec(memoryview(b"ax = 123")[1:-1], namespace)[](#l1.25)

   self.assertEqual(namespace['x'], 12)[](#l1.26)

+ class TestStackSize(unittest.TestCase): # These tests check that the computed stack size for a code object

--- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -31,7 +31,6 @@ class GeneralFloatCases(unittest.TestCas self.assertEqual(float(3.14), 3.14) self.assertEqual(float(314), 314.0) self.assertEqual(float(" 3.14 "), 3.14)

   self.assertEqual(float(b" 3.14  "), 3.14)[](#l2.7)
   self.assertRaises(ValueError, float, "  0x3.1  ")[](#l2.8)
   self.assertRaises(ValueError, float, "  -0x3.p-1  ")[](#l2.9)
   self.assertRaises(ValueError, float, "  +0x3.p-1  ")[](#l2.10)

@@ -43,7 +42,6 @@ class GeneralFloatCases(unittest.TestCas self.assertRaises(ValueError, float, "+.inf") self.assertRaises(ValueError, float, ".") self.assertRaises(ValueError, float, "-.")

   self.assertRaises(ValueError, float, b"-")[](#l2.15)
   self.assertRaises(TypeError, float, {})[](#l2.16)
   self.assertRaisesRegex(TypeError, "not 'dict'", float, {})[](#l2.17)
   # Lone surrogate[](#l2.18)

@@ -57,6 +55,42 @@ class GeneralFloatCases(unittest.TestCas float(b'.' + b'1'*1000) float('.' + '1'*1000)

def test_non_numeric_input_types(self):

   # Test possible non-numeric types for the argument x, including[](#l2.24)

   # subclasses of the explicitly documented accepted types.[](#l2.25)

```
   class CustomStr(str): pass[](#l2.26)
```

   class CustomBytes(bytes): pass[](#l2.27)

   class CustomByteArray(bytearray): pass[](#l2.28)

```
   factories = [[](#l2.30)
```
```
       bytes,[](#l2.31)
```
```
       bytearray,[](#l2.32)
```

       lambda b: CustomStr(b.decode()),[](#l2.33)

```
       CustomBytes,[](#l2.34)
```
```
       CustomByteArray,[](#l2.35)
```
```
       memoryview,[](#l2.36)
```
```
   ][](#l2.37)
```
```
   try:[](#l2.38)
```

       from array import array[](#l2.39)

```
   except ImportError:[](#l2.40)
```
```
       pass[](#l2.41)
```
```
   else:[](#l2.42)
```

       factories.append(lambda b: array('B', b))[](#l2.43)

```
   for f in factories:[](#l2.45)
```
```
       x = f(b" 3.14  ")[](#l2.46)
```

       with self.subTest(type(x)):[](#l2.47)

           self.assertEqual(float(x), 3.14)[](#l2.48)

           with self.assertRaisesRegex(ValueError, "could not convert"):[](#l2.49)

               float(f(b'A' * 0x10))[](#l2.50)

def test_float_memoryview(self):

   self.assertEqual(float(memoryview(b'12.3')[1:4]), 2.3)[](#l2.53)

   self.assertEqual(float(memoryview(b'12.3\x00')[1:4]), 2.3)[](#l2.54)

   self.assertEqual(float(memoryview(b'12.3 ')[1:4]), 2.3)[](#l2.55)

   self.assertEqual(float(memoryview(b'12.3A')[1:4]), 2.3)[](#l2.56)

   self.assertEqual(float(memoryview(b'12.34')[1:4]), 2.3)[](#l2.57)

+ def test_error_message(self): testlist = ('\xbd', '123\xbd', ' 123 456 ') for s in testlist:

--- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -276,16 +276,40 @@ class IntTestCases(unittest.TestCase): class CustomBytes(bytes): pass class CustomByteArray(bytearray): pass

```
   values = [b'100',[](#l3.7)
```

             bytearray(b'100'),[](#l3.8)

```
             CustomStr('100'),[](#l3.9)
```

             CustomBytes(b'100'),[](#l3.10)

             CustomByteArray(b'100')][](#l3.11)

```
   factories = [[](#l3.12)
```
```
       bytes,[](#l3.13)
```
```
       bytearray,[](#l3.14)
```

       lambda b: CustomStr(b.decode()),[](#l3.15)

```
       CustomBytes,[](#l3.16)
```
```
       CustomByteArray,[](#l3.17)
```
```
       memoryview,[](#l3.18)
```
```
   ][](#l3.19)
```
```
   try:[](#l3.20)
```

       from array import array[](#l3.21)

```
   except ImportError:[](#l3.22)
```
```
       pass[](#l3.23)
```
```
   else:[](#l3.24)
```

       factories.append(lambda b: array('B', b))[](#l3.25)

```
   for x in values:[](#l3.27)
```

       msg = 'x has type %s' % type(x).__name__[](#l3.28)

       self.assertEqual(int(x), 100, msg=msg)[](#l3.29)

       self.assertEqual(int(x, 2), 4, msg=msg)[](#l3.30)

```
   for f in factories:[](#l3.31)
```
```
       x = f(b'100')[](#l3.32)
```

       with self.subTest(type(x)):[](#l3.33)

           self.assertEqual(int(x), 100)[](#l3.34)

           if isinstance(x, (str, bytes, bytearray)):[](#l3.35)

               self.assertEqual(int(x, 2), 4)[](#l3.36)

```
           else:[](#l3.37)
```

               msg = "can't convert non-string"[](#l3.38)

               with self.assertRaisesRegex(TypeError, msg):[](#l3.39)

```
                   int(x, 2)[](#l3.40)
```

           with self.assertRaisesRegex(ValueError, 'invalid literal'):[](#l3.41)

               int(f(b'A' * 0x10))[](#l3.42)

def test_int_memoryview(self):

   self.assertEqual(int(memoryview(b'123')[1:3]), 23)[](#l3.45)

   self.assertEqual(int(memoryview(b'123\x00')[1:3]), 23)[](#l3.46)

   self.assertEqual(int(memoryview(b'123 ')[1:3]), 23)[](#l3.47)

   self.assertEqual(int(memoryview(b'123A')[1:3]), 23)[](#l3.48)

   self.assertEqual(int(memoryview(b'1234')[1:3]), 23)[](#l3.49)

def test_string_float(self): self.assertRaises(ValueError, int, '1.2')

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -11,6 +11,10 @@ Release date: TBA Core and Builtins ----------------- +- Issue #24802: Avoid buffer overreads when int(), float(), compile(), exec()

and eval() are passed bytes-like objects. These objects are not
necessarily terminated by a null byte, but the functions assumed they were. +

Issue #24726: Fixed a crash and leaking NULL in repr() of OrderedDict that was mutated by direct calls of dict methods.

--- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1309,12 +1309,30 @@ PyNumber_Long(PyObject o) / The below check is done in PyLong_FromUnicode(). */ return PyLong_FromUnicodeObject(o, 10);

if (PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) == 0) {

if (PyBytes_Check(o)) /* need to do extra error checking that PyLong_FromString() * doesn't do. In particular int('9\x005') must raise an * exception, not truncate at the null. */

   PyObject *result = _PyLong_FromBytes(view.buf, view.len, 10);[](#l5.13)

   return _PyLong_FromBytes(PyBytes_AS_STRING(o),[](#l5.14)

                            PyBytes_GET_SIZE(o), 10);[](#l5.15)

if (PyByteArray_Check(o))

   return _PyLong_FromBytes(PyByteArray_AS_STRING(o),[](#l5.18)

                            PyByteArray_GET_SIZE(o), 10);[](#l5.19)

if (PyObject_GetBuffer(o, &view, PyBUF_SIMPLE) == 0) {
```
   PyObject *result, *bytes;[](#l5.22)
```

   /* Copy to NUL-terminated buffer. */[](#l5.24)

   bytes = PyBytes_FromStringAndSize((const char *)view.buf, view.len);[](#l5.25)

```
   if (bytes == NULL) {[](#l5.26)
```

       PyBuffer_Release(&view);[](#l5.27)

```
       return NULL;[](#l5.28)
```
```
   }[](#l5.29)
```

   result = _PyLong_FromBytes(PyBytes_AS_STRING(bytes),[](#l5.30)

                              PyBytes_GET_SIZE(bytes), 10);[](#l5.31)

   Py_DECREF(bytes);[](#l5.32)
   PyBuffer_Release(&view);[](#l5.33)
   return result;[](#l5.34)

}

--- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -767,7 +767,6 @@ complex_subtype_from_string(PyTypeObject int got_bracket=0; PyObject *s_buffer = NULL; Py_ssize_t len;

Py_buffer view = {NULL, NULL};

if (PyUnicode_Check(v)) { s_buffer = _PyUnicode_TransformDecimalAndSpaceToASCII(v); @@ -777,10 +776,6 @@ complex_subtype_from_string(PyTypeObject if (s == NULL) goto error; }

else if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) == 0) {

   s = (const char *)view.buf;[](#l6.16)

```
   len = view.len;[](#l6.17)
```
} else { PyErr_Format(PyExc_TypeError, "complex() argument must be a string or a number, not '%.200s'",

@@ -895,7 +890,6 @@ complex_subtype_from_string(PyTypeObject if (s-start != len) goto parse_error;

PyBuffer_Release(&view); Py_XDECREF(s_buffer); return complex_subtype_from_doubles(type, x, y); @@ -903,7 +897,6 @@ complex_subtype_from_string(PyTypeObject PyErr_SetString(PyExc_ValueError, "complex() arg is a malformed string");

error:

PyBuffer_Release(&view); Py_XDECREF(s_buffer); return NULL; }

--- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -144,9 +144,24 @@ PyFloat_FromString(PyObject *v) return NULL; } }

else if (PyBytes_Check(v)) {
```
   s = PyBytes_AS_STRING(v);[](#l7.8)
```
```
   len = PyBytes_GET_SIZE(v);[](#l7.9)
```
}
else if (PyByteArray_Check(v)) {

   s = PyByteArray_AS_STRING(v);[](#l7.12)

   len = PyByteArray_GET_SIZE(v);[](#l7.13)

} else if (PyObject_GetBuffer(v, &view, PyBUF_SIMPLE) == 0) { s = (const char *)view.buf; len = view.len;

   /* Copy to NUL-terminated buffer. */[](#l7.18)

   s_buffer = PyBytes_FromStringAndSize(s, len);[](#l7.19)

```
   if (s_buffer == NULL) {[](#l7.20)
```

       PyBuffer_Release(&view);[](#l7.21)

```
       return NULL;[](#l7.22)
```
```
   }[](#l7.23)
```

   s = PyBytes_AS_STRING(s_buffer);[](#l7.24)

} else { PyErr_Format(PyExc_TypeError,

--- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -599,20 +599,37 @@ builtin_chr_impl(PyModuleDef *module, in static const char * -source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, Py_buffer *view) +source_as_string(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy) { const char *str; Py_ssize_t size;

Py_buffer view;

*cmd_copy = NULL; if (PyUnicode_Check(cmd)) { cf->cf_flags |= PyCF_IGNORE_COOKIE; str = PyUnicode_AsUTF8AndSize(cmd, &size); if (str == NULL) return NULL; }

else if (PyObject_GetBuffer(cmd, view, PyBUF_SIMPLE) == 0) {

   str = (const char *)view->buf;[](#l8.22)

```
   size = view->len;[](#l8.23)
```

else if (PyBytes_Check(cmd)) {

   str = PyBytes_AS_STRING(cmd);[](#l8.25)

   size = PyBytes_GET_SIZE(cmd);[](#l8.26)

}
else if (PyByteArray_Check(cmd)) {

   str = PyByteArray_AS_STRING(cmd);[](#l8.29)

   size = PyByteArray_GET_SIZE(cmd);[](#l8.30)

}
else if (PyObject_GetBuffer(cmd, &view, PyBUF_SIMPLE) == 0) {

   /* Copy to NUL-terminated buffer. */[](#l8.33)

   *cmd_copy = PyBytes_FromStringAndSize([](#l8.34)

       (const char *)view.buf, view.len);[](#l8.35)

```
   PyBuffer_Release(&view);[](#l8.36)
```
```
   if (*cmd_copy == NULL) {[](#l8.37)
```
```
       return NULL;[](#l8.38)
```
```
   }[](#l8.39)
```

   str = PyBytes_AS_STRING(*cmd_copy);[](#l8.40)

   size = PyBytes_GET_SIZE(*cmd_copy);[](#l8.41)

} else { PyErr_Format(PyExc_TypeError,

@@ -624,7 +641,7 @@ source_as_string(PyObject *cmd, const ch if (strlen(str) != (size_t)size) { PyErr_SetString(PyExc_ValueError, "source code string cannot contain null bytes");

```
   PyBuffer_Release(view);[](#l8.49)
```

```
   Py_CLEAR(*cmd_copy);[](#l8.50)
   return NULL;[](#l8.51)
```
} return str; @@ -660,7 +677,7 @@ builtin_compile_impl(PyModuleDef *module int dont_inherit, int optimize)

/[clinic end generated code: output=31881762c1bb90c4 input=9d53e8cfb3c86414]/ {

Py_buffer view = {NULL, NULL};

PyObject *source_copy; const char *str; int compile_mode = -1; int is_ast; @@ -732,12 +749,12 @@ builtin_compile_impl(PyModuleDef *module goto finally; }

str = source_as_string(source, "compile", "string, bytes or AST", &cf, &view);

str = source_as_string(source, "compile", "string, bytes or AST", &cf, &source_copy); if (str == NULL) goto error;

result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize);

PyBuffer_Release(&view);

Py_XDECREF(source_copy); goto finally; error: @@ -812,8 +829,7 @@ builtin_eval_impl(PyModuleDef *module, P PyObject *locals)

/[clinic end generated code: output=7284501fb7b4d666 input=11ee718a8640e527]/ {

PyObject *result, *tmp = NULL;
Py_buffer view = {NULL, NULL};

PyObject *result, *source_copy; const char *str; PyCompilerFlags cf; @@ -861,7 +877,7 @@ builtin_eval_impl(PyModuleDef *module, P } cf.cf_flags = PyCF_SOURCE_IS_UTF8;

str = source_as_string(source, "eval", "string, bytes or code", &cf, &view);

str = source_as_string(source, "eval", "string, bytes or code", &cf, &source_copy); if (str == NULL) return NULL;

@@ -870,8 +886,7 @@ builtin_eval_impl(PyModuleDef *module, P (void)PyEval_MergeCompilerFlags(&cf); result = PyRun_StringFlags(str, Py_eval_input, globals, locals, &cf);

PyBuffer_Release(&view);
Py_XDECREF(tmp);

Py_XDECREF(source_copy); return result; }

@@ -942,12 +957,13 @@ builtin_exec_impl(PyModuleDef *module, P v = PyEval_EvalCode(source, globals, locals); } else {

   Py_buffer view = {NULL, NULL};[](#l8.111)

   PyObject *source_copy;[](#l8.112)
   const char *str;[](#l8.113)
   PyCompilerFlags cf;[](#l8.114)
   cf.cf_flags = PyCF_SOURCE_IS_UTF8;[](#l8.115)
   str = source_as_string(source, "exec",[](#l8.116)

                                  "string, bytes or code", &cf, &view);[](#l8.117)

                                  "string, bytes or code", &cf,[](#l8.118)

                                  &source_copy);[](#l8.119)
   if (str == NULL)[](#l8.120)
       return NULL;[](#l8.121)
   if (PyEval_MergeCompilerFlags(&cf))[](#l8.122)

@@ -955,7 +971,7 @@ builtin_exec_impl(PyModuleDef *module, P locals, &cf); else v = PyRun_String(str, Py_file_input, globals, locals);

```
   PyBuffer_Release(&view);[](#l8.127)
```

```
   Py_XDECREF(source_copy);[](#l8.128)
```
} if (v == NULL) return NULL;