cpython: 03b2259c6cd3 (original) (raw)

Mercurial > cpython

changeset 95751:03b2259c6cd3

merge 3.4 (#24022) [#24022]

Benjamin Peterson benjamin@python.org
date	Tue, 21 Apr 2015 12:07:06 -0400
parents	475c6a4dfab3(current diff)414e08c478f4(diff)
children	03330e5edb37
files	Lib/test/test_compile.py Misc/NEWS Parser/tokenizer.c
diffstat	3 files changed, 18 insertions(+), 5 deletions(-)[+] [-] Lib/test/test_compile.py 14 Misc/NEWS 2 Parser/tokenizer.c 7

line wrap: on

line diff

--- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,9 +1,11 @@ import math +import os import unittest import sys import _ast +import tempfile import types -from test import support +from test import support, script_helper class TestSpecifics(unittest.TestCase): @@ -492,6 +494,16 @@ if 1: self.assertInvalidSingle('f()\nxy # blah\nblah()') self.assertInvalidSingle('x = 5 # comment\nx = 6\n')

def test_particularly_evil_undecodable(self):
```
   # Issue 24022[](#l1.21)
```

   src = b'0000\x00\n00000000000\n\x00\n\x9e\n'[](#l1.22)

   with tempfile.TemporaryDirectory() as tmpd:[](#l1.23)

       fn = os.path.join(tmpd, "bad.py")[](#l1.24)

       with open(fn, "wb") as fp:[](#l1.25)

```
           fp.write(src)[](#l1.26)
```

       res = script_helper.run_python_until_end(fn)[0][](#l1.27)

   self.assertIn(b"Non-UTF-8", res.err)[](#l1.28)

+ @support.cpython_only def test_compiler_recursion_limit(self): # Expected limit is sys.getrecursionlimit() * the scaling factor

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -10,6 +10,8 @@ Release date: 2015-04-24 Core and Builtins ----------------- +- Issue #24022: Fix tokenizer crash when processing undecodable source code. + Library -------

--- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1307,6 +1307,8 @@ verify_identifier(struct tok_state *tok) { PyObject *s; int result;

if (tok->decoding_erred)
```
   return 0;[](#l3.8)
```
s = PyUnicode_DecodeUTF8(tok->start, tok->cur - tok->start, NULL); if (s == NULL || PyUnicode_READY(s) == -1) { if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) {

@@ -1475,11 +1477,8 @@ tok_get(struct tok_state *tok, char **p_ c = tok_nextc(tok); } tok_backup(tok, c);

```
   if (nonascii &&[](#l3.16)
```

       !verify_identifier(tok)) {[](#l3.17)

       tok->done = E_IDENTIFIER;[](#l3.18)

   if (nonascii && !verify_identifier(tok))[](#l3.19)
       return ERRORTOKEN;[](#l3.20)

   }[](#l3.21)
   *p_start = tok->start;[](#l3.22)
   *p_end = tok->cur;[](#l3.23)
   return NAME;[](#l3.24)