[Python-Dev] [Python-checkins] cpython (merge 3.3 -> default): Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py. (original) (raw)

Brett Cannon brett at python.org
Fri Jan 11 18:08:18 CET 2013

This seems to have caused the Windows buildbots to fail.

On Fri, Jan 11, 2013 at 5:40 AM, serhiy.storchaka <python-checkins at python.org> wrote:

http://hg.python.org/cpython/rev/8452c23139c6 changeset: 81407:8452c23139c6 parent: 81399:5ec8daab477a parent: 81406:01df1f7841b2 user: Serhiy Storchaka <storchaka at gmail.com> date: Fri Jan 11 12:12:32 2013 +0200 summary: Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py. Now pindent.py works with a "with" statement. pindent.py no longer produces improper indentation. pindent.py now works with continued lines broken after "class" or "def" keywords and with continuations at the start of line. Added regression tests for pindent.py. Modernized pindent.py.

files: Lib/test/testtools.py | 326 ++++++++++++++++++++++++++- Misc/NEWS | 7 + Tools/scripts/pindent.py | 166 +++++-------- 3 files changed, 393 insertions(+), 106 deletions(-)

diff --git a/Lib/test/testtools.py b/Lib/test/testtools.py --- a/Lib/test/testtools.py +++ b/Lib/test/testtools.py @@ -9,10 +9,13 @@ import importlib.machinery import unittest from unittest import mock +import shutil +import subprocess import sysconfig import tempfile +import textwrap from test import support -from test.scripthelper import assertpythonok +from test.scripthelper import assertpythonok, tempdir if not sysconfig.ispythonbuild(): # XXX some installers do contain the tools, should we detect that @@ -36,6 +39,327 @@ self.assertGreater(err, b'') +class PindentTests(unittest.TestCase): + script = os.path.join(scriptsdir, 'pindent.py') + + def assertFileEqual(self, fn1, fn2): + with open(fn1) as f1, open(fn2) as f2: + self.assertEqual(f1.readlines(), f2.readlines()) + + def pindent(self, source, *args): + with subprocess.Popen( + (sys.executable, self.script) + args, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + universalnewlines=True) as proc: + out, err = proc.communicate(source) + self.assertIsNone(err) + return out + + def lstriplines(self, data): + return '\n'.join(line.lstrip() for line in data.splitlines()) + '\n' + + def testselftest(self): + with tempdir() as directory: + datapath = os.path.join(directory, 'test.py') + with open(self.script) as f: + closed = f.read() + with open(datapath, 'w') as f: + f.write(closed) + + rc, out, err = assertpythonok(self.script, '-d', datapath) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + backup = datapath + '~' + self.assertTrue(os.path.exists(backup)) + with open(backup) as f: + self.assertEqual(f.read(), closed) + with open(datapath) as f: + clean = f.read() + compile(clean, 'test.py', 'exec') + self.assertEqual(self.pindent(clean, '-c'), closed) + self.assertEqual(self.pindent(closed, '-d'), clean) + + rc, out, err = assertpythonok(self.script, '-c', datapath) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + with open(backup) as f: + self.assertEqual(f.read(), clean) + with open(datapath) as f: + self.assertEqual(f.read(), closed) + + broken = self.lstriplines(closed) + with open(datapath, 'w') as f: + f.write(broken) + rc, out, err = assertpythonok(self.script, '-r', datapath) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + with open(backup) as f: + self.assertEqual(f.read(), broken) + with open(datapath) as f: + indented = f.read() + compile(indented, 'test.py', 'exec') + self.assertEqual(self.pindent(broken, '-r'), indented) + + def pindenttest(self, clean, closed): + self.assertEqual(self.pindent(clean, '-c'), closed) + self.assertEqual(self.pindent(closed, '-d'), clean) + broken = self.lstriplines(closed) + self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '4'), closed) + + def teststatements(self): _+ clean = textwrap.dedent("""_ + if a: + pass + + if a: + pass + else: + pass + + if a: + pass + elif: + pass + else: + pass + + while a: + break + + while a: + break + else: + pass + + for i in a: + break + + for i in a: + break + else: + pass + + try: + pass + finally: + pass + + try: + pass + except TypeError: + pass + except ValueError: + pass + else: + pass + + try: + pass + except TypeError: + pass + except ValueError: + pass + finally: + pass + + with a: + pass + + class A: + pass + + def f(): + pass + """) + _+ closed = textwrap.dedent("""_ + if a: + pass + # end if + + if a: + pass + else: + pass + # end if + + if a: + pass + elif: + pass + else: + pass + # end if + + while a: + break + # end while + + while a: + break + else: + pass + # end while + + for i in a: + break + # end for + + for i in a: + break + else: + pass + # end for + + try: + pass + finally: + pass + # end try + + try: + pass + except TypeError: + pass + except ValueError: + pass + else: + pass + # end try + + try: + pass + except TypeError: + pass + except ValueError: + pass + finally: + pass + # end try + + with a: + pass + # end with + + class A: + pass + # end class A + + def f(): + pass + # end def f + """) + self.pindenttest(clean, closed) + + def testmultilevel(self): _+ clean = textwrap.dedent("""_ + def foobar(a, b): + if a == b: + a = a+1 + elif a < b:_ _+ b = b-1_ _+ if b > a: a = a-1 + else: + print 'oops!' + """) _+ closed = textwrap.dedent("""_ + def foobar(a, b): + if a == b: + a = a+1 + elif a < b:_ _+ b = b-1_ _+ if b > a: a = a-1 + # end if + else: + print 'oops!' + # end if + # end def foobar + """) + self.pindenttest(clean, closed) + + def testpreserveindents(self): _+ clean = textwrap.dedent("""_ + if a: + if b: + pass + """) _+ closed = textwrap.dedent("""_ + if a: + if b: + pass + # end if + # end if + """) + self.assertEqual(self.pindent(clean, '-c'), closed) + self.assertEqual(self.pindent(closed, '-d'), clean) + broken = self.lstriplines(closed) + self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '9'), closed) _+ clean = textwrap.dedent("""_ + if a: + \tif b: + \t\tpass + """) _+ closed = textwrap.dedent("""_ + if a: + \tif b: + \t\tpass + \t# end if + # end if + """) + self.assertEqual(self.pindent(clean, '-c'), closed) + self.assertEqual(self.pindent(closed, '-d'), clean) + broken = self.lstriplines(closed) + self.assertEqual(self.pindent(broken, '-r'), closed) + + def testescapednewline(self): _+ clean = textwrap.dedent("""_ + class\ + \ + A: _+ def_ + \ + f: + pass + """) _+ closed = textwrap.dedent("""_ + class\ + \ + A: _+ def_ + \ + f: + pass + # end def f + # end class A + """) + self.assertEqual(self.pindent(clean, '-c'), closed) + self.assertEqual(self.pindent(closed, '-d'), clean) + + def testemptyline(self): _+ clean = textwrap.dedent("""_ + if a: + + pass + """) _+ closed = textwrap.dedent("""_ + if a: + + pass + # end if + """) + self.pindenttest(clean, closed) + + def testoneline(self): _+ clean = textwrap.dedent("""_ + if a: pass + """) _+ closed = textwrap.dedent("""_ + if a: pass + # end if + """) + self.pindenttest(clean, closed) + + class TestSundryScripts(unittest.TestCase): # At least make sure the rest don't have syntax errors. When tests are # added for a script it should be added to the whitelist below. diff --git a/Misc/NEWS b/Misc/NEWS --- a/Misc/NEWS +++ b/Misc/NEWS @@ -621,6 +621,8 @@ Tests ----- +- Issue #15539: Added regression tests for Tools/scripts/pindent.py. + - Issue #16836: Enable IPv6 support even if IPv6 is disabled on the build host. - Issue #16925: testconfigparser now works with unittest test discovery. @@ -777,6 +779,11 @@ Tools/Demos ----------- +- Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py. Now + pindent.py works with a "with" statement. pindent.py no longer produces + improper indentation. pindent.py now works with continued lines broken after + "class" or "def" keywords and with continuations at the start of line. + - Issue #11797: Add a 2to3 fixer that maps reload() to imp.reload(). - Issue #10966: Remove the concept of unexpected skipped tests. diff --git a/Tools/scripts/pindent.py b/Tools/scripts/pindent.py --- a/Tools/scripts/pindent.py +++ b/Tools/scripts/pindent.py @@ -79,8 +79,9 @@ # Defaults STEPSIZE = 8 TABSIZE = 8 -EXPANDTABS = 0 +EXPANDTABS = False +import io import re import sys @@ -89,7 +90,8 @@ next['while'] = next['for'] = 'else', 'end' next['try'] = 'except', 'finally' next['except'] = 'except', 'else', 'finally', 'end' -next['else'] = next['finally'] = next['def'] = next['class'] = 'end' _+next['else'] = next['finally'] = next['with'] = _ + next['def'] = next['class'] = 'end' next['end'] = () start = 'if', 'while', 'for', 'try', 'with', 'def', 'class' @@ -105,11 +107,11 @@ self.expandtabs = expandtabs self.write = fpo.write self.kwprog = re.compile( - r'^\s*(?P[a-z]+)' - r'(\s+(?P[a-zA-Z]\w*))?' + r'^(?:\s|\\n)*(?P[a-z]+)' + r'((?:\s|\\n)+(?P[a-zA-Z]\w*))?' r'[^\w]') self.endprog = re.compile( - r'^\s*#?\s*end\s+(?P[a-z]+)' + r'^(?:\s|\\n)#?\send\s+(?P[a-z]+)' r'(\s+(?P[a-zA-Z]\w*))?' r'[^\w]') self.wsprog = re.compile(r'^[ \t]*') @@ -125,7 +127,7 @@ def readline(self): line = self.fpi.readline() - if line: self.lineno = self.lineno + 1 + if line: self.lineno += 1 # end if return line # end def readline @@ -143,27 +145,24 @@ line2 = self.readline() if not line2: break # end if - line = line + line2 + line += line2 # end while return line # end def getline - def putline(self, line, indent = None): - if indent is None: - self.write(line) - return + def putline(self, line, indent): + tabs, spaces = divmod(indent*self.indentsize, self.tabsize) + i = self.wsprog.match(line).end() + line = line[i:] + if line[:1] not in ('\n', '\r', ''): + line = '\t'*tabs + ' '*spaces + line # end if - tabs, spaces = divmod(indent*self.indentsize, self.tabsize) - i = 0 - m = self.wsprog.match(line) - if m: i = m.end() - # end if - self.write('\t'*tabs + ' '*spaces + line[i:]) + self.write(line) # end def putline def reformat(self): stack = [] - while 1: + while True: line = self.getline() if not line: break # EOF # end if @@ -173,10 +172,9 @@ kw2 = m.group('kw') if not stack: self.error('unexpected end') - elif stack[-1][0] != kw2: + elif stack.pop()[0] != kw2: self.error('unmatched end') # end if - del stack[-1:] self.putline(line, len(stack)) continue # end if @@ -208,23 +206,23 @@ def delete(self): begincounter = 0 endcounter = 0 - while 1: + while True: line = self.getline() if not line: break # EOF # end if m = self.endprog.match(line) if m: - endcounter = endcounter + 1 + endcounter += 1 continue # end if m = self.kwprog.match(line) if m: kw = m.group('kw') if kw in start: - begincounter = begincounter + 1 + begincounter += 1 # end if # end if - self.putline(line) + self.write(line) # end while if begincounter - endcounter < 0:_ _sys.stderr.write('Warning: input contained more end tags than expected\n')_ _@@ -234,17 +232,12 @@_ _# end def delete_ _def complete(self):_ _- self.indentsize = 1_ _stack = []_ _todo = []_ _- thisid = ''_ _- current, firstkw, lastkw, topid = 0, '', '', ''_ _- while 1:_ _+ currentws = thisid = firstkw = lastkw = topid = ''_ _+ while True:_ _line = self.getline()_ _- i = 0_ _- m = self.wsprog.match(line)_ _- if m: i = m.end()_ _- # end if_ _+ i = self.wsprog.match(line).end()_ _m = self.endprog.match(line)_ _if m:_ _thiskw = 'end'_ _@@ -269,7 +262,9 @@_ _thiskw = ''_ _# end if_ _# end if_ _- indent = len(line[:i].expandtabs(self.tabsize))_ _+ indentws = line[:i]_ _+ indent = len(indentws.expandtabs(self.tabsize))_ _+ current = len(currentws.expandtabs(self.tabsize))_ _while indent < current:_ _if firstkw:_ _if topid:_ _@@ -278,11 +273,11 @@_ _else:_ _s = '# end %s\n' % firstkw_ _# end if_ _- self.putline(s, current)_ _+ self.write(currentws + s)_ _firstkw = lastkw = ''_ _# end if_ _- current, firstkw, lastkw, topid = stack[-1]_ _- del stack[-1]_ _+ currentws, firstkw, lastkw, topid = stack.pop()_ _+ current = len(currentws.expandtabs(self.tabsize))_ _# end while_ _if indent == current and firstkw:_ _if thiskw == 'end':_ _@@ -297,18 +292,18 @@_ _else:_ _s = '# end %s\n' % firstkw_ _# end if_ _- self.putline(s, current)_ _+ self.write(currentws + s)_ _firstkw = lastkw = topid = ''_ _# end if_ _# end if_ _if indent > current: - stack.append((current, firstkw, lastkw, topid)) + stack.append((currentws, firstkw, lastkw, topid)) if thiskw and thiskw not in start: # error thiskw = '' # end if _- current, firstkw, lastkw, topid = _ - indent, thiskw, thiskw, thisid _+ currentws, firstkw, lastkw, topid = _ + indentws, thiskw, thiskw, thisid # end if if thiskw: if thiskw in start: @@ -326,7 +321,6 @@ self.write(line) # end while # end def complete - # end class PythonIndenter # Simplified user interface @@ -352,76 +346,34 @@ pi.reformat() # end def reformatfilter -class StringReader: - def init(self, buf): - self.buf = buf - self.pos = 0 - self.len = len(self.buf) - # end def init - def read(self, n = 0): - if n <= 0: - n = self.len - self.pos - else: - n = min(n, self.len - self.pos) - # end if - r = self.buf[self.pos : self.pos + n] - self.pos = self.pos + n - return r - # end def read - def readline(self): - i = self.buf.find('\n', self.pos) - return self.read(i + 1 - self.pos) - # end def readline - def readlines(self): - lines = [] - line = self.readline() - while line: - lines.append(line) - line = self.readline() - # end while - return lines - # end def readlines - # seek/tell etc. are left as an exercise for the reader -# end class StringReader - -class StringWriter: - def init(self): - self.buf = '' - # end def init - def write(self, s): - self.buf = self.buf + s - # end def write - def getvalue(self): - return self.buf - # end def getvalue -# end class StringWriter - def completestring(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - input = StringReader(source) - output = StringWriter() + input = io.StringIO(source) + output = io.StringIO() pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) pi.complete() return output.getvalue() # end def completestring def deletestring(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - input = StringReader(source) - output = StringWriter() + input = io.StringIO(source) + output = io.StringIO() pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) pi.delete() return output.getvalue() # end def deletestring def reformatstring(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - input = StringReader(source) - output = StringWriter() + input = io.StringIO(source) + output = io.StringIO() pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) pi.reformat() return output.getvalue() # end def reformatstring def completefile(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - source = open(filename, 'r').read() + with open(filename, 'r') as f: + source = f.read() + # end with result = completestring(source, stepsize, tabsize, expandtabs) if source == result: return 0 # end if @@ -429,14 +381,16 @@ try: os.rename(filename, filename + '~') except OSError: pass # end try - f = open(filename, 'w') - f.write(result) - f.close() + with open(filename, 'w') as f: + f.write(result) + # end with return 1 # end def completefile def deletefile(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - source = open(filename, 'r').read() + with open(filename, 'r') as f: + source = f.read() + # end with result = deletestring(source, stepsize, tabsize, expandtabs) if source == result: return 0 # end if @@ -444,14 +398,16 @@ try: os.rename(filename, filename + '~') except OSError: pass # end try - f = open(filename, 'w') - f.write(result) - f.close() + with open(filename, 'w') as f: + f.write(result) + # end with return 1 # end def deletefile def reformatfile(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS): - source = open(filename, 'r').read() + with open(filename, 'r') as f: + source = f.read() + # end with result = reformatstring(source, stepsize, tabsize, expandtabs) if source == result: return 0 # end if @@ -459,9 +415,9 @@ try: os.rename(filename, filename + '~') except OSError: pass # end try - f = open(filename, 'w') - f.write(result) - f.close() + with open(filename, 'w') as f: + f.write(result) + # end with return 1 # end def reformatfile @@ -474,7 +430,7 @@ -r : reformat a completed program (use #end directives) -s stepsize: indentation step (default %(STEPSIZE)d) -t tabsize : the worth in spaces of a tab (default %(TABSIZE)d) --e : expand TABs into spaces (defailt OFF) +-e : expand TABs into spaces (default OFF) [file] ... : files are changed in place, with backups in file~ If no files are specified or a single - is given, the program acts as a filter (reads stdin, writes stdout). @@ -517,7 +473,7 @@ elif o == '-t': tabsize = int(a) elif o == '-e': - expandtabs = 1 + expandtabs = True # end if # end for if not action: -- Repository URL: http://hg.python.org/cpython

Python-checkins mailing list Python-checkins at python.org http://mail.python.org/mailman/listinfo/python-checkins

More information about the Python-Dev mailing list