cpython: 8452c23139c6 (original) (raw)
--- a/Lib/test/test_tools.py +++ b/Lib/test/test_tools.py @@ -9,10 +9,13 @@ import sys import importlib.machinery import unittest from unittest import mock +import shutil +import subprocess import sysconfig import tempfile +import textwrap from test import support -from test.script_helper import assert_python_ok +from test.script_helper import assert_python_ok, temp_dir if not sysconfig.is_python_build(): # XXX some installers do contain the tools, should we detect that @@ -36,6 +39,327 @@ class ReindentTests(unittest.TestCase): self.assertGreater(err, b'') +class PindentTests(unittest.TestCase):
- def assertFileEqual(self, fn1, fn2):
with open(fn1) as f1, open(fn2) as f2:[](#l1.26)
self.assertEqual(f1.readlines(), f2.readlines())[](#l1.27)
- def pindent(self, source, *args):
with subprocess.Popen([](#l1.30)
(sys.executable, self.script) + args,[](#l1.31)
stdin=subprocess.PIPE, stdout=subprocess.PIPE,[](#l1.32)
universal_newlines=True) as proc:[](#l1.33)
out, err = proc.communicate(source)[](#l1.34)
self.assertIsNone(err)[](#l1.35)
return out[](#l1.36)
- def lstriplines(self, data):
return '\n'.join(line.lstrip() for line in data.splitlines()) + '\n'[](#l1.39)
- def test_selftest(self):
with temp_dir() as directory:[](#l1.42)
data_path = os.path.join(directory, '_test.py')[](#l1.43)
with open(self.script) as f:[](#l1.44)
closed = f.read()[](#l1.45)
with open(data_path, 'w') as f:[](#l1.46)
f.write(closed)[](#l1.47)
rc, out, err = assert_python_ok(self.script, '-d', data_path)[](#l1.49)
self.assertEqual(out, b'')[](#l1.50)
self.assertEqual(err, b'')[](#l1.51)
backup = data_path + '~'[](#l1.52)
self.assertTrue(os.path.exists(backup))[](#l1.53)
with open(backup) as f:[](#l1.54)
self.assertEqual(f.read(), closed)[](#l1.55)
with open(data_path) as f:[](#l1.56)
clean = f.read()[](#l1.57)
compile(clean, '_test.py', 'exec')[](#l1.58)
self.assertEqual(self.pindent(clean, '-c'), closed)[](#l1.59)
self.assertEqual(self.pindent(closed, '-d'), clean)[](#l1.60)
rc, out, err = assert_python_ok(self.script, '-c', data_path)[](#l1.62)
self.assertEqual(out, b'')[](#l1.63)
self.assertEqual(err, b'')[](#l1.64)
with open(backup) as f:[](#l1.65)
self.assertEqual(f.read(), clean)[](#l1.66)
with open(data_path) as f:[](#l1.67)
self.assertEqual(f.read(), closed)[](#l1.68)
broken = self.lstriplines(closed)[](#l1.70)
with open(data_path, 'w') as f:[](#l1.71)
f.write(broken)[](#l1.72)
rc, out, err = assert_python_ok(self.script, '-r', data_path)[](#l1.73)
self.assertEqual(out, b'')[](#l1.74)
self.assertEqual(err, b'')[](#l1.75)
with open(backup) as f:[](#l1.76)
self.assertEqual(f.read(), broken)[](#l1.77)
with open(data_path) as f:[](#l1.78)
indented = f.read()[](#l1.79)
compile(indented, '_test.py', 'exec')[](#l1.80)
self.assertEqual(self.pindent(broken, '-r'), indented)[](#l1.81)
- def pindent_test(self, clean, closed):
self.assertEqual(self.pindent(clean, '-c'), closed)[](#l1.84)
self.assertEqual(self.pindent(closed, '-d'), clean)[](#l1.85)
broken = self.lstriplines(closed)[](#l1.86)
self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '4'), closed)[](#l1.87)
if a:[](#l1.94)
pass[](#l1.95)
else:[](#l1.96)
pass[](#l1.97)
if a:[](#l1.99)
pass[](#l1.100)
elif:[](#l1.101)
pass[](#l1.102)
else:[](#l1.103)
pass[](#l1.104)
while a:[](#l1.106)
break[](#l1.107)
while a:[](#l1.109)
break[](#l1.110)
else:[](#l1.111)
pass[](#l1.112)
for i in a:[](#l1.114)
break[](#l1.115)
for i in a:[](#l1.117)
break[](#l1.118)
else:[](#l1.119)
pass[](#l1.120)
try:[](#l1.122)
pass[](#l1.123)
finally:[](#l1.124)
pass[](#l1.125)
try:[](#l1.127)
pass[](#l1.128)
except TypeError:[](#l1.129)
pass[](#l1.130)
except ValueError:[](#l1.131)
pass[](#l1.132)
else:[](#l1.133)
pass[](#l1.134)
try:[](#l1.136)
pass[](#l1.137)
except TypeError:[](#l1.138)
pass[](#l1.139)
except ValueError:[](#l1.140)
pass[](#l1.141)
finally:[](#l1.142)
pass[](#l1.143)
with a:[](#l1.145)
pass[](#l1.146)
class A:[](#l1.148)
pass[](#l1.149)
def f():[](#l1.151)
pass[](#l1.152)
""")[](#l1.153)
closed = textwrap.dedent("""\[](#l1.155)
if a:[](#l1.156)
pass[](#l1.157)
# end if[](#l1.158)
if a:[](#l1.160)
pass[](#l1.161)
else:[](#l1.162)
pass[](#l1.163)
# end if[](#l1.164)
if a:[](#l1.166)
pass[](#l1.167)
elif:[](#l1.168)
pass[](#l1.169)
else:[](#l1.170)
pass[](#l1.171)
# end if[](#l1.172)
while a:[](#l1.174)
break[](#l1.175)
# end while[](#l1.176)
while a:[](#l1.178)
break[](#l1.179)
else:[](#l1.180)
pass[](#l1.181)
# end while[](#l1.182)
for i in a:[](#l1.184)
break[](#l1.185)
# end for[](#l1.186)
for i in a:[](#l1.188)
break[](#l1.189)
else:[](#l1.190)
pass[](#l1.191)
# end for[](#l1.192)
try:[](#l1.194)
pass[](#l1.195)
finally:[](#l1.196)
pass[](#l1.197)
# end try[](#l1.198)
try:[](#l1.200)
pass[](#l1.201)
except TypeError:[](#l1.202)
pass[](#l1.203)
except ValueError:[](#l1.204)
pass[](#l1.205)
else:[](#l1.206)
pass[](#l1.207)
# end try[](#l1.208)
try:[](#l1.210)
pass[](#l1.211)
except TypeError:[](#l1.212)
pass[](#l1.213)
except ValueError:[](#l1.214)
pass[](#l1.215)
finally:[](#l1.216)
pass[](#l1.217)
# end try[](#l1.218)
with a:[](#l1.220)
pass[](#l1.221)
# end with[](#l1.222)
class A:[](#l1.224)
pass[](#l1.225)
# end class A[](#l1.226)
def f():[](#l1.228)
pass[](#l1.229)
# end def f[](#l1.230)
""")[](#l1.231)
self.pindent_test(clean, closed)[](#l1.232)
- def test_multilevel(self):
clean = textwrap.dedent("""\[](#l1.235)
def foobar(a, b):[](#l1.236)
if a == b:[](#l1.237)
a = a+1[](#l1.238)
elif a < b:[](#l1.239)
b = b-1[](#l1.240)
if b > a: a = a-1[](#l1.241)
else:[](#l1.242)
print 'oops!'[](#l1.243)
""")[](#l1.244)
closed = textwrap.dedent("""\[](#l1.245)
def foobar(a, b):[](#l1.246)
if a == b:[](#l1.247)
a = a+1[](#l1.248)
elif a < b:[](#l1.249)
b = b-1[](#l1.250)
if b > a: a = a-1[](#l1.251)
# end if[](#l1.252)
else:[](#l1.253)
print 'oops!'[](#l1.254)
# end if[](#l1.255)
# end def foobar[](#l1.256)
""")[](#l1.257)
self.pindent_test(clean, closed)[](#l1.258)
- def test_preserve_indents(self):
clean = textwrap.dedent("""\[](#l1.261)
if a:[](#l1.262)
if b:[](#l1.263)
pass[](#l1.264)
""")[](#l1.265)
closed = textwrap.dedent("""\[](#l1.266)
if a:[](#l1.267)
if b:[](#l1.268)
pass[](#l1.269)
# end if[](#l1.270)
# end if[](#l1.271)
""")[](#l1.272)
self.assertEqual(self.pindent(clean, '-c'), closed)[](#l1.273)
self.assertEqual(self.pindent(closed, '-d'), clean)[](#l1.274)
broken = self.lstriplines(closed)[](#l1.275)
self.assertEqual(self.pindent(broken, '-r', '-e', '-s', '9'), closed)[](#l1.276)
clean = textwrap.dedent("""\[](#l1.277)
if a:[](#l1.278)
\tif b:[](#l1.279)
\t\tpass[](#l1.280)
""")[](#l1.281)
closed = textwrap.dedent("""\[](#l1.282)
if a:[](#l1.283)
\tif b:[](#l1.284)
\t\tpass[](#l1.285)
\t# end if[](#l1.286)
# end if[](#l1.287)
""")[](#l1.288)
self.assertEqual(self.pindent(clean, '-c'), closed)[](#l1.289)
self.assertEqual(self.pindent(closed, '-d'), clean)[](#l1.290)
broken = self.lstriplines(closed)[](#l1.291)
self.assertEqual(self.pindent(broken, '-r'), closed)[](#l1.292)
- def test_escaped_newline(self):
clean = textwrap.dedent("""\[](#l1.295)
class\\[](#l1.296)
\\[](#l1.297)
A:[](#l1.298)
def\[](#l1.299)
\\[](#l1.300)
f:[](#l1.301)
pass[](#l1.302)
""")[](#l1.303)
closed = textwrap.dedent("""\[](#l1.304)
class\\[](#l1.305)
\\[](#l1.306)
A:[](#l1.307)
def\[](#l1.308)
\\[](#l1.309)
f:[](#l1.310)
pass[](#l1.311)
# end def f[](#l1.312)
# end class A[](#l1.313)
""")[](#l1.314)
self.assertEqual(self.pindent(clean, '-c'), closed)[](#l1.315)
self.assertEqual(self.pindent(closed, '-d'), clean)[](#l1.316)
pass[](#l1.322)
""")[](#l1.323)
closed = textwrap.dedent("""\[](#l1.324)
if a:[](#l1.325)
pass[](#l1.327)
# end if[](#l1.328)
""")[](#l1.329)
self.pindent_test(clean, closed)[](#l1.330)
- def test_oneline(self):
clean = textwrap.dedent("""\[](#l1.333)
if a: pass[](#l1.334)
""")[](#l1.335)
closed = textwrap.dedent("""\[](#l1.336)
if a: pass[](#l1.337)
# end if[](#l1.338)
""")[](#l1.339)
self.pindent_test(clean, closed)[](#l1.340)
+ + class TestSundryScripts(unittest.TestCase): # At least make sure the rest don't have syntax errors. When tests are # added for a script it should be added to the whitelist below.
--- a/Misc/NEWS +++ b/Misc/NEWS @@ -621,6 +621,8 @@ Extension Modules Tests ----- +- Issue #15539: Added regression tests for Tools/scripts/pindent.py. +
- Issue #16836: Enable IPv6 support even if IPv6 is disabled on the build host.
- Issue #16925: test_configparser now works with unittest test discovery. @@ -777,6 +779,11 @@ Documentation Tools/Demos ----------- +- Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py. Now
- pindent.py works with a "with" statement. pindent.py no longer produces
- improper indentation. pindent.py now works with continued lines broken after
- "class" or "def" keywords and with continuations at the start of line. +
- Issue #11797: Add a 2to3 fixer that maps reload() to imp.reload().
- Issue #10966: Remove the concept of unexpected skipped tests.
--- a/Tools/scripts/pindent.py +++ b/Tools/scripts/pindent.py @@ -79,8 +79,9 @@
Defaults
STEPSIZE = 8 TABSIZE = 8 -EXPANDTABS = 0 +EXPANDTABS = False +import io import re import sys @@ -89,7 +90,8 @@ next['if'] = next['elif'] = 'elif', 'els next['while'] = next['for'] = 'else', 'end' next['try'] = 'except', 'finally' next['except'] = 'except', 'else', 'finally', 'end' -next['else'] = next['finally'] = next['def'] = next['class'] = 'end' +next['else'] = next['finally'] = next['with'] = [](#l3.19)
next['end'] = () start = 'if', 'while', 'for', 'try', 'with', 'def', 'class' @@ -105,11 +107,11 @@ class PythonIndenter: self.expandtabs = expandtabs self._write = fpo.write self.kwprog = re.compile(
r'^\s*(?P<kw>[a-z]+)'[](#l3.28)
r'(\s+(?P<id>[a-zA-Z_]\w*))?'[](#l3.29)
r'^(?:\s|\\\n)*(?P<kw>[a-z]+)'[](#l3.30)
r'((?:\s|\\\n)+(?P<id>[a-zA-Z_]\w*))?'[](#l3.31) r'[^\w]')[](#l3.32) self.endprog = re.compile([](#l3.33)
r'^\s*#?\s*end\s+(?P<kw>[a-z]+)'[](#l3.34)
r'^(?:\s|\\\n)*#?\s*end\s+(?P<kw>[a-z]+)'[](#l3.35) r'(\s+(?P<id>[a-zA-Z_]\w*))?'[](#l3.36) r'[^\w]')[](#l3.37) self.wsprog = re.compile(r'^[ \t]*')[](#l3.38)
@@ -125,7 +127,7 @@ class PythonIndenter: def readline(self): line = self.fpi.readline()
if line: self.lineno = self.lineno + 1[](#l3.43)
if line: self.lineno += 1[](#l3.44) # end if[](#l3.45) return line[](#l3.46)
@@ -143,27 +145,24 @@ class PythonIndenter: end def readline line2 = self.readline() if not line2: break # end if
line = line + line2[](#l3.52)
- def putline(self, line, indent = None):
if indent is None:[](#l3.59)
self.write(line)[](#l3.60)
return[](#l3.61)
# end if[](#l3.62)
i = 0[](#l3.65)
m = self.wsprog.match(line)[](#l3.66)
if m: i = m.end()[](#l3.67)
i = self.wsprog.match(line).end()[](#l3.68)
line = line[i:][](#l3.69)
if line[:1] not in ('\n', '\r', ''):[](#l3.70)
line = '\t'*tabs + ' '*spaces + line[](#l3.71) # end if[](#l3.72)
self.write('\t'*tabs + ' '*spaces + line[i:])[](#l3.73)
while 1:[](#l3.79)
while True:[](#l3.80) line = self.getline()[](#l3.81) if not line: break # EOF[](#l3.82) # end if[](#l3.83)
@@ -173,10 +172,9 @@ class PythonIndenter: kw2 = m.group('kw') if not stack: self.error('unexpected end')
elif stack[-1][0] != kw2:[](#l3.88)
elif stack.pop()[0] != kw2:[](#l3.89) self.error('unmatched end')[](#l3.90) # end if[](#l3.91)
del stack[-1:][](#l3.92) self.putline(line, len(stack))[](#l3.93) continue[](#l3.94) # end if[](#l3.95)
@@ -208,23 +206,23 @@ class PythonIndenter: def delete(self): begin_counter = 0 end_counter = 0
while 1:[](#l3.100)
while True:[](#l3.101) line = self.getline()[](#l3.102) if not line: break # EOF[](#l3.103) # end if[](#l3.104) m = self.endprog.match(line)[](#l3.105) if m:[](#l3.106)
end_counter = end_counter + 1[](#l3.107)
end_counter += 1[](#l3.108) continue[](#l3.109) # end if[](#l3.110) m = self.kwprog.match(line)[](#l3.111) if m:[](#l3.112) kw = m.group('kw')[](#l3.113) if kw in start:[](#l3.114)
begin_counter = begin_counter + 1[](#l3.115)
begin_counter += 1[](#l3.116) # end if[](#l3.117) # end if[](#l3.118)
self.putline(line)[](#l3.119)
self.write(line)[](#l3.120) # end while[](#l3.121) if begin_counter - end_counter < 0:[](#l3.122) sys.stderr.write('Warning: input contained more end tags than expected\n')[](#l3.123)
@@ -234,17 +232,12 @@ class PythonIndenter: # end def delete def complete(self):
self.indentsize = 1[](#l3.128) stack = [][](#l3.129) todo = [][](#l3.130)
thisid = ''[](#l3.131)
current, firstkw, lastkw, topid = 0, '', '', ''[](#l3.132)
while 1:[](#l3.133)
currentws = thisid = firstkw = lastkw = topid = ''[](#l3.134)
while True:[](#l3.135) line = self.getline()[](#l3.136)
i = 0[](#l3.137)
m = self.wsprog.match(line)[](#l3.138)
if m: i = m.end()[](#l3.139)
# end if[](#l3.140)
i = self.wsprog.match(line).end()[](#l3.141) m = self.endprog.match(line)[](#l3.142) if m:[](#l3.143) thiskw = 'end'[](#l3.144)
@@ -269,7 +262,9 @@ class PythonIndenter: thiskw = '' # end if # end if
indent = len(line[:i].expandtabs(self.tabsize))[](#l3.149)
indentws = line[:i][](#l3.150)
indent = len(indentws.expandtabs(self.tabsize))[](#l3.151)
current = len(currentws.expandtabs(self.tabsize))[](#l3.152) while indent < current:[](#l3.153) if firstkw:[](#l3.154) if topid:[](#l3.155)
@@ -278,11 +273,11 @@ class PythonIndenter: else: s = '# end %s\n' % firstkw # end if
self.putline(s, current)[](#l3.160)
self.write(currentws + s)[](#l3.161) firstkw = lastkw = ''[](#l3.162) # end if[](#l3.163)
current, firstkw, lastkw, topid = stack[-1][](#l3.164)
del stack[-1][](#l3.165)
currentws, firstkw, lastkw, topid = stack.pop()[](#l3.166)
current = len(currentws.expandtabs(self.tabsize))[](#l3.167) # end while[](#l3.168) if indent == current and firstkw:[](#l3.169) if thiskw == 'end':[](#l3.170)
@@ -297,18 +292,18 @@ class PythonIndenter: else: s = '# end %s\n' % firstkw # end if
self.putline(s, current)[](#l3.175)
self.write(currentws + s)[](#l3.176) firstkw = lastkw = topid = ''[](#l3.177) # end if[](#l3.178) # end if[](#l3.179) if indent > current:[](#l3.180)
stack.append((current, firstkw, lastkw, topid))[](#l3.181)
stack.append((currentws, firstkw, lastkw, topid))[](#l3.182) if thiskw and thiskw not in start:[](#l3.183) # error[](#l3.184) thiskw = ''[](#l3.185) # end if[](#l3.186)
current, firstkw, lastkw, topid = \[](#l3.187)
indent, thiskw, thiskw, thisid[](#l3.188)
currentws, firstkw, lastkw, topid = \[](#l3.189)
indentws, thiskw, thiskw, thisid[](#l3.190) # end if[](#l3.191) if thiskw:[](#l3.192) if thiskw in start:[](#l3.193)
@@ -326,7 +321,6 @@ class PythonIndenter: self.write(line) # end while # end def complete -
end class PythonIndenter
Simplified user interface
@@ -352,76 +346,34 @@ def reformat_filter(input = sys.stdin, o pi.reformat()
end def reformat_filter
- def init(self, buf):
self.buf = buf[](#l3.208)
self.pos = 0[](#l3.209)
self.len = len(self.buf)[](#l3.210)
end def init
- def read(self, n = 0):
if n <= 0:[](#l3.213)
n = self.len - self.pos[](#l3.214)
else:[](#l3.215)
n = min(n, self.len - self.pos)[](#l3.216)
# end if[](#l3.217)
r = self.buf[self.pos : self.pos + n][](#l3.218)
self.pos = self.pos + n[](#l3.219)
return r[](#l3.220)
end def read
- def readline(self):
i = self.buf.find('\n', self.pos)[](#l3.223)
return self.read(i + 1 - self.pos)[](#l3.224)
end def readline
- def readlines(self):
lines = [][](#l3.227)
line = self.readline()[](#l3.228)
while line:[](#l3.229)
lines.append(line)[](#l3.230)
line = self.readline()[](#l3.231)
# end while[](#l3.232)
return lines[](#l3.233)
end def readlines
seek/tell etc. are left as an exercise for the reader
-# end class StringReader - -class StringWriter:
- def init(self):
self.buf = ''[](#l3.240)
end def init
- def write(self, s):
self.buf = self.buf + s[](#l3.243)
end def write
- def getvalue(self):
return self.buf[](#l3.246)
end def getvalue
-# end class StringWriter - def complete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
- input = io.StringIO(source)
- output = io.StringIO() pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) pi.complete() return output.getvalue()
end def complete_string
def delete_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
- input = io.StringIO(source)
- output = io.StringIO() pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) pi.delete() return output.getvalue()
end def delete_string
def reformat_string(source, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
- input = io.StringIO(source)
- output = io.StringIO() pi = PythonIndenter(input, output, stepsize, tabsize, expandtabs) pi.reformat() return output.getvalue()
end def reformat_string
def complete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
- with open(filename, 'r') as f:
source = f.read()[](#l3.283)
result = complete_string(source, stepsize, tabsize, expandtabs) end with if source == result: return 0
end if
@@ -429,14 +381,16 @@ def complete_file(filename, stepsize = S try: os.rename(filename, filename + '~') except OSError: pass # end try
end def complete_file
def delete_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
- with open(filename, 'r') as f:
source = f.read()[](#l3.304)
result = delete_string(source, stepsize, tabsize, expandtabs) end with if source == result: return 0
end if
@@ -444,14 +398,16 @@ def delete_file(filename, stepsize = STE try: os.rename(filename, filename + '~') except OSError: pass # end try
end def delete_file
def reformat_file(filename, stepsize = STEPSIZE, tabsize = TABSIZE, expandtabs = EXPANDTABS):
- with open(filename, 'r') as f:
source = f.read()[](#l3.325)
result = reformat_string(source, stepsize, tabsize, expandtabs) end with if source == result: return 0
end if
@@ -459,9 +415,9 @@ def reformat_file(filename, stepsize = S try: os.rename(filename, filename + '~') except OSError: pass # end try
end def reformat_file
@@ -474,7 +430,7 @@ usage: pindent (-c|-d|-r) [-s stepsize] -r : reformat a completed program (use #end directives) -s stepsize: indentation step (default %(STEPSIZE)d) -t tabsize : the worth in spaces of a tab (default %(TABSIZE)d) --e : expand TABs into spaces (defailt OFF) +-e : expand TABs into spaces (default OFF) [file] ... : files are changed in place, with backups in file~ If no files are specified or a single - is given, the program acts as a filter (reads stdin, writes stdout). @@ -517,7 +473,7 @@ def test(): elif o == '-t': tabsize = int(a) elif o == '-e':
expandtabs = 1[](#l3.356)