cpython: 35447332ab19 (original) (raw)

Mercurial > cpython

changeset 103402:35447332ab19

Issue #28038: Remove Tools/parser/com2ann.py and its unit test. Development is moving to https://github.com/ilevkivskyi/com2ann [#28038]

Guido van Rossum guido@dropbox.com
date	Fri, 09 Sep 2016 09:06:11 -0700
parents	801634d3c105
children	57afe873e5bf
files	Lib/test/test_tools/test_com2ann.py Tools/parser/com2ann.py
diffstat	2 files changed, 0 insertions(+), 568 deletions(-)[+] [-] Lib/test/test_tools/test_com2ann.py 260 Tools/parser/com2ann.py 308

line wrap: on

line diff

deleted file mode 100644 --- a/Lib/test/test_tools/test_com2ann.py +++ /dev/null @@ -1,260 +0,0 @@ -"""Tests for the com2ann.py script in the Tools/parser directory.""" - -import unittest -import test.support -import os -import re - -from test.test_tools import basepath, toolsdir, skip_if_missing - -skip_if_missing() - -parser_path = os.path.join(toolsdir, "parser") - -with test.support.DirsOnSysPath(parser_path):

from com2ann import *

- -class BaseTestCase(unittest.TestCase): -

def check(self, code, expected, n=False, e=False):

   self.assertEqual(com2ann(code,[](#l1.24)

                    drop_None=n, drop_Ellipsis=e, silent=True),[](#l1.25)

```
                    expected)[](#l1.26)
```

- -class SimpleTestCase(BaseTestCase):

Tests for basic conversions

def test_basics(self):

   self.check("z = 5", "z = 5")[](#l1.32)

   self.check("z: int = 5", "z: int = 5")[](#l1.33)

   self.check("z = 5 # type: int", "z: int = 5")[](#l1.34)

   self.check("z = 5 # type: int # comment",[](#l1.35)

              "z: int = 5 # comment")[](#l1.36)

def test_type_ignore(self):

   self.check("foobar = foobaz() #type: ignore",[](#l1.39)

              "foobar = foobaz() #type: ignore")[](#l1.40)

   self.check("a = 42 #type: ignore #comment",[](#l1.41)

              "a = 42 #type: ignore #comment")[](#l1.42)

def test_complete_tuple(self):

   self.check("t = 1, 2, 3 # type: Tuple[int, ...]",[](#l1.45)

              "t: Tuple[int, ...] = (1, 2, 3)")[](#l1.46)

   self.check("t = 1, # type: Tuple[int]",[](#l1.47)

              "t: Tuple[int] = (1,)")[](#l1.48)

   self.check("t = (1, 2, 3) # type: Tuple[int, ...]",[](#l1.49)

              "t: Tuple[int, ...] = (1, 2, 3)")[](#l1.50)

def test_drop_None(self):

   self.check("x = None # type: int",[](#l1.53)

```
              "x: int", True)[](#l1.54)
```

   self.check("x = None # type: int # another",[](#l1.55)

              "x: int # another", True)[](#l1.56)

   self.check("x = None # type: int # None",[](#l1.57)

              "x: int # None", True)[](#l1.58)

def test_drop_Ellipsis(self):

   self.check("x = ... # type: int",[](#l1.61)

              "x: int", False, True)[](#l1.62)

   self.check("x = ... # type: int # another",[](#l1.63)

              "x: int # another", False, True)[](#l1.64)

   self.check("x = ... # type: int # ...",[](#l1.65)

              "x: int # ...", False, True)[](#l1.66)

def test_newline(self):

   self.check("z = 5 # type: int\r\n", "z: int = 5\r\n")[](#l1.69)

   self.check("z = 5 # type: int # comment\x85",[](#l1.70)

              "z: int = 5 # comment\x85")[](#l1.71)

def test_wrong(self):

   self.check("#type : str", "#type : str")[](#l1.74)

   self.check("x==y #type: bool", "x==y #type: bool")[](#l1.75)

def test_pattern(self):

   for line in ["#type: int", "  # type:  str[:] # com"]:[](#l1.78)

       self.assertTrue(re.search(TYPE_COM, line))[](#l1.79)

   for line in ["", "#", "# comment", "#type", "type int:"]:[](#l1.80)

       self.assertFalse(re.search(TYPE_COM, line))[](#l1.81)

- -class BigTestCase(BaseTestCase):

Tests for really crazy formatting, to be sure
that script works reasonably in extreme situations

def test_crazy(self):
```
   self.maxDiff = None[](#l1.88)
```

   self.check(crazy_code, big_result, False, False)[](#l1.89)

   self.check(crazy_code, big_result_ne, True, True)[](#l1.90)

- -crazy_code = """[](#l1.92) -# -- coding: utf-8 -- # this should not be spoiled -''' -Docstring here -''' - -import testmod -x = 5 #type : int # this one is OK -ttt \

= \
```
   1.0, \\[](#l1.102)
```
```
   2.0, \\[](#l1.103)
```

   3.0, #type: Tuple[float, float, float][](#l1.104)

-with foo(x==1) as f: #type: str

print(f)

- -for i, j in my_inter(x=1): # type: ignore

i + j # type: int # what about this

- -x = y = z = 1 # type: int -x, y, z = [], [], [] # type: (List[int], List[int], List[str]) -class C: - -

l[f(x
```
   =1)] = [[](#l1.117)
```

```
    1,[](#l1.119)
```
```
    2,[](#l1.120)
```
```
    ]  # type: List[int][](#l1.121)
```

- -

(C.x[1]) = \
```
   42 == 5# type: bool[](#l1.125)
```

-lst[...] = \

-...)) # type: int # comment .. - -y = ... # type: int # comment ... -z = ... -#type: int - - -#DONE placement of annotation after target rather than before = - -TD.x[1] \

= 0 == 5# type: bool

- -TD.y[1] =5 == 5# type: bool # one more here -F[G(x == y, - -# hm... -

z)]\

 = None # type: OMG[int] # comment: None[](#l1.146)

-x = None#type:int #comment : None""" - -big_result = """[](#l1.149) -# -- coding: utf-8 -- # this should not be spoiled -''' -Docstring here -''' - -import testmod -x: int = 5 # this one is OK -ttt: Tuple[float, float, float] \

= \
```
  (1.0, \\[](#l1.159)
```
```
   2.0, \\[](#l1.160)
```
```
   3.0,)[](#l1.161)
```

-with foo(x==1) as f: #type: str

print(f)

- -for i, j in my_inter(x=1): # type: ignore

i + j # type: int # what about this

- -x = y = z = 1 # type: int -x, y, z = [], [], [] # type: (List[int], List[int], List[str]) -class C: - -

l[f(x
```
   =1)]: List[int] = [[](#l1.174)
```

```
    1,[](#l1.176)
```
```
    2,[](#l1.177)
```
```
    ][](#l1.178)
```

- -

(C.x[1]): bool = \
```
   42 == 5[](#l1.182)
```

-lst[...]: int = \

-...)) # comment .. - -y: int = ... # comment ... -z = ... -#type: int - - -#DONE placement of annotation after target rather than before = - -TD.x[1]: bool \

= 0 == 5

- -TD.y[1]: bool =5 == 5 # one more here -F[G(x == y, - -# hm... -

z)]: OMG[int]\
```
 = None # comment: None[](#l1.203)
```

-x: int = None #comment : None""" - -big_result_ne = """[](#l1.206) -# -- coding: utf-8 -- # this should not be spoiled -''' -Docstring here -''' - -import testmod -x: int = 5 # this one is OK -ttt: Tuple[float, float, float] \

= \
```
  (1.0, \\[](#l1.216)
```
```
   2.0, \\[](#l1.217)
```
```
   3.0,)[](#l1.218)
```

-with foo(x==1) as f: #type: str

print(f)

- -for i, j in my_inter(x=1): # type: ignore

i + j # type: int # what about this

- -x = y = z = 1 # type: int -x, y, z = [], [], [] # type: (List[int], List[int], List[str]) -class C: - -

l[f(x
```
   =1)]: List[int] = [[](#l1.231)
```

```
    1,[](#l1.233)
```
```
    2,[](#l1.234)
```
```
    ][](#l1.235)
```

- -

(C.x[1]): bool = \
```
   42 == 5[](#l1.239)
```

-lst[...]: int \

\
comment ..

- -y: int # comment ... -z = ... -#type: int - - -#DONE placement of annotation after target rather than before = - -TD.x[1]: bool \

= 0 == 5

- -TD.y[1]: bool =5 == 5 # one more here -F[G(x == y, - -# hm... -

z)]: OMG[int]\
```
  # comment: None[](#l1.260)
```

-x: int #comment : None""" - -if name == 'main':

unittest.main()

deleted file mode 100644 --- a/Tools/parser/com2ann.py +++ /dev/null @@ -1,308 +0,0 @@ -"""Helper module to tranlate 3.5 type comments to 3.6 variable annotations.""" -import re -import os -import ast -import argparse -import tokenize -from collections import defaultdict -from textwrap import dedent -from io import BytesIO - -all = ['com2ann', 'TYPE_COM'] - -TYPE_COM = re.compile(r'\s*#\stype\s:.$', flags=re.DOTALL) -TRAIL_OR_COM = re.compile(r'\s$|\s*#.*$', flags=re.DOTALL) - - -class _Data:

"""Internal class describing global data on file."""
def init(self, lines, tokens):
```
   self.lines = lines[](#l2.24)
```
```
   self.tokens = tokens[](#l2.25)
```

   ttab = defaultdict(list) # maps line number to token numbers[](#l2.26)

   for i, tok in enumerate(tokens):[](#l2.27)

       ttab[tok.start[0]].append(i)[](#l2.28)

```
   self.ttab = ttab[](#l2.29)
```

   self.success = [] # list of lines where type comments where processed[](#l2.30)

   self.fail = [] # list of lines where type comments where rejected[](#l2.31)

- - -def skip_blank(d, lno):

while d.lines[lno].strip() == '':
```
   lno += 1[](#l2.36)
```
return lno

- - -def find_start(d, lcom):

"""Find first char of the assignment target."""
i = d.ttab[lcom + 1][-2] # index of type comment token in tokens list
while ((d.tokens[i].exact_type != tokenize.NEWLINE) and

      (d.tokens[i].exact_type != tokenize.ENCODING)):[](#l2.44)

```
   i -= 1[](#l2.45)
```
lno = d.tokens[i].start[0]
return skip_blank(d, lno)

- - -def check_target(stmt):

if len(stmt.body):
```
   assign = stmt.body[0][](#l2.52)
```
else:
```
   return False[](#l2.54)
```
if isinstance(assign, ast.Assign) and len(assign.targets) == 1:
```
   targ = assign.targets[0][](#l2.56)
```
else:
```
   return False[](#l2.58)
```
if (isinstance(targ, ast.Name) or isinstance(targ, ast.Attribute)

   or isinstance(targ, ast.Subscript)):[](#l2.60)

```
   return True[](#l2.61)
```
return False

- - -def find_eq(d, lstart):

"""Find equal sign starting from lstart taking care about d[f(x=1)] = 5."""
col = pars = 0
lno = lstart
while d.lines[lno][col] != '=' or pars != 0:
```
   ch = d.lines[lno][col][](#l2.70)
```
```
   if ch in '([{':[](#l2.71)
```
```
       pars += 1[](#l2.72)
```
```
   elif ch in ')]}':[](#l2.73)
```
```
       pars -= 1[](#l2.74)
```

   if ch == '#' or col == len(d.lines[lno])-1:[](#l2.75)

       lno = skip_blank(d, lno+1)[](#l2.76)

```
       col = 0[](#l2.77)
```
```
   else:[](#l2.78)
```
```
       col += 1[](#l2.79)
```
return lno, col

- - -def find_val(d, poseq):

"""Find position of first char of assignment value starting from poseq."""
lno, col = poseq
while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\'):

   if col == len(d.lines[lno])-1:[](#l2.87)

```
       lno += 1[](#l2.88)
```
```
       col = 0[](#l2.89)
```
```
   else:[](#l2.90)
```
```
       col += 1[](#l2.91)
```
return lno, col

- - -def find_targ(d, poseq):

"""Find position of last char of target (annotation goes here)."""
lno, col = poseq
while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\'):
```
   if col == 0:[](#l2.99)
```
```
       lno -= 1[](#l2.100)
```

       col = len(d.lines[lno])-1[](#l2.101)

```
   else:[](#l2.102)
```
```
       col -= 1[](#l2.103)
```
return lno, col+1

- - -def trim(new_lines, string, ltarg, poseq, lcom, ccom):

"""Remove None or Ellipsis from assignment value.

Also remove parens if one has (None), (...) etc.
string -- 'None' or '...'
ltarg -- line where last char of target is located
poseq -- position of equal sign
lcom, ccom -- position of type comment
"""
nopars = lambda s: s.replace('(', '').replace(')', '')
leq, ceq = poseq
end = ccom if leq == lcom else len(new_lines[leq])
subline = new_lines[leq][:ceq]
if leq == ltarg:

   subline = subline.rstrip()[](#l2.121)

new_lines[leq] = subline + (new_lines[leq][end:] if leq == lcom

                           else new_lines[leq][ceq+1:end])[](#l2.123)

for lno in range(leq+1,lcom):

   new_lines[lno] = nopars(new_lines[lno])[](#l2.126)

if lcom != leq:

   subline = nopars(new_lines[lcom][:ccom]).replace(string, '')[](#l2.129)

   if (not subline.isspace()):[](#l2.130)

       subline = subline.rstrip()[](#l2.131)

   new_lines[lcom] = subline + new_lines[lcom][ccom:][](#l2.132)

- - -def _com2ann(d, drop_None, drop_Ellipsis):

new_lines = d.lines[:]
for lcom, line in enumerate(d.lines):

   match = re.search(TYPE_COM, line)[](#l2.138)

```
   if match:[](#l2.139)
```

       # strip " #  type  :  annotation  \n" -> "annotation  \n"[](#l2.140)

       tp = match.group().lstrip()[1:].lstrip()[4:].lstrip()[1:].lstrip()[](#l2.141)

       submatch = re.search(TRAIL_OR_COM, tp)[](#l2.142)

```
       subcom = ''[](#l2.143)
```

       if submatch and submatch.group():[](#l2.144)

           subcom = submatch.group()[](#l2.145)

           tp = tp[:submatch.start()][](#l2.146)

```
       if tp == 'ignore':[](#l2.147)
```
```
           continue[](#l2.148)
```
```
       ccom = match.start()[](#l2.149)
```

       if not any(d.tokens[i].exact_type == tokenize.COMMENT[](#l2.150)

              for i in d.ttab[lcom + 1]):[](#l2.151)

           d.fail.append(lcom)[](#l2.152)

           continue # type comment inside string[](#l2.153)

       lstart = find_start(d, lcom)[](#l2.154)

       stmt_str = dedent(''.join(d.lines[lstart:lcom+1]))[](#l2.155)

```
       try:[](#l2.156)
```

           stmt = ast.parse(stmt_str)[](#l2.157)

```
       except SyntaxError:[](#l2.158)
```

           d.fail.append(lcom)[](#l2.159)

           continue # for or with statements[](#l2.160)

       if not check_target(stmt):[](#l2.161)

           d.fail.append(lcom)[](#l2.162)

```
           continue[](#l2.163)
```

       d.success.append(lcom)[](#l2.165)

       val = stmt.body[0].value[](#l2.166)

```
       # writing output now[](#l2.168)
```

       poseq = find_eq(d, lstart)[](#l2.169)

       lval, cval = find_val(d, poseq)[](#l2.170)

       ltarg, ctarg = find_targ(d, poseq)[](#l2.171)

```
       op_par = ''[](#l2.173)
```
```
       cl_par = ''[](#l2.174)
```

       if isinstance(val, ast.Tuple):[](#l2.175)

           if d.lines[lval][cval] != '(':[](#l2.176)

```
               op_par = '('[](#l2.177)
```
```
               cl_par = ')'[](#l2.178)
```

       # write the comment first[](#l2.179)

       new_lines[lcom] = d.lines[lcom][:ccom].rstrip() + cl_par + subcom[](#l2.180)

       ccom = len(d.lines[lcom][:ccom].rstrip())[](#l2.181)

```
       string = False[](#l2.183)
```

       if isinstance(val, ast.Tuple):[](#l2.184)

       # t = 1, 2 -> t = (1, 2); only latter is allowed with annotation[](#l2.185)

           free_place = int(new_lines[lval][cval-2:cval] == '  ')[](#l2.186)

           new_lines[lval] = (new_lines[lval][:cval-free_place] +[](#l2.187)

                                  op_par + new_lines[lval][cval:])[](#l2.188)

       elif isinstance(val, ast.Ellipsis) and drop_Ellipsis:[](#l2.189)

```
           string = '...'[](#l2.190)
```

       elif (isinstance(val, ast.NameConstant) and[](#l2.191)

                   val.value is None and drop_None):[](#l2.192)

```
           string = 'None'[](#l2.193)
```
```
       if string:[](#l2.194)
```

           trim(new_lines, string, ltarg, poseq, lcom, ccom)[](#l2.195)

       # finally write an annotation[](#l2.197)

       new_lines[ltarg] = (new_lines[ltarg][:ctarg] +[](#l2.198)

                         ': ' + tp + new_lines[ltarg][ctarg:])[](#l2.199)

return ''.join(new_lines)

- - -def com2ann(code, *, drop_None=False, drop_Ellipsis=False, silent=False):

"""Translate type comments to type annotations in code.

Take code as string and return this string where::

 variable = value # type: annotation # real comment[](#l2.208)

is translated to::

 variable: annotation = value # real comment[](#l2.212)

For unsupported syntax cases, the type comments are
left intact. If drop_None is True or if drop_Ellipsis
is True translate correcpondingly::

 variable = None # type: annotation[](#l2.218)

 variable = ... # type: annotation[](#l2.219)

into::

```
 variable: annotation[](#l2.223)
```

The tool tries to preserve code formatting as much as
possible, but an exact translation is not guarateed.
A summary of translated comments id printed by default.
"""
try:

   ast.parse(code) # we want to work only with file without syntax errors[](#l2.230)

except SyntaxError:
```
   return None[](#l2.232)
```
lines = code.splitlines(keepends=True)
rl = BytesIO(code.encode('utf-8')).readline
tokens = list(tokenize.tokenize(rl))

data = _Data(lines, tokens)
new_code = _com2ann(data, drop_None, drop_Ellipsis)

if not silent:
```
   if data.success:[](#l2.241)
```

       print('Comments translated on lines:',[](#l2.242)

             ', '.join(str(lno+1) for lno in data.success))[](#l2.243)

```
   if data.fail:[](#l2.244)
```

       print('Comments rejected on lines:',[](#l2.245)

             ', '.join(str(lno+1) for lno in data.fail))[](#l2.246)

   if not data.success and not data.fail:[](#l2.247)

       print('No type comments found')[](#l2.248)

return new_code

- - -def translate_file(infile, outfile, dnone, dell, silent):

try:

   descr = tokenize.open(infile)[](#l2.255)

except SyntaxError:

   print("Cannot open", infile)[](#l2.257)

```
   return[](#l2.258)
```
with descr as f:
```
   code = f.read()[](#l2.260)
```
```
   enc = f.encoding[](#l2.261)
```
if not silent:
```
   print('File:', infile)[](#l2.263)
```
new_code = com2ann(code, drop_None=dnone,

                        drop_Ellipsis=dell,[](#l2.265)

                        silent=silent)[](#l2.266)

if new_code is None:

   print("SyntaxError in", infile)[](#l2.268)

```
   return[](#l2.269)
```
with open(outfile, 'wb') as f:

   f.write((new_code).encode(enc))[](#l2.271)

- - -if name == 'main': -

parser = argparse.ArgumentParser(description=doc)
parser.add_argument("-o", "--outfile",

                   help="output file, will be overwritten if exists,\n"[](#l2.278)

                        "defaults to input file")[](#l2.279)

parser.add_argument("infile",

                   help="input file or directory for translation, must\n"[](#l2.281)

                        "contain no syntax errors, for directory\n"[](#l2.282)

                        "the outfile is ignored and translation is\n"[](#l2.283)

                        "made in place")[](#l2.284)

parser.add_argument("-s", "--silent",

                   help="Do not print summary for line numbers of\n"[](#l2.286)

                        "translated and rejected comments",[](#l2.287)

                   action="store_true")[](#l2.288)

parser.add_argument("-n", "--drop-none",

              help="drop any None as assignment value during\n"[](#l2.290)

                   "translation if it is annotated by a type coment",[](#l2.291)

              action="store_true")[](#l2.292)

parser.add_argument("-e", "--drop-ellipsis",

              help="drop any Ellipsis (...) as assignment value during\n"[](#l2.294)

                   "translation if it is annotated by a type coment",[](#l2.295)

              action="store_true")[](#l2.296)

args = parser.parse_args()
if args.outfile is None:

   args.outfile = args.infile[](#l2.299)

if os.path.isfile(args.infile):

   translate_file(args.infile, args.outfile,[](#l2.302)

                  args.drop_none, args.drop_ellipsis, args.silent)[](#l2.303)

else:

   for root, dirs, files in os.walk(args.infile):[](#l2.305)

```
       for afile in files:[](#l2.306)
```

           _, ext = os.path.splitext(afile)[](#l2.307)

           if ext == '.py' or ext == '.pyi':[](#l2.308)

               fname = os.path.join(root, afile)[](#l2.309)

               translate_file(fname, fname,[](#l2.310)

                              args.drop_none, args.drop_ellipsis,[](#l2.311)

                              args.silent)[](#l2.312)

cpython: 35447332ab19 (original) (raw)

Mercurial > cpython

changeset 103402:35447332ab19

Tests for basic conversions

Tests for really crazy formatting, to be sure

that script works reasonably in extreme situations

comment ..