distutils2: b024621a1be6 (original) (raw)

Mercurial > distutils2

changeset 1277:b024621a1be6

Start improving 2to3 code (#13462). - Change the fixers used in tests to something not provided by lib2to3 - Test conversion of doctests in text files - Factor out test boilerplate into a common method

Éric Araujo merwok@netwok.org
date	Thu, 09 Feb 2012 17:29:38 +0100
parents	cfdf8964c3dc
children	8aca92f2fc25
files	distutils2/compat.py distutils2/tests/fixer/fix_echo.py distutils2/tests/fixer/fix_echo2.py distutils2/tests/fixer/fix_idioms.py distutils2/tests/test_mixin2to3.py distutils2/util.py
diffstat	6 files changed, 99 insertions(+), 208 deletions(-)[+] [-] distutils2/compat.py 17 distutils2/tests/fixer/fix_echo.py 16 distutils2/tests/fixer/fix_echo2.py 16 distutils2/tests/fixer/fix_idioms.py 134 distutils2/tests/test_mixin2to3.py 106 distutils2/util.py 18

line wrap: on

line diff

--- a/distutils2/compat.py +++ b/distutils2/compat.py @@ -25,7 +25,7 @@ class Mixin2to3(_KLASS): """ if _CONVERT:

   def _run_2to3(self, files, doctests=[], fixers=[]):[](#l1.7)

   def _run_2to3(self, files=[], doctests=[], fixers=[]):[](#l1.8)
       """ Takes a list of files and doctests, and performs conversion[](#l1.9)
       on those.[](#l1.10)
         - First, the files which contain the code(`files`) are converted.[](#l1.11)

@@ -35,17 +35,16 @@ class Mixin2to3(_KLASS): if fixers: self.fixer_names = fixers

       logger.info('converting Python code')[](#l1.16)

       _KLASS.run_2to3(self, files)[](#l1.17)

```
       if files:[](#l1.18)
```

           logger.info('converting Python code and doctests')[](#l1.19)

           _KLASS.run_2to3(self, files)[](#l1.20)

           _KLASS.run_2to3(self, files, doctests_only=True)[](#l1.21)

       logger.info('converting doctests in Python files')[](#l1.23)

       _KLASS.run_2to3(self, files, doctests_only=True)[](#l1.24)

```
       if doctests != []:[](#l1.26)
```

           logger.info('converting doctest in text files')[](#l1.27)

```
       if doctests:[](#l1.28)
```

           logger.info('converting doctests in text files')[](#l1.29)
           _KLASS.run_2to3(self, doctests, doctests_only=True)[](#l1.30)

else: # If run on Python 2.x, there is nothing to do.

   def _run_2to3(self, files, doctests=[], fixers=[]):[](#l1.34)

   def _run_2to3(self, files=[], doctests=[], fixers=[]):[](#l1.35)
       pass[](#l1.36)

new file mode 100644 --- /dev/null +++ b/distutils2/tests/fixer/fix_echo.py @@ -0,0 +1,16 @@ +# Example custom fixer, derived from fix_raw_input by Andre Roberge + +from lib2to3 import fixer_base +from lib2to3.fixer_util import Name + + +class FixEcho(fixer_base.BaseFix): +

BM_compatible = True
PATTERN = """

         power< name='echo' trailer< '(' [any] ')' > any* >[](#l2.15)

```
         """[](#l2.16)
```

def transform(self, node, results):
```
   name = results['name'][](#l2.19)
```

   name.replace(Name(u'print', prefix=name.prefix))[](#l2.20)

new file mode 100644 --- /dev/null +++ b/distutils2/tests/fixer/fix_echo2.py @@ -0,0 +1,16 @@ +# Example custom fixer, derived from fix_raw_input by Andre Roberge + +from lib2to3 import fixer_base +from lib2to3.fixer_util import Name + + +class FixEcho2(fixer_base.BaseFix): +

BM_compatible = True
PATTERN = """

         power< name='echo2' trailer< '(' [any] ')' > any* >[](#l3.15)

```
         """[](#l3.16)
```

def transform(self, node, results):
```
   name = results['name'][](#l3.19)
```

   name.replace(Name(u'print', prefix=name.prefix))[](#l3.20)

deleted file mode 100644 --- a/distutils2/tests/fixer/fix_idioms.py +++ /dev/null @@ -1,134 +0,0 @@ -"""Adjust some old Python 2 idioms to their modern counterparts. - -* Change some type comparisons to isinstance() calls:

type(x) == T -> isinstance(x, T)
type(x) is T -> isinstance(x, T)
type(x) != T -> not isinstance(x, T)
type(x) is not T -> not isinstance(x, T)

- -* Change "while 1:" into "while True:". - -* Change both -

v = list(EXPR)
v.sort()
foo(v)

- -and the more general -

v = EXPR
v.sort()
foo(v)

- -into -

v = sorted(EXPR)
foo(v)

-""" -# Author: Jacques Frechet, Collin Winter - -# Local imports -from lib2to3 import fixer_base -from lib2to3.fixer_util import Call, Comma, Name, Node, syms - -CMP = "(n='!=' | '==' | 'is' | n=comp_op< 'is' 'not' >)" -TYPE = "power< 'type' trailer< '(' x=any ')' > >" - -class FixIdioms(fixer_base.BaseFix): -

explicit = False # The user must ask for this fixer

PATTERN = r"""

   isinstance=comparison< %s %s T=any >[](#l4.46)

```
   |[](#l4.47)
```

   isinstance=comparison< T=any %s %s >[](#l4.48)

```
   |[](#l4.49)
```

   while_stmt< 'while' while='1' ':' any+ >[](#l4.50)

```
   |[](#l4.51)
```
```
   sorted=any<[](#l4.52)
```
```
       any*[](#l4.53)
```
```
       simple_stmt<[](#l4.54)
```

         expr_stmt< id1=any '='[](#l4.55)

                    power< list='list' trailer< '(' (not arglist<any+>) any ')' > >[](#l4.56)

```
         >[](#l4.57)
```
```
         '\n'[](#l4.58)
```
```
       >[](#l4.59)
```
```
       sort=[](#l4.60)
```
```
       simple_stmt<[](#l4.61)
```
```
         power< id2=any[](#l4.62)
```

                trailer< '.' 'sort' > trailer< '(' ')' >[](#l4.63)

```
         >[](#l4.64)
```
```
         '\n'[](#l4.65)
```
```
       >[](#l4.66)
```
```
       next=any*[](#l4.67)
```
```
   >[](#l4.68)
```
```
   |[](#l4.69)
```
```
   sorted=any<[](#l4.70)
```
```
       any*[](#l4.71)
```

       simple_stmt< expr_stmt< id1=any '=' expr=any > '\n' >[](#l4.72)

```
       sort=[](#l4.73)
```
```
       simple_stmt<[](#l4.74)
```
```
         power< id2=any[](#l4.75)
```

                trailer< '.' 'sort' > trailer< '(' ')' >[](#l4.76)

```
         >[](#l4.77)
```
```
         '\n'[](#l4.78)
```
```
       >[](#l4.79)
```
```
       next=any*[](#l4.80)
```
```
   >[](#l4.81)
```
""" % (TYPE, CMP, CMP, TYPE)

def match(self, node):

   r = super(FixIdioms, self).match(node)[](#l4.85)

   # If we've matched one of the sort/sorted subpatterns above, we[](#l4.86)

   # want to reject matches where the initial assignment and the[](#l4.87)

   # subsequent .sort() call involve different identifiers.[](#l4.88)

```
   if r and "sorted" in r:[](#l4.89)
```

       if r["id1"] == r["id2"]:[](#l4.90)

```
           return r[](#l4.91)
```
```
       return None[](#l4.92)
```
```
   return r[](#l4.93)
```

def transform(self, node, results):

   if "isinstance" in results:[](#l4.96)

       return self.transform_isinstance(node, results)[](#l4.97)

```
   elif "while" in results:[](#l4.98)
```

       return self.transform_while(node, results)[](#l4.99)

```
   elif "sorted" in results:[](#l4.100)
```

       return self.transform_sort(node, results)[](#l4.101)

```
   else:[](#l4.102)
```

       raise RuntimeError("Invalid match")[](#l4.103)

def transform_isinstance(self, node, results):

   x = results["x"].clone() # The thing inside of type()[](#l4.106)

   T = results["T"].clone() # The type being compared against[](#l4.107)

```
   x.prefix = ""[](#l4.108)
```
```
   T.prefix = " "[](#l4.109)
```

   test = Call(Name("isinstance"), [x, Comma(), T])[](#l4.110)

```
   if "n" in results:[](#l4.111)
```
```
       test.prefix = " "[](#l4.112)
```

       test = Node(syms.not_test, [Name("not"), test])[](#l4.113)

   test.prefix = getattr(node, 'prefix', ' ')[](#l4.114)

```
   return test[](#l4.115)
```

def transform_while(self, node, results):
```
   one = results["while"][](#l4.118)
```

   one.replace(Name("True", prefix=one.prefix))[](#l4.119)

def transform_sort(self, node, results):

   sort_stmt = results["sort"][](#l4.122)

   next_stmt = results["next"][](#l4.123)

   list_call = results.get("list")[](#l4.124)

   simple_expr = results.get("expr")[](#l4.125)

```
   if list_call:[](#l4.127)
```

       list_call.replace(Name("sorted", prefix=list_call.prefix))[](#l4.128)

```
   elif simple_expr:[](#l4.129)
```

       new = simple_expr.clone()[](#l4.130)

```
       new.prefix = ""[](#l4.131)
```

       simple_expr.replace(Call(Name("sorted"), [new],[](#l4.132)

                                prefix=simple_expr.prefix))[](#l4.133)

```
   else:[](#l4.134)
```

       raise RuntimeError("should not have reached here")[](#l4.135)

```
   sort_stmt.remove()[](#l4.136)
```
```
   if next_stmt:[](#l4.137)
```

       next_stmt[0].prefix = sort_stmt._prefix[](#l4.138)

--- a/distutils2/tests/test_mixin2to3.py +++ b/distutils2/tests/test_mixin2to3.py @@ -9,91 +9,85 @@ class Mixin2to3TestCase(support.TempdirM support.LoggingCatcher, unittest.TestCase):

@unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher')
@support.skip_2to3_optimize
def test_convert_code_only(self):

   # used to check if code gets converted properly.[](#l5.10)

```
   code = "print 'test'"[](#l5.11)
```

def setUp(self):

   super(Mixin2to3TestCase, self).setUp()[](#l5.13)

   self.filename = self.mktempfile().name[](#l5.14)

```
   fp = self.mktempfile()[](#l5.16)
```

def check(self, source, wanted, **kwargs):

   source = textwrap.dedent(source)[](#l5.18)

   fp = open(self.filename, 'w')[](#l5.19)
   try:[](#l5.20)

```
       fp.write(code)[](#l5.21)
```

       fp.write(source)[](#l5.22)
   finally:[](#l5.23)
       fp.close()[](#l5.24)

```
   mixin2to3 = Mixin2to3()[](#l5.26)
```

   mixin2to3._run_2to3([fp.name])[](#l5.27)

```
   expected = "print('test')"[](#l5.28)
```

   Mixin2to3()._run_2to3(**kwargs)[](#l5.29)

```
   fp = open(fp.name)[](#l5.31)
```

```
   fp = open(self.filename)[](#l5.32)
```

   wanted = textwrap.dedent(wanted)[](#l5.33)
   try:[](#l5.34)
       converted = fp.read()[](#l5.35)
   finally:[](#l5.36)
       fp.close()[](#l5.37)

   self.assertEqual(expected, converted)[](#l5.39)

   self.assertMultiLineEqual(wanted, converted)[](#l5.40)

@unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher')

def test_doctests_only(self):

   # used to check if doctests gets converted properly.[](#l5.44)

   doctest = textwrap.dedent('''\[](#l5.45)

def test_conversion(self):

   # check that code and doctests get converted[](#l5.47)

   self.check('''\[](#l5.48)
       """Example docstring.[](#l5.49)

>>> print test test It works.

```
       """''')[](#l5.55)
```

```
   fp = self.mktempfile()[](#l5.57)
```
```
   try:[](#l5.58)
```
```
       fp.write(doctest)[](#l5.59)
```
```
   finally:[](#l5.60)
```
```
       fp.close()[](#l5.61)
```

```
   mixin2to3 = Mixin2to3()[](#l5.63)
```

   mixin2to3._run_2to3([fp.name])[](#l5.64)

   expected = textwrap.dedent('''\[](#l5.65)

```
       """[](#l5.66)
```
```
       print 'test'[](#l5.67)
```
```
       ''',[](#l5.68)
```

       '''\[](#l5.69)
       """Example docstring.[](#l5.70)

>>> print(test) test It works.

```
       """\n''')[](#l5.76)
```

```
       """[](#l5.77)
```
```
       print('test')[](#l5.78)
```

       ''',  # 2to3 adds a newline here[](#l5.80)

```
       files=[self.filename])[](#l5.81)
```

@unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher')
def test_doctests_conversion(self):

   # check that doctest files are converted[](#l5.85)

```
   self.check('''\[](#l5.86)
```
```
       Welcome to the doc.[](#l5.87)
```

```
   fp = open(fp.name)[](#l5.89)
```
```
   try:[](#l5.90)
```
```
       converted = fp.read()[](#l5.91)
```
```
   finally:[](#l5.92)
```
```
       fp.close()[](#l5.93)
```

```
       >>> print test[](#l5.94)
```
```
       test[](#l5.95)
```
```
       ''',[](#l5.96)
```
```
       '''\[](#l5.97)
```
```
       Welcome to the doc.[](#l5.98)
```

   self.assertEqual(expected, converted)[](#l5.100)

```
       >>> print(test)[](#l5.101)
```
```
       test[](#l5.102)
```

```
       ''',[](#l5.104)
```

       doctests=[self.filename])[](#l5.105)

@unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_additional_fixers(self):

   # used to check if use_2to3_fixers works[](#l5.109)

```
   code = 'type(x) is not T'[](#l5.110)
```

```
   fp = self.mktempfile()[](#l5.112)
```
```
   try:[](#l5.113)
```
```
       fp.write(code)[](#l5.114)
```
```
   finally:[](#l5.115)
```
```
       fp.close()[](#l5.116)
```

```
   mixin2to3 = Mixin2to3()[](#l5.118)
```

   mixin2to3._run_2to3(files=[fp.name], doctests=[fp.name],[](#l5.119)

                       fixers=['distutils2.tests.fixer'])[](#l5.120)

   expected = 'not isinstance(x, T)'[](#l5.122)

```
   fp = open(fp.name)[](#l5.124)
```
```
   try:[](#l5.125)
```
```
       converted = fp.read()[](#l5.126)
```
```
   finally:[](#l5.127)
```
```
       fp.close()[](#l5.128)
```

   self.assertEqual(expected, converted)[](#l5.130)

   # make sure the fixers argument works[](#l5.131)

```
   self.check("""\[](#l5.132)
```
```
       echo('42')[](#l5.133)
```
```
       echo2('oh no')[](#l5.134)
```
```
       """,[](#l5.135)
```
```
       """\[](#l5.136)
```
```
       print('42')[](#l5.137)
```
```
       print('oh no')[](#l5.138)
```
```
       """,[](#l5.139)
```

       files=[self.filename],[](#l5.140)

       fixers=['distutils2.tests.fixer'])[](#l5.141)

def test_suite():

--- a/distutils2/util.py +++ b/distutils2/util.py @@ -862,13 +862,11 @@ def run_2to3(files, doctests_only=False, # Make this class local, to delay import of 2to3 from lib2to3.refactor import get_fixers_from_package, RefactoringTool

fixers = [] fixers = get_fixers_from_package('lib2to3.fixes') if fixer_names: for fixername in fixer_names:

       fixers.extend(fixer for fixer in[](#l6.12)

                     get_fixers_from_package(fixername))[](#l6.13)

```
       fixers.extend(get_fixers_from_package(fixername))[](#l6.14)
```
r = RefactoringTool(fixers, options=options) r.refactor(files, write=True, doctests_only=doctests_only) @@ -879,21 +877,23 @@ class Mixin2to3(object): the class variables, or inherit from this class to override how 2to3 is invoked. """

provide list of fixers to run.
defaults to all from lib2to3.fixers

list of fixers to run; defaults to all implicit from lib2to3.fixers
fixer_names = None -

options dictionary

dict of options
options = None -

list of fixers to invoke even though they are marked as explicit

list of extra fixers to invoke
explicit = None
TODO need a better way to add just one fixer from a package
TODO need a way to exclude individual fixers

def run_2to3(self, files, doctests_only=False): """ Issues a call to util.run_2to3. """ return run_2to3(files, doctests_only, self.fixer_names, self.options, self.explicit)

TODO provide initialize/finalize_options

+ + RICH_GLOB = re.compile(r'{([^}]*)}') _CHECK_RECURSIVE_GLOB = re.compile(r'[^/\,{]**|**[^/\,}]') _CHECK_MISMATCH_SET = re.compile(r'^[^{]}|{[^}]$')

distutils2: b024621a1be6 (original) (raw)

Mercurial > distutils2

changeset 1277:b024621a1be6

provide list of fixers to run.

defaults to all from lib2to3.fixers

list of fixers to run; defaults to all implicit from lib2to3.fixers

options dictionary

dict of options

list of fixers to invoke even though they are marked as explicit

list of extra fixers to invoke

TODO need a better way to add just one fixer from a package

TODO need a way to exclude individual fixers

TODO provide initialize/finalize_options