(original) (raw)

changeset: 1277:b024621a1be6 user: Éric Araujo merwok@netwok.org date: Thu Feb 09 17:29:38 2012 +0100 files: distutils2/compat.py distutils2/tests/fixer/fix_echo.py distutils2/tests/fixer/fix_echo2.py distutils2/tests/fixer/fix_idioms.py distutils2/tests/test_mixin2to3.py distutils2/util.py description: Start improving 2to3 code (#13462). - Change the fixers used in tests to something not provided by lib2to3 - Test conversion of doctests in text files - Factor out test boilerplate into a common method diff -r cfdf8964c3dc -r b024621a1be6 distutils2/compat.py --- a/distutils2/compat.py Thu Feb 09 02:04:02 2012 +0100 +++ b/distutils2/compat.py Thu Feb 09 17:29:38 2012 +0100 @@ -25,7 +25,7 @@ """ if _CONVERT: - def _run_2to3(self, files, doctests=[], fixers=[]): + def _run_2to3(self, files=[], doctests=[], fixers=[]): """ Takes a list of files and doctests, and performs conversion on those. - First, the files which contain the code(`files`) are converted. @@ -35,17 +35,16 @@ if fixers: self.fixer_names = fixers - logger.info('converting Python code') - _KLASS.run_2to3(self, files) + if files: + logger.info('converting Python code and doctests') + _KLASS.run_2to3(self, files) + _KLASS.run_2to3(self, files, doctests_only=True) - logger.info('converting doctests in Python files') - _KLASS.run_2to3(self, files, doctests_only=True) - - if doctests != []: - logger.info('converting doctest in text files') + if doctests: + logger.info('converting doctests in text files') _KLASS.run_2to3(self, doctests, doctests_only=True) else: # If run on Python 2.x, there is nothing to do. - def _run_2to3(self, files, doctests=[], fixers=[]): + def _run_2to3(self, files=[], doctests=[], fixers=[]): pass diff -r cfdf8964c3dc -r b024621a1be6 distutils2/tests/fixer/fix_echo.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/distutils2/tests/fixer/fix_echo.py Thu Feb 09 17:29:38 2012 +0100 @@ -0,0 +1,16 @@ +# Example custom fixer, derived from fix_raw_input by Andre Roberge + +from lib2to3 import fixer_base +from lib2to3.fixer_util import Name + + +class FixEcho(fixer_base.BaseFix): + + BM_compatible = True + PATTERN = """ + power< name='echo' trailer< '(' [any] ')' > any* > + """ + + def transform(self, node, results): + name = results['name'] + name.replace(Name(u'print', prefix=name.prefix)) diff -r cfdf8964c3dc -r b024621a1be6 distutils2/tests/fixer/fix_echo2.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/distutils2/tests/fixer/fix_echo2.py Thu Feb 09 17:29:38 2012 +0100 @@ -0,0 +1,16 @@ +# Example custom fixer, derived from fix_raw_input by Andre Roberge + +from lib2to3 import fixer_base +from lib2to3.fixer_util import Name + + +class FixEcho2(fixer_base.BaseFix): + + BM_compatible = True + PATTERN = """ + power< name='echo2' trailer< '(' [any] ')' > any* > + """ + + def transform(self, node, results): + name = results['name'] + name.replace(Name(u'print', prefix=name.prefix)) diff -r cfdf8964c3dc -r b024621a1be6 distutils2/tests/fixer/fix_idioms.py --- a/distutils2/tests/fixer/fix_idioms.py Thu Feb 09 02:04:02 2012 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,134 +0,0 @@ -"""Adjust some old Python 2 idioms to their modern counterparts. - -* Change some type comparisons to isinstance() calls: - type(x) == T -> isinstance(x, T) - type(x) is T -> isinstance(x, T) - type(x) != T -> not isinstance(x, T) - type(x) is not T -> not isinstance(x, T) - -* Change "while 1:" into "while True:". - -* Change both - - v = list(EXPR) - v.sort() - foo(v) - -and the more general - - v = EXPR - v.sort() - foo(v) - -into - - v = sorted(EXPR) - foo(v) -""" -# Author: Jacques Frechet, Collin Winter - -# Local imports -from lib2to3 import fixer_base -from lib2to3.fixer_util import Call, Comma, Name, Node, syms - -CMP = "(n='!=' | '==' | 'is' | n=comp_op< 'is' 'not' >)" -TYPE = "power< 'type' trailer< '(' x=any ')' > >" - -class FixIdioms(fixer_base.BaseFix): - - explicit = False # The user must ask for this fixer - - PATTERN = r""" - isinstance=comparison< %s %s T=any > - | - isinstance=comparison< T=any %s %s > - | - while_stmt< 'while' while='1' ':' any+ > - | - sorted=any< - any* - simple_stmt< - expr_stmt< id1=any '=' - power< list='list' trailer< '(' (not arglist<any+>) any ')' > > - > - '\n' - > - sort= - simple_stmt< - power< id2=any - trailer< '.' 'sort' > trailer< '(' ')' > - > - '\n' - > - next=any* - > - | - sorted=any< - any* - simple_stmt< expr_stmt< id1=any '=' expr=any > '\n' > - sort= - simple_stmt< - power< id2=any - trailer< '.' 'sort' > trailer< '(' ')' > - > - '\n' - > - next=any* - > - """ % (TYPE, CMP, CMP, TYPE) - - def match(self, node): - r = super(FixIdioms, self).match(node) - # If we've matched one of the sort/sorted subpatterns above, we - # want to reject matches where the initial assignment and the - # subsequent .sort() call involve different identifiers. - if r and "sorted" in r: - if r["id1"] == r["id2"]: - return r - return None - return r - - def transform(self, node, results): - if "isinstance" in results: - return self.transform_isinstance(node, results) - elif "while" in results: - return self.transform_while(node, results) - elif "sorted" in results: - return self.transform_sort(node, results) - else: - raise RuntimeError("Invalid match") - - def transform_isinstance(self, node, results): - x = results["x"].clone() # The thing inside of type() - T = results["T"].clone() # The type being compared against - x.prefix = "" - T.prefix = " " - test = Call(Name("isinstance"), [x, Comma(), T]) - if "n" in results: - test.prefix = " " - test = Node(syms.not_test, [Name("not"), test]) - test.prefix = getattr(node, 'prefix', ' ') - return test - - def transform_while(self, node, results): - one = results["while"] - one.replace(Name("True", prefix=one.prefix)) - - def transform_sort(self, node, results): - sort_stmt = results["sort"] - next_stmt = results["next"] - list_call = results.get("list") - simple_expr = results.get("expr") - - if list_call: - list_call.replace(Name("sorted", prefix=list_call.prefix)) - elif simple_expr: - new = simple_expr.clone() - new.prefix = "" - simple_expr.replace(Call(Name("sorted"), [new], - prefix=simple_expr.prefix)) - else: - raise RuntimeError("should not have reached here") - sort_stmt.remove() - if next_stmt: - next_stmt[0].prefix = sort_stmt._prefix diff -r cfdf8964c3dc -r b024621a1be6 distutils2/tests/test_mixin2to3.py --- a/distutils2/tests/test_mixin2to3.py Thu Feb 09 02:04:02 2012 +0100 +++ b/distutils2/tests/test_mixin2to3.py Thu Feb 09 17:29:38 2012 +0100 @@ -9,91 +9,85 @@ support.LoggingCatcher, unittest.TestCase): - @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') - @support.skip_2to3_optimize - def test_convert_code_only(self): - # used to check if code gets converted properly. - code = "print 'test'" + def setUp(self): + super(Mixin2to3TestCase, self).setUp() + self.filename = self.mktempfile().name - fp = self.mktempfile() + def check(self, source, wanted, **kwargs): + source = textwrap.dedent(source) + fp = open(self.filename, 'w') try: - fp.write(code) + fp.write(source) finally: fp.close() - mixin2to3 = Mixin2to3() - mixin2to3._run_2to3([fp.name]) - expected = "print('test')" + Mixin2to3()._run_2to3(**kwargs) - fp = open(fp.name) + fp = open(self.filename) + wanted = textwrap.dedent(wanted) try: converted = fp.read() finally: fp.close() - - self.assertEqual(expected, converted) + self.assertMultiLineEqual(wanted, converted) @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') - def test_doctests_only(self): - # used to check if doctests gets converted properly. - doctest = textwrap.dedent('''\ + def test_conversion(self): + # check that code and doctests get converted + self.check('''\ """Example docstring. >>> print test test It works. - """''') - - fp = self.mktempfile() - try: - fp.write(doctest) - finally: - fp.close() - - mixin2to3 = Mixin2to3() - mixin2to3._run_2to3([fp.name]) - expected = textwrap.dedent('''\ + """ + print 'test' + ''', + '''\ """Example docstring. >>> print(test) test It works. - """\n''') + """ + print('test') + + ''', # 2to3 adds a newline here + files=[self.filename]) + + @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') + def test_doctests_conversion(self): + # check that doctest files are converted + self.check('''\ + Welcome to the doc. - fp = open(fp.name) - try: - converted = fp.read() - finally: - fp.close() + >>> print test + test + ''', + '''\ + Welcome to the doc. - self.assertEqual(expected, converted) + >>> print(test) + test + + ''', + doctests=[self.filename]) @unittest.skipIf(sys.version < '2.6', 'requires Python 2.6 or higher') def test_additional_fixers(self): - # used to check if use_2to3_fixers works - code = 'type(x) is not T' - - fp = self.mktempfile() - try: - fp.write(code) - finally: - fp.close() - - mixin2to3 = Mixin2to3() - mixin2to3._run_2to3(files=[fp.name], doctests=[fp.name], - fixers=['distutils2.tests.fixer']) - - expected = 'not isinstance(x, T)' - - fp = open(fp.name) - try: - converted = fp.read() - finally: - fp.close() - - self.assertEqual(expected, converted) + # make sure the fixers argument works + self.check("""\ + echo('42') + echo2('oh no') + """, + """\ + print('42') + print('oh no') + """, + files=[self.filename], + fixers=['distutils2.tests.fixer']) def test_suite(): diff -r cfdf8964c3dc -r b024621a1be6 distutils2/util.py --- a/distutils2/util.py Thu Feb 09 02:04:02 2012 +0100 +++ b/distutils2/util.py Thu Feb 09 17:29:38 2012 +0100 @@ -862,13 +862,11 @@ # Make this class local, to delay import of 2to3 from lib2to3.refactor import get_fixers_from_package, RefactoringTool - fixers = [] fixers = get_fixers_from_package('lib2to3.fixes') if fixer_names: for fixername in fixer_names: - fixers.extend(fixer for fixer in - get_fixers_from_package(fixername)) + fixers.extend(get_fixers_from_package(fixername)) r = RefactoringTool(fixers, options=options) r.refactor(files, write=True, doctests_only=doctests_only) @@ -879,21 +877,23 @@ the class variables, or inherit from this class to override how 2to3 is invoked. """ - # provide list of fixers to run. - # defaults to all from lib2to3.fixers + # list of fixers to run; defaults to all implicit from lib2to3.fixers fixer_names = None - - # options dictionary + # dict of options options = None - - # list of fixers to invoke even though they are marked as explicit + # list of extra fixers to invoke explicit = None + # TODO need a better way to add just one fixer from a package + # TODO need a way to exclude individual fixers def run_2to3(self, files, doctests_only=False): """ Issues a call to util.run_2to3. """ return run_2to3(files, doctests_only, self.fixer_names, self.options, self.explicit) + # TODO provide initialize/finalize_options + + RICH_GLOB = re.compile(r'\{([^}]*)\}') _CHECK_RECURSIVE_GLOB = re.compile(r'[^/\\,{]\*\*|\*\*[^/\\,}]') _CHECK_MISMATCH_SET = re.compile(r'^[^{]*\}|\{[^}]*$') </any+>/merwok@netwok.org