cpython: 4b791e513c2c (original) (raw)

Mercurial > cpython

changeset 74897:4b791e513c2c

Issue #13930: Adds ability for 2to3 to write its output to a different directory tree instead of overwriting the input files. Adds three command line options: -o/--output-dir, -W/--write-unchanged-files and --add-suffix. [#13930]

line wrap: on

line diff

--- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -94,6 +94,38 @@ change can also be enabled manually with :option:-p to run fixers on code that already has had its print statements converted. +The :option:-o or :option:--output-dir option allows specification of an +alternate directory for processed output files to be written to. The +:option:-n flag is required when using this as backup files do not make sense +when not overwriting the input files. + +.. versionadded:: 3.2.3

• The :option:-o option was added. +

+The :option:-W or :option:--write-unchanged-files flag tells 2to3 to always +write output files even if no changes were required to the file. This is most +useful with :option:-o so that an entire Python source tree is copied with +translation from one directory to another. +This option implies the :option:-w flag as it would not make sense otherwise. + +.. versionadded:: 3.2.3

• The :option:-W flag was added. +

+The :option:--add-suffix option specifies a string to append to all output +filenames. The :option:-n flag is required when specifying this as backups +are not necessary when writing to different filenames. Example:: +

• $ 2to3 -n -W --add-suffix=3 example.py +

+Will cause a converted file named example.py3 to be written. + +.. versionadded:: 3.2.3

• The :option:--add-suffix option was added. +

+To translate an entire project from one directory tree to another use:: +

• $ 2to3 --output-dir=python3-version/mycode -W -n python2-version/mycode +

.. _2to3-fixers:

--- a/Lib/lib2to3/main.py +++ b/Lib/lib2to3/main.py @@ -25,12 +25,41 @@ def diff_texts(a, b, filename): class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool): """

• A refactoring tool that can avoid overwriting its input files. Prints output to stdout. +
• Output files can optionally be written to a different directory and or
• have an extra file suffix appended to their name for use in situations
• where you do not want to replace the input files. """

• def init(self, fixers, options, explicit, nobackups, show_diffs):

• def init(self, fixers, options, explicit, nobackups, show_diffs,

•             input_base_dir='', output_dir='', append_suffix=''):[](#l2.17)

•    """[](#l2.18)

•    Args:[](#l2.19)

•        fixers: A list of fixers to import.[](#l2.20)

•        options: A dict with RefactoringTool configuration.[](#l2.21)

•        explicit: A list of fixers to run even if they are explicit.[](#l2.22)

•        nobackups: If true no backup '.bak' files will be created for those[](#l2.23)

•            files that are being refactored.[](#l2.24)

•        show_diffs: Should diffs of the refactoring be printed to stdout?[](#l2.25)

•        input_base_dir: The base directory for all input files.  This class[](#l2.26)

•            will strip this path prefix off of filenames before substituting[](#l2.27)

•            it with output_dir.  Only meaningful if output_dir is supplied.[](#l2.28)

•            All files processed by refactor() must start with this path.[](#l2.29)

•        output_dir: If supplied, all converted files will be written into[](#l2.30)

•            this directory tree instead of input_base_dir.[](#l2.31)

•        append_suffix: If supplied, all files output by this tool will have[](#l2.32)

•            this appended to their filename.  Useful for changing .py to[](#l2.33)

•            .py3 for example by passing append_suffix='3'.[](#l2.34)

•    """[](#l2.35)
     self.nobackups = nobackups[](#l2.36)
     self.show_diffs = show_diffs[](#l2.37)

•    if input_base_dir and not input_base_dir.endswith(os.sep):[](#l2.38)

•        input_base_dir += os.sep[](#l2.39)

•    self._input_base_dir = input_base_dir[](#l2.40)

•    self._output_dir = output_dir[](#l2.41)

•    self._append_suffix = append_suffix[](#l2.42)
     super(StdoutRefactoringTool, self).__init__(fixers, options, explicit)[](#l2.43)

def log_error(self, msg, *args, **kwargs): @@ -38,6 +67,23 @@ class StdoutRefactoringTool(refactor.Mul self.logger.error(msg, *args, **kwargs) def write_file(self, new_text, filename, old_text, encoding):

•    orig_filename = filename[](#l2.50)

•    if self._output_dir:[](#l2.51)

•        if filename.startswith(self._input_base_dir):[](#l2.52)

•            filename = os.path.join(self._output_dir,[](#l2.53)

•                                    filename[len(self._input_base_dir):])[](#l2.54)

•        else:[](#l2.55)

•            raise ValueError('filename %s does not start with the '[](#l2.56)

•                             'input_base_dir %s' % ([](#l2.57)

•                                     filename, self._input_base_dir))[](#l2.58)

•    if self._append_suffix:[](#l2.59)

•        filename += self._append_suffix[](#l2.60)

•    if orig_filename != filename:[](#l2.61)

•      output_dir = os.path.dirname(filename)[](#l2.62)

•      if not os.path.isdir(output_dir):[](#l2.63)

•          os.makedirs(output_dir)[](#l2.64)

•      self.log_message('Writing converted %s to %s.', orig_filename,[](#l2.65)

•                       filename)[](#l2.66)
     if not self.nobackups:[](#l2.67)
         # Make backup[](#l2.68)
         backup = filename + ".bak"[](#l2.69)

@@ -55,6 +101,9 @@ class StdoutRefactoringTool(refactor.Mul write(new_text, filename, old_text, encoding) if not self.nobackups: shutil.copymode(backup, filename)

•    if orig_filename != filename:[](#l2.74)

•        # Preserve the file mode in the new output directory.[](#l2.75)

•        shutil.copymode(orig_filename, filename)[](#l2.76)

def print_output(self, old, new, filename, equal): if equal: @@ -113,11 +162,33 @@ def main(fixer_pkg, args=None): help="Write back modified files") parser.add_option("-n", "--nobackups", action="store_true", default=False, help="Don't write backups for modified files")

• parser.add_option("-o", "--output-dir", action="store", type="str",

•                  default="", help="Put output files in this directory "[](#l2.85)

•                  "instead of overwriting the input files.  Requires -n.")[](#l2.86)

• parser.add_option("-W", "--write-unchanged-files", action="store_true",

•                  help="Also write files even if no changes were required"[](#l2.88)

•                  " (useful with --output-dir); implies -w.")[](#l2.89)

• parser.add_option("--add-suffix", action="store", type="str", default="",

•                  help="Append this string to all output filenames."[](#l2.91)

•                  " Requires -n if non-empty.  "[](#l2.92)

•                  "ex: --add-suffix='3' will generate .py3 files.")[](#l2.93)

# Parse command line arguments refactor_stdin = False flags = {} options, args = parser.parse_args(args)

• if options.write_unchanged_files:

•    flags["write_unchanged_files"] = True[](#l2.100)

•    if not options.write:[](#l2.101)

•        warn("--write-unchanged-files/-W implies -w.")[](#l2.102)

•    options.write = True[](#l2.103)

• If we allowed these, the original files would be renamed to backup names

• but not replaced.

• if options.output_dir and not options.nobackups:

•    parser.error("Can't use --output-dir/-o without -n.")[](#l2.107)

• if options.add_suffix and not options.nobackups:

•    parser.error("Can't use --add-suffix without -n.")[](#l2.109)

+ if not options.write and options.no_diffs: warn("not writing files and not printing diffs; that's not very useful") if not options.write and options.nobackups: @@ -143,6 +214,7 @@ def main(fixer_pkg, args=None): # Set up logging handler level = logging.DEBUG if options.verbose else logging.INFO logging.basicConfig(format='%(name)s: %(message)s', level=level)

• logger = logging.getLogger('lib2to3.main')

# Initialize the refactoring tool avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg)) @@ -159,8 +231,23 @@ def main(fixer_pkg, args=None): else: requested = avail_fixes.union(explicit) fixer_names = requested.difference(unwanted_fixes)

• rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit),

•                           options.nobackups, not options.no_diffs)[](#l2.127)

• input_base_dir = os.path.commonprefix(args)
• if (input_base_dir and not input_base_dir.endswith(os.sep)

•    and not os.path.isdir(input_base_dir)):[](#l2.130)

•    # One or more similar names were passed, their directory is the base.[](#l2.131)

•    # os.path.commonprefix() is ignorant of path elements, this corrects[](#l2.132)

•    # for that weird API.[](#l2.133)

•    input_base_dir = os.path.dirname(input_base_dir)[](#l2.134)

• if options.output_dir:

•    input_base_dir = input_base_dir.rstrip(os.sep)[](#l2.136)

•    logger.info('Output in %r will mirror the input directory %r layout.',[](#l2.137)

•                options.output_dir, input_base_dir)[](#l2.138)

• rt = StdoutRefactoringTool(

•        sorted(fixer_names), flags, sorted(explicit),[](#l2.140)

•        options.nobackups, not options.no_diffs,[](#l2.141)

•        input_base_dir=input_base_dir,[](#l2.142)

•        output_dir=options.output_dir,[](#l2.143)

•        append_suffix=options.add_suffix)[](#l2.144)

# Refactor all files and directories passed as arguments if not rt.errors:

--- a/Lib/lib2to3/refactor.py +++ b/Lib/lib2to3/refactor.py @@ -173,7 +173,8 @@ class FixerError(Exception): class RefactoringTool(object):

• _default_options = {"print_function" : False}

• _default_options = {"print_function" : False,

•                    "write_unchanged_files" : False}[](#l3.9)

CLASS_PREFIX = "Fix" # The prefix for fixer classes FILE_PREFIX = "fix_" # The prefix for modules with a fixer within @@ -195,6 +196,10 @@ class RefactoringTool(object): self.grammar = pygram.python_grammar_no_print_statement else: self.grammar = pygram.python_grammar

•    # When this is True, the refactor*() methods will call write_file() for[](#l3.17)

•    # files processed even if they were not changed during refactoring. If[](#l3.18)

•    # and only if the refactor method's write parameter was True.[](#l3.19)

•    self.write_unchanged_files = self.options.get("write_unchanged_files")[](#l3.20)
     self.errors = [][](#l3.21)
     self.logger = logging.getLogger("RefactoringTool")[](#l3.22)
     self.fixer_log = [][](#l3.23)

@@ -341,13 +346,13 @@ class RefactoringTool(object): if doctests_only: self.log_debug("Refactoring doctests in %s", filename) output = self.refactor_docstring(input, filename)

•        if output != input:[](#l3.28)

•        if self.write_unchanged_files or output != input:[](#l3.29)
             self.processed_file(output, filename, input, write, encoding)[](#l3.30)
         else:[](#l3.31)
             self.log_debug("No doctest changes in %s", filename)[](#l3.32)
     else:[](#l3.33)
         tree = self.refactor_string(input, filename)[](#l3.34)

•        if tree and tree.was_changed:[](#l3.35)

•        if self.write_unchanged_files or (tree and tree.was_changed):[](#l3.36)
             # The [:-1] is to take off the \n we added earlier[](#l3.37)
             self.processed_file(str(tree)[:-1], filename,[](#l3.38)
                                 write=write, encoding=encoding)[](#l3.39)

@@ -386,13 +391,13 @@ class RefactoringTool(object): if doctests_only: self.log_debug("Refactoring doctests in stdin") output = self.refactor_docstring(input, "")

•        if output != input:[](#l3.44)

•        if self.write_unchanged_files or output != input:[](#l3.45)
             self.processed_file(output, "<stdin>", input)[](#l3.46)
         else:[](#l3.47)
             self.log_debug("No doctest changes in stdin")[](#l3.48)
     else:[](#l3.49)
         tree = self.refactor_string(input, "<stdin>")[](#l3.50)

•        if tree and tree.was_changed:[](#l3.51)

•        if self.write_unchanged_files or (tree and tree.was_changed):[](#l3.52)
             self.processed_file(str(tree), "<stdin>", input)[](#l3.53)
         else:[](#l3.54)
             self.log_debug("No changes in stdin")[](#l3.55)

@@ -502,7 +507,7 @@ class RefactoringTool(object): def processed_file(self, new_text, filename, old_text=None, write=False, encoding=None): """

•    Called when a file has been refactored, and there are changes.[](#l3.60)

•    Called when a file has been refactored and there may be changes.[](#l3.61)
     """[](#l3.62)
     self.files.append(filename)[](#l3.63)
     if old_text is None:[](#l3.64)

@@ -513,7 +518,8 @@ class RefactoringTool(object): self.print_output(old_text, new_text, filename, equal) if equal: self.log_debug("No changes to %s", filename)

•        return[](#l3.69)

•        if not self.write_unchanged_files:[](#l3.70)

•            return[](#l3.71)
     if write:[](#l3.72)
         self.write_file(new_text, filename, old_text, encoding)[](#l3.73)
     else:[](#l3.74)

--- a/Lib/lib2to3/tests/test_main.py +++ b/Lib/lib2to3/tests/test_main.py @@ -1,18 +1,30 @@

-- coding: utf-8 --

-import sys import codecs +import io import logging -import io +import os +import shutil +import sys +import tempfile import unittest from lib2to3 import main +TEST_DATA_DIR = os.path.join(os.path.dirname(file), "data") +PY2_TEST_MODULE = os.path.join(TEST_DATA_DIR, "py2_test_grammar.py") + + class TestMain(unittest.TestCase):

• def setUp(self):

•    self.temp_dir = None  # tearDown() will rmtree this directory if set.[](#l4.26)

+ def tearDown(self): # Clean up logging configuration down by main. del logging.root.handlers[:]

•    if self.temp_dir:[](#l4.31)

•        shutil.rmtree(self.temp_dir)[](#l4.32)

def run_2to3_capture(self, args, in_capture, out_capture, err_capture): save_stdin = sys.stdin @@ -39,3 +51,85 @@ class TestMain(unittest.TestCase): self.assertTrue("-print 'nothing'" in output) self.assertTrue("WARNING: couldn't encode 's diff for " "your terminal" in err.getvalue()) +

• def setup_test_source_trees(self):

•    """Setup a test source tree and output destination tree."""[](#l4.42)

•    self.temp_dir = tempfile.mkdtemp()  # tearDown() cleans this up.[](#l4.43)

•    self.py2_src_dir = os.path.join(self.temp_dir, "python2_project")[](#l4.44)

•    self.py3_dest_dir = os.path.join(self.temp_dir, "python3_project")[](#l4.45)

•    os.mkdir(self.py2_src_dir)[](#l4.46)

•    os.mkdir(self.py3_dest_dir)[](#l4.47)

•    # Turn it into a package with a few files.[](#l4.48)

•    self.setup_files = [][](#l4.49)

•    open(os.path.join(self.py2_src_dir, "__init__.py"), "w").close()[](#l4.50)

•    self.setup_files.append("__init__.py")[](#l4.51)

•    shutil.copy(PY2_TEST_MODULE, self.py2_src_dir)[](#l4.52)

•    self.setup_files.append(os.path.basename(PY2_TEST_MODULE))[](#l4.53)

•    self.trivial_py2_file = os.path.join(self.py2_src_dir, "trivial.py")[](#l4.54)

•    self.init_py2_file = os.path.join(self.py2_src_dir, "__init__.py")[](#l4.55)

•    with open(self.trivial_py2_file, "w") as trivial:[](#l4.56)

•        trivial.write("print 'I need a simple conversion.'")[](#l4.57)

•    self.setup_files.append("trivial.py")[](#l4.58)

+

• def test_filename_changing_on_output_single_dir(self):

•    """2to3 a single directory with a new output dir and suffix."""[](#l4.61)

•    self.setup_test_source_trees()[](#l4.62)

•    out = io.StringIO()[](#l4.63)

•    err = io.StringIO()[](#l4.64)

•    suffix = "TEST"[](#l4.65)

•    ret = self.run_2to3_capture([](#l4.66)

•            ["-n", "--add-suffix", suffix, "--write-unchanged-files",[](#l4.67)

•             "--no-diffs", "--output-dir",[](#l4.68)

•             self.py3_dest_dir, self.py2_src_dir],[](#l4.69)

•            io.StringIO(""), out, err)[](#l4.70)

•    self.assertEqual(ret, 0)[](#l4.71)

•    stderr = err.getvalue()[](#l4.72)

•    self.assertIn(" implies -w.", stderr)[](#l4.73)

•    self.assertIn([](#l4.74)

•            "Output in %r will mirror the input directory %r layout" % ([](#l4.75)

•                    self.py3_dest_dir, self.py2_src_dir), stderr)[](#l4.76)

•    self.assertEqual(set(name+suffix for name in self.setup_files),[](#l4.77)

•                     set(os.listdir(self.py3_dest_dir)))[](#l4.78)

•    for name in self.setup_files:[](#l4.79)

•        self.assertIn("Writing converted %s to %s" % ([](#l4.80)

•                os.path.join(self.py2_src_dir, name),[](#l4.81)

•                os.path.join(self.py3_dest_dir, name+suffix)), stderr)[](#l4.82)

•    self.assertRegexpMatches(stderr, r"No changes to .*/__init__\.py")[](#l4.83)

•    self.assertNotRegex(stderr, r"No changes to .*/trivial\.py")[](#l4.84)

+

• def test_filename_changing_on_output_two_files(self):

•    """2to3 two files in one directory with a new output dir."""[](#l4.87)

•    self.setup_test_source_trees()[](#l4.88)

•    err = io.StringIO()[](#l4.89)

•    py2_files = [self.trivial_py2_file, self.init_py2_file][](#l4.90)

•    expected_files = set(os.path.basename(name) for name in py2_files)[](#l4.91)

•    ret = self.run_2to3_capture([](#l4.92)

•            ["-n", "-w", "--write-unchanged-files",[](#l4.93)

•             "--no-diffs", "--output-dir", self.py3_dest_dir] + py2_files,[](#l4.94)

•            io.StringIO(""), io.StringIO(), err)[](#l4.95)

•    self.assertEqual(ret, 0)[](#l4.96)

•    stderr = err.getvalue()[](#l4.97)

•    self.assertIn([](#l4.98)

•            "Output in %r will mirror the input directory %r layout" % ([](#l4.99)

•                    self.py3_dest_dir, self.py2_src_dir), stderr)[](#l4.100)

•    self.assertEqual(expected_files, set(os.listdir(self.py3_dest_dir)))[](#l4.101)

+

• def test_filename_changing_on_output_single_file(self):

•    """2to3 a single file with a new output dir."""[](#l4.104)

•    self.setup_test_source_trees()[](#l4.105)

•    err = io.StringIO()[](#l4.106)

•    ret = self.run_2to3_capture([](#l4.107)

•            ["-n", "-w", "--no-diffs", "--output-dir", self.py3_dest_dir,[](#l4.108)

•             self.trivial_py2_file],[](#l4.109)

•            io.StringIO(""), io.StringIO(), err)[](#l4.110)

•    self.assertEqual(ret, 0)[](#l4.111)

•    stderr = err.getvalue()[](#l4.112)

•    self.assertIn([](#l4.113)

•            "Output in %r will mirror the input directory %r layout" % ([](#l4.114)

•                    self.py3_dest_dir, self.py2_src_dir), stderr)[](#l4.115)

•    self.assertEqual(set([os.path.basename(self.trivial_py2_file)]),[](#l4.116)

•                     set(os.listdir(self.py3_dest_dir)))[](#l4.117)

+ + +if name == 'main':

• unittest.main()

--- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -53,6 +53,12 @@ class TestRefactoringTool(unittest.TestC self.assertTrue(rt.driver.grammar is pygram.python_grammar_no_print_statement)

• def test_write_unchanged_files_option(self):

•    rt = self.rt()[](#l5.8)

•    self.assertFalse(rt.write_unchanged_files)[](#l5.9)

•    rt = self.rt({"write_unchanged_files" : True})[](#l5.10)

•    self.assertTrue(rt.write_unchanged_files)[](#l5.11)

+ def test_fixer_loading_helpers(self): contents = ["explicit", "first", "last", "parrot", "preorder"] non_prefixed = refactor.get_all_fix_names("myfixes") @@ -176,7 +182,9 @@ from future import print_function""" "", False] self.assertEqual(results, expected)

• def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS):

• def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS,

•                           options=None, mock_log_debug=None,[](#l5.22)

•                           actually_write=True):[](#l5.23)
     tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor")[](#l5.24)
     self.addCleanup(shutil.rmtree, tmpdir)[](#l5.25)
     # make a copy of the tested file that we can write to[](#l5.26)

@@ -189,11 +197,15 @@ from future import print_function""" return fp.read() old_contents = read_file()

•    rt = self.rt(fixers=fixers)[](#l5.31)

•    rt = self.rt(fixers=fixers, options=options)[](#l5.32)

•    if mock_log_debug:[](#l5.33)

•        rt.log_debug = mock_log_debug[](#l5.34)

rt.refactor_file(test_file) self.assertEqual(old_contents, read_file())

•    if not actually_write:[](#l5.39)

•        return[](#l5.40)
     rt.refactor_file(test_file, True)[](#l5.41)
     new_contents = read_file()[](#l5.42)
     self.assertNotEqual(old_contents, new_contents)[](#l5.43)

@@ -203,6 +215,26 @@ from future import print_function""" test_file = os.path.join(FIXER_DIR, "parrot_example.py") self.check_file_refactoring(test_file, _DEFAULT_FIXERS)

• def test_refactor_file_write_unchanged_file(self):

•    test_file = os.path.join(FIXER_DIR, "parrot_example.py")[](#l5.49)

•    debug_messages = [][](#l5.50)

•    def recording_log_debug(msg, *args):[](#l5.51)

•        debug_messages.append(msg % args)[](#l5.52)

•    self.check_file_refactoring(test_file, fixers=(),[](#l5.53)

•                                options={"write_unchanged_files": True},[](#l5.54)

•                                mock_log_debug=recording_log_debug,[](#l5.55)

•                                actually_write=False)[](#l5.56)

•    # Testing that it logged this message when write=False was passed is[](#l5.57)

•    # sufficient to see that it did not bail early after "No changes".[](#l5.58)

•    message_regex = r"Not writing changes to .*%s%s" % ([](#l5.59)

•            os.sep, os.path.basename(test_file))[](#l5.60)

•    for message in debug_messages:[](#l5.61)

•        if "Not writing changes" in message:[](#l5.62)

•            self.assertRegexpMatches(message, message_regex)[](#l5.63)

•            break[](#l5.64)

•    else:[](#l5.65)

•        self.fail("%r not matched in %r" % (message_regex, debug_messages))[](#l5.66)

+ def test_refactor_dir(self): def check(structure, expected): def mock_refactor_file(self, f, *args):

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -466,6 +466,10 @@ Core and Builtins Library ------- +- Issue #13930: lib2to3 now supports writing converted output files to another

• directory tree as well as copying unchanged files and altering the file
• suffix. +

• Issue #9750: Fix sqlite3.Connection.iterdump on tables and fields with a name that is a keyword or contains quotes. Patch by Marko Kohtala. @@ -1927,6 +1931,11 @@ IDLE Tools/Demos ----------- +- Issue #13930: 2to3 is now able to write its converted output files to another

• directory tree as well as copying unchanged files and altering the file
• suffix. See its new -o, -W and --add-suffix options. This makes it more
• useful in many automated code translation workflows. +

• Issue #13628: python-gdb.py is now able to retrieve more frames in the Python traceback if Python is optimized.