cpython: 50aa9e3ab9a4 (original) (raw)

Mercurial > cpython

changeset 88771:50aa9e3ab9a4 3.3

Issue #19077: tempfile.TemporaryDirectory cleanup is now most likely successful when called during nulling out of modules during shutdown. Misleading exception no longer raised when resource warning is emitted during shutdown. [#19077]

line wrap: on

line diff

--- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -27,11 +27,12 @@ This module also provides some data item

Imports.

+import atexit as _atexit import functools as _functools import warnings as _warnings -import sys as _sys import io as _io import os as _os +import shutil as _shutil import errno as _errno from random import Random as _Random @@ -355,10 +356,13 @@ class _TemporaryFileCloser: underlying file object, without adding a del method to the temporary file."""

• Set here since del checks it

• file = None
• close_called = False

+ def init(self, file, name, delete=True): self.file = file self.name = name

•    self.close_called = False[](#l1.28)
     self.delete = delete[](#l1.29)

# NT provides delete-on-close as a primitive, so we don't need @@ -370,14 +374,13 @@ class _TemporaryFileCloser: # that this must be referenced as self.unlink, because the # name TemporaryFileWrapper may also get None'd out before # del is called.

•    unlink = _os.unlink[](#l1.36)

•    def close(self):[](#l1.38)

•        if not self.close_called:[](#l1.39)

•    def close(self, unlink=_os.unlink):[](#l1.40)

•        if not self.close_called and self.file is not None:[](#l1.41)
             self.close_called = True[](#l1.42)
             self.file.close()[](#l1.43)
             if self.delete:[](#l1.44)

•                self.unlink(self.name)[](#l1.45)

•                unlink(self.name)[](#l1.46)

# Need to ensure the file is deleted on del def del(self): @@ -677,9 +680,11 @@ class TemporaryDirectory(object): in it are removed. """

• Handle mkdtemp raising an exception

• name = None
• _closed = False

+ def init(self, suffix="", prefix=template, dir=None):

•    self._closed = False[](#l1.59)

•    self.name = None # Handle mkdtemp raising an exception[](#l1.60)
     self.name = mkdtemp(suffix, prefix, dir)[](#l1.61)

def repr(self): @@ -688,23 +693,24 @@ class TemporaryDirectory(object): def enter(self): return self.name

• def cleanup(self, _warn=False):

• def cleanup(self, _warn=False, _warnings=_warnings): if self.name and not self._closed: try:

•            self._rmtree(self.name)[](#l1.72)

•            _shutil.rmtree(self.name)[](#l1.73)
         except (TypeError, AttributeError) as ex:[](#l1.74)

•            # Issue #10188: Emit a warning on stderr[](#l1.75)

•            # if the directory could not be cleaned[](#l1.76)

•            # up due to missing globals[](#l1.77)

•            if "None" not in str(ex):[](#l1.78)

•            if "None" not in '%s' % (ex,):[](#l1.79)
                 raise[](#l1.80)

•            print("ERROR: {!r} while cleaning up {!r}".format(ex, self,),[](#l1.81)

•                  file=_sys.stderr)[](#l1.82)

•            return[](#l1.83)

•            self._rmtree(self.name)[](#l1.84)
         self._closed = True[](#l1.85)

•        if _warn:[](#l1.86)

•            self._warn("Implicitly cleaning up {!r}".format(self),[](#l1.87)

•                       ResourceWarning)[](#l1.88)

•        if _warn and _warnings.warn:[](#l1.89)

•            try:[](#l1.90)

•                _warnings.warn("Implicitly cleaning up {!r}".format(self),[](#l1.91)

•                               ResourceWarning)[](#l1.92)

•            except:[](#l1.93)

•                if _is_running:[](#l1.94)

•                    raise[](#l1.95)

•                # Don't raise an exception if modules needed for emitting[](#l1.96)

•                # a warning are already cleaned in shutdown process.[](#l1.97)

def exit(self, exc, value, tb): self.cleanup() @@ -713,36 +719,27 @@ class TemporaryDirectory(object): # Issue a ResourceWarning if implicit cleanup needed self.cleanup(_warn=True)

• XXX (ncoghlan): The following code attempts to make

• this class tolerant of the module nulling out process

• that happens during CPython interpreter shutdown

• Alas, it doesn't actually manage it. See issue #10188

• _listdir = staticmethod(_os.listdir)
• _path_join = staticmethod(_os.path.join)
• _isdir = staticmethod(_os.path.isdir)
• _islink = staticmethod(_os.path.islink)
• _remove = staticmethod(_os.remove)
• _rmdir = staticmethod(_os.rmdir)
• _os_error = OSError
• _warn = _warnings.warn

-

• def _rmtree(self, path):

• def _rmtree(self, path, _OSError=OSError, _sep=_os.path.sep,

•            _listdir=_os.listdir, _remove=_os.remove, _rmdir=_os.rmdir):[](#l1.120)
     # Essentially a stripped down version of shutil.rmtree.  We can't[](#l1.121)
     # use globals because they may be None'ed out at shutdown.[](#l1.122)

•    for name in self._listdir(path):[](#l1.123)

•        fullname = self._path_join(path, name)[](#l1.124)

•        try:[](#l1.125)

•            isdir = self._isdir(fullname) and not self._islink(fullname)[](#l1.126)

•        except self._os_error:[](#l1.127)

•            isdir = False[](#l1.128)

•        if isdir:[](#l1.129)

•            self._rmtree(fullname)[](#l1.130)

•        else:[](#l1.131)

•    if not isinstance(path, str):[](#l1.132)

•        _sep = _sep.encode()[](#l1.133)

•    try:[](#l1.134)

•        for name in _listdir(path):[](#l1.135)

•            fullname = path + _sep + name[](#l1.136)
             try:[](#l1.137)

•                self._remove(fullname)[](#l1.138)

•            except self._os_error:[](#l1.139)

•                pass[](#l1.140)

•    try:[](#l1.141)

•        self._rmdir(path)[](#l1.142)

•    except self._os_error:[](#l1.143)

•                _remove(fullname)[](#l1.144)

•            except _OSError:[](#l1.145)

•                self._rmtree(fullname)[](#l1.146)

•        _rmdir(path)[](#l1.147)

•    except _OSError:[](#l1.148)
         pass[](#l1.149)

+ +_is_running = True + +def _on_shutdown():

• global _is_running
• _is_running = False

+ +_atexit.register(_on_shutdown)

--- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -11,7 +11,7 @@ import contextlib import weakref import unittest -from test import support +from test import support, script_helper if hasattr(os, 'stat'): @@ -1073,7 +1073,8 @@ class TestTemporaryDirectory(BaseTestCas self.nameCheck(tmp.name, dir, pre, suf) # Create a subdirectory and some files if recurse:

•        self.do_create(tmp.name, pre, suf, recurse-1)[](#l2.16)

•        d1 = self.do_create(tmp.name, pre, suf, recurse-1)[](#l2.17)

•        d1.name = None[](#l2.18)
     with open(os.path.join(tmp.name, "test.txt"), "wb") as f:[](#l2.19)
         f.write(b"Hello world!")[](#l2.20)
     return tmp[](#l2.21)

@@ -1105,7 +1106,7 @@ class TestTemporaryDirectory(BaseTestCas def test_cleanup_with_symlink_to_a_directory(self): # cleanup() should not follow symlinks to directories (issue #12464) d1 = self.do_create()

•    d2 = self.do_create()[](#l2.26)

•    d2 = self.do_create(recurse=0)[](#l2.27)

# Symlink d1/foo -> d2 os.symlink(d2.name, os.path.join(d1.name, "foo")) @@ -1135,60 +1136,49 @@ class TestTemporaryDirectory(BaseTestCas finally: os.rmdir(dir)

• @unittest.expectedFailure # See issue #10188 def test_del_on_shutdown(self): # A TemporaryDirectory may be cleaned up during shutdown

•    # Make sure it works with the relevant modules nulled out[](#l2.38)
     with self.do_create() as dir:[](#l2.39)

•        d = self.do_create(dir=dir)[](#l2.40)

•        # Mimic the nulling out of modules that[](#l2.41)

•        # occurs during system shutdown[](#l2.42)

•        modules = [os, os.path][](#l2.43)

•        if has_stat:[](#l2.44)

•            modules.append(stat)[](#l2.45)

•        # Currently broken, so suppress the warning[](#l2.46)

•        # that is otherwise emitted on stdout[](#l2.47)

•        with support.captured_stderr() as err:[](#l2.48)

•            with NulledModules(*modules):[](#l2.49)

•                d.cleanup()[](#l2.50)

•        # Currently broken, so stop spurious exception by[](#l2.51)

•        # indicating the object has already been closed[](#l2.52)

•        d._closed = True[](#l2.53)

•        # And this assert will fail, as expected by the[](#l2.54)

•        # unittest decorator...[](#l2.55)

•        self.assertFalse(os.path.exists(d.name),[](#l2.56)

•                    "TemporaryDirectory %s exists after cleanup" % d.name)[](#l2.57)

•        for mod in ('os', 'shutil', 'sys', 'tempfile', 'warnings'):[](#l2.58)

•            code = """if True:[](#l2.59)

•                import os[](#l2.60)

•                import shutil[](#l2.61)

•                import sys[](#l2.62)

•                import tempfile[](#l2.63)

•                import warnings[](#l2.64)

+

•                tmp = tempfile.TemporaryDirectory(dir={dir!r})[](#l2.66)

•                sys.stdout.buffer.write(tmp.name.encode())[](#l2.67)

+

•                tmp2 = os.path.join(tmp.name, 'test_dir')[](#l2.69)

•                os.mkdir(tmp2)[](#l2.70)

•                with open(os.path.join(tmp2, "test.txt"), "w") as f:[](#l2.71)

•                    f.write("Hello world!")[](#l2.72)

+

•                {mod}.tmp = tmp[](#l2.74)

+

•                warnings.filterwarnings("always", category=ResourceWarning)[](#l2.76)

•                """.format(dir=dir, mod=mod)[](#l2.77)

•            rc, out, err = script_helper.assert_python_ok("-c", code)[](#l2.78)

•            tmp_name = out.decode().strip()[](#l2.79)

•            self.assertFalse(os.path.exists(tmp_name),[](#l2.80)

•                        "TemporaryDirectory %s exists after cleanup" % tmp_name)[](#l2.81)

•            err = err.decode('utf-8', 'backslashreplace')[](#l2.82)

•            self.assertNotIn("Exception ", err)[](#l2.83)

def test_warnings_on_cleanup(self):

•    # Two kinds of warning on shutdown[](#l2.86)

•    #   Issue 10888: may write to stderr if modules are nulled out[](#l2.87)

•    #   ResourceWarning will be triggered by __del__[](#l2.88)

•    # ResourceWarning will be triggered by __del__[](#l2.89)
     with self.do_create() as dir:[](#l2.90)

•        if os.sep != '\\':[](#l2.91)

•            # Embed a backslash in order to make sure string escaping[](#l2.92)

•            # in the displayed error message is dealt with correctly[](#l2.93)

•            suffix = '\\check_backslash_handling'[](#l2.94)

•        else:[](#l2.95)

•            suffix = ''[](#l2.96)

•        d = self.do_create(dir=dir, suf=suffix)[](#l2.97)

-

•        #Check for the Issue 10888 message[](#l2.99)

•        modules = [os, os.path][](#l2.100)

•        if has_stat:[](#l2.101)

•            modules.append(stat)[](#l2.102)

•        with support.captured_stderr() as err:[](#l2.103)

•            with NulledModules(*modules):[](#l2.104)

•                d.cleanup()[](#l2.105)

•        message = err.getvalue().replace('\\\\', '\\')[](#l2.106)

•        self.assertIn("while cleaning up",  message)[](#l2.107)

•        self.assertIn(d.name,  message)[](#l2.108)

•        d = self.do_create(dir=dir, recurse=3)[](#l2.109)

•        name = d.name[](#l2.110)

# Check for the resource warning with support.check_warnings(('Implicitly', ResourceWarning), quiet=False): warnings.filterwarnings("always", category=ResourceWarning)

•            d.__del__()[](#l2.115)

•        self.assertFalse(os.path.exists(d.name),[](#l2.116)

•                    "TemporaryDirectory %s exists after __del__" % d.name)[](#l2.117)

•            del d[](#l2.118)

•            support.gc_collect()[](#l2.119)

•        self.assertFalse(os.path.exists(name),[](#l2.120)

•                    "TemporaryDirectory %s exists after __del__" % name)[](#l2.121)

def test_multiple_close(self): # Can be cleaned-up many times without error

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -50,6 +50,11 @@ Core and Builtins Library ------- +- Issue #19077: tempfile.TemporaryDirectory cleanup is now most likely

• successful when called during nulling out of modules during shutdown.
• Misleading exception no longer raised when resource warning is emitted
• during shutdown. +

• Issue #20367: Fix behavior of concurrent.futures.as_completed() for duplicate arguments. Patch by Glenn Langford.