cpython: bbaab666e6c7 (original) (raw)

Mercurial > cpython

changeset 75048:bbaab666e6c7

Issue #14043: Speed up importlib's _FileFinder by at least 8x, and add a new importlib.invalidate_caches() function. importlib is now often faster than imp.find_module() at finding modules. [#14043]

line wrap: on

line diff

--- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -86,6 +86,14 @@ Functions that was imported (e.g. pkg.mod), while :func:__import__ returns the top-level package or module (e.g. pkg). +.. function:: invalidate_caches() +

• Invalidate importlib's internal caches. Calling this function may be
• needed if some modules are installed while your program is running and
• you expect the program to notice the changes. +
• .. versionadded:: 3.3 +

:mod:importlib.abc -- Abstract base classes related to import ---------------------------------------------------------------

--- a/Lib/importlib/init.py +++ b/Lib/importlib/init.py @@ -18,7 +18,7 @@ References on import: http://www.python.org/dev/peps/pep-0328[](#l2.4) """ -all = ['import', 'import_module'] +all = ['import', 'import_module', 'invalidate_caches'] from . import _bootstrap @@ -37,7 +37,7 @@ import sys

Public API #########################################################

-from ._bootstrap import import +from ._bootstrap import import, invalidate_caches def import_module(name, package=None):

--- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -21,31 +21,16 @@ work. One should use importlib as the pu CASE_INSENSITIVE_PLATFORMS = 'win', 'cygwin', 'darwin' -def _case_insensitive_ok(directory, check):

• """Check if the directory contains something matching 'check' exists in the
• directory.

• If PYTHONCASEOK is a defined environment variable then skip the
• case-sensitivity check.

-

• """
• if b'PYTHONCASEOK' not in _os.environ:

•    if not directory:[](#l3.16)

•        directory = '.'[](#l3.17)

•    return check in _os.listdir(directory)[](#l3.18)

+def _relax_case():

• """True if filenames must be checked case-insensitively."""
• if any(map(sys.platform.startswith, CASE_INSENSITIVE_PLATFORMS)):

•    def _relax_case():[](#l3.22)

•        return b'PYTHONCASEOK' in _os.environ[](#l3.23)

  else:

•    return True[](#l3.25)

- -def _case_sensitive_ok(directory, check):

• """Under case-sensitive filesystems always assume the case matches.

-

• Since other code does the file existence check, that subsumes a
• case-sensitivity check.

-

• """
• return True

- -_case_ok = None

•    def _relax_case():[](#l3.37)

•        return False[](#l3.38)

• return _relax_case

TODO: Expose from marshal

@@ -172,6 +157,18 @@ code_type = type(_wrap.code)

Finder/loader utility code ##################################################

+_cache_refresh = 0 + +def invalidate_caches():

• """Invalidate importlib's internal caches.

+

• Calling this function may be needed if some modules are installed while
• your program is running and you expect the program to notice the changes.
• """
• global _cache_refresh
• _cache_refresh += 1

+ + def set_package(fxn): """Set package on the returned module.""" def set_package_wrapper(*args, **kwargs): @@ -708,7 +705,7 @@ class PathFinder: """ if path == '':

•        path = _os.getcwd()[](#l3.66)

•        path = '.'[](#l3.67)
     try:[](#l3.68)
         finder = sys.path_importer_cache[path][](#l3.69)
     except KeyError:[](#l3.70)

@@ -760,29 +757,55 @@ class _FileFinder: for suffix in detail.suffixes) self.packages = packages self.modules = modules

•    self.path = path[](#l3.75)

•    # Base (directory) path[](#l3.76)

•    self.path = path or '.'[](#l3.77)

•    self._path_mtime = -1[](#l3.78)

•    self._path_cache = set()[](#l3.79)

•    self._cache_refresh = 0[](#l3.80)

def find_module(self, fullname): """Try to find a loader for the specified module.""" tail_module = fullname.rpartition('.')[2]

•    base_path = _path_join(self.path, tail_module)[](#l3.85)

•    if _path_isdir(base_path) and _case_ok(self.path, tail_module):[](#l3.86)

•        for suffix, loader in self.packages:[](#l3.87)

•            init_filename = '__init__' + suffix[](#l3.88)

•            full_path = _path_join(base_path, init_filename)[](#l3.89)

•            if (_path_isfile(full_path) and[](#l3.90)

•                    _case_ok(base_path, init_filename)):[](#l3.91)

•                return loader(fullname, full_path)[](#l3.92)

•        else:[](#l3.93)

•            msg = "Not importing directory {}: missing __init__"[](#l3.94)

•            _warnings.warn(msg.format(base_path), ImportWarning)[](#l3.95)

•    if _relax_case():[](#l3.96)

•        tail_module = tail_module.lower()[](#l3.97)

•    try:[](#l3.98)

•        mtime = _os.stat(self.path).st_mtime[](#l3.99)

•    except OSError:[](#l3.100)

•        mtime = -1[](#l3.101)

•    if mtime != self._path_mtime or _cache_refresh != self._cache_refresh:[](#l3.102)

•        self._fill_cache()[](#l3.103)

•        self._path_mtime = mtime[](#l3.104)

•        self._cache_refresh = _cache_refresh[](#l3.105)

•    cache = self._path_cache[](#l3.106)

•    if tail_module in cache:[](#l3.107)

•        base_path = _path_join(self.path, tail_module)[](#l3.108)

•        if _path_isdir(base_path):[](#l3.109)

•            for suffix, loader in self.packages:[](#l3.110)

•                init_filename = '__init__' + suffix[](#l3.111)

•                full_path = _path_join(base_path, init_filename)[](#l3.112)

•                if _path_isfile(full_path):[](#l3.113)

•                    return loader(fullname, full_path)[](#l3.114)

•            else:[](#l3.115)

•                msg = "Not importing directory {}: missing __init__"[](#l3.116)

•                _warnings.warn(msg.format(base_path), ImportWarning)[](#l3.117)
     for suffix, loader in self.modules:[](#l3.118)
         mod_filename = tail_module + suffix[](#l3.119)

•        full_path = _path_join(self.path, mod_filename)[](#l3.120)

•        if _path_isfile(full_path) and _case_ok(self.path, mod_filename):[](#l3.121)

•            return loader(fullname, full_path)[](#l3.122)

•        if mod_filename in cache:[](#l3.123)

•            full_path = _path_join(self.path, mod_filename)[](#l3.124)

•            if _path_isfile(full_path):[](#l3.125)

•                return loader(fullname, full_path)[](#l3.126)
     return None[](#l3.127)

• def _fill_cache(self):

•    """Fill the cache of potential modules and packages for this directory."""[](#l3.130)

•    path = self.path[](#l3.131)

•    contents = _os.listdir(path)[](#l3.132)

•    if _relax_case():[](#l3.133)

•        self._path_cache = set(fn.lower() for fn in contents)[](#l3.134)

•    else:[](#l3.135)

•        self._path_cache = set(contents)[](#l3.136)

+ + class _SourceFinderDetails: loader = _SourceFileLoader @@ -1060,7 +1083,7 @@ def _setup(sys_module, imp_module): modules, those two modules must be explicitly passed in. """

• global _case_ok, imp, sys

• global imp, sys imp = imp_module sys = sys_module @@ -1093,12 +1116,8 @@ def _setup(sys_module, imp_module): raise ImportError('importlib requires posix or nt') setattr(self_module, '_os', os_module) setattr(self_module, 'path_sep', path_sep) -

• if any(sys_module.platform.startswith(x)

•        for x in CASE_INSENSITIVE_PLATFORMS):[](#l3.157)

•    _case_ok = _case_insensitive_ok[](#l3.158)

• else:

•    _case_ok = _case_sensitive_ok[](#l3.160)

• Constants

• setattr(self_module, '_relax_case', _relax_case())

def _install(sys_module, imp_module):

--- a/Lib/importlib/test/import_/test_path.py +++ b/Lib/importlib/test/import_/test_path.py @@ -78,11 +78,11 @@ class FinderTests(unittest.TestCase): path = '' module = '' importer = util.mock_modules(module)

•    hook = import_util.mock_path_hook(os.getcwd(), importer=importer)[](#l4.7)

•    hook = import_util.mock_path_hook(os.curdir, importer=importer)[](#l4.8)
     with util.import_state(path=[path], path_hooks=[hook]):[](#l4.9)
         loader = machinery.PathFinder.find_module(module)[](#l4.10)
         self.assertIs(loader, importer)[](#l4.11)

•        self.assertIn(os.getcwd(), sys.path_importer_cache)[](#l4.12)

•        self.assertIn(os.curdir, sys.path_importer_cache)[](#l4.13)

class DefaultPathFinderTests(unittest.TestCase):

--- a/Lib/test/test_import.py +++ b/Lib/test/test_import.py @@ -2,6 +2,7 @@ import builtins import imp from importlib.test.import_ import test_relative_imports from importlib.test.import_ import util as importlib_util +import importlib import marshal import os import platform @@ -34,6 +35,7 @@ class ImportTests(unittest.TestCase): def setUp(self): remove_files(TESTFN)

•    importlib.invalidate_caches()[](#l5.15)

def tearDown(self): unload(TESTFN) @@ -107,6 +109,7 @@ class ImportTests(unittest.TestCase): create_empty_file(fname) fn = imp.cache_from_source(fname) unlink(fn)

•            importlib.invalidate_caches()[](#l5.23)
             __import__(TESTFN)[](#l5.24)
             if not os.path.exists(fn):[](#l5.25)
                 self.fail("__import__ did not result in creation of "[](#l5.26)

@@ -260,6 +263,7 @@ class ImportTests(unittest.TestCase): os.remove(source) del sys.modules[TESTFN] make_legacy_pyc(source)

•        importlib.invalidate_caches()[](#l5.31)
         mod = __import__(TESTFN)[](#l5.32)
         base, ext = os.path.splitext(mod.__file__)[](#l5.33)
         self.assertIn(ext, ('.pyc', '.pyo'))[](#l5.34)

@@ -358,6 +362,7 @@ func_filename = func.code.co_filenam with open(self.file_name, "w") as f: f.write(self.module_source) sys.path.insert(0, self.dir_name)

•    importlib.invalidate_caches()[](#l5.39)

def tearDown(self): sys.path[:] = self.sys_path @@ -552,6 +557,7 @@ class PycacheTests(unittest.TestCase): with open(self.source, 'w') as fp: print('# This is a test file written by test_import.py', file=fp) sys.path.insert(0, os.curdir)

•    importlib.invalidate_caches()[](#l5.47)

def tearDown(self): assert sys.path[0] == os.curdir, 'Unexpected sys.path[0]' @@ -599,6 +605,7 @@ class PycacheTests(unittest.TestCase): pyc_file = make_legacy_pyc(self.source) os.remove(self.source) unload(TESTFN)

•    importlib.invalidate_caches()[](#l5.55)
     m = __import__(TESTFN)[](#l5.56)
     self.assertEqual(m.__file__,[](#l5.57)
                      os.path.join(os.curdir, os.path.relpath(pyc_file)))[](#l5.58)

@@ -619,6 +626,7 @@ class PycacheTests(unittest.TestCase): pyc_file = make_legacy_pyc(self.source) os.remove(self.source) unload(TESTFN)

•    importlib.invalidate_caches()[](#l5.63)
     m = __import__(TESTFN)[](#l5.64)
     self.assertEqual(m.__cached__,[](#l5.65)
                      os.path.join(os.curdir, os.path.relpath(pyc_file)))[](#l5.66)

--- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -6,6 +6,7 @@ import sys import os import shutil +import importlib import unittest from test.support import run_unittest, create_empty_file @@ -212,6 +213,7 @@ class LongReprTest(unittest.TestCase): # Remember where we are self.here = os.getcwd() sys.path.insert(0, self.here)

•    importlib.invalidate_caches()[](#l6.15)

def tearDown(self): actions = []

--- a/Misc/NEWS +++ b/Misc/NEWS @@ -469,6 +469,9 @@ Core and Builtins Library ------- +- Issue #14043: Speed up importlib's _FileFinder by at least 8x, and add a

• new importlib.invalidate_caches() function. +

• Issue #14001: CVE-2012-0845: xmlrpc: Fix an endless loop in SimpleXMLRPCServer upon malformed POST request.