util.py 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. """Utility code for constructing importers, etc."""
  2. from ._abc import Loader
  3. from ._bootstrap import module_from_spec
  4. from ._bootstrap import _resolve_name
  5. from ._bootstrap import spec_from_loader
  6. from ._bootstrap import _find_spec
  7. from ._bootstrap_external import MAGIC_NUMBER
  8. from ._bootstrap_external import _RAW_MAGIC_NUMBER
  9. from ._bootstrap_external import cache_from_source
  10. from ._bootstrap_external import decode_source
  11. from ._bootstrap_external import source_from_cache
  12. from ._bootstrap_external import spec_from_file_location
  13. import _imp
  14. import sys
  15. import types
  16. def source_hash(source_bytes):
  17. "Return the hash of *source_bytes* as used in hash-based pyc files."
  18. return _imp.source_hash(_RAW_MAGIC_NUMBER, source_bytes)
  19. def resolve_name(name, package):
  20. """Resolve a relative module name to an absolute one."""
  21. if not name.startswith('.'):
  22. return name
  23. elif not package:
  24. raise ImportError(f'no package specified for {repr(name)} '
  25. '(required for relative module names)')
  26. level = 0
  27. for character in name:
  28. if character != '.':
  29. break
  30. level += 1
  31. return _resolve_name(name[level:], package, level)
  32. def _find_spec_from_path(name, path=None):
  33. """Return the spec for the specified module.
  34. First, sys.modules is checked to see if the module was already imported. If
  35. so, then sys.modules[name].__spec__ is returned. If that happens to be
  36. set to None, then ValueError is raised. If the module is not in
  37. sys.modules, then sys.meta_path is searched for a suitable spec with the
  38. value of 'path' given to the finders. None is returned if no spec could
  39. be found.
  40. Dotted names do not have their parent packages implicitly imported. You will
  41. most likely need to explicitly import all parent packages in the proper
  42. order for a submodule to get the correct spec.
  43. """
  44. if name not in sys.modules:
  45. return _find_spec(name, path)
  46. else:
  47. module = sys.modules[name]
  48. if module is None:
  49. return None
  50. try:
  51. spec = module.__spec__
  52. except AttributeError:
  53. raise ValueError(f'{name}.__spec__ is not set') from None
  54. else:
  55. if spec is None:
  56. raise ValueError(f'{name}.__spec__ is None')
  57. return spec
  58. def find_spec(name, package=None):
  59. """Return the spec for the specified module.
  60. First, sys.modules is checked to see if the module was already imported. If
  61. so, then sys.modules[name].__spec__ is returned. If that happens to be
  62. set to None, then ValueError is raised. If the module is not in
  63. sys.modules, then sys.meta_path is searched for a suitable spec with the
  64. value of 'path' given to the finders. None is returned if no spec could
  65. be found.
  66. If the name is for submodule (contains a dot), the parent module is
  67. automatically imported.
  68. The name and package arguments work the same as importlib.import_module().
  69. In other words, relative module names (with leading dots) work.
  70. """
  71. fullname = resolve_name(name, package) if name.startswith('.') else name
  72. if fullname not in sys.modules:
  73. parent_name = fullname.rpartition('.')[0]
  74. if parent_name:
  75. parent = __import__(parent_name, fromlist=['__path__'])
  76. try:
  77. parent_path = parent.__path__
  78. except AttributeError as e:
  79. raise ModuleNotFoundError(
  80. f"__path__ attribute not found on {parent_name!r} "
  81. f"while trying to find {fullname!r}", name=fullname) from e
  82. else:
  83. parent_path = None
  84. return _find_spec(fullname, parent_path)
  85. else:
  86. module = sys.modules[fullname]
  87. if module is None:
  88. return None
  89. try:
  90. spec = module.__spec__
  91. except AttributeError:
  92. raise ValueError(f'{name}.__spec__ is not set') from None
  93. else:
  94. if spec is None:
  95. raise ValueError(f'{name}.__spec__ is None')
  96. return spec
  97. # Normally we would use contextlib.contextmanager. However, this module
  98. # is imported by runpy, which means we want to avoid any unnecessary
  99. # dependencies. Thus we use a class.
  100. class _incompatible_extension_module_restrictions:
  101. """A context manager that can temporarily skip the compatibility check.
  102. NOTE: This function is meant to accommodate an unusual case; one
  103. which is likely to eventually go away. There's is a pretty good
  104. chance this is not what you were looking for.
  105. WARNING: Using this function to disable the check can lead to
  106. unexpected behavior and even crashes. It should only be used during
  107. extension module development.
  108. If "disable_check" is True then the compatibility check will not
  109. happen while the context manager is active. Otherwise the check
  110. *will* happen.
  111. Normally, extensions that do not support multiple interpreters
  112. may not be imported in a subinterpreter. That implies modules
  113. that do not implement multi-phase init or that explicitly of out.
  114. Likewise for modules import in a subinterpeter with its own GIL
  115. when the extension does not support a per-interpreter GIL. This
  116. implies the module does not have a Py_mod_multiple_interpreters slot
  117. set to Py_MOD_PER_INTERPRETER_GIL_SUPPORTED.
  118. In both cases, this context manager may be used to temporarily
  119. disable the check for compatible extension modules.
  120. You can get the same effect as this function by implementing the
  121. basic interface of multi-phase init (PEP 489) and lying about
  122. support for mulitple interpreters (or per-interpreter GIL).
  123. """
  124. def __init__(self, *, disable_check):
  125. self.disable_check = bool(disable_check)
  126. def __enter__(self):
  127. self.old = _imp._override_multi_interp_extensions_check(self.override)
  128. return self
  129. def __exit__(self, *args):
  130. old = self.old
  131. del self.old
  132. _imp._override_multi_interp_extensions_check(old)
  133. @property
  134. def override(self):
  135. return -1 if self.disable_check else 1
  136. class _LazyModule(types.ModuleType):
  137. """A subclass of the module type which triggers loading upon attribute access."""
  138. def __getattribute__(self, attr):
  139. """Trigger the load of the module and return the attribute."""
  140. # All module metadata must be garnered from __spec__ in order to avoid
  141. # using mutated values.
  142. # Stop triggering this method.
  143. self.__class__ = types.ModuleType
  144. # Get the original name to make sure no object substitution occurred
  145. # in sys.modules.
  146. original_name = self.__spec__.name
  147. # Figure out exactly what attributes were mutated between the creation
  148. # of the module and now.
  149. attrs_then = self.__spec__.loader_state['__dict__']
  150. attrs_now = self.__dict__
  151. attrs_updated = {}
  152. for key, value in attrs_now.items():
  153. # Code that set the attribute may have kept a reference to the
  154. # assigned object, making identity more important than equality.
  155. if key not in attrs_then:
  156. attrs_updated[key] = value
  157. elif id(attrs_now[key]) != id(attrs_then[key]):
  158. attrs_updated[key] = value
  159. self.__spec__.loader.exec_module(self)
  160. # If exec_module() was used directly there is no guarantee the module
  161. # object was put into sys.modules.
  162. if original_name in sys.modules:
  163. if id(self) != id(sys.modules[original_name]):
  164. raise ValueError(f"module object for {original_name!r} "
  165. "substituted in sys.modules during a lazy "
  166. "load")
  167. # Update after loading since that's what would happen in an eager
  168. # loading situation.
  169. self.__dict__.update(attrs_updated)
  170. return getattr(self, attr)
  171. def __delattr__(self, attr):
  172. """Trigger the load and then perform the deletion."""
  173. # To trigger the load and raise an exception if the attribute
  174. # doesn't exist.
  175. self.__getattribute__(attr)
  176. delattr(self, attr)
  177. class LazyLoader(Loader):
  178. """A loader that creates a module which defers loading until attribute access."""
  179. @staticmethod
  180. def __check_eager_loader(loader):
  181. if not hasattr(loader, 'exec_module'):
  182. raise TypeError('loader must define exec_module()')
  183. @classmethod
  184. def factory(cls, loader):
  185. """Construct a callable which returns the eager loader made lazy."""
  186. cls.__check_eager_loader(loader)
  187. return lambda *args, **kwargs: cls(loader(*args, **kwargs))
  188. def __init__(self, loader):
  189. self.__check_eager_loader(loader)
  190. self.loader = loader
  191. def create_module(self, spec):
  192. return self.loader.create_module(spec)
  193. def exec_module(self, module):
  194. """Make the module load lazily."""
  195. module.__spec__.loader = self.loader
  196. module.__loader__ = self.loader
  197. # Don't need to worry about deep-copying as trying to set an attribute
  198. # on an object would have triggered the load,
  199. # e.g. ``module.__spec__.loader = None`` would trigger a load from
  200. # trying to access module.__spec__.
  201. loader_state = {}
  202. loader_state['__dict__'] = module.__dict__.copy()
  203. loader_state['__class__'] = module.__class__
  204. module.__spec__.loader_state = loader_state
  205. module.__class__ = _LazyModule