util.py 9.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359
  1. """
  2. Utility functions for
  3. - building and importing modules on test time, using a temporary location
  4. - detecting if compilers are present
  5. """
  6. import os
  7. import sys
  8. import subprocess
  9. import tempfile
  10. import shutil
  11. import atexit
  12. import textwrap
  13. import re
  14. import pytest
  15. from numpy.compat import asbytes, asstr
  16. from numpy.testing import temppath
  17. from importlib import import_module
  18. #
  19. # Maintaining a temporary module directory
  20. #
  21. _module_dir = None
  22. _module_num = 5403
  23. def _cleanup():
  24. global _module_dir
  25. if _module_dir is not None:
  26. try:
  27. sys.path.remove(_module_dir)
  28. except ValueError:
  29. pass
  30. try:
  31. shutil.rmtree(_module_dir)
  32. except (IOError, OSError):
  33. pass
  34. _module_dir = None
  35. def get_module_dir():
  36. global _module_dir
  37. if _module_dir is None:
  38. _module_dir = tempfile.mkdtemp()
  39. atexit.register(_cleanup)
  40. if _module_dir not in sys.path:
  41. sys.path.insert(0, _module_dir)
  42. return _module_dir
  43. def get_temp_module_name():
  44. # Assume single-threaded, and the module dir usable only by this thread
  45. global _module_num
  46. d = get_module_dir()
  47. name = "_test_ext_module_%d" % _module_num
  48. _module_num += 1
  49. if name in sys.modules:
  50. # this should not be possible, but check anyway
  51. raise RuntimeError("Temporary module name already in use.")
  52. return name
  53. def _memoize(func):
  54. memo = {}
  55. def wrapper(*a, **kw):
  56. key = repr((a, kw))
  57. if key not in memo:
  58. try:
  59. memo[key] = func(*a, **kw)
  60. except Exception as e:
  61. memo[key] = e
  62. raise
  63. ret = memo[key]
  64. if isinstance(ret, Exception):
  65. raise ret
  66. return ret
  67. wrapper.__name__ = func.__name__
  68. return wrapper
  69. #
  70. # Building modules
  71. #
  72. @_memoize
  73. def build_module(source_files, options=[], skip=[], only=[], module_name=None):
  74. """
  75. Compile and import a f2py module, built from the given files.
  76. """
  77. code = ("import sys; sys.path = %s; import numpy.f2py as f2py2e; "
  78. "f2py2e.main()" % repr(sys.path))
  79. d = get_module_dir()
  80. # Copy files
  81. dst_sources = []
  82. f2py_sources = []
  83. for fn in source_files:
  84. if not os.path.isfile(fn):
  85. raise RuntimeError("%s is not a file" % fn)
  86. dst = os.path.join(d, os.path.basename(fn))
  87. shutil.copyfile(fn, dst)
  88. dst_sources.append(dst)
  89. base, ext = os.path.splitext(dst)
  90. if ext in ('.f90', '.f', '.c', '.pyf'):
  91. f2py_sources.append(dst)
  92. # Prepare options
  93. if module_name is None:
  94. module_name = get_temp_module_name()
  95. f2py_opts = ['-c', '-m', module_name] + options + f2py_sources
  96. if skip:
  97. f2py_opts += ['skip:'] + skip
  98. if only:
  99. f2py_opts += ['only:'] + only
  100. # Build
  101. cwd = os.getcwd()
  102. try:
  103. os.chdir(d)
  104. cmd = [sys.executable, '-c', code] + f2py_opts
  105. p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
  106. stderr=subprocess.STDOUT)
  107. out, err = p.communicate()
  108. if p.returncode != 0:
  109. raise RuntimeError("Running f2py failed: %s\n%s"
  110. % (cmd[4:], asstr(out)))
  111. finally:
  112. os.chdir(cwd)
  113. # Partial cleanup
  114. for fn in dst_sources:
  115. os.unlink(fn)
  116. # Import
  117. return import_module(module_name)
  118. @_memoize
  119. def build_code(source_code, options=[], skip=[], only=[], suffix=None,
  120. module_name=None):
  121. """
  122. Compile and import Fortran code using f2py.
  123. """
  124. if suffix is None:
  125. suffix = '.f'
  126. with temppath(suffix=suffix) as path:
  127. with open(path, 'w') as f:
  128. f.write(source_code)
  129. return build_module([path], options=options, skip=skip, only=only,
  130. module_name=module_name)
  131. #
  132. # Check if compilers are available at all...
  133. #
  134. _compiler_status = None
  135. def _get_compiler_status():
  136. global _compiler_status
  137. if _compiler_status is not None:
  138. return _compiler_status
  139. _compiler_status = (False, False, False)
  140. # XXX: this is really ugly. But I don't know how to invoke Distutils
  141. # in a safer way...
  142. code = textwrap.dedent("""\
  143. import os
  144. import sys
  145. sys.path = %(syspath)s
  146. def configuration(parent_name='',top_path=None):
  147. global config
  148. from numpy.distutils.misc_util import Configuration
  149. config = Configuration('', parent_name, top_path)
  150. return config
  151. from numpy.distutils.core import setup
  152. setup(configuration=configuration)
  153. config_cmd = config.get_config_cmd()
  154. have_c = config_cmd.try_compile('void foo() {}')
  155. print('COMPILERS:%%d,%%d,%%d' %% (have_c,
  156. config.have_f77c(),
  157. config.have_f90c()))
  158. sys.exit(99)
  159. """)
  160. code = code % dict(syspath=repr(sys.path))
  161. tmpdir = tempfile.mkdtemp()
  162. try:
  163. script = os.path.join(tmpdir, 'setup.py')
  164. with open(script, 'w') as f:
  165. f.write(code)
  166. cmd = [sys.executable, 'setup.py', 'config']
  167. p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
  168. stderr=subprocess.STDOUT,
  169. cwd=tmpdir)
  170. out, err = p.communicate()
  171. finally:
  172. shutil.rmtree(tmpdir)
  173. m = re.search(br'COMPILERS:(\d+),(\d+),(\d+)', out)
  174. if m:
  175. _compiler_status = (bool(int(m.group(1))), bool(int(m.group(2))),
  176. bool(int(m.group(3))))
  177. # Finished
  178. return _compiler_status
  179. def has_c_compiler():
  180. return _get_compiler_status()[0]
  181. def has_f77_compiler():
  182. return _get_compiler_status()[1]
  183. def has_f90_compiler():
  184. return _get_compiler_status()[2]
  185. #
  186. # Building with distutils
  187. #
  188. @_memoize
  189. def build_module_distutils(source_files, config_code, module_name, **kw):
  190. """
  191. Build a module via distutils and import it.
  192. """
  193. from numpy.distutils.misc_util import Configuration
  194. from numpy.distutils.core import setup
  195. d = get_module_dir()
  196. # Copy files
  197. dst_sources = []
  198. for fn in source_files:
  199. if not os.path.isfile(fn):
  200. raise RuntimeError("%s is not a file" % fn)
  201. dst = os.path.join(d, os.path.basename(fn))
  202. shutil.copyfile(fn, dst)
  203. dst_sources.append(dst)
  204. # Build script
  205. config_code = textwrap.dedent(config_code).replace("\n", "\n ")
  206. code = textwrap.dedent("""\
  207. import os
  208. import sys
  209. sys.path = %(syspath)s
  210. def configuration(parent_name='',top_path=None):
  211. from numpy.distutils.misc_util import Configuration
  212. config = Configuration('', parent_name, top_path)
  213. %(config_code)s
  214. return config
  215. if __name__ == "__main__":
  216. from numpy.distutils.core import setup
  217. setup(configuration=configuration)
  218. """) % dict(config_code=config_code, syspath=repr(sys.path))
  219. script = os.path.join(d, get_temp_module_name() + '.py')
  220. dst_sources.append(script)
  221. with open(script, 'wb') as f:
  222. f.write(asbytes(code))
  223. # Build
  224. cwd = os.getcwd()
  225. try:
  226. os.chdir(d)
  227. cmd = [sys.executable, script, 'build_ext', '-i']
  228. p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
  229. stderr=subprocess.STDOUT)
  230. out, err = p.communicate()
  231. if p.returncode != 0:
  232. raise RuntimeError("Running distutils build failed: %s\n%s"
  233. % (cmd[4:], asstr(out)))
  234. finally:
  235. os.chdir(cwd)
  236. # Partial cleanup
  237. for fn in dst_sources:
  238. os.unlink(fn)
  239. # Import
  240. __import__(module_name)
  241. return sys.modules[module_name]
  242. #
  243. # Unittest convenience
  244. #
  245. class F2PyTest:
  246. code = None
  247. sources = None
  248. options = []
  249. skip = []
  250. only = []
  251. suffix = '.f'
  252. module = None
  253. module_name = None
  254. def setup(self):
  255. if sys.platform == 'win32':
  256. pytest.skip('Fails with MinGW64 Gfortran (Issue #9673)')
  257. if self.module is not None:
  258. return
  259. # Check compiler availability first
  260. if not has_c_compiler():
  261. pytest.skip("No C compiler available")
  262. codes = []
  263. if self.sources:
  264. codes.extend(self.sources)
  265. if self.code is not None:
  266. codes.append(self.suffix)
  267. needs_f77 = False
  268. needs_f90 = False
  269. for fn in codes:
  270. if fn.endswith('.f'):
  271. needs_f77 = True
  272. elif fn.endswith('.f90'):
  273. needs_f90 = True
  274. if needs_f77 and not has_f77_compiler():
  275. pytest.skip("No Fortran 77 compiler available")
  276. if needs_f90 and not has_f90_compiler():
  277. pytest.skip("No Fortran 90 compiler available")
  278. # Build the module
  279. if self.code is not None:
  280. self.module = build_code(self.code, options=self.options,
  281. skip=self.skip, only=self.only,
  282. suffix=self.suffix,
  283. module_name=self.module_name)
  284. if self.sources is not None:
  285. self.module = build_module(self.sources, options=self.options,
  286. skip=self.skip, only=self.only,
  287. module_name=self.module_name)