gnu.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550
  1. import re
  2. import os
  3. import sys
  4. import warnings
  5. import platform
  6. import tempfile
  7. import hashlib
  8. import base64
  9. import subprocess
  10. from subprocess import Popen, PIPE, STDOUT
  11. from numpy.distutils.exec_command import filepath_from_subprocess_output
  12. from numpy.distutils.fcompiler import FCompiler
  13. from distutils.version import LooseVersion
  14. compilers = ['GnuFCompiler', 'Gnu95FCompiler']
  15. TARGET_R = re.compile(r"Target: ([a-zA-Z0-9_\-]*)")
  16. # XXX: handle cross compilation
  17. def is_win64():
  18. return sys.platform == "win32" and platform.architecture()[0] == "64bit"
  19. class GnuFCompiler(FCompiler):
  20. compiler_type = 'gnu'
  21. compiler_aliases = ('g77', )
  22. description = 'GNU Fortran 77 compiler'
  23. def gnu_version_match(self, version_string):
  24. """Handle the different versions of GNU fortran compilers"""
  25. # Strip warning(s) that may be emitted by gfortran
  26. while version_string.startswith('gfortran: warning'):
  27. version_string =\
  28. version_string[version_string.find('\n') + 1:].strip()
  29. # Gfortran versions from after 2010 will output a simple string
  30. # (usually "x.y", "x.y.z" or "x.y.z-q") for ``-dumpversion``; older
  31. # gfortrans may still return long version strings (``-dumpversion`` was
  32. # an alias for ``--version``)
  33. if len(version_string) <= 20:
  34. # Try to find a valid version string
  35. m = re.search(r'([0-9.]+)', version_string)
  36. if m:
  37. # g77 provides a longer version string that starts with GNU
  38. # Fortran
  39. if version_string.startswith('GNU Fortran'):
  40. return ('g77', m.group(1))
  41. # gfortran only outputs a version string such as #.#.#, so check
  42. # if the match is at the start of the string
  43. elif m.start() == 0:
  44. return ('gfortran', m.group(1))
  45. else:
  46. # Output probably from --version, try harder:
  47. m = re.search(r'GNU Fortran\s+95.*?([0-9-.]+)', version_string)
  48. if m:
  49. return ('gfortran', m.group(1))
  50. m = re.search(
  51. r'GNU Fortran.*?\-?([0-9-.]+\.[0-9-.]+)', version_string)
  52. if m:
  53. v = m.group(1)
  54. if v.startswith('0') or v.startswith('2') or v.startswith('3'):
  55. # the '0' is for early g77's
  56. return ('g77', v)
  57. else:
  58. # at some point in the 4.x series, the ' 95' was dropped
  59. # from the version string
  60. return ('gfortran', v)
  61. # If still nothing, raise an error to make the problem easy to find.
  62. err = 'A valid Fortran version was not found in this string:\n'
  63. raise ValueError(err + version_string)
  64. def version_match(self, version_string):
  65. v = self.gnu_version_match(version_string)
  66. if not v or v[0] != 'g77':
  67. return None
  68. return v[1]
  69. possible_executables = ['g77', 'f77']
  70. executables = {
  71. 'version_cmd' : [None, "-dumpversion"],
  72. 'compiler_f77' : [None, "-g", "-Wall", "-fno-second-underscore"],
  73. 'compiler_f90' : None, # Use --fcompiler=gnu95 for f90 codes
  74. 'compiler_fix' : None,
  75. 'linker_so' : [None, "-g", "-Wall"],
  76. 'archiver' : ["ar", "-cr"],
  77. 'ranlib' : ["ranlib"],
  78. 'linker_exe' : [None, "-g", "-Wall"]
  79. }
  80. module_dir_switch = None
  81. module_include_switch = None
  82. # Cygwin: f771: warning: -fPIC ignored for target (all code is
  83. # position independent)
  84. if os.name != 'nt' and sys.platform != 'cygwin':
  85. pic_flags = ['-fPIC']
  86. # use -mno-cygwin for g77 when Python is not Cygwin-Python
  87. if sys.platform == 'win32':
  88. for key in ['version_cmd', 'compiler_f77', 'linker_so', 'linker_exe']:
  89. executables[key].append('-mno-cygwin')
  90. g2c = 'g2c'
  91. suggested_f90_compiler = 'gnu95'
  92. def get_flags_linker_so(self):
  93. opt = self.linker_so[1:]
  94. if sys.platform == 'darwin':
  95. target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None)
  96. # If MACOSX_DEPLOYMENT_TARGET is set, we simply trust the value
  97. # and leave it alone. But, distutils will complain if the
  98. # environment's value is different from the one in the Python
  99. # Makefile used to build Python. We let disutils handle this
  100. # error checking.
  101. if not target:
  102. # If MACOSX_DEPLOYMENT_TARGET is not set in the environment,
  103. # we try to get it first from sysconfig and then
  104. # fall back to setting it to 10.9 This is a reasonable default
  105. # even when using the official Python dist and those derived
  106. # from it.
  107. import sysconfig
  108. target = sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET')
  109. if not target:
  110. target = '10.9'
  111. s = f'Env. variable MACOSX_DEPLOYMENT_TARGET set to {target}'
  112. warnings.warn(s, stacklevel=2)
  113. os.environ['MACOSX_DEPLOYMENT_TARGET'] = str(target)
  114. opt.extend(['-undefined', 'dynamic_lookup', '-bundle'])
  115. else:
  116. opt.append("-shared")
  117. if sys.platform.startswith('sunos'):
  118. # SunOS often has dynamically loaded symbols defined in the
  119. # static library libg2c.a The linker doesn't like this. To
  120. # ignore the problem, use the -mimpure-text flag. It isn't
  121. # the safest thing, but seems to work. 'man gcc' says:
  122. # ".. Instead of using -mimpure-text, you should compile all
  123. # source code with -fpic or -fPIC."
  124. opt.append('-mimpure-text')
  125. return opt
  126. def get_libgcc_dir(self):
  127. try:
  128. output = subprocess.check_output(self.compiler_f77 +
  129. ['-print-libgcc-file-name'])
  130. except (OSError, subprocess.CalledProcessError):
  131. pass
  132. else:
  133. output = filepath_from_subprocess_output(output)
  134. return os.path.dirname(output)
  135. return None
  136. def get_libgfortran_dir(self):
  137. if sys.platform[:5] == 'linux':
  138. libgfortran_name = 'libgfortran.so'
  139. elif sys.platform == 'darwin':
  140. libgfortran_name = 'libgfortran.dylib'
  141. else:
  142. libgfortran_name = None
  143. libgfortran_dir = None
  144. if libgfortran_name:
  145. find_lib_arg = ['-print-file-name={0}'.format(libgfortran_name)]
  146. try:
  147. output = subprocess.check_output(
  148. self.compiler_f77 + find_lib_arg)
  149. except (OSError, subprocess.CalledProcessError):
  150. pass
  151. else:
  152. output = filepath_from_subprocess_output(output)
  153. libgfortran_dir = os.path.dirname(output)
  154. return libgfortran_dir
  155. def get_library_dirs(self):
  156. opt = []
  157. if sys.platform[:5] != 'linux':
  158. d = self.get_libgcc_dir()
  159. if d:
  160. # if windows and not cygwin, libg2c lies in a different folder
  161. if sys.platform == 'win32' and not d.startswith('/usr/lib'):
  162. d = os.path.normpath(d)
  163. path = os.path.join(d, "lib%s.a" % self.g2c)
  164. if not os.path.exists(path):
  165. root = os.path.join(d, *((os.pardir, ) * 4))
  166. d2 = os.path.abspath(os.path.join(root, 'lib'))
  167. path = os.path.join(d2, "lib%s.a" % self.g2c)
  168. if os.path.exists(path):
  169. opt.append(d2)
  170. opt.append(d)
  171. # For Macports / Linux, libgfortran and libgcc are not co-located
  172. lib_gfortran_dir = self.get_libgfortran_dir()
  173. if lib_gfortran_dir:
  174. opt.append(lib_gfortran_dir)
  175. return opt
  176. def get_libraries(self):
  177. opt = []
  178. d = self.get_libgcc_dir()
  179. if d is not None:
  180. g2c = self.g2c + '-pic'
  181. f = self.static_lib_format % (g2c, self.static_lib_extension)
  182. if not os.path.isfile(os.path.join(d, f)):
  183. g2c = self.g2c
  184. else:
  185. g2c = self.g2c
  186. if g2c is not None:
  187. opt.append(g2c)
  188. c_compiler = self.c_compiler
  189. if sys.platform == 'win32' and c_compiler and \
  190. c_compiler.compiler_type == 'msvc':
  191. opt.append('gcc')
  192. if sys.platform == 'darwin':
  193. opt.append('cc_dynamic')
  194. return opt
  195. def get_flags_debug(self):
  196. return ['-g']
  197. def get_flags_opt(self):
  198. v = self.get_version()
  199. if v and v <= '3.3.3':
  200. # With this compiler version building Fortran BLAS/LAPACK
  201. # with -O3 caused failures in lib.lapack heevr,syevr tests.
  202. opt = ['-O2']
  203. else:
  204. opt = ['-O3']
  205. opt.append('-funroll-loops')
  206. return opt
  207. def _c_arch_flags(self):
  208. """ Return detected arch flags from CFLAGS """
  209. import sysconfig
  210. try:
  211. cflags = sysconfig.get_config_vars()['CFLAGS']
  212. except KeyError:
  213. return []
  214. arch_re = re.compile(r"-arch\s+(\w+)")
  215. arch_flags = []
  216. for arch in arch_re.findall(cflags):
  217. arch_flags += ['-arch', arch]
  218. return arch_flags
  219. def get_flags_arch(self):
  220. return []
  221. def runtime_library_dir_option(self, dir):
  222. if sys.platform == 'win32':
  223. # Linux/Solaris/Unix support RPATH, Windows does not
  224. raise NotImplementedError
  225. # TODO: could use -Xlinker here, if it's supported
  226. assert "," not in dir
  227. if sys.platform == 'darwin':
  228. return f'-Wl,-rpath,{dir}'
  229. elif sys.platform[:3] == 'aix':
  230. # AIX RPATH is called LIBPATH
  231. return f'-Wl,-blibpath:{dir}'
  232. else:
  233. return f'-Wl,-rpath={dir}'
  234. class Gnu95FCompiler(GnuFCompiler):
  235. compiler_type = 'gnu95'
  236. compiler_aliases = ('gfortran', )
  237. description = 'GNU Fortran 95 compiler'
  238. def version_match(self, version_string):
  239. v = self.gnu_version_match(version_string)
  240. if not v or v[0] != 'gfortran':
  241. return None
  242. v = v[1]
  243. if LooseVersion(v) >= "4":
  244. # gcc-4 series releases do not support -mno-cygwin option
  245. pass
  246. else:
  247. # use -mno-cygwin flag for gfortran when Python is not
  248. # Cygwin-Python
  249. if sys.platform == 'win32':
  250. for key in [
  251. 'version_cmd', 'compiler_f77', 'compiler_f90',
  252. 'compiler_fix', 'linker_so', 'linker_exe'
  253. ]:
  254. self.executables[key].append('-mno-cygwin')
  255. return v
  256. possible_executables = ['gfortran', 'f95']
  257. executables = {
  258. 'version_cmd' : ["<F90>", "-dumpversion"],
  259. 'compiler_f77' : [None, "-Wall", "-g", "-ffixed-form",
  260. "-fno-second-underscore"],
  261. 'compiler_f90' : [None, "-Wall", "-g",
  262. "-fno-second-underscore"],
  263. 'compiler_fix' : [None, "-Wall", "-g","-ffixed-form",
  264. "-fno-second-underscore"],
  265. 'linker_so' : ["<F90>", "-Wall", "-g"],
  266. 'archiver' : ["ar", "-cr"],
  267. 'ranlib' : ["ranlib"],
  268. 'linker_exe' : [None, "-Wall"]
  269. }
  270. module_dir_switch = '-J'
  271. module_include_switch = '-I'
  272. if sys.platform[:3] == 'aix':
  273. executables['linker_so'].append('-lpthread')
  274. if platform.architecture()[0][:2] == '64':
  275. for key in ['compiler_f77', 'compiler_f90','compiler_fix','linker_so', 'linker_exe']:
  276. executables[key].append('-maix64')
  277. g2c = 'gfortran'
  278. def _universal_flags(self, cmd):
  279. """Return a list of -arch flags for every supported architecture."""
  280. if not sys.platform == 'darwin':
  281. return []
  282. arch_flags = []
  283. # get arches the C compiler gets.
  284. c_archs = self._c_arch_flags()
  285. if "i386" in c_archs:
  286. c_archs[c_archs.index("i386")] = "i686"
  287. # check the arches the Fortran compiler supports, and compare with
  288. # arch flags from C compiler
  289. for arch in ["ppc", "i686", "x86_64", "ppc64"]:
  290. if _can_target(cmd, arch) and arch in c_archs:
  291. arch_flags.extend(["-arch", arch])
  292. return arch_flags
  293. def get_flags(self):
  294. flags = GnuFCompiler.get_flags(self)
  295. arch_flags = self._universal_flags(self.compiler_f90)
  296. if arch_flags:
  297. flags[:0] = arch_flags
  298. return flags
  299. def get_flags_linker_so(self):
  300. flags = GnuFCompiler.get_flags_linker_so(self)
  301. arch_flags = self._universal_flags(self.linker_so)
  302. if arch_flags:
  303. flags[:0] = arch_flags
  304. return flags
  305. def get_library_dirs(self):
  306. opt = GnuFCompiler.get_library_dirs(self)
  307. if sys.platform == 'win32':
  308. c_compiler = self.c_compiler
  309. if c_compiler and c_compiler.compiler_type == "msvc":
  310. target = self.get_target()
  311. if target:
  312. d = os.path.normpath(self.get_libgcc_dir())
  313. root = os.path.join(d, *((os.pardir, ) * 4))
  314. path = os.path.join(root, "lib")
  315. mingwdir = os.path.normpath(path)
  316. if os.path.exists(os.path.join(mingwdir, "libmingwex.a")):
  317. opt.append(mingwdir)
  318. # For Macports / Linux, libgfortran and libgcc are not co-located
  319. lib_gfortran_dir = self.get_libgfortran_dir()
  320. if lib_gfortran_dir:
  321. opt.append(lib_gfortran_dir)
  322. return opt
  323. def get_libraries(self):
  324. opt = GnuFCompiler.get_libraries(self)
  325. if sys.platform == 'darwin':
  326. opt.remove('cc_dynamic')
  327. if sys.platform == 'win32':
  328. c_compiler = self.c_compiler
  329. if c_compiler and c_compiler.compiler_type == "msvc":
  330. if "gcc" in opt:
  331. i = opt.index("gcc")
  332. opt.insert(i + 1, "mingwex")
  333. opt.insert(i + 1, "mingw32")
  334. c_compiler = self.c_compiler
  335. if c_compiler and c_compiler.compiler_type == "msvc":
  336. return []
  337. else:
  338. pass
  339. return opt
  340. def get_target(self):
  341. try:
  342. output = subprocess.check_output(self.compiler_f77 + ['-v'])
  343. except (OSError, subprocess.CalledProcessError):
  344. pass
  345. else:
  346. output = filepath_from_subprocess_output(output)
  347. m = TARGET_R.search(output)
  348. if m:
  349. return m.group(1)
  350. return ""
  351. def _hash_files(self, filenames):
  352. h = hashlib.sha1()
  353. for fn in filenames:
  354. with open(fn, 'rb') as f:
  355. while True:
  356. block = f.read(131072)
  357. if not block:
  358. break
  359. h.update(block)
  360. text = base64.b32encode(h.digest())
  361. text = text.decode('ascii')
  362. return text.rstrip('=')
  363. def _link_wrapper_lib(self, objects, output_dir, extra_dll_dir,
  364. chained_dlls, is_archive):
  365. """Create a wrapper shared library for the given objects
  366. Return an MSVC-compatible lib
  367. """
  368. c_compiler = self.c_compiler
  369. if c_compiler.compiler_type != "msvc":
  370. raise ValueError("This method only supports MSVC")
  371. object_hash = self._hash_files(list(objects) + list(chained_dlls))
  372. if is_win64():
  373. tag = 'win_amd64'
  374. else:
  375. tag = 'win32'
  376. basename = 'lib' + os.path.splitext(
  377. os.path.basename(objects[0]))[0][:8]
  378. root_name = basename + '.' + object_hash + '.gfortran-' + tag
  379. dll_name = root_name + '.dll'
  380. def_name = root_name + '.def'
  381. lib_name = root_name + '.lib'
  382. dll_path = os.path.join(extra_dll_dir, dll_name)
  383. def_path = os.path.join(output_dir, def_name)
  384. lib_path = os.path.join(output_dir, lib_name)
  385. if os.path.isfile(lib_path):
  386. # Nothing to do
  387. return lib_path, dll_path
  388. if is_archive:
  389. objects = (["-Wl,--whole-archive"] + list(objects) +
  390. ["-Wl,--no-whole-archive"])
  391. self.link_shared_object(
  392. objects,
  393. dll_name,
  394. output_dir=extra_dll_dir,
  395. extra_postargs=list(chained_dlls) + [
  396. '-Wl,--allow-multiple-definition',
  397. '-Wl,--output-def,' + def_path,
  398. '-Wl,--export-all-symbols',
  399. '-Wl,--enable-auto-import',
  400. '-static',
  401. '-mlong-double-64',
  402. ])
  403. # No PowerPC!
  404. if is_win64():
  405. specifier = '/MACHINE:X64'
  406. else:
  407. specifier = '/MACHINE:X86'
  408. # MSVC specific code
  409. lib_args = ['/def:' + def_path, '/OUT:' + lib_path, specifier]
  410. if not c_compiler.initialized:
  411. c_compiler.initialize()
  412. c_compiler.spawn([c_compiler.lib] + lib_args)
  413. return lib_path, dll_path
  414. def can_ccompiler_link(self, compiler):
  415. # MSVC cannot link objects compiled by GNU fortran
  416. return compiler.compiler_type not in ("msvc", )
  417. def wrap_unlinkable_objects(self, objects, output_dir, extra_dll_dir):
  418. """
  419. Convert a set of object files that are not compatible with the default
  420. linker, to a file that is compatible.
  421. """
  422. if self.c_compiler.compiler_type == "msvc":
  423. # Compile a DLL and return the lib for the DLL as
  424. # the object. Also keep track of previous DLLs that
  425. # we have compiled so that we can link against them.
  426. # If there are .a archives, assume they are self-contained
  427. # static libraries, and build separate DLLs for each
  428. archives = []
  429. plain_objects = []
  430. for obj in objects:
  431. if obj.lower().endswith('.a'):
  432. archives.append(obj)
  433. else:
  434. plain_objects.append(obj)
  435. chained_libs = []
  436. chained_dlls = []
  437. for archive in archives[::-1]:
  438. lib, dll = self._link_wrapper_lib(
  439. [archive],
  440. output_dir,
  441. extra_dll_dir,
  442. chained_dlls=chained_dlls,
  443. is_archive=True)
  444. chained_libs.insert(0, lib)
  445. chained_dlls.insert(0, dll)
  446. if not plain_objects:
  447. return chained_libs
  448. lib, dll = self._link_wrapper_lib(
  449. plain_objects,
  450. output_dir,
  451. extra_dll_dir,
  452. chained_dlls=chained_dlls,
  453. is_archive=False)
  454. return [lib] + chained_libs
  455. else:
  456. raise ValueError("Unsupported C compiler")
  457. def _can_target(cmd, arch):
  458. """Return true if the architecture supports the -arch flag"""
  459. newcmd = cmd[:]
  460. fid, filename = tempfile.mkstemp(suffix=".f")
  461. os.close(fid)
  462. try:
  463. d = os.path.dirname(filename)
  464. output = os.path.splitext(filename)[0] + ".o"
  465. try:
  466. newcmd.extend(["-arch", arch, "-c", filename])
  467. p = Popen(newcmd, stderr=STDOUT, stdout=PIPE, cwd=d)
  468. p.communicate()
  469. return p.returncode == 0
  470. finally:
  471. if os.path.exists(output):
  472. os.remove(output)
  473. finally:
  474. os.remove(filename)
  475. return False
  476. if __name__ == '__main__':
  477. from distutils import log
  478. from numpy.distutils import customized_fcompiler
  479. log.set_verbosity(2)
  480. print(customized_fcompiler('gnu').get_version())
  481. try:
  482. print(customized_fcompiler('g95').get_version())
  483. except Exception as e:
  484. print(e)