Cythonize.py 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227
  1. #!/usr/bin/env python
  2. from __future__ import absolute_import
  3. import os
  4. import shutil
  5. import tempfile
  6. from distutils.core import setup
  7. from .Dependencies import cythonize, extended_iglob
  8. from ..Utils import is_package_dir
  9. from ..Compiler import Options
  10. try:
  11. import multiprocessing
  12. parallel_compiles = int(multiprocessing.cpu_count() * 1.5)
  13. except ImportError:
  14. multiprocessing = None
  15. parallel_compiles = 0
  16. class _FakePool(object):
  17. def map_async(self, func, args):
  18. try:
  19. from itertools import imap
  20. except ImportError:
  21. imap=map
  22. for _ in imap(func, args):
  23. pass
  24. def close(self):
  25. pass
  26. def terminate(self):
  27. pass
  28. def join(self):
  29. pass
  30. def parse_directives(option, name, value, parser):
  31. dest = option.dest
  32. old_directives = dict(getattr(parser.values, dest,
  33. Options.get_directive_defaults()))
  34. directives = Options.parse_directive_list(
  35. value, relaxed_bool=True, current_settings=old_directives)
  36. setattr(parser.values, dest, directives)
  37. def parse_options(option, name, value, parser):
  38. dest = option.dest
  39. options = dict(getattr(parser.values, dest, {}))
  40. for opt in value.split(','):
  41. if '=' in opt:
  42. n, v = opt.split('=', 1)
  43. v = v.lower() not in ('false', 'f', '0', 'no')
  44. else:
  45. n, v = opt, True
  46. options[n] = v
  47. setattr(parser.values, dest, options)
  48. def parse_compile_time_env(option, name, value, parser):
  49. dest = option.dest
  50. old_env = dict(getattr(parser.values, dest, {}))
  51. new_env = Options.parse_compile_time_env(value, current_settings=old_env)
  52. setattr(parser.values, dest, new_env)
  53. def find_package_base(path):
  54. base_dir, package_path = os.path.split(path)
  55. while os.path.isfile(os.path.join(base_dir, '__init__.py')):
  56. base_dir, parent = os.path.split(base_dir)
  57. package_path = '%s/%s' % (parent, package_path)
  58. return base_dir, package_path
  59. def cython_compile(path_pattern, options):
  60. pool = None
  61. all_paths = map(os.path.abspath, extended_iglob(path_pattern))
  62. try:
  63. for path in all_paths:
  64. if options.build_inplace:
  65. base_dir = path
  66. while not os.path.isdir(base_dir) or is_package_dir(base_dir):
  67. base_dir = os.path.dirname(base_dir)
  68. else:
  69. base_dir = None
  70. if os.path.isdir(path):
  71. # recursively compiling a package
  72. paths = [os.path.join(path, '**', '*.{py,pyx}')]
  73. else:
  74. # assume it's a file(-like thing)
  75. paths = [path]
  76. ext_modules = cythonize(
  77. paths,
  78. nthreads=options.parallel,
  79. exclude_failures=options.keep_going,
  80. exclude=options.excludes,
  81. compiler_directives=options.directives,
  82. compile_time_env=options.compile_time_env,
  83. force=options.force,
  84. quiet=options.quiet,
  85. **options.options)
  86. if ext_modules and options.build:
  87. if len(ext_modules) > 1 and options.parallel > 1:
  88. if pool is None:
  89. try:
  90. pool = multiprocessing.Pool(options.parallel)
  91. except OSError:
  92. pool = _FakePool()
  93. pool.map_async(run_distutils, [
  94. (base_dir, [ext]) for ext in ext_modules])
  95. else:
  96. run_distutils((base_dir, ext_modules))
  97. except:
  98. if pool is not None:
  99. pool.terminate()
  100. raise
  101. else:
  102. if pool is not None:
  103. pool.close()
  104. pool.join()
  105. def run_distutils(args):
  106. base_dir, ext_modules = args
  107. script_args = ['build_ext', '-i']
  108. cwd = os.getcwd()
  109. temp_dir = None
  110. try:
  111. if base_dir:
  112. os.chdir(base_dir)
  113. temp_dir = tempfile.mkdtemp(dir=base_dir)
  114. script_args.extend(['--build-temp', temp_dir])
  115. setup(
  116. script_name='setup.py',
  117. script_args=script_args,
  118. ext_modules=ext_modules,
  119. )
  120. finally:
  121. if base_dir:
  122. os.chdir(cwd)
  123. if temp_dir and os.path.isdir(temp_dir):
  124. shutil.rmtree(temp_dir)
  125. def parse_args(args):
  126. from optparse import OptionParser
  127. parser = OptionParser(usage='%prog [options] [sources and packages]+')
  128. parser.add_option('-X', '--directive', metavar='NAME=VALUE,...',
  129. dest='directives', default={}, type="str",
  130. action='callback', callback=parse_directives,
  131. help='set a compiler directive')
  132. parser.add_option('-E', '--compile-time-env', metavar='NAME=VALUE,...',
  133. dest='compile_time_env', default={}, type="str",
  134. action='callback', callback=parse_compile_time_env,
  135. help='set a compile time environment variable')
  136. parser.add_option('-s', '--option', metavar='NAME=VALUE',
  137. dest='options', default={}, type="str",
  138. action='callback', callback=parse_options,
  139. help='set a cythonize option')
  140. parser.add_option('-2', dest='language_level', action='store_const', const=2, default=None,
  141. help='use Python 2 syntax mode by default')
  142. parser.add_option('-3', dest='language_level', action='store_const', const=3,
  143. help='use Python 3 syntax mode by default')
  144. parser.add_option('--3str', dest='language_level', action='store_const', const='3str',
  145. help='use Python 3 syntax mode by default')
  146. parser.add_option('-a', '--annotate', dest='annotate', action='store_true',
  147. help='generate annotated HTML page for source files')
  148. parser.add_option('-x', '--exclude', metavar='PATTERN', dest='excludes',
  149. action='append', default=[],
  150. help='exclude certain file patterns from the compilation')
  151. parser.add_option('-b', '--build', dest='build', action='store_true',
  152. help='build extension modules using distutils')
  153. parser.add_option('-i', '--inplace', dest='build_inplace', action='store_true',
  154. help='build extension modules in place using distutils (implies -b)')
  155. parser.add_option('-j', '--parallel', dest='parallel', metavar='N',
  156. type=int, default=parallel_compiles,
  157. help=('run builds in N parallel jobs (default: %d)' %
  158. parallel_compiles or 1))
  159. parser.add_option('-f', '--force', dest='force', action='store_true',
  160. help='force recompilation')
  161. parser.add_option('-q', '--quiet', dest='quiet', action='store_true',
  162. help='be less verbose during compilation')
  163. parser.add_option('--lenient', dest='lenient', action='store_true',
  164. help='increase Python compatibility by ignoring some compile time errors')
  165. parser.add_option('-k', '--keep-going', dest='keep_going', action='store_true',
  166. help='compile as much as possible, ignore compilation failures')
  167. options, args = parser.parse_args(args)
  168. if not args:
  169. parser.error("no source files provided")
  170. if options.build_inplace:
  171. options.build = True
  172. if multiprocessing is None:
  173. options.parallel = 0
  174. if options.language_level:
  175. assert options.language_level in (2, 3, '3str')
  176. options.options['language_level'] = options.language_level
  177. return options, args
  178. def main(args=None):
  179. options, paths = parse_args(args)
  180. if options.lenient:
  181. # increase Python compatibility by ignoring compile time errors
  182. Options.error_on_unknown_names = False
  183. Options.error_on_uninitialized = False
  184. if options.annotate:
  185. Options.annotate = True
  186. for path in paths:
  187. cython_compile(path, options)
  188. if __name__ == '__main__':
  189. main()