base_subprocess.py 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285
  1. import collections
  2. import subprocess
  3. import warnings
  4. from . import protocols
  5. from . import transports
  6. from .log import logger
  7. class BaseSubprocessTransport(transports.SubprocessTransport):
  8. def __init__(self, loop, protocol, args, shell,
  9. stdin, stdout, stderr, bufsize,
  10. waiter=None, extra=None, **kwargs):
  11. super().__init__(extra)
  12. self._closed = False
  13. self._protocol = protocol
  14. self._loop = loop
  15. self._proc = None
  16. self._pid = None
  17. self._returncode = None
  18. self._exit_waiters = []
  19. self._pending_calls = collections.deque()
  20. self._pipes = {}
  21. self._finished = False
  22. if stdin == subprocess.PIPE:
  23. self._pipes[0] = None
  24. if stdout == subprocess.PIPE:
  25. self._pipes[1] = None
  26. if stderr == subprocess.PIPE:
  27. self._pipes[2] = None
  28. # Create the child process: set the _proc attribute
  29. try:
  30. self._start(args=args, shell=shell, stdin=stdin, stdout=stdout,
  31. stderr=stderr, bufsize=bufsize, **kwargs)
  32. except:
  33. self.close()
  34. raise
  35. self._pid = self._proc.pid
  36. self._extra['subprocess'] = self._proc
  37. if self._loop.get_debug():
  38. if isinstance(args, (bytes, str)):
  39. program = args
  40. else:
  41. program = args[0]
  42. logger.debug('process %r created: pid %s',
  43. program, self._pid)
  44. self._loop.create_task(self._connect_pipes(waiter))
  45. def __repr__(self):
  46. info = [self.__class__.__name__]
  47. if self._closed:
  48. info.append('closed')
  49. if self._pid is not None:
  50. info.append(f'pid={self._pid}')
  51. if self._returncode is not None:
  52. info.append(f'returncode={self._returncode}')
  53. elif self._pid is not None:
  54. info.append('running')
  55. else:
  56. info.append('not started')
  57. stdin = self._pipes.get(0)
  58. if stdin is not None:
  59. info.append(f'stdin={stdin.pipe}')
  60. stdout = self._pipes.get(1)
  61. stderr = self._pipes.get(2)
  62. if stdout is not None and stderr is stdout:
  63. info.append(f'stdout=stderr={stdout.pipe}')
  64. else:
  65. if stdout is not None:
  66. info.append(f'stdout={stdout.pipe}')
  67. if stderr is not None:
  68. info.append(f'stderr={stderr.pipe}')
  69. return '<{}>'.format(' '.join(info))
  70. def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs):
  71. raise NotImplementedError
  72. def set_protocol(self, protocol):
  73. self._protocol = protocol
  74. def get_protocol(self):
  75. return self._protocol
  76. def is_closing(self):
  77. return self._closed
  78. def close(self):
  79. if self._closed:
  80. return
  81. self._closed = True
  82. for proto in self._pipes.values():
  83. if proto is None:
  84. continue
  85. proto.pipe.close()
  86. if (self._proc is not None and
  87. # has the child process finished?
  88. self._returncode is None and
  89. # the child process has finished, but the
  90. # transport hasn't been notified yet?
  91. self._proc.poll() is None):
  92. if self._loop.get_debug():
  93. logger.warning('Close running child process: kill %r', self)
  94. try:
  95. self._proc.kill()
  96. except ProcessLookupError:
  97. pass
  98. # Don't clear the _proc reference yet: _post_init() may still run
  99. def __del__(self, _warn=warnings.warn):
  100. if not self._closed:
  101. _warn(f"unclosed transport {self!r}", ResourceWarning, source=self)
  102. self.close()
  103. def get_pid(self):
  104. return self._pid
  105. def get_returncode(self):
  106. return self._returncode
  107. def get_pipe_transport(self, fd):
  108. if fd in self._pipes:
  109. return self._pipes[fd].pipe
  110. else:
  111. return None
  112. def _check_proc(self):
  113. if self._proc is None:
  114. raise ProcessLookupError()
  115. def send_signal(self, signal):
  116. self._check_proc()
  117. self._proc.send_signal(signal)
  118. def terminate(self):
  119. self._check_proc()
  120. self._proc.terminate()
  121. def kill(self):
  122. self._check_proc()
  123. self._proc.kill()
  124. async def _connect_pipes(self, waiter):
  125. try:
  126. proc = self._proc
  127. loop = self._loop
  128. if proc.stdin is not None:
  129. _, pipe = await loop.connect_write_pipe(
  130. lambda: WriteSubprocessPipeProto(self, 0),
  131. proc.stdin)
  132. self._pipes[0] = pipe
  133. if proc.stdout is not None:
  134. _, pipe = await loop.connect_read_pipe(
  135. lambda: ReadSubprocessPipeProto(self, 1),
  136. proc.stdout)
  137. self._pipes[1] = pipe
  138. if proc.stderr is not None:
  139. _, pipe = await loop.connect_read_pipe(
  140. lambda: ReadSubprocessPipeProto(self, 2),
  141. proc.stderr)
  142. self._pipes[2] = pipe
  143. assert self._pending_calls is not None
  144. loop.call_soon(self._protocol.connection_made, self)
  145. for callback, data in self._pending_calls:
  146. loop.call_soon(callback, *data)
  147. self._pending_calls = None
  148. except (SystemExit, KeyboardInterrupt):
  149. raise
  150. except BaseException as exc:
  151. if waiter is not None and not waiter.cancelled():
  152. waiter.set_exception(exc)
  153. else:
  154. if waiter is not None and not waiter.cancelled():
  155. waiter.set_result(None)
  156. def _call(self, cb, *data):
  157. if self._pending_calls is not None:
  158. self._pending_calls.append((cb, data))
  159. else:
  160. self._loop.call_soon(cb, *data)
  161. def _pipe_connection_lost(self, fd, exc):
  162. self._call(self._protocol.pipe_connection_lost, fd, exc)
  163. self._try_finish()
  164. def _pipe_data_received(self, fd, data):
  165. self._call(self._protocol.pipe_data_received, fd, data)
  166. def _process_exited(self, returncode):
  167. assert returncode is not None, returncode
  168. assert self._returncode is None, self._returncode
  169. if self._loop.get_debug():
  170. logger.info('%r exited with return code %r', self, returncode)
  171. self._returncode = returncode
  172. if self._proc.returncode is None:
  173. # asyncio uses a child watcher: copy the status into the Popen
  174. # object. On Python 3.6, it is required to avoid a ResourceWarning.
  175. self._proc.returncode = returncode
  176. self._call(self._protocol.process_exited)
  177. self._try_finish()
  178. # wake up futures waiting for wait()
  179. for waiter in self._exit_waiters:
  180. if not waiter.cancelled():
  181. waiter.set_result(returncode)
  182. self._exit_waiters = None
  183. async def _wait(self):
  184. """Wait until the process exit and return the process return code.
  185. This method is a coroutine."""
  186. if self._returncode is not None:
  187. return self._returncode
  188. waiter = self._loop.create_future()
  189. self._exit_waiters.append(waiter)
  190. return await waiter
  191. def _try_finish(self):
  192. assert not self._finished
  193. if self._returncode is None:
  194. return
  195. if all(p is not None and p.disconnected
  196. for p in self._pipes.values()):
  197. self._finished = True
  198. self._call(self._call_connection_lost, None)
  199. def _call_connection_lost(self, exc):
  200. try:
  201. self._protocol.connection_lost(exc)
  202. finally:
  203. self._loop = None
  204. self._proc = None
  205. self._protocol = None
  206. class WriteSubprocessPipeProto(protocols.BaseProtocol):
  207. def __init__(self, proc, fd):
  208. self.proc = proc
  209. self.fd = fd
  210. self.pipe = None
  211. self.disconnected = False
  212. def connection_made(self, transport):
  213. self.pipe = transport
  214. def __repr__(self):
  215. return f'<{self.__class__.__name__} fd={self.fd} pipe={self.pipe!r}>'
  216. def connection_lost(self, exc):
  217. self.disconnected = True
  218. self.proc._pipe_connection_lost(self.fd, exc)
  219. self.proc = None
  220. def pause_writing(self):
  221. self.proc._protocol.pause_writing()
  222. def resume_writing(self):
  223. self.proc._protocol.resume_writing()
  224. class ReadSubprocessPipeProto(WriteSubprocessPipeProto,
  225. protocols.Protocol):
  226. def data_received(self, data):
  227. self.proc._pipe_data_received(self.fd, data)