diff --git a/CHANGELOG.md b/CHANGELOG.md index 91fd566ceb8695eea97d03118bed5df0eb05366c..3b366e20044a07ad4ec9fc91c6827ffb12e7310c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,3 +4,4 @@ * added `step` to `smart_enumerate` * reserve more memory in case of MemoryError * added `call_in_separate_thread` +* added `read_nowait` diff --git a/docs/processes.rst b/docs/processes.rst index fc5d914da22b1b930b277c9ab133d727494235f1..79e1a89613ec3813258b453b4cb46cfde97b707e 100644 --- a/docs/processes.rst +++ b/docs/processes.rst @@ -7,3 +7,7 @@ available, so that you don't need to worry about the buffer overflowing and such. .. autofunction:: satella.processes.call_and_return_stdout + +This is a subprocess helper: + +.. autofunction:: satella.processes.read_nowait diff --git a/satella/coding/concurrent/thread.py b/satella/coding/concurrent/thread.py index 45499f6765e24d6643c3d77ed6db264713b07967..8174022bff4d6db418590cbb937a3e3dd38f77af 100644 --- a/satella/coding/concurrent/thread.py +++ b/satella/coding/concurrent/thread.py @@ -17,7 +17,8 @@ def call_in_separate_thread(*t_args, **t_kwargs): """ Decorator to mark given routine as callable in a separate thread. - The routine will return a Future that is waitable to get the result of the function + The decorated routine will return a Future that is waitable to get the result + (or the exception) of the function The arguments given here will be passed to thread's constructor. """ diff --git a/satella/processes.py b/satella/processes.py index db49e2911e0d2247fb711224dd81b40ac16449ab..3602113f69b93e7fb416d40b8350a3a880124ecf 100644 --- a/satella/processes.py +++ b/satella/processes.py @@ -3,17 +3,25 @@ import threading import typing as tp from satella.coding.recast_exceptions import silence_excs +from .coding.concurrent import call_in_separate_thread from .exceptions import ProcessFailed -__all__ = ['call_and_return_stdout'] +__all__ = ['call_and_return_stdout', 'read_nowait'] +@call_in_separate_thread(daemon=True) @silence_excs((IOError, OSError)) -def _read_nowait(process: subprocess.Popen, output_list: tp.List[str]) -> None: +def read_nowait(process: subprocess.Popen, output_list: tp.List[str]): """ - To be launched as a daemon thread. This reads stdout and appends it's entries to a list. - This should finish as soon as the process exits or closes it's stdout. + This spawns a thread to read given process' stdout and append it to a list, in + order to prevent buffer filling up completely. + + To retrieve entire stdout after process finishes do + + >>> ''.join(list) + + This thread will terminate automatically after the process closes it's stdout or finishes. """ while True: with silence_excs(subprocess.TimeoutExpired): @@ -54,11 +62,7 @@ def call_and_return_stdout(args: tp.Union[str, tp.List[str]], stdout_list = [] proc = subprocess.Popen(args, **kwargs) - reader_thread = threading.Thread(name='stdout reader', - target=_read_nowait, - args=(proc, stdout_list), - daemon=True) - reader_thread.start() + fut = read_nowait(proc, stdout_list) try: proc.wait(timeout=timeout) @@ -67,7 +71,7 @@ def call_and_return_stdout(args: tp.Union[str, tp.List[str]], proc.wait() raise TimeoutError('Process did not complete within %s seconds' % (timeout, )) finally: - reader_thread.join() + fut.result() if encoding is None: result = b''.join(stdout_list)