From 16464a96caaa08a39996487ee54d98371ac4100c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Ma=C5=9Blanka?= <piotr.maslanka@henrietta.com.pl> Date: Tue, 8 Sep 2020 11:25:16 +0200 Subject: [PATCH] 2.11.2, added read_nowait --- CHANGELOG.md | 1 + docs/processes.rst | 4 ++++ satella/coding/concurrent/thread.py | 3 ++- satella/processes.py | 24 ++++++++++++++---------- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 91fd566c..3b366e20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,3 +4,4 @@ * added `step` to `smart_enumerate` * reserve more memory in case of MemoryError * added `call_in_separate_thread` +* added `read_nowait` diff --git a/docs/processes.rst b/docs/processes.rst index fc5d914d..79e1a896 100644 --- a/docs/processes.rst +++ b/docs/processes.rst @@ -7,3 +7,7 @@ available, so that you don't need to worry about the buffer overflowing and such. .. autofunction:: satella.processes.call_and_return_stdout + +This is a subprocess helper: + +.. autofunction:: satella.processes.read_nowait diff --git a/satella/coding/concurrent/thread.py b/satella/coding/concurrent/thread.py index 45499f67..8174022b 100644 --- a/satella/coding/concurrent/thread.py +++ b/satella/coding/concurrent/thread.py @@ -17,7 +17,8 @@ def call_in_separate_thread(*t_args, **t_kwargs): """ Decorator to mark given routine as callable in a separate thread. - The routine will return a Future that is waitable to get the result of the function + The decorated routine will return a Future that is waitable to get the result + (or the exception) of the function The arguments given here will be passed to thread's constructor. """ diff --git a/satella/processes.py b/satella/processes.py index db49e291..3602113f 100644 --- a/satella/processes.py +++ b/satella/processes.py @@ -3,17 +3,25 @@ import threading import typing as tp from satella.coding.recast_exceptions import silence_excs +from .coding.concurrent import call_in_separate_thread from .exceptions import ProcessFailed -__all__ = ['call_and_return_stdout'] +__all__ = ['call_and_return_stdout', 'read_nowait'] +@call_in_separate_thread(daemon=True) @silence_excs((IOError, OSError)) -def _read_nowait(process: subprocess.Popen, output_list: tp.List[str]) -> None: +def read_nowait(process: subprocess.Popen, output_list: tp.List[str]): """ - To be launched as a daemon thread. This reads stdout and appends it's entries to a list. - This should finish as soon as the process exits or closes it's stdout. + This spawns a thread to read given process' stdout and append it to a list, in + order to prevent buffer filling up completely. + + To retrieve entire stdout after process finishes do + + >>> ''.join(list) + + This thread will terminate automatically after the process closes it's stdout or finishes. """ while True: with silence_excs(subprocess.TimeoutExpired): @@ -54,11 +62,7 @@ def call_and_return_stdout(args: tp.Union[str, tp.List[str]], stdout_list = [] proc = subprocess.Popen(args, **kwargs) - reader_thread = threading.Thread(name='stdout reader', - target=_read_nowait, - args=(proc, stdout_list), - daemon=True) - reader_thread.start() + fut = read_nowait(proc, stdout_list) try: proc.wait(timeout=timeout) @@ -67,7 +71,7 @@ def call_and_return_stdout(args: tp.Union[str, tp.List[str]], proc.wait() raise TimeoutError('Process did not complete within %s seconds' % (timeout, )) finally: - reader_thread.join() + fut.result() if encoding is None: result = b''.join(stdout_list) -- GitLab