Geek asks and answers

End of thread detection on popen.stdout.readline

I have a python program that starts subprocesses using Popenand consumes their output in almost real time as it is created. The code for the corresponding cycle:

def run(self, output_consumer):
    self.prepare_to_run()
    popen_args = self.get_popen_args()
    logging.debug("Calling popen with arguments %s" % popen_args)
    self.popen = subprocess.Popen(**popen_args)
    while True:
        outdata = self.popen.stdout.readline()
        if not outdata and self.popen.returncode is not None:
            # Terminate when we've read all the output and the returncode is set
            break
        output_consumer.process_output(outdata)
        self.popen.poll()  # updates returncode so we can exit the loop
    output_consumer.finish(self.popen.returncode)
    self.post_run()

def get_popen_args(self):
    return {
        'args': self.command,
        'shell': False, # Just being explicit for security sake
        'bufsize': 0,   # More likely to see what being printed as it happens
                        # Not guarantted since the process itself might buffer its output
                        # run `python -u` to unbuffer output of a python processes
        'cwd': self.get_cwd(),
        'env': self.get_environment(),
        'stdout': subprocess.PIPE,
        'stderr': subprocess.STDOUT,
        'close_fds': True,  # Doesn't seem to matter
    }

This works fine on my production machines, but on my dev machine, the call .readline()hangs when some subprocesses finish. That is, it will successfully process all the output, including the final output line, saying "end of process", but then it will poll again readlineand never return. This method correctly exits to the dev machine for most of the subprocesses that I invoke, but cannot sequentially exit for one complex bash script that itself invokes many subprocesses.

, popen.returncode None ( 0) . , , , , . , , , , readline() . read() . read(1) , . popen.stdout.closed False. , ?

python 2.7.3 Ubuntu 12.04LTS. FWIW, stderr stdout stderr=subprocess.STDOUT.

? - stdout? - - , - ? , , dev-, supervisord? , , , ?

+5
6

. , , . , script , stdout, . , ?

, , .returncode . , , , . select . - , , .

+1

readline() read(), . (). , , , , , , , .

, , : * script , * script strace -f -e clone,execve,exit_group , script, - script (, script exit_group, strace , ).

+2

" bash script", , .

, , , , Python script supervisord, , stdin -, stdin - tty, ( ) supervisord /dev/null.

, , , test.sh , stdin...

import os
import subprocess

f = subprocess.Popen(args='./test.sh',
                     shell=False,
                     bufsize=0,
                     stdin=open(os.devnull, 'rb'),
                     stdout=subprocess.PIPE,
                     stderr=subprocess.STDOUT,
                     close_fds=True)

while 1:
    s = f.stdout.readline()
    if not s and f.returncode is not None:
        break
    print s.strip()
    f.poll()
print "done %d" % f.returncode

, "process complete", .

+1

, read ( readline) , , poll. select, , . , select , . select (. ).

, readline, readline , \n, stdout, stdin / . , , \n stdout_collected, .

, (pty) popen ( ), .

# handle to read from
handle = self.popen.stdout

# how many seconds to wait without data
timeout = 1

begin = datetime.now()
stdout_collected = ""

while self.popen.poll() is None:
    try:
        fds = select.select([handle], [], [], 0.01)[0]
    except select.error, exc:
        print exc
        break

    if len(fds) == 0:
        # select timed out, no new data
        delta = (datetime.now() - begin).total_seconds()
        if delta > timeout:
            return stdout_collected

        # try longer
        continue
    else:
        # have data, timeout counter resets again
        begin = datetime.now()

    for fd in fds:
        if fd == handle:
            data = os.read(handle, 1024)
            # can handle the bytes as they come in here
            # self._handle_stdout(data)
            stdout_collected += data

# process exited
# if using a pseudoterminal, close the handles here
self.popen.wait()
+1

sdterr STDOUT?

call() , , stdout, stderr.

, succsss .

, , . () , , .

0

bash, . '' readline(), - '\n'.

from subprocess import Popen, PIPE, STDOUT
p = Popen(['bash'], stdout=PIPE, stderr=STDOUT)
out = []
while True:
    outdata = p.stdout.readline()
    if not outdata:
        break
    #output_consumer.process_output(outdata)
    print "* " + repr(outdata)
    out.append(outdata)
print "* closed", repr(out)
print "* returncode", p.wait()

/: . wait() poll()

[prompt] $ python myscript.py
echo abc
* 'abc\n'
exec 1>&- # close stdout
exec 2>&- # close stderr
* closed ['abc\n']
exit
* returncode 0
[prompt] $

.

: '\n' :

echo -n abc
exit
* 'abc'
* closed ['abc']
* returncode 0
0
source

More articles:


All Articles