select() vs. pipes (was [Python-Dev] Re: PEP 324 (process module)) (original) (raw)

Chris McDonough chrism at plope.com
Wed Aug 4 22:15:50 CEST 2004

On Wed, 2004-08-04 at 16:03, Chris McDonough wrote:

I was all set to try to refute this, but after writing a minimal test program to do what I want to do, I find that you're right. That's good news! I'll need to revisit my workaroudns in the program that caused me to need to do this. Thanks for the schooling.

Ugh. I spoke a bit too soon..

The following program demonstrates that a particular usage of select (under Linux at least) always returns the output side of a pipe connected to a child process' stdout as "ready" after it gets any output from that child process, even if the child process has no further data to provide after it has provided a bit of data to the parent. This is what causes the "busywait" behavior I've experienced in the past (note that as this program runs, your CPU utilization will likely be near 100%).

Or am I doing something silly?

import select import errno import fcntl import os import stat import sys from fcntl import F_SETFL, F_GETFL

def get_path(): """Return a list corresponding to $PATH, or a default.""" path = ["/bin", "/usr/bin", "/usr/local/bin"] if os.environ.has_key("PATH"): p = os.environ["PATH"] if p: path = p.split(os.pathsep) return path

def get_execv_args(command): """Internal: turn a program name into a file name, using $PATH.""" commandargs = command.split() program = commandargs[0] if "/" in program: filename = program try: st = os.stat(filename) except os.error: return None, None else: path = get_path() for dir in path: filename = os.path.join(dir, program) try: st = os.stat(filename) except os.error: continue mode = st[stat.ST_MODE] if mode & 0111: break else: return None, None if not os.access(filename, os.X_OK): return None, None return filename, commandargs

def spawn(command): """Start the subprocess."""

filename, argv = get_execv_args(command)
if filename is None:
    raise RuntimeError, '%s is an invalid command' % command

child_stdin, stdin = os.pipe()
stdout, child_stdout = os.pipe()
stderr, child_stderr = os.pipe()

# open stderr, stdout in nonblocking mode so we can tail them
# in the mainloop without blocking
for fd in stdout, stderr:
    flags = fcntl.fcntl(fd, F_GETFL)
    fcntl.fcntl(fd, F_SETFL, flags | os.O_NDELAY)

pid = os.fork()

if pid != 0:
    # Parent
    os.close(child_stdin)
    os.close(child_stdout)
    os.close(child_stderr)
    return stdin, stdout, stderr

else:
    # Child
    try:
        os.dup2(child_stdin, 0)
        os.dup2(child_stdout, 1)
        os.dup2(child_stderr, 2)
        for i in range(3, 256):
            try:
                os.close(i)
            except:
                pass
        os.execv(filename, argv)
    finally:
        os._exit(127)

def go(out_fds): while 1: try: r, w, x = select.select(out_fds, [], [], 1) if not r: print "timed out" except select.error, err: if err[0] != errno.EINTR: raise for fd in r: sys.stdout.write(os.read(fd, 1024))

stdin, stderr, stdout = spawn('echo "foo"')

go([stderr, stdout])

More information about the Python-Dev mailing list