cpython: 61aa484a3e54 (original) (raw)

--- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1295,6 +1295,11 @@ class POSIXProcessTestCase(BaseTestCase) self.addCleanup(os.close, fds[1]) open_fds = set(fds)

```
   # add a bunch more fds[](#l1.7)
```
```
   for _ in range(9):[](#l1.8)
```

       fd = os.open("/dev/null", os.O_RDONLY)[](#l1.9)

       self.addCleanup(os.close, fd)[](#l1.10)

```
       open_fds.add(fd)[](#l1.11)
```

p = subprocess.Popen([sys.executable, fd_status], stdout=subprocess.PIPE, close_fds=False) @@ -1313,6 +1318,19 @@ class POSIXProcessTestCase(BaseTestCase) "Some fds were left open") self.assertIn(1, remaining_fds, "Subprocess failed")

   # Keep some of the fd's we opened open in the subprocess.[](#l1.19)

   # This tests _posixsubprocess.c's proper handling of fds_to_keep.[](#l1.20)

   fds_to_keep = set(open_fds.pop() for _ in range(8))[](#l1.21)

   p = subprocess.Popen([sys.executable, fd_status],[](#l1.22)

                        stdout=subprocess.PIPE, close_fds=True,[](#l1.23)

                        pass_fds=())[](#l1.24)

   output, ignored = p.communicate()[](#l1.25)

   remaining_fds = set(map(int, output.split(b',')))[](#l1.26)

   self.assertFalse(remaining_fds & fds_to_keep & open_fds,[](#l1.28)

                    "Some fds not in pass_fds were left open")[](#l1.29)

   self.assertIn(1, remaining_fds, "Subprocess failed")[](#l1.30)

+ # Mac OS X Tiger (10.4) has a kernel bug: sometimes, the file # descriptor of a pipe closed in the parent process is valid in the # child process according to fstat(), but the mode of the file

--- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -5,7 +5,26 @@ #endif #include <unistd.h> #include <fcntl.h> +#ifdef HAVE_SYS_TYPES_H +#include <sys/types.h> +#endif +#ifdef HAVE_SYS_SYSCALL_H +#include <sys/syscall.h> +#endif +#ifdef HAVE_DIRENT_H +#include <dirent.h> +#endif +#if defined(sun) && !defined(HAVE_DIRFD) +/* Some versions of Solaris lack dirfd(). */ +# define DIRFD(dirp) ((dirp)->dd_fd) +# define HAVE_DIRFD +#else +# define DIRFD(dirp) (dirfd(dirp)) +#endif + +#define LINUX_SOLARIS_FD_DIR "/proc/self/fd" +#define BSD_OSX_FD_DIR "/dev/fd" #define POSIX_CALL(call) if ((call) == -1) goto error @@ -26,6 +45,233 @@ static int _enable_gc(PyObject gc_modul } +/ Convert ASCII to a positive int, no libc call. no overflow. -1 on error. */ +static int _pos_int_from_ascii(char *name) +{

int num = 0;
while (*name >= '0' && *name <= '9') {

   num = num * 10 + (*name - '0');[](#l2.39)

```
   ++name;[](#l2.40)
```
}
if (*name)

   return -1;  /* Non digit found, not a number. */[](#l2.43)

return num;

+} + + +/* Returns 1 if there is a problem with fd_sequence, 0 otherwise. */ +static int _sanity_check_python_fd_sequence(PyObject *fd_sequence) +{

Py_ssize_t seq_idx, seq_len = PySequence_Length(fd_sequence);
long prev_fd = -1;
for (seq_idx = 0; seq_idx < seq_len; ++seq_idx) {

   PyObject* py_fd = PySequence_Fast_GET_ITEM(fd_sequence, seq_idx);[](#l2.54)

   long iter_fd = PyLong_AsLong(py_fd);[](#l2.55)

   if (iter_fd < 0 || iter_fd < prev_fd || iter_fd > INT_MAX) {[](#l2.56)

       /* Negative, overflow, not a Long, unsorted, too big for a fd. */[](#l2.57)

```
       return 1;[](#l2.58)
```
```
   }[](#l2.59)
```
}
return 0;

+} + + +/* Is fd found in the sorted Python Sequence? */ +static int _is_fd_in_sorted_fd_sequence(int fd, PyObject *fd_sequence) +{

/* Binary search. */
Py_ssize_t search_min = 0;
Py_ssize_t search_max = PySequence_Length(fd_sequence) - 1;
if (search_max < 0)
```
   return 0;[](#l2.72)
```
do {

   long middle = (search_min + search_max) / 2;[](#l2.74)

   long middle_fd = PyLong_AsLong([](#l2.75)

           PySequence_Fast_GET_ITEM(fd_sequence, middle));[](#l2.76)

```
   if (fd == middle_fd)[](#l2.77)
```
```
       return 1;[](#l2.78)
```
```
   if (fd > middle_fd)[](#l2.79)
```

       search_min = middle + 1;[](#l2.80)

```
   else[](#l2.81)
```

       search_max = middle - 1;[](#l2.82)

} while (search_min <= search_max);
return 0;

+} + + +/* Close all file descriptors in the range start_fd inclusive to

- end_fd exclusive except for those in py_fds_to_keep. If the
- range defined by [start_fd, end_fd) is large this will take a
- long time as it calls close() on EVERY possible fd.
*/ +static void _close_fds_by_brute_force(int start_fd, int end_fd,

                                 PyObject *py_fds_to_keep)[](#l2.94)

Py_ssize_t num_fds_to_keep = PySequence_Length(py_fds_to_keep);
Py_ssize_t keep_seq_idx;
int fd_num;
/* As py_fds_to_keep is sorted we can loop through the list closing

* fds inbetween any in the keep list falling within our range. */[](#l2.100)

for (keep_seq_idx = 0; keep_seq_idx < num_fds_to_keep; ++keep_seq_idx) {

   PyObject* py_keep_fd = PySequence_Fast_GET_ITEM(py_fds_to_keep,[](#l2.102)

                                                   keep_seq_idx);[](#l2.103)

   int keep_fd = PyLong_AsLong(py_keep_fd);[](#l2.104)

```
   if (keep_fd < start_fd)[](#l2.105)
```
```
       continue;[](#l2.106)
```

   for (fd_num = start_fd; fd_num < keep_fd; ++fd_num) {[](#l2.107)

       while (close(fd_num) < 0 && errno == EINTR);[](#l2.108)

```
   }[](#l2.109)
```
```
   start_fd = keep_fd + 1;[](#l2.110)
```
}
if (start_fd <= end_fd) {

   for (fd_num = start_fd; fd_num < end_fd; ++fd_num) {[](#l2.113)

       while (close(fd_num) < 0 && errno == EINTR);[](#l2.114)

```
   }[](#l2.115)
```
}

+} + + +#if defined(linux) && defined(HAVE_SYS_SYSCALL_H) +/* It doesn't matter if d_name has room for NAME_MAX chars; we're using this

- only to read a directory of short file descriptor number names. The kernel
- will return an error if we didn't give it enough space. Highly Unlikely.
- This structure is very old and stable: It will not change unless the kernel
- chooses to break compatibility with all existing binaries. Highly Unlikely.
*/ +struct linux_dirent {
unsigned long d_ino; /* Inode number */
unsigned long d_off; /* Offset to next linux_dirent */
unsigned short d_reclen; /* Length of this linux_dirent */
char d_name[256]; /* Filename (null-terminated) */ +}; +

+/* Close all open file descriptors in the range start_fd inclusive to end_fd

- exclusive. Do not close any in the sorted py_fds_to_keep list.
*
- This version is async signal safe as it does not make any unsafe C library
- calls, malloc calls or handle any locks. It is unfortunate to be forced
- to resort to making a kernel system call directly but this is the ONLY api
- available that does no harm. opendir/readdir/closedir perform memory
- allocation and locking so while they usually work they are not guaranteed
- to (especially if you have replaced your malloc implementation). A version
- of this function that uses those can be found in the _maybe_unsafe variant.
*
- This is Linux specific because that is all I am ready to test it on. It
- should be easy to add OS specific dirent or dirent64 structures and modify
- it with some cpp #define magic to work on other OSes as well if you want.
*/ +static void _close_open_fd_range_safe(int start_fd, int end_fd,

                                 PyObject* py_fds_to_keep)[](#l2.150)

int fd_dir_fd;
if (start_fd >= end_fd)
```
   return;[](#l2.154)
```
fd_dir_fd = open(LINUX_SOLARIS_FD_DIR, O_RDONLY | O_CLOEXEC, 0);
/* Not trying to open the BSD_OSX path as this is currently Linux only. */
if (fd_dir_fd == -1) {

   /* No way to get a list of open fds. */[](#l2.158)

   _close_fds_by_brute_force(start_fd, end_fd, py_fds_to_keep);[](#l2.159)

```
   return;[](#l2.160)
```
} else {

   char buffer[sizeof(struct linux_dirent)];[](#l2.162)

```
   int bytes;[](#l2.163)
```

   while ((bytes = syscall(SYS_getdents, fd_dir_fd,[](#l2.164)

                           (struct linux_dirent *)buffer,[](#l2.165)

                           sizeof(buffer))) > 0) {[](#l2.166)

       struct linux_dirent *entry;[](#l2.167)

```
       int offset;[](#l2.168)
```

       for (offset = 0; offset < bytes; offset += entry->d_reclen) {[](#l2.169)

```
           int fd;[](#l2.170)
```

           entry = (struct linux_dirent *)(buffer + offset);[](#l2.171)

           if ((fd = _pos_int_from_ascii(entry->d_name)) < 0)[](#l2.172)

               continue;  /* Not a number. */[](#l2.173)

           if (fd != fd_dir_fd && fd >= start_fd && fd < end_fd &&[](#l2.174)

               !_is_fd_in_sorted_fd_sequence(fd, py_fds_to_keep)) {[](#l2.175)

               while (close(fd) < 0 && errno == EINTR);[](#l2.176)

```
           }[](#l2.177)
```
```
       }[](#l2.178)
```
```
   }[](#l2.179)
```
```
   close(fd_dir_fd);[](#l2.180)
```
}

+} + +#define _close_open_fd_range _close_open_fd_range_safe + +#else /* NOT (defined(linux) && defined(HAVE_SYS_SYSCALL_H)) / + + +/ Close all open file descriptors in the range start_fd inclusive to end_fd

- exclusive. Do not close any in the sorted py_fds_to_keep list.
*
- This function violates the strict use of async signal safe functions. :(
- It calls opendir(), readdir64() and closedir(). Of these, the one most
- likely to ever cause a problem is opendir() as it performs an internal
- malloc(). Practically this should not be a problem. The Java VM makes the
- same calls between fork and exec in its own UNIXProcess_md.c implementation.
*
- readdir_r() is not used because it provides no benefit. It is typically
- implemented as readdir() followed by memcpy(). See also:
- http://womble.decadent.org.uk/readdir_r-advisory.html[](#l2.200)
*/ +static void _close_open_fd_range_maybe_unsafe(int start_fd, int end_fd,

                                         PyObject* py_fds_to_keep)[](#l2.203)

DIR *proc_fd_dir;

+#ifndef HAVE_DIRFD

while (_is_fd_in_sorted_fd_sequence(start_fd, py_fds_to_keep) &&
```
      (start_fd < end_fd)) {[](#l2.208)
```
```
   ++start_fd;[](#l2.209)
```
}
if (start_fd >= end_fd)
```
   return;[](#l2.212)
```
/* Close our lowest fd before we call opendir so that it is likely to

* reuse that fd otherwise we might close opendir's file descriptor in[](#l2.214)

* our loop.  This trick assumes that fd's are allocated on a lowest[](#l2.215)

```
* available basis. */[](#l2.216)
```
while (close(start_fd) < 0 && errno == EINTR);
++start_fd;

+#endif

if (start_fd >= end_fd)
```
   return;[](#l2.221)
```

proc_fd_dir = opendir(BSD_OSX_FD_DIR);
if (!proc_fd_dir)

   proc_fd_dir = opendir(LINUX_SOLARIS_FD_DIR);[](#l2.225)

if (!proc_fd_dir) {

   /* No way to get a list of open fds. */[](#l2.227)

   _close_fds_by_brute_force(start_fd, end_fd, py_fds_to_keep);[](#l2.228)

} else {

   struct dirent64 *dir_entry;[](#l2.230)

+#ifdef HAVE_DIRFD

   int fd_used_by_opendir = DIRFD(proc_fd_dir);[](#l2.232)

+#else

   int fd_used_by_opendir = start_fd - 1;[](#l2.234)

+#endif

```
   errno = 0;[](#l2.236)
```

   /* readdir64 is used to work around Solaris 9 bug 6395699. */[](#l2.237)

   while ((dir_entry = readdir64(proc_fd_dir))) {[](#l2.238)

```
       int fd;[](#l2.239)
```

       if ((fd = _pos_int_from_ascii(dir_entry->d_name)) < 0)[](#l2.240)

           continue;  /* Not a number. */[](#l2.241)

       if (fd != fd_used_by_opendir && fd >= start_fd && fd < end_fd &&[](#l2.242)

           !_is_fd_in_sorted_fd_sequence(fd, py_fds_to_keep)) {[](#l2.243)

           while (close(fd) < 0 && errno == EINTR);[](#l2.244)

```
       }[](#l2.245)
```
```
       errno = 0;[](#l2.246)
```
```
   }[](#l2.247)
```
```
   if (errno) {[](#l2.248)
```

       /* readdir error, revert behavior. Highly Unlikely. */[](#l2.249)

       _close_fds_by_brute_force(start_fd, end_fd, py_fds_to_keep);[](#l2.250)

```
   }[](#l2.251)
```
```
   closedir(proc_fd_dir);[](#l2.252)
```
}

+} + +#define _close_open_fd_range _close_open_fd_range_maybe_unsafe + +#endif /* else NOT (defined(linux) && defined(HAVE_SYS_SYSCALL_H)) / + + /

This function is code executed in the child process immediately after fork
to set things up and call exec(). @@ -46,12 +292,12 @@ static void child_exec(char *const exec_ int errread, int errwrite, int errpipe_read, int errpipe_write, int close_fds, int restore_signals,

                  int call_setsid, Py_ssize_t num_fds_to_keep,[](#l2.268)

                  int call_setsid,[](#l2.269)
                  PyObject *py_fds_to_keep,[](#l2.270)
                  PyObject *preexec_fn,[](#l2.271)
                  PyObject *preexec_fn_args_tuple)[](#l2.272)

{

int i, saved_errno, fd_num, unused;

int i, saved_errno, unused; PyObject result; const char err_msg = ""; /* Buffer large enough to hold a hex integer. We can't malloc. */ @@ -113,33 +359,8 @@ static void child_exec(char *const exec_ POSIX_CALL(close(errwrite)); }

/* close() is intentionally not checked for errors here as we are closing */
/* a large range of fds, some of which may be invalid. */
if (close_fds) {
```
   Py_ssize_t keep_seq_idx;[](#l2.286)
```
```
   int start_fd = 3;[](#l2.287)
```

   for (keep_seq_idx = 0; keep_seq_idx < num_fds_to_keep; ++keep_seq_idx) {[](#l2.288)

       PyObject* py_keep_fd = PySequence_Fast_GET_ITEM(py_fds_to_keep,[](#l2.289)

                                                       keep_seq_idx);[](#l2.290)

       int keep_fd = PyLong_AsLong(py_keep_fd);[](#l2.291)

       if (keep_fd < 0) {  /* Negative number, overflow or not a Long. */[](#l2.292)

           err_msg = "bad value in fds_to_keep.";[](#l2.293)

           errno = 0;  /* We don't want to report an OSError. */[](#l2.294)

```
           goto error;[](#l2.295)
```
```
       }[](#l2.296)
```

       if (keep_fd < start_fd)[](#l2.297)

```
           continue;[](#l2.298)
```

       for (fd_num = start_fd; fd_num < keep_fd; ++fd_num) {[](#l2.299)

```
           close(fd_num);[](#l2.300)
```
```
       }[](#l2.301)
```

       start_fd = keep_fd + 1;[](#l2.302)

```
   }[](#l2.303)
```
```
   if (start_fd <= max_fd) {[](#l2.304)
```

       for (fd_num = start_fd; fd_num < max_fd; ++fd_num) {[](#l2.305)

```
           close(fd_num);[](#l2.306)
```
```
       }[](#l2.307)
```
```
   }[](#l2.308)
```
}

if (close_fds)

   _close_open_fd_range(3, max_fd, py_fds_to_keep);[](#l2.311)

if (cwd) POSIX_CALL(chdir(cwd)); @@ -227,7 +448,7 @@ subprocess_fork_exec(PyObject* self, PyO pid_t pid; int need_to_reenable_gc = 0; char *const *exec_array, *const *argv = NULL, *const *envp = NULL;

Py_ssize_t arg_num, num_fds_to_keep;

Py_ssize_t arg_num;

if (!PyArg_ParseTuple( args, "OOOOOOiiiiiiiiiiO:fork_exec", @@ -243,9 +464,12 @@ subprocess_fork_exec(PyObject* self, PyO PyErr_SetString(PyExc_ValueError, "errpipe_write must be >= 3"); return NULL; }

num_fds_to_keep = PySequence_Length(py_fds_to_keep);
if (num_fds_to_keep < 0) {

   PyErr_SetString(PyExc_ValueError, "bad fds_to_keep");[](#l2.330)

if (PySequence_Length(py_fds_to_keep) < 0) {

   PyErr_SetString(PyExc_ValueError, "cannot get length of fds_to_keep");[](#l2.332)

```
   return NULL;[](#l2.333)
```
}
if (_sanity_check_python_fd_sequence(py_fds_to_keep)) {
```
   PyErr_SetString(PyExc_ValueError, "bad value(s) in fds_to_keep");[](#l2.336)
   return NULL;[](#l2.337)
```
} @@ -348,8 +572,7 @@ subprocess_fork_exec(PyObject* self, PyO p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, errpipe_read, errpipe_write, close_fds, restore_signals, call_setsid,

              num_fds_to_keep, py_fds_to_keep,[](#l2.344)

              preexec_fn, preexec_fn_args_tuple);[](#l2.345)

              py_fds_to_keep, preexec_fn, preexec_fn_args_tuple);[](#l2.346)
   _exit(255);[](#l2.347)
   return NULL;  /* Dead code to avoid a potential compiler warning. */[](#l2.348)

}

--- a/configure +++ b/configure @@ -6165,7 +6165,7 @@ unistd.h utime.h [](#l3.3) sys/audioio.h sys/bsdtty.h sys/epoll.h sys/event.h sys/file.h sys/loadavg.h [](#l3.4) sys/lock.h sys/mkdev.h sys/modem.h [](#l3.5) sys/param.h sys/poll.h sys/select.h sys/socket.h sys/statvfs.h sys/stat.h [](#l3.6) -sys/termio.h sys/time.h [](#l3.7) +sys/syscall.h sys/termio.h sys/time.h [](#l3.8) sys/times.h sys/types.h sys/un.h sys/utsname.h sys/wait.h pty.h libutil.h [](#l3.9) sys/resource.h netpacket/packet.h sysexits.h bluetooth.h [](#l3.10) bluetooth/bluetooth.h linux/tipc.h spawn.h util.h

--- a/configure.in +++ b/configure.in @@ -1341,7 +1341,7 @@ unistd.h utime.h [](#l4.3) sys/audioio.h sys/bsdtty.h sys/epoll.h sys/event.h sys/file.h sys/loadavg.h [](#l4.4) sys/lock.h sys/mkdev.h sys/modem.h [](#l4.5) sys/param.h sys/poll.h sys/select.h sys/socket.h sys/statvfs.h sys/stat.h [](#l4.6) -sys/termio.h sys/time.h [](#l4.7) +sys/syscall.h sys/termio.h sys/time.h [](#l4.8) sys/times.h sys/types.h sys/un.h sys/utsname.h sys/wait.h pty.h libutil.h [](#l4.9) sys/resource.h netpacket/packet.h sysexits.h bluetooth.h [](#l4.10) bluetooth/bluetooth.h linux/tipc.h spawn.h util.h)

--- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -789,6 +789,9 @@ /* Define to 1 if you have the <sys/stat.h> header file. / #undef HAVE_SYS_STAT_H +/ Define to 1 if you have the <sys/syscall.h> header file. / +#undef HAVE_SYS_SYSCALL_H + / Define to 1 if you have the <sys/termio.h> header file. */ #undef HAVE_SYS_TERMIO_H