system() using vfork() or posix_spawn()

[Jilles Tjoelker]

People sometimes use system() from large address spaces where it would
improve performance greatly to use vfork() instead of fork().

A simple approach is to change fork() to vfork(), although I have not
tried this. It seems safe enough to use sigaction and sigprocmask system
calls in the vforked process.

Alternatively, we can have posix_spawn() do the vfork() with signal
changes. This avoids possible whining from compilers and static
analyzers about using vfork() in system.c. However, I do not like the
tricky code for signals and that it adds lines of code.

This is lightly tested.

Index: lib/libc/stdlib/system.c
===================================================================
--- lib/libc/stdlib/system.c	(revision 238371)
+++ lib/libc/stdlib/system.c	(working copy)
@@ -42,16 +42,21 @@
 #include <unistd.h>
 #include <paths.h>
 #include <errno.h>
+#include <spawn.h>
 #include "un-namespace.h"
 #include "libc_private.h"
 
+extern char **environ;
+
 int
 __system(const char *command)
 {
 	pid_t pid, savedpid;
-	int pstat;
+	int error, pstat;
 	struct sigaction ign, intact, quitact;
-	sigset_t newsigblock, oldsigblock;
+	sigset_t newsigblock, oldsigblock, defmask;
+	const char *argv[4];
+	posix_spawnattr_t attr;
 
 	if (!command)		/* just checking... */
 		return(1);
@@ -65,28 +70,36 @@
 	ign.sa_flags = 0;
 	(void)_sigaction(SIGINT, &ign, &intact);
 	(void)_sigaction(SIGQUIT, &ign, &quitact);
+	(void)sigemptyset(&defmask);
+	if ((intact.sa_flags & SA_SIGINFO) != 0 ||
+	    intact.sa_handler != SIG_IGN)
+		(void)sigaddset(&defmask, SIGINT);
+	if ((quitact.sa_flags & SA_SIGINFO) != 0 ||
+	    quitact.sa_handler != SIG_IGN)
+		(void)sigaddset(&defmask, SIGQUIT);
 	(void)sigemptyset(&newsigblock);
 	(void)sigaddset(&newsigblock, SIGCHLD);
 	(void)_sigprocmask(SIG_BLOCK, &newsigblock, &oldsigblock);
-	switch(pid = fork()) {
-	case -1:			/* error */
-		break;
-	case 0:				/* child */
-		/*
-		 * Restore original signal dispositions and exec the command.
-		 */
-		(void)_sigaction(SIGINT, &intact, NULL);
-		(void)_sigaction(SIGQUIT,  &quitact, NULL);
-		(void)_sigprocmask(SIG_SETMASK, &oldsigblock, NULL);
-		execl(_PATH_BSHELL, "sh", "-c", command, (char *)NULL);
-		_exit(127);
-	default:			/* parent */
+	argv[0] = "sh";
+	argv[1] = "-c";
+	argv[2] = command;
+	argv[3] = NULL;
+	if ((error = posix_spawnattr_init(&attr)) != 0 ||
+	    (error = posix_spawnattr_setsigmask(&attr, &oldsigblock)) != 0 ||
+	    (error = posix_spawnattr_setsigdefault(&attr, &defmask)) != 0 ||
+	    (error = posix_spawnattr_setflags(&attr,
+	    POSIX_SPAWN_SETSIGDEF | POSIX_SPAWN_SETSIGMASK)) != 0 ||
+	    (error = posix_spawn(&pid, _PATH_BSHELL, NULL, &attr,
+	    __DECONST(char **, argv), environ)) != 0) {
+		pid = -1;
+		errno = error;
+	} else {
 		savedpid = pid;
 		do {
 			pid = _wait4(savedpid, &pstat, 0, (struct rusage *)0);
 		} while (pid == -1 && errno == EINTR);
-		break;
 	}
+	posix_spawnattr_destroy(&attr);
 	(void)_sigaction(SIGINT, &intact, NULL);
 	(void)_sigaction(SIGQUIT,  &quitact, NULL);
 	(void)_sigprocmask(SIG_SETMASK, &oldsigblock, NULL);

-- 
Jilles Tjoelker