jdk14 fork() problem fix

[Alexey Zelkin]

hi,

Though patch is working it has some limitations.  Under high load
java applications which utilize many parallel fork()'s may hang.

It's reproducible with testcase then least 5 threads calling Runtime.exec()

After finding a reason I had to rethink logic of 'critical section'
while calling to fork.  We can't use pthread_{suspend,resume}_all_np()
in order to not to create race conditions.

Updated version of patch to this fork problem is attached.

On Thu, Oct 23, 2003 at 12:28:39PM +0300, Alexey Zelkin wrote:
> hi,
> 
> This is an intermediate version of fix of jdk fork problem (i.e.
> Runtime.getRuntime().exec() and friends).
> 
> It does affect only people who use libc_r (kse and thr should not
> be affected).
> 
> Please try this one and report me if it fixes problems for you.
> 
> PS: If you are rebuilding already built jdk (i.e. object files are
> already compiled) remove 'control/build/bsd-i586/tmp/java/java.lang' directory
> before restarting of build.

Index: UNIXProcess_md.c.bsd
===================================================================
RCS file: /home/jdk14-cvs/jdk142-src/j2se/src/solaris/native/java/lang/UNIXProcess_md.c.bsd,v
retrieving revision 1.2
retrieving revision 1.4
diff -u -u -r1.2 -r1.4
--- UNIXProcess_md.c.bsd	15 Oct 2003 15:49:39 -0000	1.2
+++ UNIXProcess_md.c.bsd	5 Nov 2003 22:48:47 -0000	1.4
@@ -22,6 +22,12 @@
 #include <errno.h>
 #include <unistd.h>
 
+#if defined(__FreeBSD__)
+#include <dlfcn.h>
+#include <pthread.h>
+#include <pthread_np.h>
+#endif
+
 /* path in the environment */
 static char **PATH = 0;
 /* effective uid */
@@ -228,6 +234,58 @@
     }
 }
 
+#if defined(__FreeBSD__)
+
+extern pid_t   __sys_fork(void);
+
+static pid_t
+jdk_fork_wrapper()
+{
+    pid_t resultPid;
+    typedef void (*void_func)();
+    static void_func func_defer = NULL;
+    static void_func func_undefer = NULL;
+    static int is_libc_r = -1;
+
+    if (is_libc_r == -1) {
+
+	/*
+         * BSDNOTE: Check for loaded symbols.
+         *
+         * If "_thread_kern_sig_defer" symbol is found assume we are
+	 * libc_r
+         *
+         * If libc_r is loaded, use fork system call drectly to avoid
+         * problems with using protected pages. 
+         *
+         * --phantom
+	 */
+	func_defer =
+		(void_func)dlsym(RTLD_DEFAULT, "_thread_kern_sig_defer");
+	func_undefer =
+		(void_func)dlsym(RTLD_DEFAULT, "_thread_kern_sig_undefer");
+	if (func_defer != NULL)
+	    is_libc_r = 1;
+	else {
+	    is_libc_r = 0;
+	}
+    }
+
+    if (is_libc_r == 0) {
+	/* Not a libc_r */
+	resultPid = fork();
+    } else {
+        (*func_defer)();		/* call _thread_kern_sig_defer() */
+        resultPid = __sys_fork();
+        if (resultPid != 0)
+ 	    (*func_undefer)();		/* call _thread_kern_sig_undefer() */
+	/* leave child with signals disabled, but reenable in parent */
+    }
+
+    return resultPid;
+}
+#endif /* __FreeBSD__ */
+
 JNIEXPORT jint JNICALL
 Java_java_lang_UNIXProcess_forkAndExec(JNIEnv *env,
 				       jobject process,
@@ -335,8 +393,12 @@
     if (path != NULL) {
         cwd = (char *)JNU_GetStringPlatformChars(env, path, NULL);
     }
- 
+
+#if defined(__FreeBSD__)
+    resultPid = jdk_fork_wrapper();
+#else
     resultPid = fork();
+#endif
 
     if (resultPid < 0) {
 	char errmsg[128];