Samba 4.3 and 4.4 crashes on FreeBSD 11-ALPHA4

[Konstantin Belousov]

On Sat, Jun 25, 2016 at 01:03:13AM +0300, Guy Yur wrote:
> libtdb.so.1`tdb_runtime_check_for_robust_mutexes + 1475 at mutex.c:957
> ...

The pointer to tdb_runtime_check_for_robust_mutexes() appeared to be
most useful, thanks.

The two patches below should fix samba use of robustness. First,
kernel erronously reset robust lists locations on fork. Second, the
pthread_mutex_trylock() for owned errorcheck mutex must return EDEADLK
and not EBUSY.  Try that.

diff --git a/lib/libthr/thread/thr_mutex.c b/lib/libthr/thread/thr_mutex.c
index 5a99605..da71c70 100644
--- a/lib/libthr/thread/thr_mutex.c
+++ b/lib/libthr/thread/thr_mutex.c
@@ -850,9 +871,12 @@ mutex_self_trylock(struct pthread_mutex *m)
 
 	switch (PMUTEX_TYPE(m->m_flags)) {
 	case PTHREAD_MUTEX_ERRORCHECK:
-	case PTHREAD_MUTEX_NORMAL:
 	case PTHREAD_MUTEX_ADAPTIVE_NP:
-		ret = EBUSY; 
+		ret = EDEADLK;
+		break;
+
+	case PTHREAD_MUTEX_NORMAL:
+		ret = EBUSY;
 		break;
 
 	case PTHREAD_MUTEX_RECURSIVE:
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index 6d03062..6162a16 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -282,9 +282,6 @@ struct thread {
 	int		td_no_sleeping;	/* (k) Sleeping disabled count. */
 	int		td_dom_rr_idx;	/* (k) RR Numa domain selection. */
 	void		*td_su;		/* (k) FFS SU private */
-	uintptr_t	td_rb_list;	/* (k) Robust list head. */
-	uintptr_t	td_rbp_list;	/* (k) Robust priv list head. */
-	uintptr_t	td_rb_inact;	/* (k) Current in-action mutex loc. */
 #define	td_endzero td_sigmask
 
 /* Copied during fork1() or create_thread(). */
@@ -298,6 +295,9 @@ struct thread {
 	u_char		td_base_user_pri; /* (t) Base user pri */
 	u_int		td_dbg_sc_code;	/* (c) Syscall code to debugger. */
 	u_int		td_dbg_sc_narg;	/* (c) Syscall arg count to debugger.*/
+	uintptr_t	td_rb_list;	/* (k) Robust list head. */
+	uintptr_t	td_rbp_list;	/* (k) Robust priv list head. */
+	uintptr_t	td_rb_inact;	/* (k) Current in-action mutex loc. */
 #define	td_endcopy td_pcb
 
 /*