git: 4a58aca40df9 - stable/13 - Allow sleepq_signal() to drop the lock.

[Alexander Motin]

The branch stable/13 has been updated by mav:

URL: https://cgit.FreeBSD.org/src/commit/?id=4a58aca40df9a5a5e5db430f3744a2712f955ed7

commit 4a58aca40df9a5a5e5db430f3744a2712f955ed7
Author:     Alexander Motin <mav at FreeBSD.org>
AuthorDate: 2021-06-25 17:52:58 +0000
Commit:     Alexander Motin <mav at FreeBSD.org>
CommitDate: 2021-07-09 01:38:32 +0000

    Allow sleepq_signal() to drop the lock.
    
    Introduce SLEEPQ_DROP sleepq_signal() flag, allowing one to drop the
    sleep queue chain lock before returning.  Reduced lock scope allows
    significantly reduce lock contention inside taskqueue_enqueue() for
    ZFS worker threads doing ~350K disk reads/s on 40-thread system.
    
    MFC after:      2 weeks
    Sponsored by:   iXsystems, Inc.
    
    (cherry picked from commit 6df35af4d85c6311d8e762521580e7176b69394e)
---
 sys/kern/kern_synch.c      | 8 +++-----
 sys/kern/subr_sleepqueue.c | 8 ++++++--
 sys/sys/sleepqueue.h       | 1 +
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 4c0491ab6e85..b63877e26b68 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -366,8 +366,7 @@ wakeup_one(const void *ident)
 	int wakeup_swapper;
 
 	sleepq_lock(ident);
-	wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP, 0, 0);
-	sleepq_release(ident);
+	wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP | SLEEPQ_DROP, 0, 0);
 	if (wakeup_swapper)
 		kick_proc0();
 }
@@ -378,9 +377,8 @@ wakeup_any(const void *ident)
 	int wakeup_swapper;
 
 	sleepq_lock(ident);
-	wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP | SLEEPQ_UNFAIR,
-	    0, 0);
-	sleepq_release(ident);
+	wakeup_swapper = sleepq_signal(ident, SLEEPQ_SLEEP | SLEEPQ_UNFAIR |
+	    SLEEPQ_DROP, 0, 0);
 	if (wakeup_swapper)
 		kick_proc0();
 }
diff --git a/sys/kern/subr_sleepqueue.c b/sys/kern/subr_sleepqueue.c
index 0718f01fa48a..b146a978a60c 100644
--- a/sys/kern/subr_sleepqueue.c
+++ b/sys/kern/subr_sleepqueue.c
@@ -927,8 +927,11 @@ sleepq_signal(const void *wchan, int flags, int pri, int queue)
 	KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
 	MPASS((queue >= 0) && (queue < NR_SLEEPQS));
 	sq = sleepq_lookup(wchan);
-	if (sq == NULL)
+	if (sq == NULL) {
+		if (flags & SLEEPQ_DROP)
+			sleepq_release(wchan);
 		return (0);
+	}
 	KASSERT(sq->sq_type == (flags & SLEEPQ_TYPE),
 	    ("%s: mismatch between sleep/wakeup and cv_*", __func__));
 
@@ -961,7 +964,8 @@ sleepq_signal(const void *wchan, int flags, int pri, int queue)
 		}
 	}
 	MPASS(besttd != NULL);
-	wakeup_swapper = sleepq_resume_thread(sq, besttd, pri, SRQ_HOLD);
+	wakeup_swapper = sleepq_resume_thread(sq, besttd, pri,
+	    (flags & SLEEPQ_DROP) ? 0 : SRQ_HOLD);
 	return (wakeup_swapper);
 }
 
diff --git a/sys/sys/sleepqueue.h b/sys/sys/sleepqueue.h
index 18c7568777b6..6715894ed1f3 100644
--- a/sys/sys/sleepqueue.h
+++ b/sys/sys/sleepqueue.h
@@ -85,6 +85,7 @@ struct thread;
 #define	SLEEPQ_LK		0x04		/* Used by a lockmgr. */
 #define	SLEEPQ_INTERRUPTIBLE	0x100		/* Sleep is interruptible. */
 #define	SLEEPQ_UNFAIR		0x200		/* Unfair wakeup order. */
+#define	SLEEPQ_DROP		0x400		/* Return without lock held. */
 
 void	init_sleepqueues(void);
 int	sleepq_abort(struct thread *td, int intrval);