svn commit: r254816 - in projects/camlock/sys/cddl: compat/opensolaris/kern contrib/opensolaris/uts/common/sys

[Alexander Motin]

Author: mav
Date: Sat Aug 24 22:42:19 2013
New Revision: 254816
URL: http://svnweb.freebsd.org/changeset/base/254816

Log:
  Make taskqueue wrapper used for ZFS more SMP-scalable.
  
  For some purposes ZFS requests to create taskqueues with number of threads
  equal to number of CPUs.  As result, on 24-core system we may get situation
  when interoperation between 5 CAM completion threads and 24 ZFS taskqueue
  threads protected with single global lock.  When number of IOPS reaches
  hundreds of thousands, that causes significant lock congestion.
  
  To workaround that, create several FreeBSD taskqueues to emulate single
  Solaris taskqueue and distribute tasks between them in round-robin fashion.
  Using 4 taskqueues of 6 threads for the 24-core system almost completely
  removed spinning on these locks.

Modified:
  projects/camlock/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c
  projects/camlock/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h

Modified: projects/camlock/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c
==============================================================================
--- projects/camlock/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c	Sat Aug 24 21:30:35 2013	(r254815)
+++ projects/camlock/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c	Sat Aug 24 22:42:19 2013	(r254816)
@@ -66,14 +66,26 @@ taskq_create(const char *name, int nthre
     int maxalloc __unused, uint_t flags)
 {
 	taskq_t *tq;
+	int i, nqueues;
 
 	if ((flags & TASKQ_THREADS_CPU_PCT) != 0)
 		nthreads = MAX((mp_ncpus * nthreads) / 100, 1);
+	nqueues = MAX((nthreads + 4) / 6, 1);
+	nthreads = MAX((nthreads + nqueues / 2) / nqueues, 1);
 
-	tq = kmem_alloc(sizeof(*tq), KM_SLEEP);
-	tq->tq_queue = taskqueue_create(name, M_WAITOK, taskqueue_thread_enqueue,
-	    &tq->tq_queue);
-	(void) taskqueue_start_threads(&tq->tq_queue, nthreads, pri, "%s", name);
+	tq = kmem_alloc(sizeof(*tq) + sizeof(tq->tq_queue[0]) * nqueues, KM_SLEEP);
+	tq->tq_num = nqueues;
+	tq->tq_last = 0;
+	for (i = 0; i < nqueues; i++) {
+		tq->tq_queue[i] = taskqueue_create(name, M_WAITOK,
+		    taskqueue_thread_enqueue, &tq->tq_queue[i]);
+		if (nqueues == 1)
+			(void) taskqueue_start_threads(&tq->tq_queue[i],
+			    nthreads, pri, "%s", name);
+		else
+			(void) taskqueue_start_threads(&tq->tq_queue[i],
+			    nthreads, pri, "%s_%d", name, i);
+	}
 
 	return ((taskq_t *)tq);
 }
@@ -89,16 +101,22 @@ taskq_create_proc(const char *name, int 
 void
 taskq_destroy(taskq_t *tq)
 {
+	int i;
 
-	taskqueue_free(tq->tq_queue);
-	kmem_free(tq, sizeof(*tq));
+	for (i = 0; i < tq->tq_num; i++)
+		taskqueue_free(tq->tq_queue[i]);
+	kmem_free(tq, sizeof(*tq) + sizeof(tq->tq_queue[0]) * tq->tq_num);
 }
 
 int
 taskq_member(taskq_t *tq, kthread_t *thread)
 {
+	int i, j;
 
-	return (taskqueue_member(tq->tq_queue, thread));
+	for (i = 0; i < tq->tq_num; i++)
+		if (taskqueue_member(tq->tq_queue[i], thread))
+			return (1);
+	return (0);
 }
 
 static void
@@ -115,7 +133,7 @@ taskqid_t
 taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags)
 {
 	struct ostask *task;
-	int mflag, prio;
+	int i, mflag, prio;
 
 	if ((flags & (TQ_SLEEP | TQ_NOQUEUE)) == TQ_SLEEP)
 		mflag = M_WAITOK;
@@ -135,7 +153,11 @@ taskq_dispatch(taskq_t *tq, task_func_t 
 	task->ost_arg = arg;
 
 	TASK_INIT(&task->ost_task, prio, taskq_run, task);
-	taskqueue_enqueue(tq->tq_queue, &task->ost_task);
+	if (tq->tq_num > 1)
+		i = atomic_fetchadd_int(&tq->tq_last, 1) % tq->tq_num;
+	else
+		i = 0;
+	taskqueue_enqueue(tq->tq_queue[i], &task->ost_task);
 
 	return ((taskqid_t)(void *)task);
 }
@@ -154,7 +176,7 @@ taskqid_t
 taskq_dispatch_safe(taskq_t *tq, task_func_t func, void *arg, u_int flags,
     struct ostask *task)
 {
-	int prio;
+	int i, prio;
 
 	/* 
 	 * If TQ_FRONT is given, we want higher priority for this task, so it
@@ -166,7 +188,11 @@ taskq_dispatch_safe(taskq_t *tq, task_fu
 	task->ost_arg = arg;
 
 	TASK_INIT(&task->ost_task, prio, taskq_run_safe, task);
-	taskqueue_enqueue(tq->tq_queue, &task->ost_task);
+	if (tq->tq_num > 1)
+		i = atomic_fetchadd_int(&tq->tq_last, 1) % tq->tq_num;
+	else
+		i = 0;
+	taskqueue_enqueue(tq->tq_queue[i], &task->ost_task);
 
 	return ((taskqid_t)(void *)task);
 }

Modified: projects/camlock/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h
==============================================================================
--- projects/camlock/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h	Sat Aug 24 21:30:35 2013	(r254815)
+++ projects/camlock/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h	Sat Aug 24 22:42:19 2013	(r254816)
@@ -38,7 +38,9 @@ extern "C" {
 
 struct taskqueue;
 struct taskq {
-	struct taskqueue	*tq_queue;
+	int			 tq_num;
+	int			 tq_last;
+	struct taskqueue	*tq_queue[];
 };
 
 typedef struct taskq taskq_t;