svn commit: r291664 - projects/cxl_iscsi/sys/dev/cxgbe/cxgbei

[Navdeep Parhar]

Author: np
Date: Wed Dec  2 23:54:59 2015
New Revision: 291664
URL: https://svnweb.freebsd.org/changeset/base/291664

Log:
  Make full use of the pool of worker threads instead of using the first one all
  the time.

Modified:
  projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.c
  projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.h
  projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/icl_cxgbei.c

Modified: projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.c
==============================================================================
--- projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.c	Wed Dec  2 23:44:29 2015	(r291663)
+++ projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.c	Wed Dec  2 23:54:59 2015	(r291664)
@@ -90,6 +90,10 @@ __FBSDID("$FreeBSD$");
 #include "cxgbei.h"
 #include "cxgbei_ulp2_ddp.h"
 
+static int worker_thread_count;
+static struct cxgbei_worker_thread_softc *cwt_softc;
+static struct proc *cxgbei_proc;
+
 /* XXXNP some header instead. */
 struct icl_pdu *icl_cxgbei_new_pdu(int);
 void icl_cxgbei_new_pdu_set_conn(struct icl_pdu *, struct icl_conn *);
@@ -718,7 +722,7 @@ do_rx_iscsi_ddp(struct sge_iq *iq, const
 
 	STAILQ_INSERT_TAIL(&icc->rcvd_pdus, ip, ip_next);
 	if ((icc->rx_flags & RXF_ACTIVE) == 0) {
-		struct cxgbei_worker_thread_softc *cwt = icc->cwt;
+		struct cxgbei_worker_thread_softc *cwt = &cwt_softc[icc->cwt];
 
 		mtx_lock(&cwt->cwt_lock);
 		icc->rx_flags |= RXF_ACTIVE;
@@ -884,10 +888,6 @@ static struct uld_info cxgbei_uld_info =
 	.deactivate = cxgbei_deactivate,
 };
 
-static int worker_thread_count;
-static struct cxgbei_worker_thread_softc *cwt_softc;
-static struct proc *cxgbei_proc;
-
 static void
 cwt_main(void *arg)
 {
@@ -1036,6 +1036,25 @@ stop_worker_threads(void)
 	free(cwt_softc, M_CXGBE);
 }
 
+/* Select a worker thread for a connection. */
+u_int
+cxgbei_select_worker_thread(struct icl_cxgbei_conn *icc)
+{
+	struct adapter *sc = icc->sc;
+	struct toepcb *toep = icc->toep;
+	u_int i, n;
+
+	n = worker_thread_count / sc->sge.nofldrxq;
+	if (n > 0)
+		i = toep->port->port_id * n + arc4random() % n;
+	else
+		i = arc4random() % worker_thread_count;
+
+	CTR3(KTR_CXGBE, "%s: tid %u, cwt %u", __func__, toep->tid, i);
+
+	return (i);
+}
+
 static int
 cxgbei_mod_load(void)
 {

Modified: projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.h
==============================================================================
--- projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.h	Wed Dec  2 23:44:29 2015	(r291663)
+++ projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/cxgbei.h	Wed Dec  2 23:54:59 2015	(r291664)
@@ -61,9 +61,9 @@ struct icl_cxgbei_conn {
 
 	/* Receive related. */
 	u_int rx_flags;				/* protected by so_rcv lock */
+	u_int cwt;
 	STAILQ_HEAD(, icl_pdu) rcvd_pdus;	/* protected by so_rcv lock */
 	TAILQ_ENTRY(icl_cxgbei_conn) rx_link;	/* protected by cwt lock */
-	struct cxgbei_worker_thread_softc *cwt;
 };
 
 static inline struct icl_cxgbei_conn *
@@ -153,6 +153,7 @@ struct cxgbei_data {
 void cxgbei_conn_task_reserve_itt(void *, void **, void *, unsigned int *);
 void cxgbei_conn_transfer_reserve_ttt(void *, void **, void *, unsigned int *);
 void cxgbei_cleanup_task(void *, void *);
+u_int cxgbei_select_worker_thread(struct icl_cxgbei_conn *);
 
 struct cxgbei_ulp2_pagepod_hdr;
 int t4_ddp_set_map(struct cxgbei_data *, void *,

Modified: projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
==============================================================================
--- projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/icl_cxgbei.c	Wed Dec  2 23:44:29 2015	(r291663)
+++ projects/cxl_iscsi/sys/dev/cxgbe/cxgbei/icl_cxgbei.c	Wed Dec  2 23:54:59 2015	(r291664)
@@ -597,9 +597,6 @@ set_ulp_mode_iscsi(struct adapter *sc, s
 	t4_set_tcb_field(sc, toep, 1, 0, 0xfff, val);
 }
 
-/* XXXNP */
-extern struct cxgbei_worker_thread_softc *cwt_softc;
-
 /*
  * XXXNP: Who is responsible for cleaning up the socket if this returns with an
  * error?  Review all error paths.
@@ -681,7 +678,7 @@ icl_cxgbei_conn_handoff(struct icl_conn 
 		toep = tp->t_toe;
 		MPASS(toep->port->adapter == icc->sc);
 		icc->toep = toep;
-		icc->cwt = &cwt_softc[0]; /* XXXNP */
+		icc->cwt = cxgbei_select_worker_thread(icc);
 		icc->ulp_submode = 0;
 		if (ic->ic_header_crc32c)
 			icc->ulp_submode |= ULP_CRC_HEADER;