[PATCH 4/4] sfxge: Support tunable to control Tx deferred packet list limits

[Andrew Rybchenko]

Support tunable to control Tx deferred packet list limits

Also increase default for Tx queue get-list limit.
Too small limit results in TCP packets drops especiall when many
streams are running simultaneously.
Put list may be kept small enought since it is just a temporary
location if transmit function can't get Tx queue lock.
The information contained in this message is confidential and is intended for the addressee(s) only. If you have received this message in error, please notify the sender immediately and delete the message. Unless you are an addressee (or authorized to receive for an addressee), you may not use, copy or disclose to anyone this message or any information contained in this message. The unauthorized use, disclosure, copying or alteration of this message is strictly prohibited.
-------------- next part --------------
Support tunable to control Tx deferred packet list limits

Also increase default for Tx queue get-list limit.
Too small limit results in TCP packets drops especiall when many
streams are running simultaneously.
Put list may be kept small enought since it is just a temporary
location if transmit function can't get Tx queue lock.

Submitted by:	Andrew Rybchenko <arybchenko at solarflare.com>
Sponsored by:	Solarflare Communications, Inc.

diff -r 0c812cd721d6 sys/dev/sfxge/sfxge_tx.c
--- a/sys/dev/sfxge/sfxge_tx.c	Thu Sep 25 16:06:49 2014 +0400
+++ b/sys/dev/sfxge/sfxge_tx.c	Thu Sep 25 16:06:56 2014 +0400
@@ -50,6 +50,7 @@
 #include <sys/smp.h>
 #include <sys/socket.h>
 #include <sys/sysctl.h>
+#include <sys/syslog.h>
 
 #include <net/bpf.h>
 #include <net/ethernet.h>
@@ -77,6 +78,25 @@
 #define	SFXGE_TSO_MAX_DESC ((65535 / 512) * 2 + SFXGE_TX_MAPPING_MAX_SEG - 1)
 #define	SFXGE_TXQ_BLOCK_LEVEL(_entries)	((_entries) - SFXGE_TSO_MAX_DESC)
 
+#ifdef SFXGE_HAVE_MQ
+
+#define	SFXGE_PARAM_TX_DPL_GET_MAX	SFXGE_PARAM(tx_dpl_get_max)
+static int sfxge_tx_dpl_get_max = SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT;
+TUNABLE_INT(SFXGE_PARAM_TX_DPL_GET_MAX, &sfxge_tx_dpl_get_max);
+SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_get_max, CTLFLAG_RDTUN,
+	   &sfxge_tx_dpl_get_max, 0,
+	   "Maximum number of packets in deferred packet get-list");
+
+#define	SFXGE_PARAM_TX_DPL_PUT_MAX	SFXGE_PARAM(tx_dpl_put_max)
+static int sfxge_tx_dpl_put_max = SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT;
+TUNABLE_INT(SFXGE_PARAM_TX_DPL_PUT_MAX, &sfxge_tx_dpl_put_max);
+SYSCTL_INT(_hw_sfxge, OID_AUTO, tx_dpl_put_max, CTLFLAG_RDTUN,
+	   &sfxge_tx_dpl_put_max, 0,
+	   "Maximum number of packets in deferred packet put-list");
+
+#endif
+
+
 /* Forward declarations. */
 static inline void sfxge_tx_qdpl_service(struct sfxge_txq *txq);
 static void sfxge_tx_qlist_post(struct sfxge_txq *txq);
@@ -476,7 +496,7 @@
 
 		sfxge_tx_qdpl_swizzle(txq);
 
-		if (stdp->std_get_count >= SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT)
+		if (stdp->std_get_count >= stdp->std_get_max)
 			return (ENOBUFS);
 
 		*(stdp->std_getp) = mbuf;
@@ -498,7 +518,7 @@
 				old_len = mp->m_pkthdr.csum_data;
 			} else
 				old_len = 0;
-			if (old_len >= SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT)
+			if (old_len >= stdp->std_put_max)
 				return (ENOBUFS);
 			mbuf->m_pkthdr.csum_data = old_len + 1;
 			mbuf->m_nextpkt = (void *)old;
@@ -1384,8 +1404,23 @@
 		goto fail3;
 
 #ifdef SFXGE_HAVE_MQ
+	if (sfxge_tx_dpl_get_max <= 0) {
+		log(LOG_ERR, "%s=%d must be greater than 0",
+		    SFXGE_PARAM_TX_DPL_GET_MAX, sfxge_tx_dpl_get_max);
+		rc = EINVAL;
+		goto fail_tx_dpl_get_max;
+	}
+	if (sfxge_tx_dpl_put_max < 0) {
+		log(LOG_ERR, "%s=%d must be greater or equal to 0",
+		    SFXGE_PARAM_TX_DPL_PUT_MAX, sfxge_tx_dpl_put_max);
+		rc = EINVAL;
+		goto fail_tx_dpl_put_max;
+	}
+
 	/* Initialize the deferred packet list. */
 	stdp = &txq->dpl;
+	stdp->std_put_max = sfxge_tx_dpl_put_max;
+	stdp->std_get_max = sfxge_tx_dpl_get_max;
 	stdp->std_getp = &stdp->std_get;
 
 	mtx_init(&txq->lock, "txq", NULL, MTX_DEF);
@@ -1403,6 +1438,8 @@
 
 	return (0);
 
+fail_tx_dpl_put_max:
+fail_tx_dpl_get_max:
 fail3:
 fail_txq_node:
 	free(txq->pend_desc, M_SFXGE);
diff -r 0c812cd721d6 sys/dev/sfxge/sfxge_tx.h
--- a/sys/dev/sfxge/sfxge_tx.h	Thu Sep 25 16:06:49 2014 +0400
+++ b/sys/dev/sfxge/sfxge_tx.h	Thu Sep 25 16:06:56 2014 +0400
@@ -75,13 +75,17 @@
 	enum sfxge_tx_buf_flags	flags;
 };
 
-#define	SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT	64
+#define	SFXGE_TX_DPL_GET_PKT_LIMIT_DEFAULT	1024
 #define	SFXGE_TX_DPL_PUT_PKT_LIMIT_DEFAULT	64
 
 /*
  * Deferred packet list.
  */
 struct sfxge_tx_dpl {
+	unsigned int		std_get_max;	/* Maximum number of packets
+						 * in get list */
+	unsigned int		std_put_max;	/* Maximum number of packets
+						 * in put list */
 	uintptr_t		std_put;	/* Head of put list. */
 	struct mbuf		*std_get;	/* Head of get list. */
 	struct mbuf		**std_getp;	/* Tail of get list. */