svn commit: r267600 - head/sys/dev/cxgbe

[Navdeep Parhar]

Author: np
Date: Wed Jun 18 00:16:35 2014
New Revision: 267600
URL: http://svnweb.freebsd.org/changeset/base/267600

Log:
  cxgbe(4):  Fix bug in the fast rx buffer recycle path.  In some cases rx
  buffers were getting recycled when they should have been left alone.
  
  MFC after:	3 days

Modified:
  head/sys/dev/cxgbe/adapter.h
  head/sys/dev/cxgbe/t4_sge.c

Modified: head/sys/dev/cxgbe/adapter.h
==============================================================================
--- head/sys/dev/cxgbe/adapter.h	Tue Jun 17 22:23:36 2014	(r267599)
+++ head/sys/dev/cxgbe/adapter.h	Wed Jun 18 00:16:35 2014	(r267600)
@@ -290,7 +290,8 @@ struct cluster_metadata {
 
 struct fl_sdesc {
 	caddr_t cl;
-	uint8_t nmbuf;
+	uint8_t nimbuf;		/* # of inline mbufs with ref on the cluster */
+	uint8_t nembuf;		/* # of allocated mbufs with ref */
 	struct cluster_layout cll;
 };
 

Modified: head/sys/dev/cxgbe/t4_sge.c
==============================================================================
--- head/sys/dev/cxgbe/t4_sge.c	Tue Jun 17 22:23:36 2014	(r267599)
+++ head/sys/dev/cxgbe/t4_sge.c	Wed Jun 18 00:16:35 2014	(r267600)
@@ -1561,22 +1561,22 @@ get_scatter_segment(struct adapter *sc, 
 		/* copy data to mbuf */
 		bcopy(payload, mtod(m, caddr_t), len);
 
-	} else if (sd->nmbuf * MSIZE < cll->region1) {
+	} else if (sd->nimbuf * MSIZE < cll->region1) {
 
 		/*
 		 * There's spare room in the cluster for an mbuf.  Create one
-		 * and associate it with the payload that's in the cluster too.
+		 * and associate it with the payload that's in the cluster.
 		 */
 
 		MPASS(clm != NULL);
-		m = (struct mbuf *)(sd->cl + sd->nmbuf * MSIZE);
+		m = (struct mbuf *)(sd->cl + sd->nimbuf * MSIZE);
 		/* No bzero required */
 		if (m_init(m, NULL, 0, M_NOWAIT, MT_DATA, flags | M_NOFREE))
 			return (NULL);
 		fl->mbuf_inlined++;
 		m_extaddref(m, payload, padded_len, &clm->refcount, rxb_free,
 		    swz->zone, sd->cl);
-		sd->nmbuf++;
+		sd->nimbuf++;
 
 	} else {
 
@@ -1590,10 +1590,11 @@ get_scatter_segment(struct adapter *sc, 
 		if (m == NULL)
 			return (NULL);
 		fl->mbuf_allocated++;
-		if (clm != NULL)
+		if (clm != NULL) {
 			m_extaddref(m, payload, padded_len, &clm->refcount,
 			    rxb_free, swz->zone, sd->cl);
-		else {
+			sd->nembuf++;
+		} else {
 			m_cljset(m, sd->cl, swz->type);
 			sd->cl = NULL;	/* consumed, not a recycle candidate */
 		}
@@ -3253,7 +3254,7 @@ refill_fl(struct adapter *sc, struct sge
 
 		if (sd->cl != NULL) {
 
-			if (sd->nmbuf == 0) {
+			if (sd->nimbuf + sd->nembuf == 0) {
 				/*
 				 * Fast recycle without involving any atomics on
 				 * the cluster's metadata (if the cluster has
@@ -3262,6 +3263,11 @@ refill_fl(struct adapter *sc, struct sge
 				 * fit within a single mbuf each.
 				 */
 				fl->cl_fast_recycled++;
+#ifdef INVARIANTS
+				clm = cl_metadata(sc, fl, &sd->cll, sd->cl);
+				if (clm != NULL)
+					MPASS(clm->refcount == 1);
+#endif
 				goto recycled_fast;
 			}
 
@@ -3307,7 +3313,8 @@ recycled:
 #endif
 			clm->refcount = 1;
 		}
-		sd->nmbuf = 0;
+		sd->nimbuf = 0;
+		sd->nembuf = 0;
 recycled_fast:
 		fl->pending++;
 		fl->needed--;
@@ -3376,7 +3383,7 @@ free_fl_sdesc(struct adapter *sc, struct
 
 		cll = &sd->cll;
 		clm = cl_metadata(sc, fl, cll, sd->cl);
-		if (sd->nmbuf == 0 ||
+		if (sd->nimbuf + sd->nembuf == 0 ||
 		    (clm && atomic_fetchadd_int(&clm->refcount, -1) == 1)) {
 			uma_zfree(sc->sge.sw_zone_info[cll->zidx].zone, sd->cl);
 		}