svn commit: r184727 - user/kmacy/HEAD_multi_tx/sys/dev/mxge

[Andrew Gallatin]

Author: gallatin
Date: Thu Nov  6 15:32:02 2008
New Revision: 184727
URL: http://svn.freebsd.org/changeset/base/184727

Log:
  - Fix a bug in mxge's multi-queue transmit support
  which could cause the firmware to not see transmits.
  
  - Add some debug counters to diagnose further problems
  with multiple transmit queues

Modified:
  user/kmacy/HEAD_multi_tx/sys/dev/mxge/if_mxge.c
  user/kmacy/HEAD_multi_tx/sys/dev/mxge/if_mxge_var.h

Modified: user/kmacy/HEAD_multi_tx/sys/dev/mxge/if_mxge.c
==============================================================================
--- user/kmacy/HEAD_multi_tx/sys/dev/mxge/if_mxge.c	Thu Nov  6 15:26:09 2008	(r184726)
+++ user/kmacy/HEAD_multi_tx/sys/dev/mxge/if_mxge.c	Thu Nov  6 15:32:02 2008	(r184727)
@@ -1269,6 +1269,8 @@ mxge_reset(mxge_softc_t *sc, int interru
 		ss->tx.done = 0;
 		ss->tx.pkt_done = 0;
 		ss->tx.queue_active = 0;
+		ss->tx.activate = 0;
+		ss->tx.deactivate = 0;
 		ss->tx.wake = 0;
 		ss->tx.defrag = 0;
 		ss->tx.stall = 0;
@@ -1634,6 +1636,18 @@ mxge_add_sysctls(mxge_softc_t *sc)
 			       "tx_defrag",
 			       CTLFLAG_RD, &ss->tx.defrag,
 			       0, "tx_defrag");
+		SYSCTL_ADD_INT(ctx, children, OID_AUTO, 
+			       "tx_queue_active",
+			       CTLFLAG_RD, &ss->tx.queue_active,
+			       0, "tx_queue_active");
+		SYSCTL_ADD_INT(ctx, children, OID_AUTO, 
+			       "tx_activate",
+			       CTLFLAG_RD, &ss->tx.activate,
+			       0, "tx_activate");
+		SYSCTL_ADD_INT(ctx, children, OID_AUTO, 
+			       "tx_deactivate",
+			       CTLFLAG_RD, &ss->tx.deactivate,
+			       0, "tx_deactivate");
 	}
 }
 
@@ -1860,6 +1874,7 @@ mxge_encap_tso(struct mxge_slice_state *
 		/* tell the NIC to start polling this slice */
 		*tx->send_go = 1;
 		tx->queue_active = 1;
+		tx->activate++;
 		mb();
 	}
 	return;
@@ -2068,6 +2083,7 @@ mxge_encap(struct mxge_slice_state *ss, 
 		/* tell the NIC to start polling this slice */
 		*tx->send_go = 1;
 		tx->queue_active = 1;
+		tx->activate++;
 		mb();
 	}
 	return;
@@ -2547,9 +2563,12 @@ mxge_tx_done(struct mxge_slice_state *ss
 	    mtx_trylock(&ss->tx.mtx)) {
 		/* let the NIC stop polling this queue, since there
 		 * are no more transmits pending */
-		*tx->send_stop = 1;
-		tx->queue_active = 0;
-		mb();
+		if (tx->req == tx->done) {
+			*tx->send_stop = 1;
+			tx->queue_active = 0;
+			tx->deactivate++;
+			mb();
+		}
 		mtx_unlock(&ss->tx.mtx);
 	}
 }
@@ -3451,9 +3470,10 @@ mxge_read_reboot(mxge_softc_t *sc)
 }
 
 static int
-mxge_watchdog_reset(mxge_softc_t *sc)
+mxge_watchdog_reset(mxge_softc_t *sc, int slice)
 {
 	struct pci_devinfo *dinfo;
+	mxge_tx_ring_t *tx;
 	int err;
 	uint32_t reboot;
 	uint16_t cmd;
@@ -3500,11 +3520,14 @@ mxge_watchdog_reset(mxge_softc_t *sc)
 			err = mxge_open(sc);
 		}
 	} else {
-		device_printf(sc->dev, "NIC did not reboot, ring state:\n");
-		device_printf(sc->dev, "tx.req=%d tx.done=%d\n",
-			      sc->ss->tx.req, sc->ss->tx.done);
+		tx = &sc->ss[slice].tx;
+		device_printf(sc->dev, "NIC did not reboot, slice %d ring state:\n", slice);
+		device_printf(sc->dev, "tx.req=%d tx.done=%d, tx.queue_active=%d\n",
+			      tx->req, tx->done, tx->queue_active);
+		device_printf(sc->dev, "tx.activate=%d tx.deactivate=%d\n",
+			      tx->activate, tx->deactivate);
 		device_printf(sc->dev, "pkt_done=%d fw=%d\n",
-			      sc->ss->tx.pkt_done,
+			      tx->pkt_done,
 			      be32toh(sc->ss->fw_stats->send_done_count));
 		device_printf(sc->dev, "not resetting\n");
 	}
@@ -3514,26 +3537,29 @@ mxge_watchdog_reset(mxge_softc_t *sc)
 static int
 mxge_watchdog(mxge_softc_t *sc)
 {
-	mxge_tx_ring_t *tx = &sc->ss->tx;
+	mxge_tx_ring_t *tx;
 	uint32_t rx_pause = be32toh(sc->ss->fw_stats->dropped_pause);
-	int err = 0;
+	int i, err = 0;
 
 	/* see if we have outstanding transmits, which
 	   have been pending for more than mxge_ticks */
-	if (tx->req != tx->done &&
-	    tx->watchdog_req != tx->watchdog_done &&
-	    tx->done == tx->watchdog_done) {
-		/* check for pause blocking before resetting */
-		if (tx->watchdog_rx_pause == rx_pause)
-			err = mxge_watchdog_reset(sc);
-		else
-			device_printf(sc->dev, "Flow control blocking "
-				      "xmits, check link partner\n");
-	}
+	for (i = 0; (i < sc->num_slices) && (err == 0); i++) {
+		tx = &sc->ss[i].tx;		
+		if (tx->req != tx->done &&
+		    tx->watchdog_req != tx->watchdog_done &&
+		    tx->done == tx->watchdog_done) {
+			/* check for pause blocking before resetting */
+			if (tx->watchdog_rx_pause == rx_pause)
+				err = mxge_watchdog_reset(sc, i);
+			else
+				device_printf(sc->dev, "Flow control blocking "
+					      "xmits, check link partner\n");
+		}
 
-	tx->watchdog_req = tx->req;
-	tx->watchdog_done = tx->done;
-	tx->watchdog_rx_pause = rx_pause;
+		tx->watchdog_req = tx->req;
+		tx->watchdog_done = tx->done;
+		tx->watchdog_rx_pause = rx_pause;
+	}
 
 	if (sc->need_media_probe)
 		mxge_media_probe(sc);

Modified: user/kmacy/HEAD_multi_tx/sys/dev/mxge/if_mxge_var.h
==============================================================================
--- user/kmacy/HEAD_multi_tx/sys/dev/mxge/if_mxge_var.h	Thu Nov  6 15:26:09 2008	(r184726)
+++ user/kmacy/HEAD_multi_tx/sys/dev/mxge/if_mxge_var.h	Thu Nov  6 15:32:02 2008	(r184727)
@@ -139,6 +139,8 @@ typedef struct
 	int pkt_done;			/* packets completed */
 	int max_desc;			/* max descriptors per xmit */
 	int queue_active;		/* fw currently polling this queue*/
+	int activate;
+	int deactivate;
 	int stall;			/* #times hw queue exhausted */
 	int wake;			/* #times irq re-enabled xmit */
 	int watchdog_req;		/* cache of req */