Intel 82574L interface wedging on em 7.1.9/7.2.3 when MSIX enabled

[Hooman Fazaeli]

I finally managed to re-produce an affect similar to Jason's case. It
may not be the exact same issue, but it is a serious problem and must
be addressed.

1. Push out packet on em/igb with high rate.
2. Disconnect cable and wait for a few seconds. "netstat -ind" shows that
    Drops are increasing.
3. Re-connect the cable. Both sides of like re-negotiate and the links comes up.
4. But ..., no packets is ever transmitted again and Drops still increasing!

This is because em/lem/igb and some other interfaces (i.e., bce) have
a check at the very beginning of their _start function
which checks link status and immediately returns if it is inactive.
This behavior causes if_snd to fills up in step 2 and when this happens,
IFQ_HANDOFF never calls if_start again, even when the link becomes
active again.

A cable unplug is not necessary to trigger the issue. Any temporary
link loss (e.i., during re-negotiation) can potentially lead to
aforementioned problem.

IMHO, this is not a driver issue and the real fix would be to change
IFQ_HANDOFF to call if_start when the queue is full.

Jason, If you are interested, I can prepare a patch for you
to address this issue in if_em and see if it helps.





-------------- next part --------------
--- if_em.c.orig	2011-10-27 21:09:33.000000000 +0330
+++ if_em.c	2011-10-27 21:46:18.000000000 +0330
@@ -85,6 +85,14 @@
 #include "e1000_82571.h"
 #include "if_em.h"
 
+#if !defined(DISABLE_FIXUPS) && __FreeBSD_version < 800000
+static __inline int
+pci_find_cap(device_t dev, int capability, int *capreg)
+{
+        return (PCI_FIND_EXTCAP(device_get_parent(dev), dev, capability, capreg));
+}
+#endif
+
 /*********************************************************************
  *  Set this to one to display debug statistics
  *********************************************************************/
@@ -399,6 +407,12 @@
 /* Global used in WOL setup with multiport cards */
 static int global_quad_port_a = 0;
 
+#ifndef DISABLE_FIXUPS
+static int em_rx_hang_fixup = 0;
+SYSCTL_INT(_hw_em, OID_AUTO, rx_hang_fixup, CTLFLAG_RW, &em_rx_hang_fixup, 0,
+    "Enable/disable r1.69 RX hang fixup code");
+#endif
+
 /*********************************************************************
  *  Device identification routine
  *
@@ -864,7 +878,11 @@
         int             err = 0, enq = 0;
 
 	if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
+#ifdef DISABLE_FIXUPS	
 	    IFF_DRV_RUNNING || adapter->link_active == 0) {
+#else
+	    IFF_DRV_RUNNING) {
+#endif
 		if (m != NULL)
 			err = drbr_enqueue(ifp, txr->br, m);
 		return (err);
@@ -963,8 +981,10 @@
 	    IFF_DRV_RUNNING)
 		return;
 
+#ifdef DISABLE_FIXUPS
 	if (!adapter->link_active)
 		return;
+#endif
 
 	while (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) {
         	/* Call cleanup if number of TX descriptors low */
@@ -1414,7 +1434,11 @@
  *  Legacy polling routine: note this only works with single queue
  *
  *********************************************************************/
+#if !defined(DISABLE_FIXUPS) && __FreeBSD_version < 800000
+static void
+#else
 static int
+#endif
 em_poll(struct ifnet *ifp, enum poll_cmd cmd, int count)
 {
 	struct adapter *adapter = ifp->if_softc;
@@ -1426,7 +1450,11 @@
 	EM_CORE_LOCK(adapter);
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
 		EM_CORE_UNLOCK(adapter);
+#if !defined(DISABLE_FIXUPS) && __FreeBSD_version < 800000
+		return;
+#else
 		return (0);
+#endif
 	}
 
 	if (cmd == POLL_AND_CHECK_STATUS) {
@@ -1452,8 +1480,11 @@
 	em_start_locked(ifp, txr);
 #endif
 	EM_TX_UNLOCK(txr);
-
+#if !defined(DISABLE_FIXUPS) && __FreeBSD_version < 800000
+	return;
+#else
 	return (rx_done);
+#endif
 }
 #endif /* DEVICE_POLLING */
 
@@ -2213,6 +2244,16 @@
 	    e1000_get_laa_state_82571(&adapter->hw))
 		e1000_rar_set(&adapter->hw, adapter->hw.mac.addr, 0);
 
+#ifndef DISABLE_FIXUPS
+	if (em_rx_hang_fixup) {
+		/* trigger tq to refill rx ring queue if it is empty */
+		for (int i = 0; i < adapter->num_queues; i++, rxr++) {
+			if (rxr->next_to_check == rxr->next_to_refresh) {
+				taskqueue_enqueue(rxr->tq, &rxr->rx_task);
+			}
+		}
+	}
+#endif
 	/* Mask to use in the irq trigger */
 	if (adapter->msix_mem)
 		trigger = rxr->ims; /* RX for 82574 */
@@ -3766,7 +3807,7 @@
          * If we have a minimum free, clear IFF_DRV_OACTIVE
          * to tell the stack that it is OK to send packets.
          */
-        if (txr->tx_avail > EM_MAX_SCATTER)
+        if (txr->tx_avail >= EM_MAX_SCATTER)
                 ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 
 	/* Disable watchdog if all clean */
@@ -5553,4 +5594,8 @@
 	    rxr->rx_discarded);
 	device_printf(dev, "RX Next to Check = %d\n", rxr->next_to_check);
 	device_printf(dev, "RX Next to Refresh = %d\n", rxr->next_to_refresh);
+#ifndef DISABLE_FIXUPS
+	device_printf(dev, "Link state: %s\n", 
+		adapter->link_active? "active": "inactive");
+#endif
 }