svn commit: r305027 - head/sys/dev/ioat

[Conrad E. Meyer]

Author: cem
Date: Mon Aug 29 20:46:33 2016
New Revision: 305027
URL: https://svnweb.freebsd.org/changeset/base/305027

Log:
  ioat(4): Don't "complete" DMA descriptors prematurely
  
  In r304602, I mistakenly removed the ioat_process_events check that we weren't
  processing events before the hardware had completed the descriptor
  ("last_seen").  Reinstate that logic.
  
  Keep the defensive loop condition and additionally make sure we've actually
  completed a descriptor before blindly chasing the ring around.
  
  In reset, queue and finish the startup command before allowing any event
  processing or submission to occur.  Avoid potential missed callouts by
  requeueing the poll later.

Modified:
  head/sys/dev/ioat/ioat.c

Modified: head/sys/dev/ioat/ioat.c
==============================================================================
--- head/sys/dev/ioat/ioat.c	Mon Aug 29 20:01:53 2016	(r305026)
+++ head/sys/dev/ioat/ioat.c	Mon Aug 29 20:46:33 2016	(r305027)
@@ -683,7 +683,16 @@ ioat_process_events(struct ioat_softc *i
 	    __func__, ioat->chan_idx, comp_update, ioat->last_seen);
 	status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK;
 
-	while (ioat_get_active(ioat) > 0) {
+	if (status == ioat->last_seen) {
+		/*
+		 * If we landed in process_events and nothing has been
+		 * completed, check for a timeout due to channel halt.
+		 */
+		goto out;
+	}
+
+	desc = ioat_get_ring_entry(ioat, ioat->tail - 1);
+	while (desc->hw_desc_bus_addr != status && ioat_get_active(ioat) > 0) {
 		desc = ioat_get_ring_entry(ioat, ioat->tail);
 		dmadesc = &desc->bus_dmadesc;
 		CTR4(KTR_IOAT, "channel=%u completing desc %u ok  cb %p(%p)",
@@ -695,8 +704,6 @@ ioat_process_events(struct ioat_softc *i
 
 		completed++;
 		ioat->tail++;
-		if (desc->hw_desc_bus_addr == status)
-			break;
 	}
 
 	if (completed != 0) {
@@ -704,6 +711,7 @@ ioat_process_events(struct ioat_softc *i
 		ioat->stats.descriptors_processed += completed;
 	}
 
+out:
 	ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
 
 	/* Perform a racy check first; only take the locks if it passes. */
@@ -1913,19 +1921,17 @@ ioat_reset_hw(struct ioat_softc *ioat)
 	error = 0;
 
 out:
+	/* Enqueues a null operation and ensures it completes. */
+	if (error == 0)
+		error = ioat_start_channel(ioat);
+
 	/*
 	 * Resume completions now that ring state is consistent.
-	 * ioat_start_channel will add a pending completion and if we are still
-	 * blocking completions, we may livelock.
 	 */
 	mtx_lock(&ioat->cleanup_lock);
 	ioat->resetting_cleanup = FALSE;
 	mtx_unlock(&ioat->cleanup_lock);
 
-	/* Enqueues a null operation and ensures it completes. */
-	if (error == 0)
-		error = ioat_start_channel(ioat);
-
 	/* Unblock submission of new work */
 	mtx_lock(IOAT_REFLK);
 	ioat->quiescing = FALSE;
@@ -1933,6 +1939,10 @@ out:
 
 	ioat->resetting = FALSE;
 	wakeup(&ioat->resetting);
+
+	if (ioat->is_completion_pending)
+		callout_reset(&ioat->poll_timer, 1, ioat_poll_timer_callback,
+		    ioat);
 	mtx_unlock(IOAT_REFLK);
 
 	return (error);