git: 86721e606c51 - stable/13 - nvme: Use adaptive spinning when polling for completion or state change

The branch stable/13 has been updated by mav:

URL: https://cgit.FreeBSD.org/src/commit/?id=86721e606c51f1012e851f2c930694eae7890cff

commit 86721e606c51f1012e851f2c930694eae7890cff
Author:     Warner Losh <imp@FreeBSD.org>
AuthorDate: 2021-10-01 17:32:48 +0000
Commit:     Alexander Motin <mav@FreeBSD.org>
CommitDate: 2022-01-21 02:07:31 +0000

    nvme: Use adaptive spinning when polling for completion or state change
    
    We only use nvme_completion_poll in the initialization path. The
    commands they queue and wait for finish quickly as they involve no I/O
    to the drive's media. These command take about 20-200 microsecnds
    each. Set the wait time to 1us and then increase it by 1.5 each
    successive iteration (max 1ms). This reduces initialization time by
    80ms in cpervica's tests.
    
    Use this same technique waiting for RDY state transitions. This saves
    another 20ms. In total we're down from ~330ms to ~2ms.
    
    Tested by:              cperciva
    Sponsored by:           Netflix
    Reviewed by:            mav
    Differential Review:    https://reviews.freebsd.org/D32259
    
    (cherry picked from commit 83581511d9476ef5084f47e3cc379be7191ae866)
---
 sys/dev/nvme/nvme_ctrlr.c   | 11 ++++++++++-
 sys/dev/nvme/nvme_private.h | 35 +++++++++++++++++++++--------------
 2 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c
index ca89e99d2934..aa41c8fe5540 100644
--- a/sys/dev/nvme/nvme_ctrlr.c
+++ b/sys/dev/nvme/nvme_ctrlr.c
@@ -260,10 +260,17 @@ nvme_ctrlr_fail_req_task(void *arg, int pending)
 	mtx_unlock(&ctrlr->lock);
 }
 
+/*
+ * Wait for RDY to change.
+ *
+ * Starts sleeping for 1us and geometrically increases it the longer we wait,
+ * capped at 1ms.
+ */
 static int
 nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val)
 {
 	int timeout = ticks + MSEC_2_TICKS(ctrlr->ready_timeout_in_ms);
+	sbintime_t delta_t = SBT_1US;
 	uint32_t csts;
 
 	while (1) {
@@ -278,7 +285,9 @@ nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val)
 			    "within %d ms\n", desired_val, ctrlr->ready_timeout_in_ms);
 			return (ENXIO);
 		}
-		pause("nvmerdy", 1);
+
+		pause_sbt("nvmerdy", delta_t, 0, C_PREL(1));
+		delta_t = min(SBT_1MS, delta_t * 3 / 2);
 	}
 
 	return (0);
diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h
index b00130910a82..9c7c1b46eab4 100644
--- a/sys/dev/nvme/nvme_private.h
+++ b/sys/dev/nvme/nvme_private.h
@@ -455,25 +455,32 @@ int	nvme_shutdown(device_t dev);
 int	nvme_detach(device_t dev);
 
 /*
- * Wait for a command to complete using the nvme_completion_poll_cb.
- * Used in limited contexts where the caller knows it's OK to block
- * briefly while the command runs. The ISR will run the callback which
- * will set status->done to true, usually within microseconds. If not,
- * then after one second timeout handler should reset the controller
- * and abort all outstanding requests including this polled one. If
- * still not after ten seconds, then something is wrong with the driver,
- * and panic is the only way to recover.
+ * Wait for a command to complete using the nvme_completion_poll_cb.  Used in
+ * limited contexts where the caller knows it's OK to block briefly while the
+ * command runs. The ISR will run the callback which will set status->done to
+ * true, usually within microseconds. If not, then after one second timeout
+ * handler should reset the controller and abort all outstanding requests
+ * including this polled one. If still not after ten seconds, then something is
+ * wrong with the driver, and panic is the only way to recover.
+ *
+ * Most commands using this interface aren't actual I/O to the drive's media so
+ * complete within a few microseconds. Adaptively spin for one tick to catch the
+ * vast majority of these without waiting for a tick plus scheduling delays. Since
+ * these are on startup, this drastically reduces startup time.
  */
 static __inline
 void
 nvme_completion_poll(struct nvme_completion_poll_status *status)
 {
-	int sanity = hz * 10;
-
-	while (!atomic_load_acq_int(&status->done) && --sanity > 0)
-		pause("nvme", 1);
-	if (sanity <= 0)
-		panic("NVME polled command failed to complete within 10s.");
+	int timeout = ticks + 10 * hz;
+	sbintime_t delta_t = SBT_1US;
+
+	while (!atomic_load_acq_int(&status->done)) {
+		if (timeout - ticks < 0)
+			panic("NVME polled command failed to complete within 10s.");
+		pause_sbt("nvme", delta_t, 0, C_PREL(1));
+		delta_t = min(SBT_1MS, delta_t * 3 / 2);
+	}
 }
 
 static __inline void